diff --git a/.github/workflows/spdk.yml b/.github/workflows/spdk.yml index 12be0ee..beac6ff 100644 --- a/.github/workflows/spdk.yml +++ b/.github/workflows/spdk.yml @@ -18,13 +18,17 @@ on: paths: - "images/spdk/**" - "module/bdev/cbt/**" + - "module/bdev/tier/**" - "patches/**" - ".github/workflows/spdk.yml" pull_request: - types: [labeled, synchronize] + # TS3: `opened` so lint + unit tests run on EVERY PR from the first push; + # image builds stay gated by the authorize job (label + allowlist). + types: [opened, labeled, synchronize] paths: - "images/spdk/**" - "module/bdev/cbt/**" + - "module/bdev/tier/**" - "patches/**" - ".github/workflows/spdk.yml" workflow_dispatch: @@ -74,6 +78,13 @@ jobs: dockerfile: images/spdk/Dockerfile config: images/spdk/.hadolint.yaml + # TS2: host-compiled unit tests of the PRODUCTION tier code (mock SPDK + # headers — no SPDK/Docker build involved, runs in seconds). Kept in this + # job to avoid spinning an extra runner; a failure blocks the image build + # via the compute-tag dependency below. + - name: Tier module unit tests (T1, ASAN+UBSAN) + run: make -C module/bdev/tier/test + # ──────────────────────────────────────────────────────────────── # 2. Compute image tag (single source of truth) # ──────────────────────────────────────────────────────────────── diff --git a/.gitignore b/.gitignore index 98a9af2..f48b14d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ module/bdev/cbt/test_cbt_props module/bdev/cbt/test_cbt_resilience module/bdev/cbt/test_cbt_postfix module/bdev/cbt/*.dSYM/ +module/bdev/tier/test/test_tier_sb +module/bdev/tier/test/*.dSYM/ + +# Audit reports & plans (local only) +docs/audits/ diff --git a/docs/FORMAT-tier-superblock.md b/docs/FORMAT-tier-superblock.md new file mode 100644 index 0000000..eb31b70 --- /dev/null +++ b/docs/FORMAT-tier-superblock.md @@ -0,0 +1,126 @@ +# bdev_tier on-disk superblock — format v2 + +Authoritative description of the `struct tier_superblock` written by `vbdev_tier_sb.c`. +The C structs in `module/bdev/tier/vbdev_tier.h` are the source of truth; this +document explains the layout and the invariants. `SPDK_STATIC_ASSERT`s in the +header fail the build if the layout drifts from what is described here. + +## Scope & compatibility + +**Clean break — there is NO migration from v1 and none is planned.** The project +predates any public deployment; every environment is redeploy-by-wipe. A v1 +superblock (`magic "TIERSB01"`, `version 1`) is deliberately **not** readable by +v2 code (`tier_sb_valid` rejects any `version != 2`). Upgrading = wipe the disks +and re-provision. + +## Placement + +One superblock reserve lives at the very start of **every** base bdev of a +composite (LBA 0). The reserve is `TIER_SB_RESERVE_BYTES` = 256 KiB, per disk, +outside the composite's usable address space (`phys_offset >= sb_blocks`). + +Every copy self-describes the **whole** composite (all bands). There is no SPDK +`examine` path: assembly is driven by the CSI agent, which reads each disk's +superblock (`bdev_tier_read_sb`), picks the highest-seq valid copy across disks, +and replays `bdev_tier_create` + `bdev_tier_assemble_band` at the stored +geometry. + +## A/B slots (F-5) + +The 256 KiB reserve is split into **two 128 KiB slots**: + +``` + disk LBA 0 128 KiB 256 KiB + |------------- slot A --------|------------- slot B --------| + seq 0,2,4,... seq 1,3,5,... +``` + +Generation `seq = N` is written to slot `N % 2` (`tier_sb_slot_for_seq`). A torn +write (crash mid-write) therefore damages **at most one slot**; the other still +holds the previous generation. Readers validate both slots and take the valid +one with the highest `seq` (`tier_sb_select`). This is the standard mdadm/LVM +double-copy technique and closes the v1 single-copy hole (F-5): a mid-write crash +no longer bricks a disk's superblock. + +Only the slot for the current generation is written and flushed each update +(128 KiB I/O), not the whole reserve. + +## Header layout (256 bytes) + +| offset | size | field | notes | +|-------:|-----:|:------|:------| +| 0 | 8 | `magic` | `0x5449455253423032` = "TIERSB02" (LE) | +| 8 | 4 | `version` | `2` | +| 12 | 4 | `crc` | CRC32c over the whole struct with `crc = 0` | +| 16 | 8 | `seq` | monotone generation; highest valid wins | +| 24 | 8 | `created_epoch_sec` | wall clock at serialization (informative) | +| 32 | 16 | `generation_uuid` | composite **instance** id (fencing, F-2) | +| 48 | 64 | `composite_name` | | +| 112 | 8 | `md_num_blocks` | size of the mirrored md region (composite blocks) | +| 120 | 8 | `cluster_blocks` | blobstore cluster grain; **u64 since v2 (F-3)** | +| 128 | 4 | `md_mirror_a` | band slot id of md RAID1 leg A | +| 132 | 4 | `md_mirror_b` | band slot id of md RAID1 leg B | +| 136 | 4 | `num_bands` | | +| 140 | 4 | `this_band_id` | which slot this copy physically sits on (`UINT32_MAX` if unknown) | +| 144 | 4 | `blocklen` | common block size | +| 148 | 4 | `reserved0` | | +| 152 | 104 | `reserved[104]` | zero-filled; future header fields (F-2) | +| 256 | … | `bands[64]` | 64 × 192-byte band descriptors | + +Total = 256 + 64 × 192 = **12544 bytes** (fits one 128 KiB slot with room to spare). + +## Band descriptor layout (192 bytes) + +| offset | size | field | +|-------:|-----:|:------| +| 0 | 4 | `band_id` | +| 4 | 4 | `tier` (enum tier_class) | +| 8 | 4 | `state` (enum tier_band_state) | +| 12 | 4 | `reserved0` | +| 16 | 8 | `lba_start` | +| 24 | 8 | `num_blocks` | +| 32 | 64 | `wwn` | +| 96 | 64 | `serial` | +| 160 | 32 | `reserved[32]` (F-2) | + +## Invariants + +- **Endianness (F-4)**: little-endian only (amd64/arm64). `tier_sb_valid` rejects + a byte-swapped magic explicitly (a big-endian writer) rather than failing the + CRC silently. +- **Generation uniqueness (M5)**: `seq` is reserved (`t->seq++`) at the entry of + `tier_sb_write_all`, before any I/O. Two concurrent fan-outs can never share a + `seq`; gaps are harmless ("highest seq wins"), duplicates would be fatal. +- **Cross-restart monotonicity (R2)**: `bdev_tier_register` re-reads every band's + on-disk SB and seeds `t->seq` to the highest seq found **before** the first + persist. A fresh in-RAM composite starts at `seq 0` (the CSI replays + `create`+`assemble` without threading seq); without rehydration the first persist + writes `seq 1`, which a pre-restart SB at a high seq out-votes forever, so the + composite would reassemble the STALE geometry. Rehydration makes the first + post-register persist `max_on_disk + 1`, which wins. The generation_uuid still + changes each `create` (F-2), but after rehydration the current instance always + holds the highest seq, so its slot wins per disk. +- **Durability (F-6)**: each written slot is FLUSHed before the generation is + considered committed. +- **DEGRADED exclusion (M5b)**: only `ACTIVE` bands are written; a DEGRADED + disk's stale copy is out-voted by `seq` at reassembly. +- **Fencing (F-2)**: `generation_uuid` is minted once at `bdev_tier_create` and + copied into every superblock. A re-created composite gets a fresh uuid, so + disks left over from a previous instance cannot be silently cross-assembled. + The CSI compares uuids across a candidate disk set before assembling. + +## Reserved space + +104 header bytes + 32 bytes/band are zero-filled and CRC-covered. New fields are +added by shrinking a `reserved` array (keeping the surrounding offsets fixed) — +which changes the meaning of previously-zero bytes but NOT the struct size, so a +reader that predates the field sees zero (a safe default) and the static asserts +still pass. A field that changes size or reorders existing fields is a v3 break. + +## Test coverage + +`module/bdev/tier/test/test_tier_sb.c` (host-compiled, ASAN+UBSAN, links the +PRODUCTION `vbdev_tier_sb.c`): ABI offsets/sizes, serialize/validate roundtrip, +CRC/version/byte-swapped-magic rejection, u64 cluster_blocks, A/B slot selection +(highest-seq, torn-slot fallback, short-buffer safety), and a binary golden +header vector that catches a field reorder even when `sizeof` is unchanged. diff --git a/docs/RPC-CONTRACT.md b/docs/RPC-CONTRACT.md new file mode 100644 index 0000000..e792db8 --- /dev/null +++ b/docs/RPC-CONTRACT.md @@ -0,0 +1,243 @@ +# Evariops fork — RPC contract (SPEC-73 / SPEC-66) + +Contract sheet for the JSON-RPC surface the CSI control-plane drives (PR1). For +each RPC: preconditions, idempotence, and crash behavior. The invariants P1–P5 +and the decisions D5/CBT-7 that the control-plane must honor are stated at the +end. This is the fork-side half; the paired control-plane guards live in +`spdk-csi/docs/reports/2026-07-04_remediations-appariees-fork-spdk.md`. + +Probe support with `evariops_get_capabilities` (below) before assuming any of +these exist — on vanilla SPDK they return JSON-RPC -32601. + +## Global assumptions + +- **D5 — single reactor (`-m 0x1`).** The standing-pause registry (0003), + `g_relocate_inflight` (0005), `band->state`, and the heat counters assume all + RPC handlers run on one reactor, lock-free. `nvmf_subsystem_pause` logs an + error if it ever sees >1 reactor. Widening the CPU mask requires a concurrency + pass first. `evariops_get_capabilities` reports `single_reactor_assumed: true`. +- **Volatile state dies with the process.** Standing pauses, in-flight + relocations, cbt epochs and rebuilds are RAM-only. A target restart loses them + all — detect it via `boot_id` (below) and reconcile. +- **SEC1 — destructive RPCs are audited.** Every mutation handler + (`bdev_tier_delete`, `bdev_tier_retire_band`, `bdev_tier_resync_md`, + `bdev_lvol_relocate_cluster` / `_clusters` / `remap_cluster`, + `bdev_raid_rebuild_ranges`, `nvmf_subsystem_pause`) emits an + `audit rpc= peer=pid:…,uid:…,gid:… ` NOTICELOG line, with the + caller's Unix-socket credentials (SO_PEERCRED, patch 0011). `peer=unknown` over + a TCP transport. This is an audit trail, **not** an authorization gate — access + control is still the socket mode (0600, patch 0010) + a NetworkPolicy (D4). The + socket is created owner-only **atomically** — `umask(077)` wraps the `bind()` + (R14), closing the TOCTOU window that a chmod-after-listen left open; the chmod + stays as belt-and-braces. + +## evariops_get_capabilities + +- **Params**: none. +- **Returns**: `boot_id` (per-process uuid), `tier_sb_version`, + `capabilities_schema`, `single_reactor_assumed`, `methods[]`. +- **Use**: a changed `boot_id` across polls ⇒ the target restarted ⇒ treat all + volatile state as lost. `methods[]` membership is the capability probe. +- `methods[]` is **filtered through the live RPC registry** (deferred #3): a fork + method left in the candidate list but not built into this binary is NOT emitted, + so `methods[]` never yields a false positive (a method reported present that then + fails -32601). A method built in but absent from the list is a harmless false + negative — the control-plane's per-call -32601 probe remains the ground truth. +- Idempotent, read-only. + +## Lifecycle — tier + +| RPC | Preconditions | Idempotence | Crash behavior | +|:----|:--------------|:------------|:---------------| +| `bdev_tier_create` | name free; `md_num_blocks>0`; `cluster_blocks` is 0 (legacy, no alignment) or ≥2 — **1 is rejected** (R13); `md_num_blocks` bounded (no align overflow, R17) | -EEXIST if name taken | in-RAM only until first SB write | +| `bdev_tier_add_band` | tier exists, **not registered**; unique wwn; disk ≥ sb+md; blocklen divides the 128 KiB SB slot (R7 — T10-DIF 520/4160 rejected) | band_id auto-assigned; caller replays deterministically | — | +| `bdev_tier_assemble_band` | **not registered (R8)**; band_id<64; state≤RETIRED; no overlap; fits disk; unique wwn; blocklen divides SB slot (R7) | -EEXIST on duplicate band_id; **-EBUSY after register (R8)** | places at stored geometry | +| `bdev_tier_register` | ≥1 band, cluster-aligned geometry | **-EEXIST if already registered (W1)** | **rehydrates `t->seq` from the on-disk SBs (R2), then persists SB** on success | +| `bdev_tier_retire_band` | not an md-mirror band (-EBUSY, T-7) | **idempotent**: re-run re-persists + re-closes | **async: acks only after SB durable (MJ6)**; rc≠0 ⇒ retry | +| `bdev_tier_resync_md` | target is a DEGRADED md leg; healthy source leg exists | re-runnable (leg stays DEGRADED on failure) | copy under md-range quiesce; acks after activate+persist | +| `bdev_tier_delete` | — | -ENODEV if absent | unregister → destruct | +| `bdev_tier_get_bands` / `bdev_tier_read_sb` | — | read-only | read_sb returns highest-seq valid slot + `generation_uuid` | + +`bdev_tier_read_sb` exposes `version`, `seq`, `generation_uuid`, +`created_epoch_sec`. **Assembly (control-plane) must**: read every candidate +disk's SB, group by `generation_uuid` (fence stale disks from a previous +instance), take the highest `seq` per band, and if the two md legs disagree on +`seq`, assemble the higher one ACTIVE and the other DEGRADED then +`bdev_tier_resync_md` (G-CSI-2). The fork persists DEGRADED but cannot arbitrate +a split-brain across disks. + +- **Seq monotonicity across a restart is fork-owned (R2).** The control-plane + does NOT thread `seq` back through `create`/`assemble`/`register` (it reads seq + only to pick the authoritative SB). So `bdev_tier_register` **re-reads the bands' + on-disk SBs and seeds `t->seq` above the highest seq found** before its first + persist. Without this the fresh instance would write `seq 1`, which a pre-restart + SB at a high seq out-votes forever (highest-seq-wins), reassembling the STALE + geometry and silently undoing every retire/relocate persisted at the high seq. No + CSI change is required; this is the behavior the contract already assumed. + +## Data-plane — relocate / remap (patch 0005) + +- `bdev_lvol_relocate_cluster {name, tier_name, cluster_num, dst_lba_start, dst_lba_count}` + - **P-2**: the lvol must live on `tier_name`'s composite (-EINVAL else). + - **F2**: refuses snapshot blobs (-EBUSY). + - **F4**: one relocate/remap in flight per blob (-EBUSY else). + - **C1 + C1-DRAIN**: runs under an **lvol-bdev quiesce** (drains outstanding + host I/O, then holds new I/O above the L2P translation) plus an inner + **blob freeze**, for copy+commit; ALL of the lvol's I/O stalls + ~drain + 3× one cluster. The drain is what makes the copy's source stable — + the freeze alone never covered writes already in flight. **N-2**: the blob + is pinned by an own open-ref for the whole chain. + - Crash-safe (invariants A/B): a crash mid-op leaves an orphan cluster the + native blobstore replay reclaims; never a lost ACKed write. + - **H4**: an ambiguous commit failure (extent write dispatched, error + completion) QUARANTINES the destination cluster until restart instead of + releasing it — the durable extent may reference it; replay reconciles. +- `bdev_lvol_relocate_clusters {…, clusters:[…], verify?}` — **PF3 batch**: one + quiesce+freeze amortized over N clusters (≤4096). Correctness identical to the + single form (same C1-DRAIN and H4 contracts). `verify` (**PF4**, default true) + forwards to each copy; false skips the C5 read-back on trusted media. Returns + `{relocated, requested, error}` — **partial success is a 200** (caller retries + the tail from `relocated`). +- `bdev_lvol_remap_cluster {…}` — **N-7/W6**: source band must be DEGRADED, + destination band ACTIVE. Relaxes invariant A intentionally (the cluster is + already lost; the subsequent `bdev_raid_rebuild_ranges` fills the new one). The + control-plane MUST journal the remap durably BEFORE calling and re-drive the + range rebuild at restart until confirmed (**PR3**, remap-before-rebuild). + - **R11 — the old cluster is QUARANTINED, not freed.** A remap re-homes a cluster + whose old copy is on the DEGRADED band; the commit keeps that old cluster + marked-allocated (`release_old=false`) rather than returning it to the thin + pool. Otherwise the allocator could re-serve that LBA to a normal host write, + routing it back into the dead band → `-EIO` on a healthy volume. The quarantine + is in-RAM for the running instance; a reboot rebuilds `used_clusters` from the + blob extents (now pointing at the new cluster) and the control-plane reassembles + the dead band DEGRADED/RETIRED so its range is not re-served. Relocate (copy, + healthy source) still frees the old cluster normally (`release_old=true`). +- `bdev_lvol_remap_clusters {name, tier_name, clusters:[{cluster_num, dst_lba_start, dst_lba_count}]}` + — **batch no-copy remap**: the no-copy analogue of `bdev_lvol_relocate_clusters`. + Re-homes N lost clusters (DEGRADED source → ACTIVE dst) under the **same per-item + guards** as the single `bdev_lvol_remap_cluster` (shared `remap_one_precheck`); + **no freeze, no copy, no `verify`** (the source is dead — nothing to drain or + read back). Sequential; **stops at the first per-item error** releasing that + item's un-committed claim, and returns `{remapped, requested, error}` — + **partial success is a 200** (caller retries the tail from `remapped`). Bounded + to 4096 items. Old clusters are QUARANTINED per item (`release_old=false`, R11). + Crash-safety per item is identical to the single remap (invariant B); the + control-plane journals the remap set and re-drives the tail + range rebuild at + restart (**PR3**). Re-issuing a partially-committed batch is safe: already-moved + clusters no longer sit on a DEGRADED band, so their per-item guard cleanly + `-EINVAL`s rather than double-moving. +- **Cluster claim performance (deferred #4, patch 0004).** The claim under both + batch RPCs (`spdk_blob_claim_cluster_in_range`) is **word-wise** (skips + fully-allocated 64-bit words of the `used_clusters` bit-pool in one step) with an + optional **resume cursor** (`spdk_blob_claim_cluster_in_range_from`): a batch that + fills one band claims in **O(bandsize + N)** rather than O(N²), so a long + demotion/evac campaign shows flat per-cluster claim cost. Semantics are + **unchanged** (first free cluster in the window, `-ENOSPC` when full); the cursor + is a hint only — never a correctness input — so a stale/over-shot cursor is always + safe (the live windowed scan bounds every result). The batch threads one cursor + per dst band window and resets it when the window changes. + +## Capacity / ENOSPC (patch 0009) + +- A raid1 write to a **thin** member that is full returns NVMe + `CAPACITY_EXCEEDED` to the host **without** failing the member (**G3** — no + silent degradation). The write is NACKed. +- **Divergence, by design.** Both legs stay ONLINE, but the leg whose write + failed and the leg whose write succeeded now **disagree on that block**. This + is correct block semantics (a NACKed write has indeterminate content) but the + fork does **NOT** auto-reconverge. The control-plane must treat that LBA's + content as undefined until it is rewritten, and must keep thin reservations + symmetric across legs (placement-side) so the case stays rare. + +## Repair / rebuild + +- `bdev_raid_rebuild_ranges {name, ranges:[{start_lba,num_blocks}]}` — **C2**: + each chunk repaired under a channel-owned LBA lock (host writes held + + replayed). **P-3**: parity raids require **full-stripe-aligned** ranges + (-EINVAL else); the caller must align. **N-6**: a REMOVE aborts at the next + chunk (-ENODEV) — re-drive. + - **C3 — geometry is published, not guessed.** Read `full_stripe_blocks` from + the raid bdev's `bdev_get_bdevs` → `driver_specific.raid` (emitted for any + striped raid). It is EXACTLY the alignment `rebuild_ranges` validates: + `strip_size × min_base_bdevs_operational` (for raid5f, `min == num-1 == k`, + the data-chunk count). Align every range to it. Do **not** re-derive `k` + from `num_base_bdevs_operational` — for a healthy raid5f that field is `n` + (all members), not `k`; only `full_stripe_blocks` (or `min_base_bdevs_operational`) + gives the data-chunk count. `strip_size_kb`, `num_base_bdevs`, + `num_base_bdevs_operational` remain available for cross-checks. + - **EC reconstruct precondition (load-bearing).** The repair reconstructs a + lost chunk only because the degraded member returns **read errors** (present + band DEGRADED, patch 0006) or is **absent** (NULL channel, upstream) — either + triggers a parity reconstruct-read that the write-back then re-lays with fresh + parity. A member replaced by a **healthy, zeroed** disk reads its zeros + *successfully*, so no reconstruct fires and the repair would rewrite zeros. + The control-plane MUST therefore drive `rebuild_ranges` while the target + member is still **DEGRADED/absent** (the `remap → rebuild` order, PR3), NOT + after swapping in a fresh member. See + `docs/audits/2026-07-04_revue-ec-rebuild-ranges.md`. + - An unrecoverable stripe (>1 fault) fails the whole call with -EIO; the target + logs the offending chunk LBA/range (`raid repair: unrecoverable read at + chunk …`). Repair is idempotent — re-drive the tail after fixing redundancy. +- `bdev_raid_add_base_bdev {…, skip_rebuild}` — **CBT-3/P5**: skip_rebuild is a + "trust me" primitive; the control-plane MUST prove the residual delta is zero + (garde résidu-nul + INV-37) before calling. **CBT-6**: a channel-promotion + failure now unwinds fully and returns an error (the member is not left + half-wired) — treat as retryable. **CBT-7**: the SB flips to CONFIGURED only + after the unquiesce; a crash between the two costs a full rebuild at reboot + (conservative, safe). + +## CBT epochs / rebuild + +- `bdev_cbt_epoch_freeze` / `epoch_close` / `epoch_open` (higher generation): + return **-EBUSY** while a rebuild is RUNNING on the epoch (**CBT-1/2/c5**) — + the control-plane must `bdev_cbt_cancel_rebuild` first. +- `bdev_cbt_partial_rebuild` / `bdev_cbt_start_rebuild`: one rebuild per + (cbt, epoch); a second returns -EBUSY (interpret as "already running", not a + failure). **CBT-4**: the rebuild FLUSHes the target before COMPLETED. + - **R1 — an aborted rebuild is NOT `completed`.** A source/target/cbt hot-remove + mid-rebuild yields `bdev_cbt_get_rebuild_status` state **`aborted`** (distinct + from `failed` = an I/O error, and `completed`) with `completed: false`. The + delta was NOT fully copied, so the control-plane must **resume** the rebuild, + never treat the member as synced. (Previously a mid-rebuild abort with no failed + I/O reported `completed` → silent under-replication.) +- `bdev_cbt_reset`: refused while any epoch is active. Bitmap clearing is + reset-driven (**D3** — no automatic healthy-clear). +- **Breaking (D3)**: `bdev_cbt_epoch_list` no longer emits + `healthy_clear_suspended` nor `backends_healthy` — both fields' backing state + was removed with the dead healthy-clear poller (`backends_healthy` was + constant `false`: `bdev_cbt_set_backends_healthy` never had a caller). Strict + parsers must drop these keys. +- `bdev_cbt_epoch_invalidate`: refused with `-EBUSY` while a rebuild is RUNNING + on the epoch (**C3** — an INVALID epoch is evictable; evicting it under the + rebuild would free the frozen bitmap the rebuild is scanning). Cancel first. +- **H1 (delta preservation)**: a frozen delta that was exchanged out of the + live bitmap and never proven copied (rebuild aborted/failed/never run) is + merged back into the live bitmap before its buffer is discarded (re-freeze, + close, evict). A freeze retry therefore captures (unconsumed delta ∪ new + writes) — a failed iteration never loses chunks under `skip_rebuild`. +- After a base-bdev hot-remove the cbt vbdev is NOT silently recreated with a + virgin bitmap (**c4**); recreation is an explicit `bdev_cbt_create` and the + delta history is considered lost (epoch_invalidate → full rebuild if needed). + +## Pause barrier (patch 0003, SPEC-66 H10) + +- `nvmf_subsystem_pause {nqn, nsid?, ttl_ms?}` — a **standing, drain-certified** + barrier: the 200 OK certifies the drain. Returns an opaque `token` + (`:`) and the applied `ttl_ms` (clamped to 60 s, logged if + clamped). Idempotent re-pause refreshes the TTL. TTL auto-resumes a leaked + pause. **The registry entry is preallocated before the pause dispatch** — a + Paused subsystem always gets its entry+TTL (no orphan pause on OOM). +- `nvmf_subsystem_resume {nqn, token?}` — reports `barrier_intact`: false means + the freeze broke (TTL/crash/other) between pause and resume, so the controller + must drop any snapshot taken under it. A stale `boot_id` implies the token is + dead. + +## Invariants the control-plane owns (P1–P5) + +- **P1** geometry is SB-authoritative (highest seq); CRD is intent. +- **P2** relocate/remap target the lvol's own composite (enforced fork-side). +- **P3** remap journaled before the call; range rebuild re-driven at restart. +- **P4** skip_rebuild only after a proven-zero residual delta. +- **P5** the reintegration order is `pause → final freeze → copy delta → + add_base_bdev(skip_rebuild) → await callback → resume`; verify by conformance + test. diff --git a/images/spdk/Dockerfile b/images/spdk/Dockerfile index bcf05db..2f4c04e 100644 --- a/images/spdk/Dockerfile +++ b/images/spdk/Dockerfile @@ -33,6 +33,10 @@ FROM docker.io/library/fedora:43 AS builder SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG SPDK_VERSION=v26.01 +# W4 (supply chain): a git TAG is mutable — pin the exact upstream commit and +# verify it after clone, so a moved/replaced tag fails the build instead of +# silently shipping different sources. Must match vendor/spdk (v26.01). +ARG SPDK_COMMIT_SHA=2ef883e ARG TARGETARCH ARG BUILD_TYPE=release @@ -43,19 +47,26 @@ RUN dnf install -y --setopt=install_weak_deps=False --nodocs git && \ dnf clean all && rm -rf /var/cache/dnf RUN git clone --branch "${SPDK_VERSION}" --depth 1 --recurse-submodules \ - https://github.com/spdk/spdk.git + https://github.com/spdk/spdk.git && \ + ACTUAL="$(git -C spdk rev-parse --short=7 HEAD)" && \ + if [ "${ACTUAL}" != "${SPDK_COMMIT_SHA}" ]; then \ + echo "FATAL: tag ${SPDK_VERSION} resolves to ${ACTUAL}, expected ${SPDK_COMMIT_SHA} (tag moved?)" >&2; \ + exit 1; \ + fi WORKDIR /build/spdk -# ── Inject CBT module and RAID patch ── +# ── Inject CBT + tier modules and patches (SPEC-73 M1-M5) ── COPY module/bdev/cbt/ module/bdev/cbt/ +COPY module/bdev/tier/ module/bdev/tier/ COPY patches/ /build/patches/ -# Register CBT in the bdev module build, link list, and deps & apply RAID patch +# Register CBT + tier in the bdev module build, link list, and deps & apply patches # hadolint ignore=SC2016 -RUN sed -i '/^DIRS-y += delay/s/$/ cbt/' module/bdev/Makefile && \ - sed -i '/^BLOCKDEV_MODULES_LIST += bdev_zone_block/a BLOCKDEV_MODULES_LIST += bdev_cbt' mk/spdk.modules.mk && \ - sed -i '/^DEPDIRS-bdev_passthru/a DEPDIRS-bdev_cbt := $(BDEV_DEPS_THREAD)' mk/spdk.lib_deps.mk && \ +RUN sed -i '/^DIRS-y += delay/s/$/ cbt tier/' module/bdev/Makefile && \ + sed -i '/^BLOCKDEV_MODULES_LIST += bdev_zone_block/a BLOCKDEV_MODULES_LIST += bdev_cbt\nBLOCKDEV_MODULES_LIST += bdev_tier' mk/spdk.modules.mk && \ + sed -i '/^DEPDIRS-bdev_passthru/a DEPDIRS-bdev_cbt := $(BDEV_DEPS_THREAD)\nDEPDIRS-bdev_tier := $(BDEV_DEPS_THREAD)' mk/spdk.lib_deps.mk && \ + sed -i '/^DEPDIRS-bdev_lvol/s/$/ bdev_tier/' mk/spdk.lib_deps.mk && \ for p in /build/patches/*.patch; do echo "Applying ${p}" && git apply "${p}" || exit 1; done # Fedora's CMake rejects cmake_minimum_required(<3.5) in SPDK subprojects (ISA-L, etc.) diff --git a/module/bdev/cbt/README.md b/module/bdev/cbt/README.md index a6863bf..8bf3756 100644 --- a/module/bdev/cbt/README.md +++ b/module/bdev/cbt/README.md @@ -26,9 +26,13 @@ The module exposes its functionality through JSON-RPC. The orchestrator drives t An *epoch* represents a single backend outage and its associated recovery. The orchestrator opens an epoch when it detects that a backend is stale, identifying which backend and at which generation. Up to four epochs may coexist simultaneously — allowing independent tracking when multiple backends fail at different times. -**Opening** (`bdev_cbt_epoch_open`) records the stale backend identity and suspends the healthy-clear poller. The bitmap continues accumulating all writes as before — the epoch is purely an administrative marker. +**Opening** (`bdev_cbt_epoch_open`) records the stale backend identity. The bitmap continues accumulating all writes as before — the epoch is purely an administrative marker. -**Freezing** (`bdev_cbt_epoch_freeze`) takes a point-in-time snapshot of the live bitmap into the epoch. An atomic fence ensures visibility of all prior relaxed stores from IO threads. After freeze, the live bitmap continues accumulating new writes independently of the frozen copy. Re-freezing the same epoch replaces the previous snapshot — useful for iterative convergence. Freeze is accepted from OPEN, FROZEN, or REBUILDING states (the latter enables the convergence loop: freeze → partial_rebuild → re-freeze → partial_rebuild → ...). +**Freezing** (`bdev_cbt_epoch_freeze`) captures the DELTA since the previous freeze: when no other epoch is active, the live bitmap is atomically exchanged (snapshot-AND-clear) into the epoch's frozen bitmap, so iterative partial rebuilds converge geometrically. An atomic fence ensures visibility of all prior relaxed stores from IO threads. After freeze, the live bitmap accumulates only NEW writes. Freeze is accepted from OPEN, FROZEN, or REBUILDING states (the latter enables the convergence loop: freeze → partial_rebuild → re-freeze → partial_rebuild → ...), and is refused with `-EBUSY` while a rebuild is RUNNING on the epoch (CBT-1). + +Delta-preservation invariant (H1): bits exchanged out of the live bitmap exist only in the frozen buffer until a rebuild COMPLETES. Any path that discards that buffer without a completed rebuild — re-freeze after an aborted/failed rebuild, close, eviction — first ORs it back into the live bitmap, so a retry snapshot is exactly (unconsumed delta ∪ new writes). A failed iteration loses nothing; already-copied chunks are pessimistically re-copied. + +In-flight writes (H2): a write's dirty bit is set at submission AND re-set at completion (before the host ack). A freeze that consumes the submit-time bit of a still-in-flight write is therefore harmless — the chunk lands in the next delta. No host-I/O drain is required around freeze. **Reading ranges** (`bdev_cbt_epoch_get_dirty_ranges`) walks the frozen bitmap and coalesces contiguous dirty chunks into offset+length pairs. The result is capped to avoid unbounded allocations, and a `truncated` flag signals when the caller must paginate. @@ -80,13 +84,13 @@ On completion, the response includes: - `duration_ms`: wall-clock time for the operation - `residual_dirty_ratio`: fraction of the bitmap that is now dirty (writes that arrived during the copy). This ratio drives the orchestrator's convergence loop — when it drops below a threshold, a final quiesce+freeze+copy achieves zero delta. -**Closing** (`bdev_cbt_epoch_close`) discards the epoch and its frozen bitmap. If no epochs remain active, the healthy-clear poller is re-enabled. +**Closing** (`bdev_cbt_epoch_close`) discards the epoch and its frozen bitmap — after merging an unconsumed delta back into the live bitmap (H1), so abandoning an epoch never loses dirty history. Refused with `-EBUSY` while a rebuild is RUNNING (CBT-2). -**Invalidation** (`bdev_cbt_epoch_invalidate`) marks an epoch as unrecoverable. The orchestrator knows it must fall back to a full rebuild for that backend. +**Invalidation** (`bdev_cbt_epoch_invalidate`) marks an epoch as unrecoverable. The orchestrator knows it must fall back to a full rebuild for that backend. Refused with `-EBUSY` while a rebuild is RUNNING (C3 — an INVALID epoch is evictable, and evicting it under a running rebuild would free the bitmap the rebuild is scanning). -### Health signaling +### Bitmap clearing is reset-driven -The orchestrator must explicitly call `bdev_cbt_set_backends_healthy(true)` to confirm that all backends are synchronized. Only then will the poller clear the bitmap. This double-guard prevents premature clearing — even if all epochs are closed, the bitmap persists until health is confirmed. +There is no automatic clearing (the former healthy-clear poller and its `bdev_cbt_set_backends_healthy()` signal were dead code and have been REMOVED — D3). Once the orchestrator has confirmed all backends are synchronized (after `skip_rebuild` promotion), it MUST call `bdev_cbt_reset` explicitly; otherwise the bitmap grows monotonically and "partial" rebuilds degrade toward full-surface copies. ### Reset @@ -108,7 +112,8 @@ A typical orchestrator sequence for partial rebuild under sustained write load: 8. ANA drain (2s quiesce at the NVMe-oF target level). No more writes arrive. 9. Final freeze. Delta is zero by construction. 10. Final `bdev_cbt_partial_rebuild` (nothing or near-nothing). Close epoch. -11. Re-add the backend with `skip_rebuild=true`. Signal healthy. +11. Re-add the backend with `skip_rebuild=true`. Then `bdev_cbt_reset` (explicit — + there is no automatic clear; see "Bitmap clearing is reset-driven"). The state machine for a single epoch through the convergence loop: @@ -132,9 +137,9 @@ This is modeled on SPDK's own `raid_bdev_process_finish` sequence (the normal re ## Concurrency model -All epoch operations and the healthy-clear poller run on the SPDK app thread. They are not thread-safe against each other — SPDK guarantees single-threaded execution on that thread. The `assert(spdk_get_thread() == spdk_thread_get_app_thread())` guards enforce this. +All epoch operations run on the SPDK app thread. They are not thread-safe against each other — SPDK guarantees single-threaded execution on that thread. The `assert(spdk_get_thread() == spdk_thread_get_app_thread())` guards enforce this. -IO submission (`cbt_mark_dirty`) runs on any reactor thread. It uses `__atomic_fetch_or` with relaxed ordering for individual bitmap bytes, and `memset(0xFF)` for full bytes in large ranges. The combination is safe because setting a bit to 1 is idempotent — concurrent ORs cannot lose information. +IO submission (`cbt_mark_dirty`) runs on any reactor thread. It uses `__atomic_fetch_or` with relaxed ordering for individual bitmap bytes, and `memset(0xFF)` for full bytes in large ranges. The combination is safe because setting a bit to 1 is idempotent — concurrent ORs cannot lose information. Completions RE-mark the written range before acking the host (H2), so a snapshot-and-clear freeze racing an in-flight write never loses the chunk. The `total_writes_tracked` counter uses relaxed atomics. It is a statistical counter with no correctness semantics. diff --git a/module/bdev/cbt/test_cbt_postfix.c b/module/bdev/cbt/test_cbt_postfix.c index a1ef775..975d82a 100644 --- a/module/bdev/cbt/test_cbt_postfix.c +++ b/module/bdev/cbt/test_cbt_postfix.c @@ -341,6 +341,21 @@ cbt_epoch_open(struct cbt_device *dev, const char *epoch_id, return 0; } +/* Mirrors vbdev_cbt.c: snapshot-and-clear is only safe when no OTHER epoch + * still needs the shared accumulated live view. */ +static bool +cbt_has_other_active_epoch(struct cbt_device *dev, struct cbt_epoch *self) +{ + for (struct cbt_epoch *ep = dev->epochs_head; ep; ep = ep->next) { + if (ep == self) continue; + if (ep->state == CBT_EPOCH_OPEN || ep->state == CBT_EPOCH_FROZEN || + ep->state == CBT_EPOCH_REBUILDING) { + return true; + } + } + return false; +} + static int cbt_epoch_freeze(struct cbt_device *dev, const char *epoch_id) { @@ -353,7 +368,15 @@ cbt_epoch_freeze(struct cbt_device *dev, const char *epoch_id) if (!ep->bitmap_frozen) return -ENOMEM; __atomic_thread_fence(__ATOMIC_ACQUIRE); - memcpy(ep->bitmap_frozen, dev->bitmap, dev->bitmap_size_bytes); + if (!cbt_has_other_active_epoch(dev, ep)) { + /* Delta semantics: atomic per-byte exchange (no lost concurrent ORs). */ + for (uint64_t i = 0; i < dev->bitmap_size_bytes; i++) { + ep->bitmap_frozen[i] = __atomic_exchange_n(&dev->bitmap[i], 0, + __ATOMIC_ACQ_REL); + } + } else { + memcpy(ep->bitmap_frozen, dev->bitmap, dev->bitmap_size_bytes); + } ep->state = CBT_EPOCH_FROZEN; return 0; } diff --git a/module/bdev/cbt/test_cbt_resilience.c b/module/bdev/cbt/test_cbt_resilience.c index 97be06b..cfadb1b 100644 --- a/module/bdev/cbt/test_cbt_resilience.c +++ b/module/bdev/cbt/test_cbt_resilience.c @@ -120,6 +120,12 @@ struct cbt_epoch { uint64_t generation; enum cbt_epoch_state state; uint8_t *bitmap_frozen; + /* H1 (mirrors vbdev_cbt.h): true while bitmap_frozen holds bits exchanged + * OUT of the live bitmap and not yet proven copied by a COMPLETED rebuild. */ + bool frozen_live_consumed; + /* Model equivalent of production's g_rebuild_registry RUNNING lookup + * (cbt_rebuild_find_active_for_epoch). */ + bool rebuild_running; struct cbt_epoch *next; }; @@ -203,6 +209,51 @@ cbt_mark_dirty(struct cbt_device *dev, uint64_t offset_blocks, uint64_t num_bloc } } +/* H2 (mirrors vbdev_cbt.c two-phase marking): the dirty bit is set at SUBMIT + * (crash conservatism) and RE-set at COMPLETION before the host ack. A freeze + * exchanging the submit-time bit of a still-in-flight write is therefore + * harmless: the completion re-mark lands the chunk in the next delta. */ +static void +cbt_write_submit(struct cbt_device *dev, uint64_t offset_blocks, uint64_t num_blocks) +{ + cbt_mark_dirty(dev, offset_blocks, num_blocks); +} + +static void +cbt_write_complete(struct cbt_device *dev, uint64_t offset_blocks, uint64_t num_blocks) +{ + cbt_mark_dirty(dev, offset_blocks, num_blocks); +} + +/* M1 (mirrors cbt_rebuild_finish + cbt_rebuild_finalize classification, R1): + * the terminal flush is only submitted when the rebuild is a genuine success + * candidate — error==0, not cancelled, NOT ABORTED, chunks copied, target + * present. A flush failure maps to FAILED; the R1 order is + * cancelled → error → aborted → completed. */ +enum cbt_rebuild_final_state { + REB_COMPLETED = 0, + REB_FAILED = 1, + REB_ABORTED = 2, + REB_CANCELLED = 3, +}; + +static enum cbt_rebuild_final_state +cbt_rebuild_finish_classify(int error, bool cancelled, bool aborted, + uint64_t chunks_copied, bool dst_present, + bool flush_fails) +{ + if (error == 0 && !cancelled && !aborted && chunks_copied > 0 && dst_present) { + if (flush_fails) { + error = -EIO; + aborted = true; + } + } + if (cancelled) return REB_CANCELLED; + if (error != 0) return REB_FAILED; + if (aborted) return REB_ABORTED; + return REB_COMPLETED; +} + static struct cbt_epoch * cbt_find_epoch(struct cbt_device *dev, const char *epoch_id) { @@ -228,6 +279,9 @@ cbt_any_epoch_active(struct cbt_device *dev) return false; } +static void cbt_epoch_restore_unconsumed_delta(struct cbt_device *dev, + struct cbt_epoch *ep); + static int cbt_epoch_open(struct cbt_device *dev, const char *epoch_id, const char *stale_backend_id, uint64_t generation) @@ -250,14 +304,22 @@ cbt_epoch_open(struct cbt_device *dev, const char *epoch_id, } if (dev->epoch_count >= CBT_MAX_EPOCHS) { - /* Evict oldest. */ + /* Mirrors production: evict the oldest ONLY if safe — never an + * active epoch (OPEN/FROZEN/REBUILDING) and never one a rebuild + * still points at (C3 defense-in-depth). */ struct cbt_epoch *oldest = dev->epochs_head; - if (oldest) { - dev->epochs_head = oldest->next; - dev->epoch_count--; - free(oldest->bitmap_frozen); - free(oldest); + if (!oldest || oldest->state == CBT_EPOCH_OPEN || + oldest->state == CBT_EPOCH_FROZEN || + oldest->state == CBT_EPOCH_REBUILDING || + oldest->rebuild_running) { + return -ENOSPC; } + /* H1: an INVALID epoch may still hold an unconsumed exchanged delta. */ + cbt_epoch_restore_unconsumed_delta(dev, oldest); + dev->epochs_head = oldest->next; + dev->epoch_count--; + free(oldest->bitmap_frozen); + free(oldest); } struct cbt_epoch *ep = fi_calloc(1, sizeof(*ep)); @@ -283,6 +345,37 @@ cbt_epoch_open(struct cbt_device *dev, const char *epoch_id, return 0; } +/* Mirrors vbdev_cbt.c cbt_has_other_active_epoch: epochs share the live bitmap, + * so snapshot-and-clear is only safe when no OTHER epoch needs the accumulated view. */ +static bool +cbt_has_other_active_epoch(struct cbt_device *dev, struct cbt_epoch *self) +{ + for (struct cbt_epoch *ep = dev->epochs_head; ep; ep = ep->next) { + if (ep == self) continue; + if (ep->state == CBT_EPOCH_OPEN || ep->state == CBT_EPOCH_FROZEN || + ep->state == CBT_EPOCH_REBUILDING) { + return true; + } + } + return false; +} + +/* H1 (mirrors vbdev_cbt.c cbt_epoch_restore_unconsumed_delta): OR an + * unconsumed exchanged delta back into the live bitmap before its buffer is + * discarded. Pessimistic (already-copied chunks get re-copied), never lossy. */ +static void +cbt_epoch_restore_unconsumed_delta(struct cbt_device *dev, struct cbt_epoch *ep) +{ + if (!ep->frozen_live_consumed || ep->bitmap_frozen == NULL) return; + for (uint64_t i = 0; i < dev->bitmap_size_bytes; i++) { + uint8_t b = ep->bitmap_frozen[i]; + if (b != 0) { + __atomic_fetch_or(&dev->bitmap[i], b, __ATOMIC_RELAXED); + } + } + ep->frozen_live_consumed = false; +} + static int cbt_epoch_freeze(struct cbt_device *dev, const char *epoch_id) { @@ -294,13 +387,38 @@ cbt_epoch_freeze(struct cbt_device *dev, const char *epoch_id) ep->state != CBT_EPOCH_REBUILDING) { return -EINVAL; } - - free(ep->bitmap_frozen); - - ep->bitmap_frozen = fi_malloc(dev->bitmap_size_bytes); - if (!ep->bitmap_frozen) return -ENOMEM; - - memcpy(ep->bitmap_frozen, dev->bitmap, dev->bitmap_size_bytes); + /* CBT-1 (mirrors production): never free/realloc the frozen bitmap a + * RUNNING rebuild is scanning. */ + if (ep->rebuild_running) return -EBUSY; + + /* H1: allocate BEFORE touching the old snapshot — ENOMEM must leave the + * epoch (and the previous delta) exactly as they were. */ + uint8_t *new_frozen = fi_malloc(dev->bitmap_size_bytes); + if (!new_frozen) return -ENOMEM; + + /* H1: an unconsumed previous delta (rebuild aborted/failed/never run) is + * merged back first, so the exchange below re-captures it: new snapshot = + * old unconsumed delta ∪ writes since last freeze. */ + if (ep->bitmap_frozen != NULL) { + cbt_epoch_restore_unconsumed_delta(dev, ep); + free(ep->bitmap_frozen); + } + ep->bitmap_frozen = new_frozen; + + if (!cbt_has_other_active_epoch(dev, ep)) { + /* Snapshot-AND-CLEAR (atomic per-byte exchange): each freeze captures + * the DELTA since the previous freeze — iterative rebuilds converge. + * Exchange, not memcpy+memset: a concurrent OR between copy and clear + * would be lost (missed chunk under skip_rebuild = silent divergence). */ + for (uint64_t i = 0; i < dev->bitmap_size_bytes; i++) { + ep->bitmap_frozen[i] = __atomic_exchange_n(&dev->bitmap[i], 0, + __ATOMIC_ACQ_REL); + } + ep->frozen_live_consumed = true; + } else { + memcpy(ep->bitmap_frozen, dev->bitmap, dev->bitmap_size_bytes); + ep->frozen_live_consumed = false; + } ep->state = CBT_EPOCH_FROZEN; return 0; } @@ -312,9 +430,33 @@ cbt_epoch_rebuild_start(struct cbt_device *dev, const char *epoch_id) struct cbt_epoch *ep = cbt_find_epoch(dev, epoch_id); if (!ep) return -ENOENT; - if (ep->state != CBT_EPOCH_FROZEN) return -EINVAL; + /* Mirrors production: FROZEN or REBUILDING (retry), frozen bitmap + * required, one rebuild per epoch. */ + if (ep->state != CBT_EPOCH_FROZEN && ep->state != CBT_EPOCH_REBUILDING) { + return -EINVAL; + } + if (!ep->bitmap_frozen) return -EINVAL; + if (ep->rebuild_running) return -EBUSY; ep->state = CBT_EPOCH_REBUILDING; + ep->rebuild_running = true; + return 0; +} + +/* Model of the rebuild terminating (cbt_rebuild_finalize): on COMPLETED the + * exchanged delta is proven copied (H1 flag cleared); on abort/failure the + * delta stays owned by bitmap_frozen until merged back. */ +static int +cbt_epoch_rebuild_finish(struct cbt_device *dev, const char *epoch_id, bool completed) +{ + struct cbt_epoch *ep = cbt_find_epoch(dev, epoch_id); + if (!ep) return -ENOENT; + if (!ep->rebuild_running) return -EINVAL; + + ep->rebuild_running = false; + if (completed) { + ep->frozen_live_consumed = false; + } return 0; } @@ -326,6 +468,11 @@ cbt_epoch_close(struct cbt_device *dev, const char *epoch_id) struct cbt_epoch *ep = cbt_find_epoch(dev, epoch_id); if (!ep) return -ENOENT; if (ep->state == CBT_EPOCH_OPEN) return -EINVAL; + /* CBT-2 (mirrors production): a RUNNING rebuild writes into the epoch. */ + if (ep->rebuild_running) return -EBUSY; + + /* H1: closing must not lose un-copied dirty history. */ + cbt_epoch_restore_unconsumed_delta(dev, ep); /* Remove from list. */ struct cbt_epoch **pp = &dev->epochs_head; @@ -349,11 +496,24 @@ cbt_epoch_invalidate(struct cbt_device *dev, const char *epoch_id) struct cbt_epoch *ep = cbt_find_epoch(dev, epoch_id); if (!ep) return -ENOENT; + /* C3 (mirrors production): invalidating a REBUILDING epoch would make it + * evictable under the RUNNING rebuild → UAF. Cancel the rebuild first. */ + if (ep->rebuild_running) return -EBUSY; ep->state = CBT_EPOCH_INVALID; return 0; } +static uint64_t +count_bits(const uint8_t *bm, uint64_t bytes) +{ + uint64_t n = 0; + for (uint64_t i = 0; i < bytes; i++) { + n += (uint64_t)__builtin_popcount(bm[i]); + } + return n; +} + struct dirty_range { uint64_t offset_blocks; uint64_t length_blocks; @@ -534,7 +694,12 @@ static void test_freeze_non_open_epoch(void) ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); - /* Re-freeze from REBUILDING is now allowed (convergence loop). */ + /* CBT-1: re-freeze refused while the rebuild is RUNNING. */ + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), -EBUSY); + + /* Re-freeze from REBUILDING is allowed once no rebuild runs + * (convergence loop). */ + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", true), 0); ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); /* Invalidate, then try to freeze — INVALID is rejected. */ @@ -800,7 +965,11 @@ static void test_max_epochs_eviction(void) ASSERT_RC(cbt_epoch_open(dev, "ep4", "b4", 4), 0); ASSERT_EQ(dev->epoch_count, CBT_MAX_EPOCHS); - /* One more → evicts ep1. */ + /* All epochs are OPEN (active) → production refuses to evict. */ + ASSERT_RC(cbt_epoch_open(dev, "ep5", "b5", 5), -ENOSPC); + + /* Invalidate the oldest → now evictable. */ + ASSERT_RC(cbt_epoch_invalidate(dev, "ep1"), 0); ASSERT_RC(cbt_epoch_open(dev, "ep5", "b5", 5), 0); ASSERT_EQ(dev->epoch_count, CBT_MAX_EPOCHS); ASSERT(cbt_find_epoch(dev, "ep1") == NULL); @@ -825,10 +994,15 @@ static void test_max_epochs_eviction_with_frozen_bitmap(void) ASSERT_RC(cbt_epoch_open(dev, "ep3", "b3", 3), 0); ASSERT_RC(cbt_epoch_open(dev, "ep4", "b4", 4), 0); - /* Evict ep1 (which has bitmap_frozen): must not leak. */ + /* FROZEN is active → refused; invalidate first, then evict. */ + ASSERT_RC(cbt_epoch_open(dev, "ep5", "b5", 5), -ENOSPC); + ASSERT_RC(cbt_epoch_invalidate(dev, "ep1"), 0); ASSERT_RC(cbt_epoch_open(dev, "ep5", "b5", 5), 0); ASSERT(cbt_find_epoch(dev, "ep1") == NULL); - /* If we get here without ASan complaining, no leak. */ + /* If we get here without ASan complaining, no leak. And H1: ep1's + * exchanged-out delta must have been merged back into the live bitmap + * by the eviction, not silently destroyed. */ + ASSERT(count_bits(dev->bitmap, dev->bitmap_size_bytes) > 0); cbt_destroy(dev); PASS(); @@ -848,15 +1022,20 @@ static void test_double_freeze(void) cbt_mark_dirty(dev, 0, 128); ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); - /* Mark more, re-freeze → should succeed, updating snapshot. */ + /* Mark more, re-freeze → succeeds. H1: the first delta was consumed from + * the live bitmap but NEVER copied (no rebuild ran) — the re-freeze must + * merge it back and re-capture it, so the new snapshot holds BOTH ranges. + * (Only a COMPLETED rebuild licenses dropping a consumed delta; see + * test_freeze_clears_live_when_sole_epoch for the pure-delta flow.) */ cbt_mark_dirty(dev, 256, 128); ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); - /* Verify new snapshot has both ranges. */ struct dirty_range *ranges = NULL; uint32_t count = 0; ASSERT_RC(cbt_epoch_get_dirty_ranges(dev, "ep1", 0, &ranges, &count), 0); ASSERT_EQ(count, 2); + ASSERT_EQ(ranges[0].offset_blocks, 0); + ASSERT_EQ(ranges[1].offset_blocks, 256); free(ranges); cbt_destroy(dev); @@ -959,6 +1138,9 @@ static void test_full_lifecycle_sequence(void) ASSERT(count > 0); free(ranges); + /* CBT-2: close waits for the rebuild to terminate. */ + ASSERT_RC(cbt_epoch_close(dev, "ep1"), -EBUSY); + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", true), 0); ASSERT_RC(cbt_epoch_close(dev, "ep1"), 0); ASSERT_EQ(dev->epoch_count, 0); ASSERT_EQ(dev->healthy_clear_suspended, false); @@ -1329,8 +1511,9 @@ static void test_rebuild_start_requires_frozen_state(void) ASSERT(ep != NULL); ASSERT_EQ(ep->state, CBT_EPOCH_REBUILDING); - /* Double rebuild_start should fail (already REBUILDING, not FROZEN). */ - ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), -EINVAL); + /* Double rebuild_start should fail: one rebuild per epoch (-EBUSY, + * mirrors production's registry guard). */ + ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), -EBUSY); cbt_destroy(dev); PASS(); @@ -1424,7 +1607,11 @@ static void test_rebuild_close_after_rebuild(void) ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); - /* Close should succeed from REBUILDING state. */ + /* CBT-2: close is refused while the rebuild is RUNNING. */ + ASSERT_RC(cbt_epoch_close(dev, "ep1"), -EBUSY); + + /* Once the rebuild terminates, close succeeds from REBUILDING state. */ + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", true), 0); ASSERT_RC(cbt_epoch_close(dev, "ep1"), 0); /* Epoch should be gone. */ @@ -1447,7 +1634,13 @@ static void test_rebuild_invalidate_during_rebuild(void) ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); - /* Invalidation should work from any state. */ + /* C3: invalidation is REFUSED while the rebuild is RUNNING — an INVALID + * epoch is evictable, and evicting it would free the bitmap under the + * rebuild (UAF). */ + ASSERT_RC(cbt_epoch_invalidate(dev, "ep1"), -EBUSY); + + /* After the rebuild terminates (aborted here), invalidation is legal. */ + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", false), 0); ASSERT_RC(cbt_epoch_invalidate(dev, "ep1"), 0); struct cbt_epoch *ep = cbt_find_epoch(dev, "ep1"); @@ -1498,13 +1691,15 @@ static void test_rebuild_convergence_refreeze(void) ASSERT(ep != NULL); ASSERT_EQ(ep->state, CBT_EPOCH_FROZEN); - /* Simulate partial_rebuild completing (FROZEN→REBUILDING) */ + /* Simulate partial_rebuild running then COMPLETING (FROZEN→REBUILDING) */ ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); ASSERT_EQ(ep->state, CBT_EPOCH_REBUILDING); /* New writes arrive during rebuild */ cbt_mark_dirty(dev, 512, 64); + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", true), 0); + /* Pass 2: re-freeze (REBUILDING→FROZEN) — the key fix */ ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); ASSERT_EQ(ep->state, CBT_EPOCH_FROZEN); @@ -1514,14 +1709,383 @@ static void test_rebuild_convergence_refreeze(void) ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); ASSERT_EQ(ep->state, CBT_EPOCH_REBUILDING); - /* Close from REBUILDING state */ + /* Close from REBUILDING state (after the rebuild terminates) */ + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", true), 0); + ASSERT_RC(cbt_epoch_close(dev, "ep1"), 0); + ASSERT(cbt_find_epoch(dev, "ep1") == NULL); + + cbt_destroy(dev); + PASS(); +} + +/* ================================================================== */ +/* SECTION 12: Freeze delta semantics (snapshot-and-clear) */ +/* ================================================================== */ + +static void test_freeze_clears_live_when_sole_epoch(void) +{ + TEST(test_freeze_clears_live_when_sole_epoch); + fi_reset(); + struct cbt_device *dev = cbt_create(2048, 64, 512); + ASSERT(dev != NULL); + + ASSERT_RC(cbt_epoch_open(dev, "ep1", "b", 1), 0); + cbt_mark_dirty(dev, 0, 512); + + /* Freeze #1: captures the marks AND clears the live bitmap. */ + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + struct cbt_epoch *ep = cbt_find_epoch(dev, "ep1"); + ASSERT(ep != NULL); + ASSERT(count_bits(ep->bitmap_frozen, dev->bitmap_size_bytes) > 0); + ASSERT_EQ(count_bits(dev->bitmap, dev->bitmap_size_bytes), 0); + + /* Delta #1 is proven copied by a COMPLETED rebuild — only then does the + * next freeze produce a PURE delta (H1: without a completed rebuild the + * un-copied delta is merged back and re-captured, see the H1 tests). */ + ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", true), 0); + + /* New writes land AFTER the freeze. */ + cbt_mark_dirty(dev, 1024, 128); + uint64_t delta_chunks = count_bits(dev->bitmap, dev->bitmap_size_bytes); + ASSERT(delta_chunks > 0); + + /* Freeze #2 (re-freeze after COMPLETED rebuild): frozen must contain ONLY + * the new delta — geometric convergence depends on this. */ + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + ASSERT_EQ(count_bits(ep->bitmap_frozen, dev->bitmap_size_bytes), delta_chunks); + ASSERT_EQ(count_bits(dev->bitmap, dev->bitmap_size_bytes), 0); + + /* Freeze #3 with no new writes (delta #2 copied): empty delta → converged. */ + ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", true), 0); + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + ASSERT_EQ(count_bits(ep->bitmap_frozen, dev->bitmap_size_bytes), 0); + + cbt_destroy(dev); + PASS(); +} + +static void test_freeze_preserves_live_with_other_active_epoch(void) +{ + TEST(test_freeze_preserves_live_with_other_active_epoch); + fi_reset(); + struct cbt_device *dev = cbt_create(2048, 64, 512); + ASSERT(dev != NULL); + + ASSERT_RC(cbt_epoch_open(dev, "ep1", "b1", 1), 0); + ASSERT_RC(cbt_epoch_open(dev, "ep2", "b2", 2), 0); + cbt_mark_dirty(dev, 0, 512); + + uint64_t before = count_bits(dev->bitmap, dev->bitmap_size_bytes); + ASSERT(before > 0); + + /* ep2 is still OPEN and needs the accumulated view → freeze must NOT clear. */ + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + ASSERT_EQ(count_bits(dev->bitmap, dev->bitmap_size_bytes), before); + + cbt_destroy(dev); + PASS(); +} + +/* Property: under a concurrent writer, the union of all frozen snapshots plus + * the live bitmap must equal exactly the set of chunks the writer marked — + * the per-byte atomic exchange must never lose (or invent) a bit. */ +struct delta_race_ctx { + struct cbt_device *dev; + uint8_t *shadow; /* same layout as dev->bitmap */ + _Atomic bool stop; + _Atomic uint64_t marks; +}; + +static void * +delta_marker_thread(void *arg) +{ + struct delta_race_ctx *ctx = arg; + uint64_t seed = 0x9e3779b97f4a7c15ull; + while (!atomic_load(&ctx->stop)) { + seed = seed * 6364136223846793005ull + 1442695040888963407ull; + uint64_t block = seed % ctx->dev->total_blocks; + cbt_mark_dirty(ctx->dev, block, 1); + uint64_t chunk = block >> ctx->dev->chunk_shift; + __atomic_fetch_or(&ctx->shadow[chunk >> 3], (uint8_t)(1u << (chunk & 7)), + __ATOMIC_RELAXED); + atomic_fetch_add(&ctx->marks, 1); + } + return NULL; +} + +static void test_concurrent_refreeze_never_loses_bits(void) +{ + TEST(test_concurrent_refreeze_never_loses_bits); + fi_reset(); + struct cbt_device *dev = cbt_create(65536, 64, 512); + ASSERT(dev != NULL); + + uint8_t *shadow = calloc(1, dev->bitmap_size_bytes); + uint8_t *acc = calloc(1, dev->bitmap_size_bytes); + ASSERT(shadow != NULL && acc != NULL); + + ASSERT_RC(cbt_epoch_open(dev, "ep1", "b", 1), 0); + struct cbt_epoch *ep = cbt_find_epoch(dev, "ep1"); + ASSERT(ep != NULL); + + struct delta_race_ctx ctx = { .dev = dev, .shadow = shadow, .stop = false, .marks = 0 }; + pthread_t t; + pthread_create(&t, NULL, delta_marker_thread, &ctx); + + /* Re-freeze repeatedly under fire, accumulating every captured delta. */ + for (int pass = 0; pass < 200; pass++) { + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + for (uint64_t i = 0; i < dev->bitmap_size_bytes; i++) { + acc[i] |= ep->bitmap_frozen[i]; + } + } + + atomic_store(&ctx.stop, true); + pthread_join(t, NULL); + ASSERT(atomic_load(&ctx.marks) > 0); + + /* Tail: whatever the writer marked after the last freeze is still live. */ + for (uint64_t i = 0; i < dev->bitmap_size_bytes; i++) { + acc[i] |= dev->bitmap[i]; + } + + /* Exact equality: no lost bits (missed chunk = silent divergence under + * skip_rebuild) and no phantom bits. */ + ASSERT_EQ(memcmp(acc, shadow, dev->bitmap_size_bytes), 0); + + free(shadow); + free(acc); + cbt_destroy(dev); + PASS(); +} + +/* ================================================================== */ +/* SECTION 13: Audit findings framing (H1/H2/C3/M1) */ +/* Each test pins the exact failure scenario from the SPEC-73 audit. */ +/* ================================================================== */ + +/* H1 — the core loss scenario: freeze consumes delta D1, the rebuild ABORTS + * (hot-remove at 10%), the orchestrator retries with a re-freeze. The old + * code freed the only copy of D1's un-copied 90%; the fix merges it back so + * the retry snapshot = D1 ∪ (writes since freeze #1). */ +static void test_h1_refreeze_after_aborted_rebuild_preserves_delta(void) +{ + TEST(test_h1_refreeze_after_aborted_rebuild_preserves_delta); + fi_reset(); + struct cbt_device *dev = cbt_create(2048, 64, 512); + ASSERT(dev != NULL); + + ASSERT_RC(cbt_epoch_open(dev, "ep1", "b", 1), 0); + cbt_mark_dirty(dev, 0, 128); /* chunk 0 — delta D1 */ + cbt_mark_dirty(dev, 256, 128); /* chunk 2 — delta D1 */ + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + ASSERT_EQ(count_bits(dev->bitmap, dev->bitmap_size_bytes), 0); + + ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); + /* Target hot-removed mid-rebuild: ABORTED, D1 not (fully) copied. */ + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", false), 0); + + /* Writes since freeze #1. */ + cbt_mark_dirty(dev, 512, 128); /* chunk 4 — delta D2 */ + + /* Retry: re-freeze. Fixed semantics: snapshot = D1 ∪ D2 (3 chunks). */ + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + struct cbt_epoch *ep = cbt_find_epoch(dev, "ep1"); + ASSERT(ep != NULL); + ASSERT_EQ(count_bits(ep->bitmap_frozen, dev->bitmap_size_bytes), 3); + + cbt_destroy(dev); + PASS(); +} + +/* H1 — closing (abandoning) an epoch whose delta was consumed but never + * copied must return the delta to the live bitmap, not destroy it. */ +static void test_h1_close_merges_back_unconsumed_delta(void) +{ + TEST(test_h1_close_merges_back_unconsumed_delta); + fi_reset(); + struct cbt_device *dev = cbt_create(2048, 64, 512); + ASSERT(dev != NULL); + + ASSERT_RC(cbt_epoch_open(dev, "ep1", "b", 1), 0); + cbt_mark_dirty(dev, 0, 128); + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + ASSERT_EQ(count_bits(dev->bitmap, dev->bitmap_size_bytes), 0); + + ASSERT_RC(cbt_epoch_close(dev, "ep1"), 0); + + /* The delta survived the abandon: a later epoch for the same stale + * backend recaptures it. */ + ASSERT_EQ(count_bits(dev->bitmap, dev->bitmap_size_bytes), 1); + + cbt_destroy(dev); + PASS(); +} + +/* H1 — a COMPLETED rebuild consumes the delta legitimately: close must NOT + * re-inject it (that would force a spurious re-copy of everything). */ +static void test_h1_completed_rebuild_does_not_merge_back(void) +{ + TEST(test_h1_completed_rebuild_does_not_merge_back); + fi_reset(); + struct cbt_device *dev = cbt_create(2048, 64, 512); + ASSERT(dev != NULL); + + ASSERT_RC(cbt_epoch_open(dev, "ep1", "b", 1), 0); + cbt_mark_dirty(dev, 0, 128); + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", true), 0); + ASSERT_RC(cbt_epoch_close(dev, "ep1"), 0); + ASSERT_EQ(count_bits(dev->bitmap, dev->bitmap_size_bytes), 0); + + cbt_destroy(dev); + PASS(); +} + +/* H1 — ENOMEM on re-freeze: the old code free'd the previous delta BEFORE + * malloc, so a failed allocation bricked the epoch (bitmap_frozen == NULL) + * AND lost the delta. Fixed: allocate first — ENOMEM leaves everything + * exactly as it was and the retry succeeds. */ +static void test_h1_refreeze_enomem_preserves_previous_delta(void) +{ + TEST(test_h1_refreeze_enomem_preserves_previous_delta); + fi_reset(); + struct cbt_device *dev = cbt_create(2048, 64, 512); + ASSERT(dev != NULL); + + ASSERT_RC(cbt_epoch_open(dev, "ep1", "b", 1), 0); + cbt_mark_dirty(dev, 0, 128); + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + struct cbt_epoch *ep = cbt_find_epoch(dev, "ep1"); + ASSERT(ep != NULL); + uint8_t *frozen_before = ep->bitmap_frozen; + + /* Next allocation fails. */ + atomic_store(&g_malloc_fail_countdown, 0); + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), -ENOMEM); + fi_reset(); + + /* Previous delta untouched — not bricked, not lost. */ + ASSERT(ep->bitmap_frozen == frozen_before); + ASSERT_EQ(count_bits(ep->bitmap_frozen, dev->bitmap_size_bytes), 1); + + /* Retry works and still holds the delta. */ + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + ASSERT_EQ(count_bits(ep->bitmap_frozen, dev->bitmap_size_bytes), 1); + + cbt_destroy(dev); + PASS(); +} + +/* H2 — a write in flight across a clearing freeze: the submit-time bit is + * consumed by the exchange, but the completion re-mark (before the host ack) + * puts the chunk back in the live bitmap → captured by the next delta. + * Lock-free: no drain, no host-I/O freeze anywhere. */ +static void test_h2_inflight_write_survives_clearing_freeze(void) +{ + TEST(test_h2_inflight_write_survives_clearing_freeze); + fi_reset(); + struct cbt_device *dev = cbt_create(2048, 64, 512); + ASSERT(dev != NULL); + + ASSERT_RC(cbt_epoch_open(dev, "ep1", "b", 1), 0); + + /* Write W submitted (bit set) but not yet completed. */ + cbt_write_submit(dev, 0, 128); + + /* Freeze consumes the submit-time bit. */ + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + ASSERT_EQ(count_bits(dev->bitmap, dev->bitmap_size_bytes), 0); + /* Rebuild of this delta may read the chunk BEFORE W lands — that copy + * is stale. Completed + closed: consumed delta legitimately dropped. */ + ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", true), 0); + + /* W completes AFTER the rebuild read: the re-mark lands the chunk in + * the live bitmap — the next delta re-copies it with W's data. */ + cbt_write_complete(dev, 0, 128); + ASSERT_EQ(count_bits(dev->bitmap, dev->bitmap_size_bytes), 1); + + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + struct cbt_epoch *ep = cbt_find_epoch(dev, "ep1"); + ASSERT(ep != NULL); + ASSERT_EQ(count_bits(ep->bitmap_frozen, dev->bitmap_size_bytes), 1); + + cbt_destroy(dev); + PASS(); +} + +/* C3 — invalidate is refused while a rebuild is RUNNING (would make the + * epoch evictable under the rebuild → UAF), and eviction refuses an epoch a + * rebuild still points at even if its state says INVALID. */ +static void test_c3_invalidate_and_evict_guards_running_rebuild(void) +{ + TEST(test_c3_invalidate_and_evict_guards_running_rebuild); + fi_reset(); + struct cbt_device *dev = cbt_create(2048, 64, 512); + ASSERT(dev != NULL); + + ASSERT_RC(cbt_epoch_open(dev, "ep1", "b", 1), 0); + cbt_mark_dirty(dev, 0, 128); + ASSERT_RC(cbt_epoch_freeze(dev, "ep1"), 0); + ASSERT_RC(cbt_epoch_rebuild_start(dev, "ep1"), 0); + + /* Guard #1: invalidate refused while RUNNING. */ + ASSERT_RC(cbt_epoch_invalidate(dev, "ep1"), -EBUSY); + + /* Guard #2 (defense-in-depth): even with state forced to INVALID, the + * eviction path must refuse while the rebuild holds the epoch. */ + struct cbt_epoch *ep = cbt_find_epoch(dev, "ep1"); + ASSERT(ep != NULL); + ep->state = CBT_EPOCH_INVALID; /* test backdoor: simulate the race */ + ASSERT_RC(cbt_epoch_open(dev, "ep2", "b2", 2), 0); + ASSERT_RC(cbt_epoch_open(dev, "ep3", "b3", 3), 0); + ASSERT_RC(cbt_epoch_open(dev, "ep4", "b4", 4), 0); + ASSERT_RC(cbt_epoch_open(dev, "ep5", "b5", 5), -ENOSPC); + ASSERT(cbt_find_epoch(dev, "ep1") != NULL); /* NOT freed under the rebuild */ + + /* Rebuild terminates → epoch becomes truly evictable. */ + ASSERT_RC(cbt_epoch_rebuild_finish(dev, "ep1", false), 0); + ASSERT_RC(cbt_epoch_open(dev, "ep5", "b5", 5), 0); ASSERT(cbt_find_epoch(dev, "ep1") == NULL); cbt_destroy(dev); PASS(); } +/* M1 — terminal-state classification (R1): a hot-remove abort (aborted only, + * error==0) must classify ABORTED even when a flush WOULD fail — the fixed + * gate no longer submits the flush on an aborted run. The old gate (no + * !aborted) turned the abort into FAILED via the failing flush. */ +static void test_m1_hotremove_abort_classified_aborted_not_failed(void) +{ + TEST(test_m1_hotremove_abort_classified_aborted_not_failed); + fi_reset(); + + /* The audit scenario: aborted=true, error==0, chunks copied, target + * still open (dst_desc != NULL after hot-remove), flush would fail. */ + ASSERT_EQ(cbt_rebuild_finish_classify(0, false, true, 5, true, true), + REB_ABORTED); + + /* Sanity: the full R1 matrix. */ + ASSERT_EQ(cbt_rebuild_finish_classify(0, false, false, 5, true, false), + REB_COMPLETED); + ASSERT_EQ(cbt_rebuild_finish_classify(0, false, false, 5, true, true), + REB_FAILED); /* genuine flush failure on a clean run */ + ASSERT_EQ(cbt_rebuild_finish_classify(-EIO, false, true, 5, true, false), + REB_FAILED); /* I/O error sets both → FAILED wins */ + ASSERT_EQ(cbt_rebuild_finish_classify(0, true, false, 5, true, false), + REB_CANCELLED); + ASSERT_EQ(cbt_rebuild_finish_classify(0, false, false, 0, true, true), + REB_COMPLETED); /* zero chunks: no flush needed */ + + PASS(); +} + /* ================================================================== */ /* Main */ /* ================================================================== */ @@ -1608,6 +2172,20 @@ main(void) test_rebuild_get_ranges_during_rebuild(); test_rebuild_convergence_refreeze(); + printf("\n── Freeze delta semantics (snapshot-and-clear) ──\n"); + test_freeze_clears_live_when_sole_epoch(); + test_freeze_preserves_live_with_other_active_epoch(); + test_concurrent_refreeze_never_loses_bits(); + + /* SECTION 13: audit findings framing */ + test_h1_refreeze_after_aborted_rebuild_preserves_delta(); + test_h1_close_merges_back_unconsumed_delta(); + test_h1_completed_rebuild_does_not_merge_back(); + test_h1_refreeze_enomem_preserves_previous_delta(); + test_h2_inflight_write_survives_clearing_freeze(); + test_c3_invalidate_and_evict_guards_running_rebuild(); + test_m1_hotremove_abort_classified_aborted_not_failed(); + printf("\n========================================================\n"); printf("Results: %d passed, %d failed\n", g_passed, g_failed); diff --git a/module/bdev/cbt/vbdev_cbt.c b/module/bdev/cbt/vbdev_cbt.c index a55cd14..b0df97d 100644 --- a/module/bdev/cbt/vbdev_cbt.c +++ b/module/bdev/cbt/vbdev_cbt.c @@ -10,6 +10,12 @@ * write/unmap/write_zeroes that flows through it. The bitmap is used to * drive incremental (partial) RAID rebuilds after backend outages. * + * Bitmap lifecycle is RESET-DRIVEN (D3): the bitmap only shrinks when the + * orchestrator explicitly calls bdev_cbt_reset after certifying all backends + * are healthy and in-sync. There is NO automatic healthy-clear poller — an + * in-target timer cannot know distributed backend health, and a wrong clear + * silently destroys the delta needed for partial rebuild. + * * Design reference: SPEC-52 §2. */ @@ -35,6 +41,7 @@ static void vbdev_cbt_finish(void); static int vbdev_cbt_get_ctx_size(void); static void vbdev_cbt_examine(struct spdk_bdev *bdev); static int vbdev_cbt_config_json(struct spdk_json_write_ctx *w); +static uint64_t cbt_count_dirty_bits(const uint8_t *bitmap, uint64_t size_bytes); /* ================================================================== */ /* Module registration */ @@ -144,6 +151,57 @@ cbt_any_epoch_open(struct vbdev_cbt *cbt) return false; } +/* Epochs share the single live bitmap: snapshot-and-clear at freeze is only + * safe when no OTHER epoch still needs the accumulated view. */ +static bool +cbt_has_other_active_epoch(struct vbdev_cbt *cbt, struct cbt_epoch *self) +{ + struct cbt_epoch *ep; + + TAILQ_FOREACH(ep, &cbt->epochs, link) { + if (ep == self) { + continue; + } + if (ep->state == CBT_EPOCH_OPEN || ep->state == CBT_EPOCH_FROZEN || + ep->state == CBT_EPOCH_REBUILDING) { + return true; + } + } + return false; +} + +/* H1: OR an unconsumed frozen delta back into the live bitmap before its + * buffer is discarded (re-freeze, close, evict). Bits exchanged out of the + * live bitmap at freeze exist ONLY in bitmap_frozen until a rebuild COMPLETES; + * freeing that buffer without this merge-back permanently loses the chunks — + * a later "successful" delta rebuild then promotes a silently divergent + * member under skip_rebuild. + * + * Pessimistic by design: chunks the aborted rebuild DID copy are re-marked + * too and get re-copied on the next iteration — wasted bandwidth, never lost + * data. Lock-free: per-byte atomic OR, same discipline as the IO-thread + * markers; caller is the app thread and all call sites are guarded against a + * concurrently RUNNING rebuild, so bitmap_frozen has no concurrent reader. */ +static void +cbt_epoch_restore_unconsumed_delta(struct vbdev_cbt *cbt, struct cbt_epoch *ep) +{ + uint64_t restored = 0; + + if (!ep->frozen_live_consumed || ep->bitmap_frozen == NULL) { + return; + } + for (uint64_t i = 0; i < cbt->bitmap_size_bytes; i++) { + uint8_t b = ep->bitmap_frozen[i]; + if (b != 0) { + __atomic_fetch_or(&cbt->bitmap[i], b, __ATOMIC_RELAXED); + restored += (uint64_t)__builtin_popcount(b); + } + } + ep->frozen_live_consumed = false; + SPDK_NOTICELOG("CBT: epoch '%s' unconsumed delta merged back into live bitmap " + "(%lu chunks)\n", ep->epoch_id, (unsigned long)restored); +} + /* ================================================================== */ /* Bitmap operations (hot path — may run on any reactor thread) */ /* Uses atomic OR so concurrent IO threads cannot lose bits. */ @@ -155,8 +213,10 @@ cbt_any_epoch_open(struct vbdev_cbt *cbt) /* - total_writes_tracked uses relaxed add (stats only) */ /* ================================================================== */ +/* Core bit-setter, shared by the submit-time mark and the completion-time + * re-mark (H2). No statistics side effect. */ static inline void -cbt_mark_dirty(struct vbdev_cbt *cbt, uint64_t offset_blocks, uint64_t num_blocks) +cbt_mark_dirty_bits(struct vbdev_cbt *cbt, uint64_t offset_blocks, uint64_t num_blocks) { uint64_t chunk_start, chunk_end; @@ -202,43 +262,25 @@ cbt_mark_dirty(struct vbdev_cbt *cbt, uint64_t offset_blocks, uint64_t num_block uint8_t last_mask = (uint8_t)(0xFF >> (7 - (chunk_end & 7))); __atomic_fetch_or(&cbt->bitmap[byte_end], last_mask, __ATOMIC_RELAXED); } - - __atomic_fetch_add(&cbt->total_writes_tracked, 1, __ATOMIC_RELAXED); } -/* ================================================================== */ -/* Healthy-clear poller */ -/* ================================================================== */ - -static int -cbt_healthy_clear_poller_fn(void *ctx) +static inline void +cbt_mark_dirty(struct vbdev_cbt *cbt, uint64_t offset_blocks, uint64_t num_blocks) { - struct vbdev_cbt *cbt = ctx; - - /* If any epoch is open/frozen/rebuilding, do NOT clear. */ - if (cbt->healthy_clear_suspended || cbt_any_epoch_open(cbt)) { - return SPDK_POLLER_IDLE; - } - - /* Only clear if the orchestrator has explicitly confirmed all - * backends are healthy and in-sync. - */ - if (!cbt->backends_healthy) { - return SPDK_POLLER_IDLE; - } + cbt_mark_dirty_bits(cbt, offset_blocks, num_blocks); + __atomic_fetch_add(&cbt->total_writes_tracked, 1, __ATOMIC_RELAXED); +} - /* Clear the bitmap — all backends are confirmed healthy. - * This runs on the owner thread (same as epoch ops). IO threads - * may concurrently set bits via atomic OR; a few bits set between - * our check and the memset are acceptable (they will be re-set on - * the next write and caught by the next clear cycle). - */ - if (cbt_popcount_bitmap(cbt) > 0) { - memset(cbt->bitmap, 0, cbt->bitmap_size_bytes); - } +/* D3: the healthy-clear poller was removed — it required an explicit + * bdev_cbt_set_backends_healthy() signal that no caller ever sent (dead code), + * so the bitmap only tended towards 100% dirty. Clearing is reset-driven: + * the orchestrator calls bdev_cbt_reset (refused while any epoch is active). */ - return SPDK_POLLER_IDLE; -} +/* Forward declaration (rebuild registry, defined below) — used by the epoch + * state guards CBT-1/CBT-2/c5. */ +struct cbt_rebuild_ctx; +static struct cbt_rebuild_ctx *cbt_rebuild_find_active_for_epoch(const struct vbdev_cbt *cbt, + const char *epoch_id); /* ================================================================== */ /* IO forwarding (passthrough + tracking) */ @@ -263,6 +305,35 @@ _cbt_complete_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) { struct spdk_bdev_io *orig_io = cb_arg; + /* H2: RE-mark the dirty bit at completion, BEFORE acking the host. + * + * The submit-time mark alone is not enough under snapshot-and-clear + * freeze: a freeze can exchange-and-consume the bit while this write is + * still in flight to the base. The rebuild may then read the chunk + * before the write lands, and the bit exists in no bitmap afterwards — + * the chunk silently diverges forever. Re-marking here closes that + * window without any drain or freeze of host I/O: + * - write landed before the rebuild read → the consumed bit is fine, + * the rebuild copies the new data; + * - write lands after → this re-mark puts the bit + * in the (new) live bitmap → captured by the next delta. + * Re-mark also on FAILURE: a failed write may have partially reached + * media. The submit-time mark stays for crash conservatism. + * Cost: one relaxed atomic OR on a byte whose cacheline the submit-time + * mark touched moments ago. */ + switch (orig_io->type) { + case SPDK_BDEV_IO_TYPE_WRITE: + case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: + case SPDK_BDEV_IO_TYPE_UNMAP: + case SPDK_BDEV_IO_TYPE_COPY: + cbt_mark_dirty_bits(SPDK_CONTAINEROF(orig_io->bdev, struct vbdev_cbt, cbt_bdev), + orig_io->u.bdev.offset_blocks, + orig_io->u.bdev.num_blocks); + break; + default: + break; + } + spdk_bdev_io_complete_base_io_status(orig_io, bdev_io); spdk_bdev_free_io(bdev_io); } @@ -519,10 +590,6 @@ vbdev_cbt_destruct(void *ctx) TAILQ_REMOVE(&g_cbt_nodes, cbt_node, link); - if (cbt_node->healthy_poller) { - spdk_poller_unregister(&cbt_node->healthy_poller); - } - spdk_bdev_module_release_bdev(cbt_node->base_bdev); if (cbt_node->thread && cbt_node->thread != spdk_get_thread()) { @@ -582,9 +649,24 @@ vbdev_cbt_base_bdev_event_cb(enum spdk_bdev_event_type type, { if (type == SPDK_BDEV_EVENT_REMOVE) { struct vbdev_cbt *node, *tmp; + struct cbt_bdev_name *name, *ntmp; TAILQ_FOREACH_SAFE(node, &g_cbt_nodes, link, tmp) { if (bdev == node->base_bdev) { + /* c4: drop the deferred-create entry too — otherwise a + * reappearing base bdev silently recreates the cbt vbdev + * with a VIRGIN bitmap that masquerades as continuous + * tracking history. Recreation must be an explicit + * bdev_cbt_create from the orchestrator. */ + TAILQ_FOREACH_SAFE(name, &g_bdev_names, link, ntmp) { + if (strcmp(name->vbdev_name, + spdk_bdev_get_name(&node->cbt_bdev)) == 0) { + TAILQ_REMOVE(&g_bdev_names, name, link); + free(name->bdev_name); + free(name->vbdev_name); + free(name); + } + } spdk_bdev_unregister(&node->cbt_bdev, NULL, NULL); } } @@ -736,11 +818,6 @@ vbdev_cbt_register(const char *bdev_name) return rc; } - /* ── Register healthy-clear poller (always-on) ── */ - cbt_node->healthy_poller = SPDK_POLLER_REGISTER( - cbt_healthy_clear_poller_fn, cbt_node, - CBT_HEALTHY_CLEAR_INTERVAL_US); - SPDK_NOTICELOG("CBT: created vbdev '%s' over '%s' " "(chunk=%u KB, bitmap=%lu bytes, %lu chunks)\n", name->vbdev_name, bdev_name, @@ -889,6 +966,19 @@ bdev_cbt_epoch_open(const char *cbt_name, const char *epoch_id, ep = cbt_find_epoch(cbt, epoch_id); if (ep) { if (generation > ep->generation) { + /* c5 (requalified): a higher-generation takeover must NOT rip the + * epoch away from a rebuild that is scanning/writing its frozen + * bitmap — that corrupts the state machine and opens the CBT-1 UAF. + * Gate on an ACTUALLY-RUNNING rebuild, not on ep->state: a COMPLETED + * rebuild deliberately leaves the epoch in REBUILDING (finalize keeps + * it there), so keying on the state would refuse every later takeover + * forever (cancel_rebuild returns -EINVAL — no running rebuild — so + * the flow would deadlock). */ + if (cbt_rebuild_find_active_for_epoch(cbt, epoch_id) != NULL) { + SPDK_ERRLOG("CBT: epoch_open gen=%lu refused: epoch '%s' has an " + "active rebuild\n", (unsigned long)generation, epoch_id); + return -EBUSY; + } /* Replace with higher generation. */ ep->generation = generation; snprintf(ep->stale_backend_id, sizeof(ep->stale_backend_id), @@ -913,9 +1003,20 @@ bdev_cbt_epoch_open(const char *cbt_name, const char *epoch_id, oldest ? (int)oldest->state : -1); return -ENOSPC; } + /* C3 (defense-in-depth): with the epoch_invalidate rebuild guard an + * INVALID epoch cannot have a RUNNING rebuild (rebuild_start requires + * FROZEN/REBUILDING), but never free an epoch a rebuild ctx still + * points at — that is a UAF read (ctx->bitmap) AND write (finalize). */ + if (cbt_rebuild_find_active_for_epoch(cbt, oldest->epoch_id) != NULL) { + SPDK_ERRLOG("CBT: max epochs reached, epoch '%s' has an active " + "rebuild — refusing eviction\n", oldest->epoch_id); + return -ENOSPC; + } /* Safe to evict: COMPLETED or INVALID. */ SPDK_WARNLOG("CBT: max epochs reached, evicting '%s'\n", oldest->epoch_id); + /* H1: an INVALID epoch may still hold an unconsumed exchanged delta. */ + cbt_epoch_restore_unconsumed_delta(cbt, oldest); TAILQ_REMOVE(&cbt->epochs, oldest, link); cbt->epoch_count--; free(oldest->bitmap_frozen); @@ -936,9 +1037,6 @@ bdev_cbt_epoch_open(const char *cbt_name, const char *epoch_id, TAILQ_INSERT_TAIL(&cbt->epochs, ep, link); cbt->epoch_count++; - /* Suspend healthy-clear while any epoch is open. */ - cbt->healthy_clear_suspended = true; - SPDK_NOTICELOG("CBT: epoch_open '%s' for stale backend '%s' gen=%lu\n", epoch_id, stale_backend_id, (unsigned long)generation); return 0; @@ -965,36 +1063,83 @@ bdev_cbt_epoch_freeze(const char *cbt_name, const char *epoch_id) ep->state != CBT_EPOCH_REBUILDING) { return -EINVAL; } + /* CBT-1: a RUNNING rebuild holds ctx->bitmap = ep->bitmap_frozen and scans it + * asynchronously — freeing/reallocating it here is a use-after-free read. + * Same guard start_rebuild already applies, made symmetric. */ + if (cbt_rebuild_find_active_for_epoch(cbt, epoch_id) != NULL) { + SPDK_ERRLOG("CBT: epoch_freeze '%s' refused: rebuild in progress\n", epoch_id); + return -EBUSY; + } - /* Free previous frozen bitmap if re-freezing. */ - free(ep->bitmap_frozen); + /* H1: allocate the new snapshot BEFORE touching the old one — an ENOMEM + * must leave the epoch exactly as it was (the old code freed the only + * copy of the previous delta first, so a failed malloc bricked the epoch + * AND lost the un-copied chunks). */ + uint8_t *new_frozen = malloc(cbt->bitmap_size_bytes); + if (!new_frozen) { + return -ENOMEM; + } + + /* H1: re-freeze after a FAILED/ABORTED rebuild — the previous delta was + * exchanged out of the live bitmap and its un-copied chunks exist only in + * the old bitmap_frozen. Merge it back first: the exchange below then + * re-captures (old unconsumed delta ∪ writes since last freeze), which is + * exactly the correct retry set. */ + if (ep->bitmap_frozen != NULL) { + cbt_epoch_restore_unconsumed_delta(cbt, ep); + free(ep->bitmap_frozen); + } + ep->bitmap_frozen = new_frozen; - /* Allocate and snapshot the current bitmap into this epoch. + /* Snapshot the current bitmap into this epoch. * * Thread safety: IO threads write individual bits with atomic OR. * We read the bitmap here on the app thread. On x86/arm64, each byte - * read is atomic, so we never see a torn byte. The semantic guarantee - * is: any write that completed its IO callback before this function - * was called is guaranteed to be in the snapshot. Writes that are - * in-flight may or may not be captured. This is acceptable because - * the orchestrator must ensure all writes are drained before calling - * freeze if it needs a precise point-in-time snapshot. - * - * An atomic fence ensures visibility of all relaxed stores before - * we read. - */ - ep->bitmap_frozen = malloc(cbt->bitmap_size_bytes); - if (!ep->bitmap_frozen) { - return -ENOMEM; - } + * read is atomic, so we never see a torn byte. Any write that completed + * its IO callback before this function was called is guaranteed to be + * in the snapshot. A write still in flight may have its submit-time bit + * consumed by the exchange below, but that is harmless (H2): the + * completion path RE-marks the bit before acking the host, so the chunk + * either was copied with the new data (write landed before the rebuild + * read) or lands in the next delta (bit re-set in the live bitmap). No + * host-I/O drain is required around freeze. */ __atomic_thread_fence(__ATOMIC_ACQUIRE); - memcpy(ep->bitmap_frozen, cbt->bitmap, cbt->bitmap_size_bytes); + + if (!cbt_has_other_active_epoch(cbt, ep)) { + /* Snapshot-AND-CLEAR (atomic per-byte exchange): each freeze captures the + * DELTA since the previous freeze, so iterative partial rebuilds converge + * geometrically once copy_rate > dirty_rate. The previous accumulate-only + * semantics recopied the union of everything dirtied since epoch open on + * every iteration — residual was monotonically non-decreasing and the + * convergence loop could not terminate by design. + * + * Exchange (not memcpy+memset): a bit OR'd by an IO thread between the + * copy and the clear would be LOST — a missed chunk under skip_rebuild + * is silent data divergence. The exchange makes each concurrent OR land + * either before (captured in this snapshot) or after (tracked for the + * next delta). */ + for (uint64_t i = 0; i < cbt->bitmap_size_bytes; i++) { + ep->bitmap_frozen[i] = + __atomic_exchange_n(&cbt->bitmap[i], 0, __ATOMIC_ACQ_REL); + } + /* H1: these bits now exist ONLY here until a rebuild COMPLETES. */ + ep->frozen_live_consumed = true; + } else { + /* Another epoch still consumes the accumulated live view — snapshot only. + * Convergence is degraded (residual includes prior deltas) but correct. */ + memcpy(ep->bitmap_frozen, cbt->bitmap, cbt->bitmap_size_bytes); + ep->frozen_live_consumed = false; /* live bitmap untouched */ + SPDK_NOTICELOG("CBT: epoch_freeze '%s' without clear (other active epochs)\n", + epoch_id); + } ep->state = CBT_EPOCH_FROZEN; - SPDK_NOTICELOG("CBT: epoch_freeze '%s' (dirty=%lu/%lu)\n", + SPDK_NOTICELOG("CBT: epoch_freeze '%s' (dirty=%lu/%lu, live_after=%lu)\n", epoch_id, - (unsigned long)cbt_popcount_bitmap(cbt), - (unsigned long)cbt->bitmap_size_bits); + (unsigned long)cbt_count_dirty_bits(ep->bitmap_frozen, + cbt->bitmap_size_bytes), + (unsigned long)cbt->bitmap_size_bits, + (unsigned long)cbt_popcount_bitmap(cbt)); return 0; } @@ -1020,6 +1165,18 @@ bdev_cbt_epoch_close(const char *cbt_name, const char *epoch_id) if (ep->state == CBT_EPOCH_OPEN) { return -EINVAL; } + /* CBT-2: a RUNNING rebuild writes ctx->epoch->bitmap_frozen and + * ctx->epoch->state at completion — freeing the epoch under it is a + * use-after-free WRITE. Cancel the rebuild first. */ + if (cbt_rebuild_find_active_for_epoch(cbt, epoch_id) != NULL) { + SPDK_ERRLOG("CBT: epoch_close '%s' refused: rebuild in progress\n", epoch_id); + return -EBUSY; + } + + /* H1: closing an epoch must not lose dirty history. If the frozen delta + * was exchanged out of the live bitmap and never proven copied (rebuild + * aborted/failed/never run), merge it back before discarding. */ + cbt_epoch_restore_unconsumed_delta(cbt, ep); ep->state = CBT_EPOCH_COMPLETED; TAILQ_REMOVE(&cbt->epochs, ep, link); @@ -1027,11 +1184,6 @@ bdev_cbt_epoch_close(const char *cbt_name, const char *epoch_id) free(ep->bitmap_frozen); free(ep); - /* Resume healthy-clear if no more active epochs. */ - if (!cbt_any_epoch_open(cbt)) { - cbt->healthy_clear_suspended = false; - } - SPDK_NOTICELOG("CBT: epoch_close '%s'\n", epoch_id); return 0; } @@ -1054,6 +1206,16 @@ bdev_cbt_epoch_invalidate(const char *cbt_name, const char *epoch_id) return -ENOENT; } + /* C3: same guard as freeze (CBT-1) and close (CBT-2). Invalidating a + * REBUILDING epoch makes it evictable by epoch_open's max-epochs path, + * which would free ep->bitmap_frozen and ep under the RUNNING rebuild — + * UAF read in the chunk scanner, UAF write in finalize. */ + if (cbt_rebuild_find_active_for_epoch(cbt, epoch_id) != NULL) { + SPDK_ERRLOG("CBT: epoch_invalidate '%s' refused: rebuild in progress " + "(cancel it first)\n", epoch_id); + return -EBUSY; + } + ep->state = CBT_EPOCH_INVALID; SPDK_WARNLOG("CBT: epoch_invalidate '%s' → full rebuild required\n", epoch_id); return 0; @@ -1200,6 +1362,14 @@ struct cbt_rebuild_ctx { struct cbt_epoch *epoch; struct spdk_bdev_desc *src_desc; struct spdk_bdev_desc *dst_desc; + /* Lifetime pin (UAF guard): `cbt` and `epoch`/`bitmap` are BARE pointers into + * the cbt vbdev, but the rebuild reads its geometry/bitmap on every chunk. A + * base hot-remove unregisters+frees the cbt vbdev; when the read source is the + * cbt bdev itself (default) src_desc already pins it, but with a source_bdev_name + * OVERRIDE nothing did, so the free raced the rebuild. This extra descriptor on + * the cbt bdev makes SPDK defer the cbt destruct until the rebuild's cleanup + * closes it. NULL when src_desc already covers the cbt bdev. */ + struct spdk_bdev_desc *cbt_pin_desc; struct spdk_io_channel *src_ch; struct spdk_io_channel *dst_ch; @@ -1276,12 +1446,14 @@ cbt_rebuild_find_by_id(const char *rebuild_id) return NULL; } +/* New: discriminate by cbt node too — two cbt vbdevs may legitimately use the + * same epoch_id (the old name-only match made them block each other). */ static struct cbt_rebuild_ctx * -cbt_rebuild_find_active_for_epoch(const char *epoch_id) +cbt_rebuild_find_active_for_epoch(const struct vbdev_cbt *cbt, const char *epoch_id) { struct cbt_rebuild_ctx *ctx; TAILQ_FOREACH(ctx, &g_rebuild_registry, registry_link) { - if (ctx->state == CBT_REBUILD_RUNNING && + if (ctx->state == CBT_REBUILD_RUNNING && ctx->cbt == cbt && strcmp(ctx->epoch->epoch_id, epoch_id) == 0) { return ctx; } @@ -1307,6 +1479,12 @@ cbt_rebuild_registry_cleanup(struct cbt_rebuild_ctx *ctx) if (ctx->dst_desc) { spdk_bdev_close(ctx->dst_desc); } + if (ctx->cbt_pin_desc) { + /* Releasing the lifetime pin: lets a cbt destruct deferred by a hot-remove + * finally proceed (see cbt_pin_desc). */ + spdk_bdev_close(ctx->cbt_pin_desc); + ctx->cbt_pin_desc = NULL; + } for (int i = 0; i < ctx->num_slots; i++) { if (ctx->slots[i].buf) { spdk_dma_free(ctx->slots[i].buf); @@ -1383,9 +1561,20 @@ static void cbt_rebuild_base_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) { - /* If the bdev is removed mid-rebuild, the IO callbacks will - * return failure and the rebuild will abort gracefully. - */ + struct cbt_rebuild_ctx *ctx = event_ctx; + + /* A REMOVE of the source, target, OR the pinned cbt bdev aborts the rebuild. + * Marking aborted stops it at the next chunk boundary; the in-flight I/O + * drains and the terminal path closes every descriptor (including the cbt + * lifetime pin) — only then does a hot-remove-driven cbt destruct proceed. + * When the read source is the cbt bdev itself and no I/O is in flight the old + * "let the I/O fail" heuristic could stall, so set the flag explicitly. */ + if (type == SPDK_BDEV_EVENT_REMOVE && ctx != NULL && + ctx->state == CBT_REBUILD_RUNNING) { + SPDK_WARNLOG("CBT rebuild: bdev '%s' removed — aborting\n", + spdk_bdev_get_name(bdev)); + ctx->aborted = true; + } } /* Maximum number of chunks to coalesce into a single I/O. @@ -1573,10 +1762,18 @@ cbt_rebuild_throttle_poller_fn(void *arg) ctx->window_start_tsc = now; if (ctx->throttled) { ctx->throttled = false; + /* R10: on the legacy path cbt_rebuild_submit_next may drain the last + * outstanding I/O and free(ctx) (submit_next → finish → finalize → + * TAILQ_REMOVE + free). Do NOT dereference ctx afterward. We un-throttled + * and drove work, so report BUSY without touching ctx; finalize already + * unregistered this poller, so the return value is only ever consumed + * while ctx is still alive. */ cbt_rebuild_submit_next(ctx); + return SPDK_POLLER_BUSY; } } + /* Not reached via submit_next → ctx is alive here. */ return ctx->throttled ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; } @@ -1661,8 +1858,56 @@ cbt_rebuild_submit_next(struct cbt_rebuild_ctx *ctx) } } +static void cbt_rebuild_finalize(struct cbt_rebuild_ctx *ctx); + +static void +cbt_rebuild_flush_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct cbt_rebuild_ctx *ctx = cb_arg; + + spdk_bdev_free_io(bdev_io); + if (!success) { + SPDK_ERRLOG("CBT rebuild: target flush failed — member NOT durably synced\n"); + ctx->error = -EIO; + ctx->aborted = true; + } + cbt_rebuild_finalize(ctx); +} + +/* CBT-4: all chunk writes landed — FLUSH the target before declaring the rebuild + * COMPLETED. Without it the copied chunks may sit in a volatile write cache; a + * power cut then leaves a lacunary member that the control-plane believes synced. */ static void cbt_rebuild_finish(struct cbt_rebuild_ctx *ctx) +{ + /* M1: !aborted — a hot-remove abort must reach finalize with error == 0 + * so R1 classifies it ABORTED (resumable), not FAILED. Flushing the + * (possibly removed) target here would fail, overwrite the + * classification, and durability of a partial copy is moot anyway: the + * resume re-copies from the merged-back delta (H1). */ + if (ctx->error == 0 && !ctx->cancelled && !ctx->aborted && + ctx->chunks_copied > 0 && + ctx->dst_desc != NULL && ctx->dst_ch != NULL) { + struct spdk_bdev *dst = spdk_bdev_desc_get_bdev(ctx->dst_desc); + + if (spdk_bdev_io_type_supported(dst, SPDK_BDEV_IO_TYPE_FLUSH)) { + int rc = spdk_bdev_flush_blocks(ctx->dst_desc, ctx->dst_ch, 0, + spdk_bdev_get_num_blocks(dst), + cbt_rebuild_flush_cb, ctx); + if (rc == 0) { + return; /* finalize from the flush callback */ + } + SPDK_ERRLOG("CBT rebuild: target flush submit failed rc=%d\n", rc); + ctx->error = rc; + ctx->aborted = true; + } + /* No FLUSH support ⇒ no volatile cache to drain — fall through. */ + } + cbt_rebuild_finalize(ctx); +} + +static void +cbt_rebuild_finalize(struct cbt_rebuild_ctx *ctx) { struct cbt_rebuild_result result = {0}; uint64_t elapsed_tsc = spdk_get_ticks() - ctx->start_tsc; @@ -1674,7 +1919,10 @@ cbt_rebuild_finish(struct cbt_rebuild_ctx *ctx) result.bytes_copied = ctx->bytes_copied; result.duration_ms = elapsed_tsc * 1000 / hz; result.error = ctx->error; - result.completed = (ctx->error == 0 && !ctx->cancelled); + /* R1: an ABORTED rebuild (hot-remove mid-flight) is neither a success nor a + * plain I/O error. It must NOT report completed, or the control-plane marks the + * member fully synced after a mid-rebuild abort → silent under-replication. */ + result.completed = (ctx->error == 0 && !ctx->cancelled && !ctx->aborted); /* ── Instrumentation report ── */ avg_read_us = ctx->chunks_copied ? @@ -1706,6 +1954,11 @@ cbt_rebuild_finish(struct cbt_rebuild_ctx *ctx) /* Compute residual dirty ratio on successful completion. */ if (result.completed && ctx->cbt->bitmap_size_bits > 0) { + /* H1: every chunk of the exchanged delta is now proven copied — the + * frozen buffer no longer holds the only copy of anything, and the + * memcpy below repurposes it as a residual snapshot of the live + * bitmap. It must NOT be merged back on a later discard. */ + ctx->epoch->frozen_live_consumed = false; __atomic_thread_fence(__ATOMIC_ACQUIRE); memcpy(ctx->epoch->bitmap_frozen, ctx->cbt->bitmap, ctx->cbt->bitmap_size_bytes); @@ -1723,11 +1976,16 @@ cbt_rebuild_finish(struct cbt_rebuild_ctx *ctx) /* Cleanup I/O resources. */ cbt_rebuild_registry_cleanup(ctx); - /* ── Finalize state for the registry ── */ + /* ── Finalize state for the registry ── + * R1: order matters. An I/O failure sets BOTH error and aborted (read/write_cb), + * so check error before aborted → it maps to FAILED. A hot-remove abort sets + * ONLY aborted (error == 0) → ABORTED. Neither is COMPLETED. */ if (ctx->cancelled) { ctx->state = CBT_REBUILD_CANCELLED; } else if (ctx->error != 0) { ctx->state = CBT_REBUILD_FAILED; + } else if (ctx->aborted) { + ctx->state = CBT_REBUILD_ABORTED; } else { ctx->state = CBT_REBUILD_COMPLETED; } @@ -1751,14 +2009,19 @@ cbt_rebuild_finish(struct cbt_rebuild_ctx *ctx) */ } -int -bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, - const char *target_bdev_name, - const char *source_bdev_name, - uint64_t max_bw_mb_sec, uint32_t queue_depth, - const struct cbt_rebuild_range *override_ranges, - uint32_t num_ranges, - cbt_rebuild_done_cb cb_fn, void *cb_arg) +/* Shared engine start for both the legacy (deferred-response) and async + * (rebuild_id) paths. rebuild_id, when non-NULL, is stamped on the context + * BEFORE the first I/O can complete — a zero-dirty bitmap finishes + * SYNCHRONOUSLY, and the old tag-after-return dance freed the context before + * the id was set (get_rebuild_status then returned -ENOENT forever). */ +static int +cbt_rebuild_start(const char *cbt_name, const char *epoch_id, + const char *target_bdev_name, + const char *source_bdev_name, + uint64_t max_bw_mb_sec, uint32_t queue_depth, + const struct cbt_rebuild_range *override_ranges, + uint32_t num_ranges, const char *rebuild_id, + cbt_rebuild_done_cb cb_fn, void *cb_arg) { struct vbdev_cbt *cbt; struct cbt_epoch *ep; @@ -1784,6 +2047,11 @@ bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, if (!ep->bitmap_frozen) { return -EINVAL; } + /* New: the legacy RPC path had NO anti-double-rebuild guard — two concurrent + * rebuilds would share ep->bitmap_frozen. One rebuild per (cbt, epoch). */ + if (cbt_rebuild_find_active_for_epoch(cbt, epoch_id) != NULL) { + return -EBUSY; + } /* Validate queue_depth. */ if (queue_depth == 0) { @@ -1809,6 +2077,11 @@ bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, ctx->start_tsc = spdk_get_ticks(); ctx->window_start_tsc = ctx->start_tsc; ctx->bitmap = ep->bitmap_frozen; + if (rebuild_id != NULL) { + /* Stamp the id NOW (see function comment): a synchronous finish must + * already see it so the ctx survives in the registry for get_status. */ + snprintf(ctx->rebuild_id, sizeof(ctx->rebuild_id), "%s", rebuild_id); + } /* Copy override ranges if provided. */ if (override_ranges && num_ranges > 0) { @@ -1828,17 +2101,34 @@ bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, src_name = source_bdev_name ? source_bdev_name : spdk_bdev_get_name(&cbt->cbt_bdev); rc = spdk_bdev_open_ext(src_name, false, - cbt_rebuild_base_event_cb, NULL, &ctx->src_desc); + cbt_rebuild_base_event_cb, ctx, &ctx->src_desc); if (rc != 0) { free(ctx->override_ranges); free(ctx); return rc; } + /* UAF guard: pin the cbt vbdev's lifetime for the whole rebuild. `ctx->cbt` + * / `ctx->epoch` / `ctx->bitmap` are bare pointers into it, dereferenced on + * every chunk; a base hot-remove would otherwise free the cbt vbdev underneath + * a running rebuild. When the read source IS the cbt bdev, src_desc already + * pins it — only a source override needs a dedicated pin descriptor. */ + if (strcmp(src_name, spdk_bdev_get_name(&cbt->cbt_bdev)) != 0) { + rc = spdk_bdev_open_ext(spdk_bdev_get_name(&cbt->cbt_bdev), false, + cbt_rebuild_base_event_cb, ctx, &ctx->cbt_pin_desc); + if (rc != 0) { + spdk_bdev_close(ctx->src_desc); + free(ctx->override_ranges); + free(ctx); + return rc; + } + } + /* Open target bdev. */ rc = spdk_bdev_open_ext(target_bdev_name, true, - cbt_rebuild_base_event_cb, NULL, &ctx->dst_desc); + cbt_rebuild_base_event_cb, ctx, &ctx->dst_desc); if (rc != 0) { + if (ctx->cbt_pin_desc) spdk_bdev_close(ctx->cbt_pin_desc); spdk_bdev_close(ctx->src_desc); free(ctx->override_ranges); free(ctx); @@ -1851,6 +2141,7 @@ bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, if (!ctx->src_ch || !ctx->dst_ch) { if (ctx->src_ch) spdk_put_io_channel(ctx->src_ch); if (ctx->dst_ch) spdk_put_io_channel(ctx->dst_ch); + if (ctx->cbt_pin_desc) spdk_bdev_close(ctx->cbt_pin_desc); spdk_bdev_close(ctx->src_desc); spdk_bdev_close(ctx->dst_desc); free(ctx->override_ranges); @@ -1868,6 +2159,7 @@ bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, if (!ctx->slots) { spdk_put_io_channel(ctx->src_ch); spdk_put_io_channel(ctx->dst_ch); + if (ctx->cbt_pin_desc) spdk_bdev_close(ctx->cbt_pin_desc); spdk_bdev_close(ctx->src_desc); spdk_bdev_close(ctx->dst_desc); free(ctx->override_ranges); @@ -1885,6 +2177,7 @@ bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, free(ctx->slots); spdk_put_io_channel(ctx->src_ch); spdk_put_io_channel(ctx->dst_ch); + if (ctx->cbt_pin_desc) spdk_bdev_close(ctx->cbt_pin_desc); spdk_bdev_close(ctx->src_desc); spdk_bdev_close(ctx->dst_desc); free(ctx->override_ranges); @@ -1913,9 +2206,10 @@ bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, cbt_rebuild_gc_poller_fn, NULL, 10000000); /* 10s */ } - SPDK_NOTICELOG("CBT: partial_rebuild started for '%s' epoch '%s' → '%s' " - "(qd=%d, bw_limit=%lu MB/s, coalesce=%d chunks/io)\n", + SPDK_NOTICELOG("CBT: rebuild started for '%s' epoch '%s' → '%s' " + "(id=%s, qd=%d, bw_limit=%lu MB/s, coalesce=%d chunks/io)\n", cbt_name, epoch_id, target_bdev_name, + rebuild_id != NULL ? rebuild_id : "-", ctx->max_outstanding, (unsigned long)max_bw_mb_sec, CBT_REBUILD_MAX_COALESCE_CHUNKS); @@ -1925,6 +2219,20 @@ bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, return 0; } +int +bdev_cbt_partial_rebuild(const char *cbt_name, const char *epoch_id, + const char *target_bdev_name, + const char *source_bdev_name, + uint64_t max_bw_mb_sec, uint32_t queue_depth, + const struct cbt_rebuild_range *override_ranges, + uint32_t num_ranges, + cbt_rebuild_done_cb cb_fn, void *cb_arg) +{ + return cbt_rebuild_start(cbt_name, epoch_id, target_bdev_name, source_bdev_name, + max_bw_mb_sec, queue_depth, override_ranges, num_ranges, + NULL, cb_fn, cb_arg); +} + /* ================================================================== */ /* Async rebuild API (Phase 2) */ /* ================================================================== */ @@ -1936,54 +2244,26 @@ bdev_cbt_start_rebuild(const char *cbt_name, const char *epoch_id, uint64_t max_bw_mb_sec, uint32_t queue_depth, char *out_rebuild_id) { - struct vbdev_cbt *cbt; - struct cbt_epoch *ep; - struct cbt_rebuild_ctx *ctx; uint64_t id; int rc; assert(spdk_get_thread() == spdk_thread_get_app_thread()); - /* Validate early — same checks as bdev_cbt_partial_rebuild. */ - cbt = cbt_find_by_name(cbt_name); - if (!cbt) { - return -ENODEV; - } - ep = cbt_find_epoch(cbt, epoch_id); - if (!ep) { - return -ENOENT; - } - if (ep->state != CBT_EPOCH_FROZEN && ep->state != CBT_EPOCH_REBUILDING) { - return -EINVAL; - } - if (cbt_rebuild_find_active_for_epoch(epoch_id)) { - return -EBUSY; - } - - /* Generate a unique rebuild_id. */ + /* Generate the rebuild_id FIRST: cbt_rebuild_start stamps it on the context + * before any I/O, so even a synchronous finish (zero dirty chunks) leaves a + * queryable registry entry (the old post-hoc tagging raced exactly that). */ id = ++g_rebuild_id_counter; snprintf(out_rebuild_id, CBT_REBUILD_ID_MAX, "rebuild-%lu", (unsigned long)id); - /* Start the rebuild using the existing engine (no callback — async model). */ - rc = bdev_cbt_partial_rebuild(cbt_name, epoch_id, target_bdev_name, - source_bdev_name, max_bw_mb_sec, - queue_depth, NULL, 0, NULL, NULL); + rc = cbt_rebuild_start(cbt_name, epoch_id, target_bdev_name, + source_bdev_name, max_bw_mb_sec, + queue_depth, NULL, 0, out_rebuild_id, NULL, NULL); if (rc != 0) { + /* No rebuild was created — clear the pre-stamped id so a caller that + * ignores rc cannot query/persist a phantom "rebuild-N". */ + out_rebuild_id[0] = '\0'; return rc; } - - /* Tag the just-created context with its rebuild_id. - * It's the last entry in the registry (we just inserted it). - */ - TAILQ_FOREACH_REVERSE(ctx, &g_rebuild_registry, cbt_rebuild_registry_head, - registry_link) { - if (ctx->state == CBT_REBUILD_RUNNING && ctx->rebuild_id[0] == '\0') { - snprintf(ctx->rebuild_id, sizeof(ctx->rebuild_id), - "%s", out_rebuild_id); - break; - } - } - return 0; } @@ -2141,19 +2421,6 @@ bdev_cbt_reset(const char *cbt_name) return 0; } -void -bdev_cbt_set_backends_healthy(const char *cbt_name, bool healthy) -{ - struct vbdev_cbt *cbt = cbt_find_by_name(cbt_name); - - if (!cbt) { - return; - } - - cbt->backends_healthy = healthy; - SPDK_NOTICELOG("CBT: '%s' backends_healthy=%d\n", cbt_name, (int)healthy); -} - /* ================================================================== */ /* Module lifecycle */ /* ================================================================== */ diff --git a/module/bdev/cbt/vbdev_cbt.h b/module/bdev/cbt/vbdev_cbt.h index 723bbe0..ba3ecf2 100644 --- a/module/bdev/cbt/vbdev_cbt.h +++ b/module/bdev/cbt/vbdev_cbt.h @@ -21,7 +21,6 @@ extern "C" { #define CBT_CHUNK_SIZE_MAX_KB 65536 /* 64 MB */ #define CBT_MAX_EPOCHS 4 #define CBT_MAX_RANGES_LIMIT 65536 -#define CBT_HEALTHY_CLEAR_INTERVAL_US 5000000 /* 5 s */ #define CBT_EPOCH_ID_MAX 64 #define CBT_BACKEND_ID_MAX 128 #define CBT_REBUILD_DEFAULT_QD 16 @@ -36,6 +35,10 @@ enum cbt_rebuild_state { CBT_REBUILD_COMPLETED = 1, CBT_REBUILD_FAILED = 2, CBT_REBUILD_CANCELLED = 3, + /* R1: a rebuild aborted by a source/target/cbt hot-remove mid-flight. Distinct + * from FAILED (an I/O error) and COMPLETED: the delta was NOT fully copied, so + * the control-plane must RESUME it, not treat the member as synced. */ + CBT_REBUILD_ABORTED = 4, }; /* ── Epoch lifecycle ───────────────────────────────────────────────── */ @@ -57,6 +60,13 @@ struct cbt_epoch { /* Per-epoch frozen bitmap (allocated on freeze, freed on close). */ uint8_t *bitmap_frozen; + /* H1: true while bitmap_frozen holds bits that were exchanged OUT of the + * live bitmap (snapshot-and-clear freeze) and not yet proven copied by a + * COMPLETED rebuild. Discarding such a buffer (re-freeze, close, evict) + * must first OR it back into the live bitmap — those chunks exist nowhere + * else and dropping them is silent divergence under skip_rebuild. */ + bool frozen_live_consumed; + TAILQ_ENTRY(cbt_epoch) link; }; @@ -223,12 +233,6 @@ int bdev_cbt_get_dirty_ranges(const char *cbt_name, uint32_t max_ranges, bool *out_truncated); int bdev_cbt_reset(const char *cbt_name); -/** - * Notify CBT that all backends are confirmed healthy and in-sync. - * Only after this call will the healthy-clear poller clear the bitmap. - */ -void bdev_cbt_set_backends_healthy(const char *cbt_name, bool healthy); - #ifdef __cplusplus } #endif diff --git a/module/bdev/cbt/vbdev_cbt_internal.h b/module/bdev/cbt/vbdev_cbt_internal.h index 8334526..982133e 100644 --- a/module/bdev/cbt/vbdev_cbt_internal.h +++ b/module/bdev/cbt/vbdev_cbt_internal.h @@ -29,11 +29,10 @@ struct vbdev_cbt { uint32_t chunk_shift; /* log2(chunk_size_blocks) for fast path */ uint64_t total_blocks; - /* ── Healthy-clear poller ── */ - struct spdk_poller *healthy_poller; - bool healthy_clear_suspended; - bool backends_healthy; /* explicit health signal from orchestrator */ - bool dirty_history_valid; /* false after restart until first epoch completes */ + /* D3: bitmap clearing is RESET-DRIVEN (bdev_cbt_reset, refused while an + * epoch is active). The automatic healthy-clear poller and its + * backends_healthy / dirty_history_valid signals were dead code (no + * caller ever set them) and were removed. */ /* ── Epoch management ── */ TAILQ_HEAD(, cbt_epoch) epochs; diff --git a/module/bdev/cbt/vbdev_cbt_rpc.c b/module/bdev/cbt/vbdev_cbt_rpc.c index 1d6c2d3..88e4503 100644 --- a/module/bdev/cbt/vbdev_cbt_rpc.c +++ b/module/bdev/cbt/vbdev_cbt_rpc.c @@ -465,8 +465,8 @@ rpc_bdev_cbt_epoch_list(struct spdk_jsonrpc_request *request, spdk_json_write_named_uint64(w, "total_chunks", cbt->bitmap_size_bits); spdk_json_write_named_double(w, "dirty_ratio", dirty_ratio); spdk_json_write_named_uint64(w, "total_writes_tracked", writes_tracked); - spdk_json_write_named_bool(w, "healthy_clear_suspended", cbt->healthy_clear_suspended); - spdk_json_write_named_bool(w, "backends_healthy", cbt->backends_healthy); + /* D3: healthy_clear_suspended / backends_healthy removed with the dead + * healthy-clear poller (bitmap clearing is reset-driven). */ spdk_json_write_named_array_begin(w, "epochs"); TAILQ_FOREACH(ep, &cbt->epochs, link) { @@ -679,8 +679,6 @@ struct rpc_cbt_partial_rebuild { char *source_bdev_name; uint64_t max_bandwidth_mb_sec; uint32_t queue_depth; - struct cbt_rebuild_range *override_ranges; - uint32_t num_override_ranges; }; static void @@ -690,77 +688,11 @@ free_rpc_cbt_partial_rebuild(struct rpc_cbt_partial_rebuild *r) free(r->epoch_id); free(r->target_bdev_name); free(r->source_bdev_name); - free(r->override_ranges); } -static int -decode_override_ranges(const struct spdk_json_val *val, void *out) -{ - struct rpc_cbt_partial_rebuild *req = out; - struct spdk_json_val *array_val = (struct spdk_json_val *)val; - uint32_t count, i; - struct cbt_rebuild_range *ranges; - - if (val->type != SPDK_JSON_VAL_ARRAY_BEGIN) { - return -EINVAL; - } - - count = val->len; - if (count == 0) { - req->override_ranges = NULL; - req->num_override_ranges = 0; - return 0; - } - - ranges = calloc(count, sizeof(*ranges)); - if (!ranges) { - return -ENOMEM; - } - - /* Step into array items. */ - array_val++; - for (i = 0; i < count; i++) { - /* Each element is an object with offset_blocks and length_blocks. */ - struct spdk_json_val *obj = array_val; - if (obj->type != SPDK_JSON_VAL_OBJECT_BEGIN) { - free(ranges); - return -EINVAL; - } - - uint32_t obj_size = obj->len; - struct spdk_json_val *key = obj + 1; - uint32_t j; - - for (j = 0; j < obj_size; j++) { - if (key->type != SPDK_JSON_VAL_NAME) { - break; - } - struct spdk_json_val *value = key + 1; - - if (spdk_json_strequal(key, "offset_blocks")) { - if (spdk_json_number_to_uint64(value, &ranges[i].offset_blocks)) { - free(ranges); - return -EINVAL; - } - } else if (spdk_json_strequal(key, "length_blocks")) { - if (spdk_json_number_to_uint64(value, &ranges[i].length_blocks)) { - free(ranges); - return -EINVAL; - } - } - key = value + spdk_json_val_len(value); - } - - /* Advance past this object. */ - array_val = obj + 1 + spdk_json_val_len(obj) - 1; - /* Actually skip entire object. */ - array_val = (struct spdk_json_val *)obj + spdk_json_val_len(obj); - } - - req->override_ranges = ranges; - req->num_override_ranges = count; - return 0; -} +/* Note: the C engine supports override_ranges, but this RPC exposes only the + * bitmap-driven mode (the former decode_override_ranges was ~70 lines of dead, + * never-wired parsing — removed; re-add via a proper decoder if ever needed). */ static const struct spdk_json_object_decoder rpc_cbt_partial_rebuild_decoders[] = { {"name", offsetof(struct rpc_cbt_partial_rebuild, name), spdk_json_decode_string}, @@ -815,12 +747,6 @@ rpc_bdev_cbt_partial_rebuild(struct spdk_jsonrpc_request *request, goto cleanup; } - /* Manually decode optional override_ranges array if present. */ - /* For simplicity, override_ranges are not decoded here — we only - * support bitmap-driven rebuild via this RPC. Override ranges can - * be added later if needed by extending the decoder. - */ - cb_arg = calloc(1, sizeof(*cb_arg)); if (!cb_arg) { spdk_jsonrpc_send_error_response(request, -ENOMEM, @@ -834,8 +760,7 @@ rpc_bdev_cbt_partial_rebuild(struct spdk_jsonrpc_request *request, req.source_bdev_name, req.max_bandwidth_mb_sec, req.queue_depth, - req.override_ranges, - req.num_override_ranges, + NULL, 0, rpc_rebuild_done_cb, cb_arg); if (rc != 0) { spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc)); @@ -966,6 +891,7 @@ rpc_bdev_cbt_get_rebuild_status(struct spdk_jsonrpc_request *request, case CBT_REBUILD_COMPLETED: state_str = "completed"; break; case CBT_REBUILD_FAILED: state_str = "failed"; break; case CBT_REBUILD_CANCELLED: state_str = "cancelled"; break; + case CBT_REBUILD_ABORTED: state_str = "aborted"; break; /* R1 */ default: state_str = "unknown"; break; } spdk_json_write_named_string(w, "state", state_str); diff --git a/module/bdev/tier/Makefile b/module/bdev/tier/Makefile new file mode 100644 index 0000000..277ee29 --- /dev/null +++ b/module/bdev/tier/Makefile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Evariops. +# All rights reserved. + +# SPDK_ROOT_DIR: use parent if module is in-tree, else use vendor/ path. +SPDK_ROOT_DIR ?= $(abspath $(CURDIR)/../../..) +ifeq ($(wildcard $(SPDK_ROOT_DIR)/mk/spdk.common.mk),) + SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../vendor/spdk) +endif +include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + +SO_VER := 1 +SO_MINOR := 0 + +CFLAGS += -I$(SPDK_ROOT_DIR)/lib/bdev/ + +C_SRCS = vbdev_tier.c vbdev_tier_sb.c vbdev_tier_rpc.c +LIBNAME = bdev_tier + +SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map + +include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk diff --git a/module/bdev/tier/test/Makefile b/module/bdev/tier/test/Makefile new file mode 100644 index 0000000..415ac9c --- /dev/null +++ b/module/bdev/tier/test/Makefile @@ -0,0 +1,23 @@ +# T1: host-compilable unit tests of the PRODUCTION tier code (see test_tier_sb.c). +# No SPDK build required — mock headers in mock/spdk. ASAN+UBSAN always on. + +CC ?= cc +CFLAGS ?= -O1 -g +CFLAGS += -std=gnu11 -Wall -Wextra -Werror -Wno-unused-function +CFLAGS += -fsanitize=address,undefined -fno-sanitize-recover=all +CFLAGS += -Imock -I.. + +BIN := test_tier_sb + +.PHONY: all run clean + +all: run + +$(BIN): test_tier_sb.c ../vbdev_tier_sb.c ../vbdev_tier.h $(wildcard mock/spdk/*.h) + $(CC) $(CFLAGS) -o $@ test_tier_sb.c + +run: $(BIN) + ./$(BIN) + +clean: + rm -rf $(BIN) $(BIN).dSYM diff --git a/module/bdev/tier/test/mock/spdk/assert.h b/module/bdev/tier/test/mock/spdk/assert.h new file mode 100644 index 0000000..e7ff9ec --- /dev/null +++ b/module/bdev/tier/test/mock/spdk/assert.h @@ -0,0 +1,9 @@ +/* Host-test mock of spdk/assert.h (T1). */ +#ifndef SPDK_ASSERT_MOCK_H +#define SPDK_ASSERT_MOCK_H + +#include + +#define SPDK_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg) + +#endif diff --git a/module/bdev/tier/test/mock/spdk/bdev.h b/module/bdev/tier/test/mock/spdk/bdev.h new file mode 100644 index 0000000..9450492 --- /dev/null +++ b/module/bdev/tier/test/mock/spdk/bdev.h @@ -0,0 +1,55 @@ +/* Host-test mock of spdk/bdev.h (T1) — just enough surface for vbdev_tier.h and + * vbdev_tier_sb.c to compile. I/O entry points are declared here and defined by + * the test harness (test_tier_sb.c), which records or fails them as the + * scenario requires. */ +#ifndef SPDK_BDEV_MOCK_H +#define SPDK_BDEV_MOCK_H + +#include "spdk/stdinc.h" + +struct spdk_io_channel; +struct spdk_bdev_io; +struct spdk_bdev_fn_table; +struct spdk_bdev_module; +struct spdk_bdev_desc; +struct spdk_json_write_ctx; + +struct spdk_bdev { + char *name; + const char *product_name; + uint32_t blocklen; + uint64_t blockcnt; + int write_cache; + void *ctxt; + const struct spdk_bdev_fn_table *fn_table; + struct spdk_bdev_module *module; +}; + +typedef void (*spdk_bdev_io_completion_cb)(struct spdk_bdev_io *bdev_io, bool success, + void *cb_arg); + +int spdk_bdev_write_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + void *buf, uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg); +int spdk_bdev_read_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + void *buf, uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg); +int spdk_bdev_flush_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg); +void spdk_bdev_free_io(struct spdk_bdev_io *bdev_io); +struct spdk_io_channel *spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc); +void spdk_put_io_channel(struct spdk_io_channel *ch); + +enum spdk_bdev_io_type { + SPDK_BDEV_IO_TYPE_INVALID = 0, + SPDK_BDEV_IO_TYPE_READ, + SPDK_BDEV_IO_TYPE_WRITE, + SPDK_BDEV_IO_TYPE_UNMAP, + SPDK_BDEV_IO_TYPE_FLUSH, +}; + +struct spdk_bdev *spdk_bdev_desc_get_bdev(struct spdk_bdev_desc *desc); +bool spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type); + +#endif diff --git a/module/bdev/tier/test/mock/spdk/bdev_module.h b/module/bdev/tier/test/mock/spdk/bdev_module.h new file mode 100644 index 0000000..21c91f3 --- /dev/null +++ b/module/bdev/tier/test/mock/spdk/bdev_module.h @@ -0,0 +1,7 @@ +/* Host-test mock of spdk/bdev_module.h (T1). */ +#ifndef SPDK_BDEV_MODULE_MOCK_H +#define SPDK_BDEV_MODULE_MOCK_H + +#include "spdk/bdev.h" + +#endif diff --git a/module/bdev/tier/test/mock/spdk/crc32.h b/module/bdev/tier/test/mock/spdk/crc32.h new file mode 100644 index 0000000..e21cb1f --- /dev/null +++ b/module/bdev/tier/test/mock/spdk/crc32.h @@ -0,0 +1,27 @@ +/* Host-test mock of spdk/crc32.h (T1) — real software CRC32C (Castagnoli), + * bit-reflected, same convention as SPDK's spdk_crc32c_update (init ~0, + * NOT post-inverted). Any self-consistent implementation validates the + * serialize/validate roundtrip; using the actual CRC32C polynomial keeps the + * fixture bytes meaningful. */ +#ifndef SPDK_CRC32_MOCK_H +#define SPDK_CRC32_MOCK_H + +#include "spdk/stdinc.h" + +static inline uint32_t +spdk_crc32c_update(const void *buf, size_t len, uint32_t crc) +{ + const uint8_t *p = buf; + size_t i; + int b; + + for (i = 0; i < len; i++) { + crc ^= p[i]; + for (b = 0; b < 8; b++) { + crc = (crc >> 1) ^ (0x82F63B78u & (0u - (crc & 1u))); + } + } + return crc; +} + +#endif diff --git a/module/bdev/tier/test/mock/spdk/env.h b/module/bdev/tier/test/mock/spdk/env.h new file mode 100644 index 0000000..3992d8c --- /dev/null +++ b/module/bdev/tier/test/mock/spdk/env.h @@ -0,0 +1,29 @@ +/* Host-test mock of spdk/env.h (T1) — DMA alloc maps to plain heap. */ +#ifndef SPDK_ENV_MOCK_H +#define SPDK_ENV_MOCK_H + +#include "spdk/stdinc.h" + +static inline void * +spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) +{ + (void)align; + (void)phys_addr; + return calloc(1, size); +} + +static inline void * +spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) +{ + (void)align; + (void)phys_addr; + return malloc(size); +} + +static inline void +spdk_dma_free(void *buf) +{ + free(buf); +} + +#endif diff --git a/module/bdev/tier/test/mock/spdk/log.h b/module/bdev/tier/test/mock/spdk/log.h new file mode 100644 index 0000000..da0668f --- /dev/null +++ b/module/bdev/tier/test/mock/spdk/log.h @@ -0,0 +1,11 @@ +/* Host-test mock of spdk/log.h (T1). */ +#ifndef SPDK_LOG_MOCK_H +#define SPDK_LOG_MOCK_H + +#include "spdk/stdinc.h" + +#define SPDK_ERRLOG(...) fprintf(stderr, "ERR: " __VA_ARGS__) +#define SPDK_WARNLOG(...) fprintf(stderr, "WARN: " __VA_ARGS__) +#define SPDK_NOTICELOG(...) fprintf(stderr, "NOTICE: " __VA_ARGS__) + +#endif diff --git a/module/bdev/tier/test/mock/spdk/stdinc.h b/module/bdev/tier/test/mock/spdk/stdinc.h new file mode 100644 index 0000000..f5969ff --- /dev/null +++ b/module/bdev/tier/test/mock/spdk/stdinc.h @@ -0,0 +1,25 @@ +/* Host-test mock of spdk/stdinc.h (T1) — standard C only. */ +#ifndef SPDK_STDINC_MOCK_H +#define SPDK_STDINC_MOCK_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* glibc's sys/queue.h lacks a few BSD macros; provide what the tier code uses. */ +#ifndef TAILQ_FOREACH_SAFE +#define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = TAILQ_FIRST((head)); \ + (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) +#endif + +#endif diff --git a/module/bdev/tier/test/mock/spdk/string.h b/module/bdev/tier/test/mock/spdk/string.h new file mode 100644 index 0000000..62587c3 --- /dev/null +++ b/module/bdev/tier/test/mock/spdk/string.h @@ -0,0 +1,13 @@ +/* Host-test mock of spdk/string.h (T1). */ +#ifndef SPDK_STRING_MOCK_H +#define SPDK_STRING_MOCK_H + +#include "spdk/stdinc.h" + +static inline const char * +spdk_strerror(int errnum) +{ + return strerror(errnum); +} + +#endif diff --git a/module/bdev/tier/test/mock/spdk/util.h b/module/bdev/tier/test/mock/spdk/util.h new file mode 100644 index 0000000..6e605f6 --- /dev/null +++ b/module/bdev/tier/test/mock/spdk/util.h @@ -0,0 +1,12 @@ +/* Host-test mock of spdk/util.h (T1). */ +#ifndef SPDK_UTIL_MOCK_H +#define SPDK_UTIL_MOCK_H + +#include "spdk/stdinc.h" + +#define SPDK_COUNTOF(a) (sizeof(a) / sizeof(*(a))) +#define spdk_divide_round_up(n, d) (((n) + (d) - 1) / (d)) +#define spdk_min(a, b) (((a) < (b)) ? (a) : (b)) +#define spdk_max(a, b) (((a) > (b)) ? (a) : (b)) + +#endif diff --git a/module/bdev/tier/test/test_tier_sb.c b/module/bdev/tier/test/test_tier_sb.c new file mode 100644 index 0000000..b83c8de --- /dev/null +++ b/module/bdev/tier/test/test_tier_sb.c @@ -0,0 +1,463 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 Evariops. + * + * T1: host-compilable unit tests of the PRODUCTION tier code (not a copy): + * this translation unit #includes vbdev_tier_sb.c directly, compiled against + * the minimal mock SPDK headers in mock/. Covers the v2 on-disk format: + * - ABI (sizeof/offsetof, doubling the SPDK_STATIC_ASSERTs at runtime) + * - tier_sb_serialize / tier_sb_valid roundtrip + corruption + version + + * byte-swapped-magic (F-4) rejection + * - F-5 A/B slot selection (tier_sb_select): highest valid seq wins, + * torn/invalid slot tolerated + * - F-2 generation_uuid + created_epoch_sec + u64 cluster_blocks (F-3) + * - band table serialization content, geometry inlines + * - a binary GOLDEN vector: the serialized header bytes are pinned so an + * accidental field reorder is caught even if sizeof stays equal + * + * NOT covered here (needs the full bdev runtime — see docs/audits T3 bench): + * I/O routing, fan-out, hot-remove, resync, write_all channel plumbing. + */ + +/* M2 framing: countdown-armed calloc fault injection for the production code + * compiled below (-1 = pass-through). test_calloc's own call binds to the real + * libc calloc (resolved before the macro exists). */ +#include +static int g_calloc_fail_countdown = -1; +static void * +test_calloc(size_t n, size_t sz) +{ + if (g_calloc_fail_countdown == 0) { + g_calloc_fail_countdown = -1; + return NULL; + } + if (g_calloc_fail_countdown > 0) { + g_calloc_fail_countdown--; + } + return calloc(n, sz); +} +#define calloc(n, sz) test_calloc((n), (sz)) + +#include "vbdev_tier_sb.c" /* PRODUCTION code under test */ + +static int g_failures; + +#define CHECK(cond) do { \ + if (!(cond)) { \ + g_failures++; \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + } \ +} while (0) + +/* ---- mock bdev I/O stubs (unused by the serialize/valid/select tests) ------- */ + +int +spdk_bdev_write_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + void *buf, uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + (void)desc; (void)ch; (void)buf; (void)offset_blocks; (void)num_blocks; + (void)cb; (void)cb_arg; + return -ENOTSUP; +} + +struct spdk_bdev * +spdk_bdev_desc_get_bdev(struct spdk_bdev_desc *desc) +{ + return (struct spdk_bdev *)desc; /* opaque round-trip for the mock */ +} + +bool +spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type) +{ + (void)bdev; (void)io_type; + return false; /* mirrors the -ENOTSUP flush stub: no-FLUSH base (bdev_uring) */ +} + +int +spdk_bdev_read_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + void *buf, uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + (void)desc; (void)ch; (void)buf; (void)offset_blocks; (void)num_blocks; + (void)cb; (void)cb_arg; + return -ENOTSUP; +} + +int +spdk_bdev_flush_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + (void)desc; (void)ch; (void)offset_blocks; (void)num_blocks; (void)cb; (void)cb_arg; + return -ENOTSUP; +} + +void +spdk_bdev_free_io(struct spdk_bdev_io *bdev_io) +{ + (void)bdev_io; +} + +struct spdk_io_channel * +spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc) +{ + (void)desc; + return NULL; +} + +/* T-4b: lives in vbdev_tier.c (not host-compilable). A no-op that reports + * "nothing deferred" (false) satisfies the link; the M2 framing test counts + * invocations to pin "every fan-out termination resolves deferred teardown". */ +static int g_fanout_idle_calls; +bool +vbdev_tier_sb_fanout_idle(struct vbdev_tier *t) +{ + (void)t; + g_fanout_idle_calls++; + return false; +} + +void +spdk_put_io_channel(struct spdk_io_channel *ch) +{ + (void)ch; +} + +/* ---- fixtures ---------------------------------------------------------------- */ + +static struct tier_band * +add_band(struct vbdev_tier *t, uint32_t id, enum tier_class cls, enum tier_band_state st, + uint64_t lba_start, uint64_t num_blocks, const char *wwn) +{ + struct tier_band *b = calloc(1, sizeof(*b)); + + assert(b != NULL); + b->band_id = id; + b->tier = cls; + b->state = st; + b->lba_start = lba_start; + b->num_blocks = num_blocks; + snprintf(b->wwn, sizeof(b->wwn), "%s", wwn); + snprintf(b->serial, sizeof(b->serial), "serial-%u", id); + snprintf(b->base_bdev_name, sizeof(b->base_bdev_name), "nvme%un1", id); + b->t = t; + TAILQ_INSERT_TAIL(&t->bands, b, link); + t->num_bands++; + return b; +} + +static void +make_composite(struct vbdev_tier *t) +{ + uint32_t i; + + memset(t, 0, sizeof(*t)); + TAILQ_INIT(&t->bands); + TAILQ_INIT(&t->sb_pending_cbs); + t->bdev.name = (char *)"tier0"; + t->blocklen = 4096; + t->sb_blocks = TIER_SB_RESERVE_BYTES / 4096; /* 64 */ + t->cluster_blocks = 256; /* 1 MiB clusters */ + t->md_num_blocks = 4096; /* cluster-aligned */ + t->md_mirror_a = 0; + t->md_mirror_b = 1; + t->seq = 41; + for (i = 0; i < TIER_SB_GEN_UUID_LEN; i++) { + t->gen_uuid[i] = (uint8_t)(0xA0 + i); + } + add_band(t, 0, TIER_ULTRA, TIER_BAND_ACTIVE, 4096, 100352, "wwn-a"); + add_band(t, 1, TIER_PREMIUM, TIER_BAND_ACTIVE, 104448, 50176, "wwn-b"); + add_band(t, 2, TIER_ECONOMY, TIER_BAND_DEGRADED, 154624, 25088, "wwn-c"); + t->total_num_blocks = 4096 + 100352 + 50176 + 25088; +} + +/* Free the bands make_composite/add_band allocated (the composite `t` itself is + * stack-owned; only the band nodes are heap). Keeps the ASAN+LSAN build clean. */ +static void +free_composite(struct vbdev_tier *t) +{ + struct tier_band *b; + + while ((b = TAILQ_FIRST(&t->bands)) != NULL) { + TAILQ_REMOVE(&t->bands, b, link); + free(b); + } + t->num_bands = 0; +} + +/* ---- tests ------------------------------------------------------------------- */ + +static void +test_abi(void) +{ + /* F-1: runtime double-check of the compile-time asserts. */ + CHECK(sizeof(struct tier_sb_band) == 192); + CHECK(offsetof(struct tier_superblock, bands) == 256); + CHECK(sizeof(struct tier_superblock) == 12544); + CHECK(offsetof(struct tier_superblock, magic) == 0); + CHECK(offsetof(struct tier_superblock, version) == 8); + CHECK(offsetof(struct tier_superblock, crc) == 12); + CHECK(offsetof(struct tier_superblock, seq) == 16); + CHECK(offsetof(struct tier_superblock, created_epoch_sec) == 24); + CHECK(offsetof(struct tier_superblock, generation_uuid) == 32); + CHECK(offsetof(struct tier_superblock, composite_name) == 48); + /* v2 constants. */ + CHECK(TIER_SB_MAGIC == 0x5449455253423032ULL); + CHECK(TIER_SB_VERSION == 2u); + CHECK(TIER_SB_SLOT_BYTES * 2 == TIER_SB_RESERVE_BYTES); + /* A full superblock must fit in one slot (write_all precondition). */ + CHECK(sizeof(struct tier_superblock) <= TIER_SB_SLOT_BYTES); +} + +static void +test_serialize_roundtrip(void) +{ + struct vbdev_tier t; + struct tier_superblock sb; + struct tier_band *self; + + make_composite(&t); + self = TAILQ_FIRST(&t.bands); + + tier_sb_serialize(&t, self, 42, 1751630000ULL, &sb); + CHECK(tier_sb_valid(&sb)); + CHECK(sb.magic == TIER_SB_MAGIC); + CHECK(sb.version == TIER_SB_VERSION); + CHECK(sb.seq == 42); + CHECK(sb.created_epoch_sec == 1751630000ULL); + CHECK(sb.generation_uuid[0] == 0xA0 && sb.generation_uuid[15] == 0xAF); + CHECK(strcmp(sb.composite_name, "tier0") == 0); + CHECK(sb.md_num_blocks == 4096); + CHECK(sb.cluster_blocks == 256); /* u64 field */ + CHECK(sb.md_mirror_a == 0 && sb.md_mirror_b == 1); + CHECK(sb.num_bands == 3); + CHECK(sb.this_band_id == 0); + CHECK(sb.blocklen == 4096); + CHECK(sb.bands[0].band_id == 0 && sb.bands[0].state == TIER_BAND_ACTIVE); + CHECK(strcmp(sb.bands[0].wwn, "wwn-a") == 0); + CHECK(sb.bands[1].lba_start == 104448 && sb.bands[1].num_blocks == 50176); + CHECK(sb.bands[2].state == TIER_BAND_DEGRADED); + CHECK(strcmp(sb.bands[2].serial, "serial-2") == 0); + CHECK(sb.bands[3].band_id == 0 && sb.bands[3].num_blocks == 0); + + /* u64 cluster_blocks must survive a value > UINT32_MAX (F-3). */ + t.cluster_blocks = 0x100000001ULL; + tier_sb_serialize(&t, self, 42, 0, &sb); + CHECK(sb.cluster_blocks == 0x100000001ULL); + CHECK(tier_sb_valid(&sb)); + + /* NULL self => this_band_id sentinel. */ + tier_sb_serialize(&t, NULL, 43, 0, &sb); + CHECK(sb.this_band_id == UINT32_MAX); + CHECK(tier_sb_valid(&sb)); + free_composite(&t); +} + +static void +test_reject_corruption(void) +{ + struct vbdev_tier t; + struct tier_superblock sb; + + make_composite(&t); + tier_sb_serialize(&t, TAILQ_FIRST(&t.bands), 42, 0, &sb); + + ((uint8_t *)&sb)[sizeof(sb) - 1] ^= 0xFF; + CHECK(!tier_sb_valid(&sb)); + ((uint8_t *)&sb)[sizeof(sb) - 1] ^= 0xFF; + CHECK(tier_sb_valid(&sb)); + + sb.seq++; + CHECK(!tier_sb_valid(&sb)); + sb.seq--; + CHECK(tier_sb_valid(&sb)); + + /* A tampered generation_uuid must invalidate the CRC (fencing integrity). */ + sb.generation_uuid[7] ^= 0x55; + CHECK(!tier_sb_valid(&sb)); + sb.generation_uuid[7] ^= 0x55; + CHECK(tier_sb_valid(&sb)); + + sb.version = TIER_SB_VERSION + 1; /* v1/v3 not accepted (clean break) */ + CHECK(!tier_sb_valid(&sb)); + sb.version = 1u; /* the abandoned v1 must NOT read */ + CHECK(!tier_sb_valid(&sb)); + sb.version = TIER_SB_VERSION; + + sb.magic ^= 1; + CHECK(!tier_sb_valid(&sb)); + sb.magic ^= 1; + + sb.magic = __builtin_bswap64(TIER_SB_MAGIC); /* F-4 big-endian writer */ + CHECK(!tier_sb_valid(&sb)); + free_composite(&t); +} + +/* F-5: two-slot selection. */ +static void +test_slot_select(void) +{ + struct vbdev_tier t; + uint8_t *reserve; + struct tier_superblock *a, *b; + + make_composite(&t); + reserve = calloc(1, TIER_SB_RESERVE_BYTES); + assert(reserve != NULL); + a = (struct tier_superblock *)reserve; + b = (struct tier_superblock *)(reserve + TIER_SB_SLOT_BYTES); + + /* slot layout matches the writer: seq N -> slot N%2. */ + CHECK(tier_sb_slot_for_seq(42) == 0); + CHECK(tier_sb_slot_for_seq(43) == 1); + + /* Both valid: higher seq wins regardless of which slot holds it. */ + tier_sb_serialize(&t, NULL, 44, 0, a); /* slot A: even seq */ + tier_sb_serialize(&t, NULL, 45, 0, b); /* slot B: odd seq */ + CHECK(tier_sb_select(reserve, TIER_SB_RESERVE_BYTES) == b); + tier_sb_serialize(&t, NULL, 46, 0, a); /* now A newer */ + CHECK(tier_sb_select(reserve, TIER_SB_RESERVE_BYTES) == a); + + /* Torn newer slot (B) -> fall back to the older valid slot (A). */ + tier_sb_serialize(&t, NULL, 100, 0, a); + tier_sb_serialize(&t, NULL, 101, 0, b); + ((uint8_t *)b)[64] ^= 0xFF; /* corrupt B's CRC-covered bytes */ + CHECK(tier_sb_select(reserve, TIER_SB_RESERVE_BYTES) == a); + + /* Both torn -> NULL. */ + ((uint8_t *)a)[64] ^= 0xFF; + CHECK(tier_sb_select(reserve, TIER_SB_RESERVE_BYTES) == NULL); + + /* Short buffer -> NULL (no OOB read). */ + tier_sb_serialize(&t, NULL, 1, 0, a); + CHECK(tier_sb_select(reserve, TIER_SB_RESERVE_BYTES - 1) == NULL); + CHECK(tier_sb_select(NULL, TIER_SB_RESERVE_BYTES) == NULL); + + free(reserve); + free_composite(&t); +} + +static void +test_geometry_inlines(void) +{ + struct vbdev_tier t; + + make_composite(&t); /* cluster_blocks = 256, md = 4096 */ + + CHECK(tier_align_down(&t, 255) == 0); + CHECK(tier_align_down(&t, 256) == 256); + CHECK(tier_align_up(&t, 1) == 256); + CHECK(tier_align_up(&t, 257) == 512); + t.cluster_blocks = 0; + CHECK(tier_align_up(&t, 257) == 257); + t.cluster_blocks = 256; + + CHECK(vbdev_tier_is_md_range(&t, 0, 4096)); + CHECK(vbdev_tier_is_md_range(&t, 4095, 1)); + CHECK(!vbdev_tier_is_md_range(&t, 4095, 2)); + CHECK(!vbdev_tier_is_md_range(&t, 4096, 1)); + t.md_num_blocks = 0; + CHECK(!vbdev_tier_is_md_range(&t, 0, 1)); + free_composite(&t); +} + +/* Binary golden vector: pin the serialized header bytes so a field reorder that + * preserves sizeof is still caught. Only deterministic header fields are pinned + * (crc/created_epoch excluded — crc is derived, epoch is wall-clock). */ +static void +test_golden_header(void) +{ + struct vbdev_tier t; + struct tier_superblock sb; + const uint8_t *p = (const uint8_t *)&sb; + + make_composite(&t); + tier_sb_serialize(&t, TAILQ_FIRST(&t.bands), 0x42, 0, &sb); + + /* magic "TIERSB02" little-endian at offset 0. */ + CHECK(p[0] == 0x32 && p[1] == 0x30 && p[2] == 0x42 && p[3] == 0x53); + CHECK(p[4] == 0x52 && p[5] == 0x45 && p[6] == 0x49 && p[7] == 0x54); + /* version = 2 at offset 8 (LE u32). */ + CHECK(p[8] == 0x02 && p[9] == 0 && p[10] == 0 && p[11] == 0); + /* seq = 0x42 at offset 16 (LE u64). */ + CHECK(p[16] == 0x42 && p[17] == 0 && p[23] == 0); + /* generation_uuid at offset 32. */ + CHECK(p[32] == 0xA0 && p[47] == 0xAF); + /* composite_name "tier0" at offset 48. */ + CHECK(p[48] == 't' && p[49] == 'i' && p[52] == '0' && p[53] == '\0'); + free_composite(&t); +} + +/* ---- audit findings framing (M2/M3) ----------------------------------------- */ + +static void +persist_rc_cb(void *cb_arg, int rc) +{ + *(int *)cb_arg = rc; +} + +/* M3: a fan-out where EVERY band is skipped (DEGRADED / RETIRED / desc-less) + * writes zero superblock copies and must NOT complete rc == 0 — the callers' + * MJ6 contract reads rc == 0 as "durably persisted", and a retire acked on a + * zero-copy persist resurrects after reboot from the old highest-seq SBs. */ +static void +test_m3_zero_copy_persist_not_durable(void) +{ + struct vbdev_tier t; + struct tier_band *b; + int rc = 12345; + + make_composite(&t); + /* All bands non-ACTIVE (fixture descs are NULL anyway — both skip legs). */ + TAILQ_FOREACH(b, &t.bands, link) { + b->state = TIER_BAND_DEGRADED; + } + CHECK(tier_sb_write_all(&t, persist_rc_cb, &rc) == 0); + /* Zero writes launch → the fan-out completes synchronously. */ + CHECK(rc == -ENODEV); + CHECK(t.sb_write_inflight == false); + free_composite(&t); +} + +/* M2: the ctx-calloc ENOMEM path is a fan-out TERMINATION — it must resolve + * teardown deferred behind the fan-out (vbdev_tier_sb_fanout_idle), or a + * delete_pending composite whose drained callbacks hold no async_inflight + * reference leaks forever (bands, claims, -EEXIST on re-create). */ +static void +test_m2_enomem_termination_resolves_deferred_teardown(void) +{ + struct vbdev_tier t; + int rc = 12345; + + make_composite(&t); + t.delete_pending = true; + g_fanout_idle_calls = 0; + /* calloc #1 = the pending-cb wrapper (pass), #2 = the write ctx (fail). */ + g_calloc_fail_countdown = 1; + CHECK(tier_sb_write_all(&t, persist_rc_cb, &rc) == 0); + g_calloc_fail_countdown = -1; + CHECK(rc == -ENOMEM); /* queued cb drained with the error */ + CHECK(t.sb_write_inflight == false); + CHECK(g_fanout_idle_calls == 1); /* the fix: termination → idle hook */ + free_composite(&t); +} + +int +main(void) +{ + test_abi(); + test_serialize_roundtrip(); + test_reject_corruption(); + test_slot_select(); + test_geometry_inlines(); + test_golden_header(); + test_m3_zero_copy_persist_not_durable(); + test_m2_enomem_termination_resolves_deferred_teardown(); + + if (g_failures != 0) { + fprintf(stderr, "test_tier_sb: %d FAILURE(S)\n", g_failures); + return 1; + } + printf("test_tier_sb: all tests passed\n"); + return 0; +} diff --git a/module/bdev/tier/vbdev_tier.c b/module/bdev/tier/vbdev_tier.c new file mode 100644 index 0000000..add01c1 --- /dev/null +++ b/module/bdev/tier/vbdev_tier.c @@ -0,0 +1,2520 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 Evariops. All rights reserved. + * + * bdev_tier — composite tier-mapped vbdev (SPEC-73A/D, M1). See vbdev_tier.h. + * + * Design notes: + * - One vbdev per node. Bands appended fast->slow; band == one base bdev. + * - The low LBA range [0, md_num_blocks) is MIRRORED (RAID1) across two + * bands (md_mirror_a / md_mirror_b) so blobstore metadata (the L2P) + * survives a single disk loss (D1). The rest is a pure CONCAT. + * - Per-band failure isolation (C-FAIL-1): an I/O addressed to a DEGRADED + * band's range completes -EIO; the vbdev never reports a global failure. + * - The band table IS persisted on disk (INV-T1): one superblock copy per + * band, "highest seq wins" (vbdev_tier_sb.c). The SB is authoritative for + * geometry; the CSI control-plane (CRD = intent) replays create + + * assemble_band from the highest-seq SB on agent startup (SPEC-73 A2). + */ + +#include "vbdev_tier.h" + +#include "spdk/rpc.h" +#include "spdk/env.h" +#include "spdk/string.h" +#include "spdk/log.h" +#include "spdk/util.h" +#include "spdk/likely.h" +#include "spdk/crc32.h" +#include "spdk/uuid.h" + +static int vbdev_tier_init(void); +static void vbdev_tier_finish(void); +static int vbdev_tier_get_ctx_size(void); +static int vbdev_tier_config_json(struct spdk_json_write_ctx *w); + +static struct spdk_bdev_module tier_if = { + .name = "tier", + .module_init = vbdev_tier_init, + .module_fini = vbdev_tier_finish, + .config_json = vbdev_tier_config_json, + .get_ctx_size = vbdev_tier_get_ctx_size, +}; +SPDK_BDEV_MODULE_REGISTER(tier, &tier_if) + +/* All composites on this node. */ +static TAILQ_HEAD(, vbdev_tier) g_tier_nodes = TAILQ_HEAD_INITIALIZER(g_tier_nodes); + +/* Per-IO context. For a mirrored md write we fan out to 2 bands and complete the + * original only when the LAST leg is done (M1: never while a submitted leg is in + * flight — the leg callback holds cb_arg == orig_io). */ +struct tier_bdev_io { + struct spdk_io_channel *ch; + int remaining; /* outstanding legs (1 for concat, 2 for md mirror write) */ + uint8_t good_legs; /* legs completed successfully (md mirror fan-out) */ + bool md_retry_done; /* M2: at most one mirror-failover retry per md read */ + bool submit_failed; /* a leg SUBMISSION failed (no degrade; orig fails) */ + enum spdk_bdev_io_status status; /* worst-of across legs */ +}; + +static void vbdev_tier_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io); +static int vbdev_tier_destruct(void *ctx); +static void tier_sb_persist_cb(void *cb_arg, int rc); + +/* -------------------------------------------------------------------------- + * Band table lookups + * -------------------------------------------------------------------------- */ + +struct vbdev_tier * +vbdev_tier_get_by_name(const char *name) +{ + struct vbdev_tier *t; + + TAILQ_FOREACH(t, &g_tier_nodes, link) { + if (t->bdev.name != NULL && strcmp(t->bdev.name, name) == 0) { + return t; + } + } + return NULL; +} + +struct tier_band * +vbdev_tier_band_by_id(struct vbdev_tier *t, uint32_t band_id) +{ + struct tier_band *b; + + TAILQ_FOREACH(b, &t->bands, link) { + if (b->band_id == band_id) { + return b; + } + } + return NULL; +} + +struct tier_band * +vbdev_tier_band_of_lba(struct vbdev_tier *t, uint64_t lba, uint64_t *band_offset) +{ + struct tier_band *b; + + TAILQ_FOREACH(b, &t->bands, link) { + if (b->state == TIER_BAND_RETIRED) { + continue; + } + if (lba >= b->lba_start && lba < b->lba_start + b->num_blocks) { + if (band_offset) { + *band_offset = lba - b->lba_start; + } + return b; + } + } + return NULL; +} + +/* -------------------------------------------------------------------------- + * I/O completion + * -------------------------------------------------------------------------- */ + +static int tier_submit_leg(struct vbdev_tier *t, struct tier_io_channel *tch, + struct spdk_bdev_io *bdev_io, struct tier_band *band, + uint64_t base_phys); + +/* Map a base bdev back to its band (leg completions only carry the base bdev_io). */ +static struct tier_band * +tier_band_by_base_bdev(struct vbdev_tier *t, struct spdk_bdev *base) +{ + struct tier_band *b; + + TAILQ_FOREACH(b, &t->bands, link) { + if (b->desc != NULL && spdk_bdev_desc_get_bdev(b->desc) == base) { + return b; + } + } + return NULL; +} + +/* The md-mirror leg that is NOT `b` (NULL if unmirrored / not found). */ +static struct tier_band * +tier_md_other_leg(struct vbdev_tier *t, struct tier_band *b) +{ + uint32_t other = (b != NULL && b->band_id == t->md_mirror_a) ? t->md_mirror_b + : t->md_mirror_a; + + if (other == UINT32_MAX) { + return NULL; + } + return vbdev_tier_band_by_id(t, other); +} + +static void tier_band_drain_release(struct tier_band *band); + +/* C2: account a leg completion against this reactor's in-flight counter, and + * perform the channel put a band drain deferred to the LAST completion. Must + * run AFTER spdk_bdev_free_io(leg_io) — the io returns to the base channel's + * cache, which must still exist. Runs on the submitting reactor (leg + * completions arrive on their submitting thread), so inflight[] needs no + * atomics. */ +static void +tier_leg_channel_release(struct vbdev_tier *t, struct tier_io_channel *tch, + struct spdk_bdev *leg_bdev) +{ + struct tier_band *b = tier_band_by_base_bdev(t, leg_bdev); + + if (b == NULL || b->band_id >= TIER_MAX_BANDS) { + return; + } + assert(tch->inflight[b->band_id] > 0); + if (tch->inflight[b->band_id] == 0 || --tch->inflight[b->band_id] > 0) { + return; + } + if (tch->drain_deferred[b->band_id]) { + tch->drain_deferred[b->band_id] = false; + if (tch->base_ch[b->band_id] != NULL) { + spdk_put_io_channel(tch->base_ch[b->band_id]); + tch->base_ch[b->band_id] = NULL; + } + tier_band_drain_release(b); + } +} + +/* H3: the DEGRADED persist mutates composite-global SB state (t->seq, + * sb_pending_cbs, sb_write_inflight/queued) that is owned by t->thread. A leg + * completion runs on the submitting reactor, so the persist is funneled through + * spdk_thread_send_msg. The message re-resolves the composite through + * g_tier_nodes — never through captured pointers — so a composite torn down + * while the message was in flight is simply skipped. */ +struct tier_degrade_msg { + struct vbdev_tier *t; + uint32_t band_id; +}; + +static bool +tier_node_is_live(const struct vbdev_tier *t) +{ + struct vbdev_tier *n; + + TAILQ_FOREACH(n, &g_tier_nodes, link) { + if (n == t) { + return true; + } + } + return false; +} + +static void +tier_degrade_persist_msg(void *arg) +{ + struct tier_degrade_msg *m = arg; + struct vbdev_tier *t = m->t; + + if (tier_node_is_live(t) && t->registered) { + tier_sb_write_all(t, tier_sb_persist_cb, NULL); + } + free(m); +} + +/* R4/R5/M3: a WRITE/UNMAP/WRITE_ZEROES/FLUSH to a mirrored-md leg that does NOT + * land (submission OR completion failure) while the sibling leg DID leaves the two + * L2P copies divergent and both ACTIVE — resync_md (which only targets DEGRADED) + * would never repair them, and a reboot could prefer the stale copy. Degrade the + * failing leg (reads stop preferring it) and persist DEGRADED (M5(b) excludes it + * from the SB fan-out). Idempotent: a leg already non-ACTIVE is a no-op. + * + * May run on any reactor (leg completion context). The state write is a + * monotone single-word store the submit paths already read racily by design; + * the SB persist itself is funneled to t->thread (H3). */ +static void +tier_degrade_md_leg(struct vbdev_tier *t, struct tier_band *leg) +{ + if (leg == NULL || leg->state != TIER_BAND_ACTIVE) { + return; + } + SPDK_ERRLOG("tier '%s': md write failed on band %u — degrading leg\n", + t->bdev.name, leg->band_id); + leg->state = TIER_BAND_DEGRADED; + if (!t->registered) { + return; + } + if (spdk_get_thread() == t->thread) { + tier_sb_write_all(t, tier_sb_persist_cb, NULL); + return; + } + struct tier_degrade_msg *m = calloc(1, sizeof(*m)); + if (m == NULL) { + /* The RAM state is DEGRADED (reads/fan-outs already avoid the leg); + * the next SB persist from any path records it durably. */ + SPDK_ERRLOG("tier '%s': cannot queue DEGRADED persist for band %u (out " + "of memory) — will persist with the next SB write\n", + t->bdev.name, leg->band_id); + return; + } + m->t = t; + m->band_id = leg->band_id; + spdk_thread_send_msg(t->thread, tier_degrade_persist_msg, m); +} + +/* True iff `band` is an md-mirror leg and the base-physical offset `base_phys` + * lands inside the mirrored md region [sb_blocks, sb_blocks+md). Used to classify + * a completing/failing leg per-leg (NOT from the whole orig_io range, which + * mis-classifies a whole-device op — R5) without a per-leg context allocation. */ +static inline bool +tier_leg_is_md(const struct vbdev_tier *t, const struct tier_band *band, uint64_t base_phys) +{ + return band != NULL && + (band->band_id == t->md_mirror_a || band->band_id == t->md_mirror_b) && + base_phys >= t->sb_blocks && base_phys < t->sb_blocks + t->md_num_blocks; +} + +static void +_tier_leg_complete(struct spdk_bdev_io *leg_io, bool success, void *cb_arg) +{ + struct spdk_bdev_io *orig_io = cb_arg; + struct tier_bdev_io *io_ctx = (struct tier_bdev_io *)orig_io->driver_ctx; + struct vbdev_tier *t = SPDK_CONTAINEROF(orig_io->bdev, struct vbdev_tier, bdev); + struct tier_io_channel *leg_tch = spdk_io_channel_get_ctx(io_ctx->ch); + struct spdk_bdev *leg_bdev = leg_io->bdev; /* C2: for accounting after free_io */ + bool md_range = vbdev_tier_is_md_range(t, orig_io->u.bdev.offset_blocks, + orig_io->u.bdev.num_blocks); + + if (success) { + io_ctx->good_legs++; + } else { + struct tier_band *fb = tier_band_by_base_bdev(t, leg_io->bdev); + /* R5: classify THIS leg from its own base offset, not orig_io's range. */ + bool leg_md = tier_leg_is_md(t, fb, leg_io->u.bdev.offset_blocks); + + if (leg_md && orig_io->type == SPDK_BDEV_IO_TYPE_READ && !io_ctx->md_retry_done) { + /* M2: async media error on one md leg — the mirror holds a healthy + * copy; fail over instead of failing the read. */ + struct tier_band *alt = tier_md_other_leg(t, fb); + + if (alt != NULL && alt != fb && alt->state == TIER_BAND_ACTIVE && + alt->desc != NULL) { + struct tier_io_channel *tch = spdk_io_channel_get_ctx(io_ctx->ch); + + io_ctx->md_retry_done = true; + spdk_bdev_free_io(leg_io); + tier_leg_channel_release(t, leg_tch, leg_bdev); + if (tier_submit_leg(t, tch, orig_io, alt, + t->sb_blocks + orig_io->u.bdev.offset_blocks) == 0) { + return; /* retry in flight; completion re-enters here */ + } + /* C-M2: the retry submission failed and leg_io is ALREADY freed. + * Complete the (single-leg) md read as failed here and return — + * falling through would free leg_io a second time (double-free). */ + io_ctx->status = SPDK_BDEV_IO_STATUS_FAILED; + if (--io_ctx->remaining == 0) { + spdk_bdev_io_complete(orig_io, io_ctx->status); + } + return; + } + } + if (leg_md && orig_io->type != SPDK_BDEV_IO_TYPE_READ) { + /* M3/R5: a failed md-mirror WRITE/mgmt leg diverges — degrade + persist. */ + tier_degrade_md_leg(t, fb); + } + io_ctx->status = SPDK_BDEV_IO_STATUS_FAILED; + } + spdk_bdev_free_io(leg_io); + tier_leg_channel_release(t, leg_tch, leg_bdev); + + if (--io_ctx->remaining == 0) { + /* An md-mirror WRITE/mgmt op succeeds when at least one leg persisted it + * (the failed leg was degraded above — raid1 semantics). A leg whose + * SUBMISSION failed got no data and no degrade, so the orig must fail. */ + if (md_range && orig_io->type != SPDK_BDEV_IO_TYPE_READ && + io_ctx->good_legs > 0 && !io_ctx->submit_failed) { + spdk_bdev_io_complete(orig_io, SPDK_BDEV_IO_STATUS_SUCCESS); + return; + } + spdk_bdev_io_complete(orig_io, io_ctx->status); + } +} + +static void +tier_init_ext_io_opts(struct spdk_bdev_io *bdev_io, struct spdk_bdev_ext_io_opts *opts) +{ + memset(opts, 0, sizeof(*opts)); + opts->size = sizeof(*opts); + opts->memory_domain = bdev_io->u.bdev.memory_domain; + opts->memory_domain_ctx = bdev_io->u.bdev.memory_domain_ctx; + opts->metadata = bdev_io->u.bdev.md_buf; + opts->dif_check_flags_exclude_mask = ~bdev_io->u.bdev.dif_check_flags; +} + +/* -------------------------------------------------------------------------- + * I/O submission — translate composite LBA -> band(s) + * -------------------------------------------------------------------------- */ + +/* Submit one leg of the original I/O (any supported type) to a band at the given + * base-physical offset. */ +static int +tier_submit_leg(struct vbdev_tier *t, struct tier_io_channel *tch, struct spdk_bdev_io *bdev_io, + struct tier_band *band, uint64_t base_phys) +{ + struct spdk_io_channel *base_ch = tch->base_ch[band->band_id]; + struct spdk_bdev_ext_io_opts io_opts; + int rc; + + if (spdk_unlikely(band->state != TIER_BAND_ACTIVE || band->desc == NULL || + base_ch == NULL)) { + return -EIO; /* per-band isolation: caller fails THIS io, not the chunk */ + } + + switch (bdev_io->type) { + case SPDK_BDEV_IO_TYPE_READ: + tier_init_ext_io_opts(bdev_io, &io_opts); + rc = spdk_bdev_readv_blocks_ext(band->desc, base_ch, bdev_io->u.bdev.iovs, + bdev_io->u.bdev.iovcnt, base_phys, + bdev_io->u.bdev.num_blocks, _tier_leg_complete, + bdev_io, &io_opts); + break; + case SPDK_BDEV_IO_TYPE_WRITE: + tier_init_ext_io_opts(bdev_io, &io_opts); + rc = spdk_bdev_writev_blocks_ext(band->desc, base_ch, bdev_io->u.bdev.iovs, + bdev_io->u.bdev.iovcnt, base_phys, + bdev_io->u.bdev.num_blocks, _tier_leg_complete, + bdev_io, &io_opts); + break; + case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: + rc = spdk_bdev_write_zeroes_blocks(band->desc, base_ch, base_phys, + bdev_io->u.bdev.num_blocks, + _tier_leg_complete, bdev_io); + break; + case SPDK_BDEV_IO_TYPE_UNMAP: + rc = spdk_bdev_unmap_blocks(band->desc, base_ch, base_phys, + bdev_io->u.bdev.num_blocks, + _tier_leg_complete, bdev_io); + break; + case SPDK_BDEV_IO_TYPE_FLUSH: + rc = spdk_bdev_flush_blocks(band->desc, base_ch, base_phys, + bdev_io->u.bdev.num_blocks, + _tier_leg_complete, bdev_io); + break; + default: + return -EINVAL; + } + if (rc == 0) { + /* C2: gate this reactor's channel put on the leg's completion. */ + tch->inflight[band->band_id]++; + } + return rc; +} + +/* Fan an md-region WRITE / WRITE_ZEROES / UNMAP / FLUSH out to every ACTIVE md + * leg. M1: once one leg is submitted, the orig_io completes ONLY from + * _tier_leg_complete — a later submission failure just drops that leg's count. + * Returns 0 if at least one leg is in flight, -errno if none was submitted. */ +static int +tier_route_md_fanout(struct vbdev_tier *t, struct tier_io_channel *tch, + struct spdk_bdev_io *bdev_io) +{ + struct tier_bdev_io *io_ctx = (struct tier_bdev_io *)bdev_io->driver_ctx; + struct tier_band *legs[2]; + struct tier_band *failed[2]; + struct tier_band *a = vbdev_tier_band_by_id(t, t->md_mirror_a); + struct tier_band *b = vbdev_tier_band_by_id(t, t->md_mirror_b); + int nlegs = 0, submitted = 0, nfailed = 0, i, rc = -EIO; + + if (a != NULL && a->state == TIER_BAND_ACTIVE) { + legs[nlegs++] = a; + } + if (b != NULL && b->state == TIER_BAND_ACTIVE) { + legs[nlegs++] = b; + } + if (nlegs == 0) { + return -EIO; + } + io_ctx->remaining = nlegs; + for (i = 0; i < nlegs; i++) { + rc = tier_submit_leg(t, tch, bdev_io, legs[i], + t->sb_blocks + bdev_io->u.bdev.offset_blocks); + if (rc != 0) { + io_ctx->status = SPDK_BDEV_IO_STATUS_FAILED; + io_ctx->submit_failed = true; + io_ctx->remaining--; + failed[nfailed++] = legs[i]; + } else { + submitted++; + } + } + if (submitted == 0) { + return rc; /* nothing in flight; caller completes/requeues, no leg wrote */ + } + /* R4: a sibling leg is in flight and WILL write the new md data, so any md leg + * that failed to submit now diverges from it — degrade it (+ persist). Without + * this both legs stay ACTIVE and resync_md (DEGRADED-only) never repairs the + * silent divergence; a reboot could then prefer the stale copy. */ + for (i = 0; i < nfailed; i++) { + tier_degrade_md_leg(t, failed[i]); + } + return 0; +} + +/* Route a read or write to the band(s) owning [offset, offset+num). Handles the + * mirrored md region (write -> 2 legs, read -> primary leg) and the concat data + * region (1 leg). Returns 0 on success (legs submitted), -errno otherwise. */ +static int +tier_route_rw(struct vbdev_tier *t, struct tier_io_channel *tch, struct spdk_bdev_io *bdev_io, + bool is_write) +{ + struct tier_bdev_io *io_ctx = (struct tier_bdev_io *)bdev_io->driver_ctx; + uint64_t offset = bdev_io->u.bdev.offset_blocks; + uint64_t num = bdev_io->u.bdev.num_blocks; + struct tier_band *band, *band_b; + uint64_t band_off; + int rc; + + io_ctx->status = SPDK_BDEV_IO_STATUS_SUCCESS; + + /* Metadata region: RAID1-mirrored across two bands when the composite has ≥2 disks; + * single-copy on the one band when the node has a single disk (band_b == NULL). The md + * maps to base-physical [sb_blocks, sb_blocks+md) on each md band. */ + if (vbdev_tier_is_md_range(t, offset, num)) { + if (!is_write) { + band = vbdev_tier_band_by_id(t, t->md_mirror_a); + band_b = vbdev_tier_band_by_id(t, t->md_mirror_b); /* may be NULL: single-band composite */ + if (band == NULL) { + return -EIO; + } + /* Prefer an ACTIVE leg; with a mirror, fall back to the secondary. + * (An ASYNC media error on the chosen leg is retried on the mirror + * by _tier_leg_complete — M2.) */ + struct tier_band *src = band; + if (band->state != TIER_BAND_ACTIVE && band_b != NULL && band_b->state == TIER_BAND_ACTIVE) { + src = band_b; + } + io_ctx->remaining = 1; + rc = tier_submit_leg(t, tch, bdev_io, src, t->sb_blocks + offset); + if (rc != 0 && src == band && band_b != NULL && band_b->state == TIER_BAND_ACTIVE) { + rc = tier_submit_leg(t, tch, bdev_io, band_b, t->sb_blocks + offset); + } + return rc; + } + /* Write: fan out to every ACTIVE md leg (M1-safe). */ + return tier_route_md_fanout(t, tch, bdev_io); + } + + /* Data region: single band. Reject a straddle of band boundary (defensive; + * blobstore cluster I/O is band-aligned). */ + band = vbdev_tier_band_of_lba(t, offset, &band_off); + if (band == NULL) { + return -EIO; + } + if (offset + num > band->lba_start + band->num_blocks) { + SPDK_ERRLOG("tier: I/O straddles band boundary (off=%" PRIu64 " num=%" PRIu64 ")\n", + offset, num); + return -EINVAL; + } + io_ctx->remaining = 1; + return tier_submit_leg(t, tch, bdev_io, band, band->phys_offset + band_off); +} + +/* Submit a FLUSH/UNMAP/WRITE_ZEROES to one band over an explicit sub-range. */ +static int +tier_submit_mgmt_range(struct tier_band *band, struct spdk_io_channel *base_ch, + enum spdk_bdev_io_type type, uint64_t base_phys, uint64_t num, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + switch (type) { + case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: + return spdk_bdev_write_zeroes_blocks(band->desc, base_ch, base_phys, num, cb, cb_arg); + case SPDK_BDEV_IO_TYPE_UNMAP: + return spdk_bdev_unmap_blocks(band->desc, base_ch, base_phys, num, cb, cb_arg); + case SPDK_BDEV_IO_TYPE_FLUSH: + return spdk_bdev_flush_blocks(band->desc, base_ch, base_phys, num, cb, cb_arg); + default: + return -EINVAL; + } +} + +/* R6: a RETIRED band keeps its composite LBA range as an unreclaimable HOLE + * (vbdev_tier_band_of_lba skips RETIRED). For a FLUSH/UNMAP/WRITE_ZEROES a hole is + * a durable NO-OP (reads of the range already -EIO), so the mgmt router skips it + * instead of failing the whole op. Return the end LBA of the retired band covering + * `lba` so the caller can advance past the hole; 0 if `lba` is not inside any + * retired band (a genuine unmapped gap → still an error). */ +static uint64_t +tier_retired_hole_end(struct vbdev_tier *t, uint64_t lba) +{ + struct tier_band *b; + + TAILQ_FOREACH(b, &t->bands, link) { + if (b->state == TIER_BAND_RETIRED && + lba >= b->lba_start && lba < b->lba_start + b->num_blocks) { + return b->lba_start + b->num_blocks; + } + } + return 0; +} + +/* Route a FLUSH/UNMAP/WRITE_ZEROES over ANY composite range, splitting it into + * one leg per covered region: the mirrored md portion fans out to both md legs + * (m6 — a single-leg unmap would desync the L2P copies), and each data band the + * range crosses gets its own leg. This is what makes a whole-device FLUSH (the + * canonical NVMe Flush, offset 0..blockcnt) work — it spans md + every band. + * Reuses the io_ctx leg-count machinery; the orig completes only from the last + * _tier_leg_complete. Returns 0 if any leg is in flight (or the op no-ops over a + * hole and was completed here), -errno otherwise. */ +static int +tier_route_mgmt(struct vbdev_tier *t, struct tier_io_channel *tch, struct spdk_bdev_io *bdev_io) +{ + struct tier_bdev_io *io_ctx = (struct tier_bdev_io *)bdev_io->driver_ctx; + uint64_t offset = bdev_io->u.bdev.offset_blocks; + uint64_t end = offset + bdev_io->u.bdev.num_blocks; + struct tier_mgmt_seg { + struct tier_band *band; + uint64_t base_phys; + uint64_t num; + bool is_md; /* R4/R5: md-region leg (degrade sibling on divergence) */ + } segs[2 + TIER_MAX_BANDS]; + struct tier_band *md_failed[2]; + int nseg = 0, submitted = 0, md_submitted = 0, md_nfailed = 0, i, rc = -EIO; + uint64_t pos; + + /* md-covered portion [offset, min(end, md)) — mirrored across the active md legs. */ + if (offset < t->md_num_blocks) { + uint64_t md_num = spdk_min(end, t->md_num_blocks) - offset; + struct tier_band *legs[2]; + int n = 0, k; + + legs[0] = vbdev_tier_band_by_id(t, t->md_mirror_a); + legs[1] = vbdev_tier_band_by_id(t, t->md_mirror_b); + for (k = 0; k < 2; k++) { + if (legs[k] != NULL && legs[k]->state == TIER_BAND_ACTIVE) { + segs[nseg].band = legs[k]; + segs[nseg].base_phys = t->sb_blocks + offset; + segs[nseg].num = md_num; + segs[nseg].is_md = true; + nseg++; + n++; + } + } + if (n == 0) { + return -EIO; /* md region has no active leg */ + } + } + + /* data portion [max(offset, md), end) — one leg per owning band segment. + * R6: a RETIRED band's range is a hole — skip it (mgmt no-op) instead of + * failing, so a whole-device FLUSH/UNMAP/WRITE_ZEROES survives a retired band. */ + pos = spdk_max(offset, t->md_num_blocks); + while (pos < end) { + uint64_t band_off, seg_num; + struct tier_band *band = vbdev_tier_band_of_lba(t, pos, &band_off); + + if (band == NULL) { + uint64_t hole_end = tier_retired_hole_end(t, pos); + + if (hole_end > pos) { + pos = spdk_min(hole_end, end); /* skip the retired hole */ + continue; + } + return -EIO; /* a genuinely unmapped gap is still an error */ + } + if (nseg >= (int)SPDK_COUNTOF(segs)) { + return -EIO; /* too many segments */ + } + seg_num = spdk_min(end - pos, band->num_blocks - band_off); + segs[nseg].band = band; + segs[nseg].base_phys = band->phys_offset + band_off; + segs[nseg].num = seg_num; + segs[nseg].is_md = false; + nseg++; + pos += seg_num; + } + + if (nseg == 0) { + /* R6: the whole range fell inside retired hole(s) (offset >= md, all data + * retired) — a mgmt op over a hole is a durable no-op. Complete SUCCESS here + * (returning 0; the single caller does nothing more on rc == 0). */ + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + return 0; + } + int flush_skipped = 0; + + io_ctx->remaining = nseg; + for (i = 0; i < nseg; i++) { + struct tier_band *band = segs[i].band; + struct spdk_io_channel *base_ch = band->band_id < TIER_MAX_BANDS ? + tch->base_ch[band->band_id] : NULL; + + if (band->state != TIER_BAND_ACTIVE || band->desc == NULL || base_ch == NULL) { + /* H6: a FLUSH over a DEGRADED *data* band is a no-op, not a + * failure. The band's data is already unreachable (reads and + * writes to the range return -EIO — the consumer has been told), + * and no flush can make an unreachable range durable. Failing + * the segment made every WHOLE-DEVICE flush barrier fail for the + * entire evacuation window after a single-disk loss — filesystem + * remounts read-only, a full-volume outage that defeats the + * per-band isolation design. Mutations (UNMAP/WRITE_ZEROES) and + * md legs keep failing honestly. */ + if (bdev_io->type == SPDK_BDEV_IO_TYPE_FLUSH && !segs[i].is_md) { + SPDK_WARNLOG("tier '%s': flush skips degraded band %u " + "(range already unreachable)\n", + t->bdev.name, band->band_id); + flush_skipped++; + io_ctx->remaining--; + continue; + } + rc = -EIO; /* per-band isolation: this leg fails, op fails */ + } else { + rc = tier_submit_mgmt_range(band, base_ch, bdev_io->type, + segs[i].base_phys, segs[i].num, + _tier_leg_complete, bdev_io); + } + if (rc != 0) { + io_ctx->status = SPDK_BDEV_IO_STATUS_FAILED; + io_ctx->submit_failed = true; + io_ctx->remaining--; + if (segs[i].is_md && md_nfailed < (int)SPDK_COUNTOF(md_failed)) { + md_failed[md_nfailed++] = band; + } + } else { + /* C2: gate this reactor's channel put on the leg's completion. */ + tch->inflight[band->band_id]++; + submitted++; + if (segs[i].is_md) { + md_submitted++; + } + } + } + if (submitted == 0) { + if (flush_skipped > 0 && !io_ctx->submit_failed) { + /* H6: every segment was a degraded-band flush no-op. */ + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); + return 0; + } + return rc; /* nothing in flight; caller completes the orig_io */ + } + /* R4: if at least one md leg took the md-region write, any md leg whose md-region + * seg failed to submit now diverges — degrade it (+ persist). If NO md leg took it + * (md_submitted == 0), the two legs are still identical, so leave them ACTIVE. */ + if (md_submitted > 0) { + for (i = 0; i < md_nfailed; i++) { + tier_degrade_md_leg(t, md_failed[i]); + } + } + return 0; +} + +/* RESET fan-out: reset every distinct active base disk and complete the composite + * reset only when all legs finish. Uses a dedicated completion (not + * _tier_leg_complete) because a reset carries no LBA range — is_md_range() would + * misclassify it and wrongly degrade an md leg on failure. */ +static void +_tier_reset_leg_complete(struct spdk_bdev_io *leg_io, bool success, void *cb_arg) +{ + struct spdk_bdev_io *orig_io = cb_arg; + struct tier_bdev_io *io_ctx = (struct tier_bdev_io *)orig_io->driver_ctx; + struct vbdev_tier *t = SPDK_CONTAINEROF(orig_io->bdev, struct vbdev_tier, bdev); + struct spdk_bdev *leg_bdev = leg_io->bdev; + + if (!success) { + io_ctx->status = SPDK_BDEV_IO_STATUS_FAILED; + } + spdk_bdev_free_io(leg_io); + tier_leg_channel_release(t, spdk_io_channel_get_ctx(io_ctx->ch), leg_bdev); + if (--io_ctx->remaining == 0) { + spdk_bdev_io_complete(orig_io, io_ctx->status); + } +} + +static int +tier_route_reset(struct vbdev_tier *t, struct tier_io_channel *tch, struct spdk_bdev_io *bdev_io) +{ + struct tier_bdev_io *io_ctx = (struct tier_bdev_io *)bdev_io->driver_ctx; + struct tier_band *b; + int nlegs = 0, submitted = 0, rc = -EIO; + + TAILQ_FOREACH(b, &t->bands, link) { + if (b->state == TIER_BAND_ACTIVE && b->desc != NULL && + b->band_id < TIER_MAX_BANDS && tch->base_ch[b->band_id] != NULL) { + nlegs++; + } + } + if (nlegs == 0) { + return -EIO; + } + io_ctx->remaining = nlegs; + TAILQ_FOREACH(b, &t->bands, link) { + struct spdk_io_channel *base_ch; + + if (b->state != TIER_BAND_ACTIVE || b->desc == NULL || + b->band_id >= TIER_MAX_BANDS || tch->base_ch[b->band_id] == NULL) { + continue; + } + base_ch = tch->base_ch[b->band_id]; + rc = spdk_bdev_reset(b->desc, base_ch, _tier_reset_leg_complete, bdev_io); + if (rc != 0) { + io_ctx->status = SPDK_BDEV_IO_STATUS_FAILED; + io_ctx->remaining--; + } else { + /* C2: gate this reactor's channel put on the leg's completion. */ + tch->inflight[b->band_id]++; + submitted++; + } + } + if (submitted == 0) { + return rc; + } + return 0; +} + +static void +tier_read_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success) +{ + struct vbdev_tier *t = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_tier, bdev); + struct tier_io_channel *tch = spdk_io_channel_get_ctx(ch); + int rc; + + if (!success) { + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; + } + + rc = tier_route_rw(t, tch, bdev_io, false); + if (rc != 0) { + /* M4: -ENOMEM completes NOMEM so the bdev core requeues + retries the + * whole submit (works for md reads too — the old queue_io path resolved + * the band via band_of_lba, which never covers [0, md)). */ + spdk_bdev_io_complete(bdev_io, rc == -ENOMEM ? SPDK_BDEV_IO_STATUS_NOMEM : + SPDK_BDEV_IO_STATUS_FAILED); + } +} + +static void +vbdev_tier_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) +{ + struct vbdev_tier *t = SPDK_CONTAINEROF(bdev_io->bdev, struct vbdev_tier, bdev); + struct tier_io_channel *tch = spdk_io_channel_get_ctx(ch); + struct tier_bdev_io *io_ctx = (struct tier_bdev_io *)bdev_io->driver_ctx; + int rc = 0; + + io_ctx->ch = ch; + io_ctx->status = SPDK_BDEV_IO_STATUS_SUCCESS; + io_ctx->good_legs = 0; + io_ctx->md_retry_done = false; + io_ctx->submit_failed = false; + + switch (bdev_io->type) { + case SPDK_BDEV_IO_TYPE_READ: + spdk_bdev_io_get_buf(bdev_io, tier_read_get_buf_cb, + bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); + return; + case SPDK_BDEV_IO_TYPE_WRITE: + rc = tier_route_rw(t, tch, bdev_io, true); + break; + case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: + case SPDK_BDEV_IO_TYPE_UNMAP: + case SPDK_BDEV_IO_TYPE_FLUSH: + /* m6: a mgmt op is a mutation/barrier — route it to EVERY covered leg + * (both md legs for the mirrored portion, one per data band). This is the + * only correct handling for a range spanning the md/data boundary or + * multiple bands, including the whole-device flush an NVMe Flush issues. */ + rc = tier_route_mgmt(t, tch, bdev_io); + break; + case SPDK_BDEV_IO_TYPE_RESET: + rc = tier_route_reset(t, tch, bdev_io); + break; + default: + SPDK_ERRLOG("tier: unsupported I/O type %d\n", bdev_io->type); + spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); + return; + } + + if (rc != 0) { + /* M4: propagate -ENOMEM as NOMEM (bdev-core retry), everything else fails. */ + spdk_bdev_io_complete(bdev_io, rc == -ENOMEM ? SPDK_BDEV_IO_STATUS_NOMEM : + SPDK_BDEV_IO_STATUS_FAILED); + } +} + +static bool +vbdev_tier_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) +{ + struct vbdev_tier *t = (struct vbdev_tier *)ctx; + struct tier_band *b; + + switch (io_type) { + case SPDK_BDEV_IO_TYPE_READ: + case SPDK_BDEV_IO_TYPE_WRITE: + case SPDK_BDEV_IO_TYPE_RESET: + return true; + case SPDK_BDEV_IO_TYPE_WRITE_ZEROES: + case SPDK_BDEV_IO_TYPE_UNMAP: + case SPDK_BDEV_IO_TYPE_FLUSH: + /* Every leg is submitted to a base band, so the composite can only + * honor these if EVERY active band's base bdev does — advertising a + * type a band cannot execute makes the leg complete FAILED (a base + * lacking UNMAP would fail every discard the consumer was told worked). */ + TAILQ_FOREACH(b, &t->bands, link) { + if (b->state == TIER_BAND_RETIRED || b->desc == NULL) { + continue; + } + if (!spdk_bdev_io_type_supported(spdk_bdev_desc_get_bdev(b->desc), + io_type)) { + return false; + } + } + return true; + default: + return false; + } +} + +/* -------------------------------------------------------------------------- + * Channels + * -------------------------------------------------------------------------- */ + +static int +tier_ch_create_cb(void *io_device, void *ctx_buf) +{ + struct tier_io_channel *tch = ctx_buf; + struct vbdev_tier *t = io_device; + struct tier_band *b; + + memset(tch, 0, sizeof(*tch)); /* base_ch + C2 inflight/drain_deferred */ + TAILQ_FOREACH(b, &t->bands, link) { + if (b->state != TIER_BAND_RETIRED && b->desc != NULL && + b->band_id < TIER_MAX_BANDS) { + tch->base_ch[b->band_id] = spdk_bdev_get_io_channel(b->desc); + if (tch->base_ch[b->band_id] == NULL) { + /* An ACTIVE band with a NULL base channel would return -EIO for + * every I/O on this reactor (an unexplained per-core failure). + * Fail channel creation instead, releasing the channels already + * opened for this tch. */ + uint32_t j; + + SPDK_ERRLOG("tier '%s': cannot open base channel for band %u\n", + t->bdev.name, b->band_id); + for (j = 0; j < TIER_MAX_BANDS; j++) { + if (tch->base_ch[j] != NULL) { + spdk_put_io_channel(tch->base_ch[j]); + tch->base_ch[j] = NULL; + } + } + return -ENOMEM; + } + } + } + return 0; +} + +static void +tier_ch_destroy_cb(void *io_device, void *ctx_buf) +{ + struct tier_io_channel *tch = ctx_buf; + int i; + + for (i = 0; i < TIER_MAX_BANDS; i++) { + if (tch->base_ch[i] != NULL) { + spdk_put_io_channel(tch->base_ch[i]); + tch->base_ch[i] = NULL; + } + } +} + +static struct spdk_io_channel * +vbdev_tier_get_io_channel(void *ctx) +{ + struct vbdev_tier *t = (struct vbdev_tier *)ctx; + + return spdk_get_io_channel(t); +} + +/* -------------------------------------------------------------------------- + * dump / config json + * -------------------------------------------------------------------------- */ + +static int +vbdev_tier_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) +{ + struct vbdev_tier *t = (struct vbdev_tier *)ctx; + struct tier_band *b; + + spdk_json_write_named_object_begin(w, "tier"); + spdk_json_write_named_string(w, "name", spdk_bdev_get_name(&t->bdev)); + spdk_json_write_named_uint64(w, "md_num_blocks", t->md_num_blocks); + spdk_json_write_named_uint32(w, "md_mirror_a", t->md_mirror_a); + spdk_json_write_named_uint32(w, "md_mirror_b", t->md_mirror_b); + spdk_json_write_named_array_begin(w, "bands"); + TAILQ_FOREACH(b, &t->bands, link) { + spdk_json_write_object_begin(w); + spdk_json_write_named_uint32(w, "band_id", b->band_id); + spdk_json_write_named_string(w, "base_bdev_name", b->base_bdev_name); + spdk_json_write_named_uint32(w, "tier", b->tier); + spdk_json_write_named_uint32(w, "state", b->state); + spdk_json_write_named_uint64(w, "lba_start", b->lba_start); + spdk_json_write_named_uint64(w, "num_blocks", b->num_blocks); + spdk_json_write_named_string(w, "wwn", b->wwn); + spdk_json_write_named_string(w, "serial", b->serial); + spdk_json_write_object_end(w); + } + spdk_json_write_array_end(w); + spdk_json_write_object_end(w); + return 0; +} + +static void +vbdev_tier_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) +{ + /* The CSI control-plane replays create/add_band from CRD; no per-bdev config emitted. */ +} + +static const struct spdk_bdev_fn_table vbdev_tier_fn_table = { + .destruct = vbdev_tier_destruct, + .submit_request = vbdev_tier_submit_request, + .io_type_supported = vbdev_tier_io_type_supported, + .get_io_channel = vbdev_tier_get_io_channel, + .dump_info_json = vbdev_tier_dump_info_json, + .write_config_json = vbdev_tier_write_config_json, +}; + +/* -------------------------------------------------------------------------- + * destruct + * -------------------------------------------------------------------------- */ + +static void +_tier_device_unregister_cb(void *io_device) +{ + struct vbdev_tier *t = io_device; + struct tier_band *b; + + while ((b = TAILQ_FIRST(&t->bands))) { + TAILQ_REMOVE(&t->bands, b, link); + if (b->desc != NULL) { + spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(b->desc)); + spdk_bdev_close(b->desc); + } + free(b); + } + free(t->bdev.name); + free(t); +} + +static int +vbdev_tier_destruct(void *ctx) +{ + struct vbdev_tier *t = (struct vbdev_tier *)ctx; + + TAILQ_REMOVE(&g_tier_nodes, t, link); + spdk_io_device_unregister(t, _tier_device_unregister_cb); + return 0; +} + +/* -------------------------------------------------------------------------- + * Band drain + close (C3 / T-4): remove the band's per-reactor base channels, + * THEN close its desc. Closing without the drain violates the SPDK ownership + * contract (channels outlive the desc) and, on hot-remove, leaves the base + * bdev's unregister pending forever. + * -------------------------------------------------------------------------- */ + +struct tier_band_drain_ctx { + struct vbdev_tier *t; + struct tier_band *band; +}; + +/* C2: final step of a band drain — runs on the app thread once every reactor + * has put its base channel (drain_refs == 0). The desc close itself is still + * gated on desc_pins: a relocate/resync engine submitting on this desc defers + * it to the engine's terminal (tier_band_desc_unpin). */ +static void +tier_band_try_close(struct tier_band *band) +{ + assert(spdk_get_thread() == band->t->thread); + if (band->desc_pins > 0) { + band->close_deferred = true; + return; + } + if (band->desc != NULL) { + spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(band->desc)); + spdk_bdev_close(band->desc); + band->desc = NULL; + } + band->draining = false; + if (band->drain_cb) { + void (*cb)(void *, int) = band->drain_cb; + void *cb_arg = band->drain_cb_arg; + + band->drain_cb = NULL; + band->drain_cb_arg = NULL; + cb(cb_arg, 0); + } +} + +static void +tier_band_try_close_msg(void *arg) +{ + tier_band_try_close(arg); +} + +/* C2: release one drain reference (a reactor that finished its deferred put, + * or the drain fan-out itself). The LAST release performs the close — on the + * app thread, because spdk_bdev_close must run on the opening thread. */ +static void +tier_band_drain_release(struct tier_band *band) +{ + if (__atomic_sub_fetch(&band->drain_refs, 1, __ATOMIC_ACQ_REL) != 0) { + return; + } + if (spdk_get_thread() == band->t->thread) { + tier_band_try_close(band); + } else { + spdk_thread_send_msg(band->t->thread, tier_band_try_close_msg, band); + } +} + +static void +tier_band_drain_ch_iter(struct spdk_io_channel_iter *i) +{ + struct tier_band_drain_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); + struct tier_io_channel *tch = spdk_io_channel_get_ctx(_ch); + uint32_t id = ctx->band->band_id; + + if (id < TIER_MAX_BANDS && tch->base_ch[id] != NULL) { + if (tch->inflight[id] > 0) { + /* C2: legs in flight on this reactor — putting the channel now + * would destroy it with io_outstanding > 0 (assert/UAF). Defer + * the put to the last leg completion. */ + __atomic_fetch_add(&ctx->band->drain_refs, 1, __ATOMIC_RELAXED); + tch->drain_deferred[id] = true; + } else { + spdk_put_io_channel(tch->base_ch[id]); + tch->base_ch[id] = NULL; + } + } + spdk_for_each_channel_continue(i, 0); +} + +static void +tier_band_drain_done(struct spdk_io_channel_iter *i, int status) +{ + struct tier_band_drain_ctx *ctx = spdk_io_channel_iter_get_ctx(i); + struct tier_band *band = ctx->band; + + (void)status; + free(ctx); + /* Release the fan-out's own reference; if no reactor deferred, this is + * the last one and the close runs now (we are on the app thread). */ + tier_band_drain_release(band); +} + +static int +tier_band_drain_and_close(struct vbdev_tier *t, struct tier_band *band, + void (*cb)(void *cb_arg, int rc), void *cb_arg) +{ + struct tier_band_drain_ctx *ctx; + + if (band->draining) { + /* One drain at a time per band (initiations are app-thread-only: + * hot-remove event, retire RPC, fanout_idle). The in-flight drain + * finishes the job; a second explicit-cb caller must not lose its + * completion, so reject it (retire retries idempotently). */ + return cb != NULL ? -EALREADY : 0; + } + if (!t->registered) { + /* No io_device / channels yet: close directly (respecting pins). */ + band->draining = true; + band->drain_cb = cb; + band->drain_cb_arg = cb_arg; + tier_band_try_close(band); + return 0; + } + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return -ENOMEM; + } + ctx->t = t; + ctx->band = band; + band->draining = true; + band->drain_cb = cb; + band->drain_cb_arg = cb_arg; + __atomic_store_n(&band->drain_refs, 1, __ATOMIC_RELAXED); + spdk_for_each_channel(t, tier_band_drain_ch_iter, ctx, tier_band_drain_done); + return 0; +} + +/* C2: bracket an engine's (relocate / md-resync) use of a band's desc. App + * thread only. A drain that lands while pinned closes the desc at unpin. */ +static void +tier_band_desc_pin(struct tier_band *band) +{ + band->desc_pins++; +} + +static void +tier_band_desc_unpin(struct tier_band *band) +{ + assert(band->desc_pins > 0); + if (--band->desc_pins == 0 && band->close_deferred) { + band->close_deferred = false; + tier_band_try_close(band); + } +} + +/* Base bdev hot-remove (C3): degrade the band (per-band isolation, do NOT tear + * down the composite), PERSIST the degradation, and honor the SPDK REMOVE + * contract (drain channels + close desc). The CSI brain reacts via tier events + * + rebuild-by-range. */ +static void +tier_base_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx) +{ + struct tier_band *band = event_ctx; + struct vbdev_tier *t = band->t; + + if (type != SPDK_BDEV_EVENT_REMOVE) { + return; + } + SPDK_WARNLOG("tier: base bdev '%s' removed, degrading band %u\n", + bdev->name, band->band_id); + band->state = TIER_BAND_DEGRADED; + /* C3(2): persist DEGRADED to the surviving bands (M5(b) excludes this band + * from the fan-out) — otherwise a reboot reassembles the band ACTIVE and md + * reads can prefer its STALE L2P copy (silent corruption). */ + if (t->registered) { + tier_sb_write_all(t, tier_sb_persist_cb, NULL); + } + /* C3(1): close the desc (after the channel drain) or the removed base + * bdev's unregister pends forever. T-4b: if an SB fan-out is in flight it may + * hold an in-flight write on THIS band's desc (the band was ACTIVE when the + * fan-out launched, and the DEGRADED persist above may itself be that + * fan-out). Closing the desc now would violate the channel-before-desc + * ownership contract, so defer the drain+close to vbdev_tier_sb_fanout_idle. */ + if (t->sb_write_inflight || t->sb_write_queued) { + band->close_pending = true; + } else if (tier_band_drain_and_close(t, band, NULL, NULL) != 0) { + SPDK_ERRLOG("tier: cannot drain band %u after hot-remove (out of memory); " + "desc left open\n", band->band_id); + } +} + +/* -------------------------------------------------------------------------- + * Lifecycle: create / add_band / retire_band + * -------------------------------------------------------------------------- */ + +struct vbdev_tier * +vbdev_tier_create(const char *name, uint64_t md_num_blocks, uint64_t cluster_blocks) +{ + struct vbdev_tier *t; + + t = calloc(1, sizeof(*t)); + if (t == NULL) { + return NULL; + } + TAILQ_INIT(&t->bands); + TAILQ_INIT(&t->sb_pending_cbs); + t->next_band_id = 0; + /* H3: composite-global SB state is owned by this (the app/RPC) thread. */ + t->thread = spdk_get_thread(); + /* F1: the md region is the FIRST boundary the blobstore crosses; round it UP to the cluster + * grain so the md/data boundary is cluster-aligned. Band boundaries are aligned in add_band. */ + t->cluster_blocks = cluster_blocks; + t->md_num_blocks = tier_align_up(t, md_num_blocks); + t->md_mirror_a = UINT32_MAX; + t->md_mirror_b = UINT32_MAX; + t->blocklen = 0; + t->total_num_blocks = 0; + /* F-2: mint the composite INSTANCE uuid — stored in every SB copy. A + * re-created composite gets a fresh uuid, so disks left over from a + * previous life can never be cross-assembled with the new instance. */ + { + struct spdk_uuid u; + + SPDK_STATIC_ASSERT(sizeof(u) == TIER_SB_GEN_UUID_LEN, "uuid size"); + spdk_uuid_generate(&u); + memcpy(t->gen_uuid, &u, sizeof(t->gen_uuid)); + } + + t->bdev.name = strdup(name); + if (t->bdev.name == NULL) { + free(t); + return NULL; + } + t->bdev.product_name = "tier"; + t->bdev.write_cache = 0; + t->bdev.ctxt = t; + t->bdev.fn_table = &vbdev_tier_fn_table; + t->bdev.module = &tier_if; + + TAILQ_INSERT_TAIL(&g_tier_nodes, t, link); + return t; +} + +int +vbdev_tier_add_band(struct vbdev_tier *t, const char *base_bdev_name, enum tier_class tier, + const char *wwn, const char *serial, uint32_t *out_band_id) +{ + struct tier_band *band; + struct spdk_bdev *base_bdev; + uint64_t usable_blocks; + int rc; + + /* Geometry is frozen at register (blockcnt, per-reactor base channels): a band + * added afterwards would not be addressable (stale blockcnt, no base_ch on the + * live channels) yet would be persisted to the SB — a reboot then reassembles a + * geometry the running composite never served. The CSI contract adds all bands + * before register; enforce it. */ + if (t->registered) { + SPDK_ERRLOG("tier '%s': add_band after register is not allowed\n", t->bdev.name); + return -EBUSY; + } + /* Bound the slot BEFORE it is assigned: band_id indexes the fixed-size + * base_ch[TIER_MAX_BANDS] and is stored in a 64-slot superblock. assemble_band + * bounds it too; add_band must match or an out-of-range id reads past base_ch[]. */ + if (t->next_band_id >= TIER_MAX_BANDS) { + SPDK_ERRLOG("tier '%s': cannot add band, max %d reached\n", t->bdev.name, + TIER_MAX_BANDS); + return -ENOSPC; + } + + /* SPEC-73 (sb-validate): reject a disk identity already present in this composite. + * A duplicate wwn means the same physical disk was enumerated into two bands — that + * would silently double-count capacity and corrupt the concat geometry. Cheap in-memory + * guard; the on-disk superblock additionally guards swap/replacement at assembly. */ + if (wwn != NULL && wwn[0] != '\0') { + struct tier_band *existing; + TAILQ_FOREACH(existing, &t->bands, link) { + if (strncmp(existing->wwn, wwn, sizeof(existing->wwn)) == 0) { + SPDK_ERRLOG("tier: band wwn '%s' already present (duplicate disk '%s')\n", + wwn, base_bdev_name); + return -EEXIST; + } + } + } + + band = calloc(1, sizeof(*band)); + if (band == NULL) { + return -ENOMEM; + } + band->t = t; + + rc = spdk_bdev_open_ext(base_bdev_name, true, tier_base_event_cb, band, &band->desc); + if (rc != 0) { + SPDK_ERRLOG("tier: cannot open base bdev '%s' rc=%d\n", base_bdev_name, rc); + free(band); + return rc; + } + base_bdev = spdk_bdev_desc_get_bdev(band->desc); + + /* All bands must share the block size (mixing 512/4096 corrupts geometry). */ + if (t->blocklen == 0) { + t->blocklen = base_bdev->blocklen; + /* R7: the superblock is written as whole TIER_SB_SLOT_BYTES slots, so the + * slot must be an integral number of blocks. A T10-DIF blocklen (520/4160) + * that does not divide the slot would make the FIRST SB write fail -EINVAL at + * register — leaving a registered-but-superblock-less composite that no + * reassembly can find (read_sb → -EILSEQ). Reject the incompatible base here. */ + if (TIER_SB_SLOT_BYTES % t->blocklen != 0) { + SPDK_ERRLOG("tier: band '%s' blocklen %u cannot host the superblock slot " + "(%d not a multiple)\n", base_bdev_name, t->blocklen, + TIER_SB_SLOT_BYTES); + spdk_bdev_close(band->desc); + free(band); + return -EINVAL; + } + } else if (base_bdev->blocklen != t->blocklen) { + SPDK_ERRLOG("tier: band '%s' blocklen %u != composite %u\n", + base_bdev_name, base_bdev->blocklen, t->blocklen); + spdk_bdev_close(band->desc); + free(band); + return -EINVAL; + } + /* Reserve a superblock region at the start of EACH base bdev (INV-T1). */ + if (t->sb_blocks == 0) { + t->sb_blocks = spdk_divide_round_up(TIER_SB_RESERVE_BYTES, t->blocklen); + } + + rc = spdk_bdev_module_claim_bdev(base_bdev, band->desc, &tier_if); + if (rc != 0) { + SPDK_ERRLOG("tier: cannot claim base bdev '%s' rc=%d\n", base_bdev_name, rc); + spdk_bdev_close(band->desc); + free(band); + return rc; + } + + band->band_id = t->next_band_id++; + band->tier = tier; + band->state = TIER_BAND_ACTIVE; + snprintf(band->base_bdev_name, sizeof(band->base_bdev_name), "%s", base_bdev_name); + if (wwn) { + snprintf(band->wwn, sizeof(band->wwn), "%s", wwn); + } + if (serial) { + snprintf(band->serial, sizeof(band->serial), "%s", serial); + } + + /* Geometry. Each disk reserves [0, sb_blocks) for the superblock; usable = + * blockcnt - sb_blocks. The first two bands additionally host the mirrored md + * region [0, md_num_blocks) of the composite at base-physical + * [sb_blocks, sb_blocks+md_num_blocks); their DATA contribution follows. */ + if (base_bdev->blockcnt <= t->sb_blocks) { + SPDK_ERRLOG("tier: band '%s' too small for superblock reserve\n", base_bdev_name); + spdk_bdev_module_release_bdev(base_bdev); + spdk_bdev_close(band->desc); + free(band); + return -ENOSPC; + } + usable_blocks = base_bdev->blockcnt - t->sb_blocks; + + if (t->md_mirror_a == UINT32_MAX) { + /* Band A: hosts the md region (counted ONCE in the composite) + a data tail. */ + if (usable_blocks <= t->md_num_blocks) { + SPDK_ERRLOG("tier: band '%s' too small for md region\n", base_bdev_name); + spdk_bdev_module_release_bdev(base_bdev); + spdk_bdev_close(band->desc); + free(band); + return -ENOSPC; + } + t->md_mirror_a = band->band_id; + t->total_num_blocks = t->md_num_blocks; /* md region occupies [0, md), cluster-aligned */ + band->lba_start = t->md_num_blocks; /* this band's data tail follows md (aligned) */ + /* F1: round the data contribution DOWN to the cluster grain (the trailing remainder is an + * unusable hole) so the NEXT band starts cluster-aligned and no cluster straddles. */ + band->num_blocks = tier_align_down(t, usable_blocks - t->md_num_blocks); + band->phys_offset = t->sb_blocks + t->md_num_blocks; + t->total_num_blocks += band->num_blocks; + } else if (t->md_mirror_b == UINT32_MAX) { + /* Band B: hosts the md MIRROR (not re-counted) + a data tail. */ + if (usable_blocks <= t->md_num_blocks) { + SPDK_ERRLOG("tier: band '%s' too small for md mirror\n", base_bdev_name); + spdk_bdev_module_release_bdev(base_bdev); + spdk_bdev_close(band->desc); + free(band); + return -ENOSPC; + } + t->md_mirror_b = band->band_id; + band->lba_start = t->total_num_blocks; /* aligned (prior boundaries aligned) */ + band->num_blocks = tier_align_down(t, usable_blocks - t->md_num_blocks); /* F1 */ + band->phys_offset = t->sb_blocks + t->md_num_blocks; + t->total_num_blocks += band->num_blocks; + } else { + /* Plain concat band. */ + band->lba_start = t->total_num_blocks; /* aligned */ + band->num_blocks = tier_align_down(t, usable_blocks); /* F1 */ + band->phys_offset = t->sb_blocks; + t->total_num_blocks += band->num_blocks; + } + + if (band->num_blocks == 0) { + SPDK_ERRLOG("tier: band '%s' has no cluster-aligned capacity (cluster_blocks=%" PRIu64 ")\n", + base_bdev_name, t->cluster_blocks); + spdk_bdev_module_release_bdev(base_bdev); + spdk_bdev_close(band->desc); + free(band); + return -ENOSPC; + } + + TAILQ_INSERT_TAIL(&t->bands, band, link); + t->num_bands++; + + if (out_band_id) { + *out_band_id = band->band_id; + } + SPDK_NOTICELOG("tier '%s': added band %u ('%s', tier=%d) lba_start=%" PRIu64 + " num_blocks=%" PRIu64 "\n", t->bdev.name, band->band_id, + base_bdev_name, tier, band->lba_start, band->num_blocks); + return 0; +} + +/* SPEC-73 A2: place a band at an EXPLICIT stored geometry (from the on-disk superblock), instead of the + * add_band auto-layout. The CSI agent uses this to reassemble a composite IDENTICALLY across reboots + * (stable slot→lba_start, regardless of disk enumeration order) and to detect a swapped disk (its live + * wwn won't match the slot's stored wwn). is_md ⇒ this band hosts the mirrored md region. */ +int +vbdev_tier_assemble_band(struct vbdev_tier *t, const char *base_bdev_name, uint32_t band_id, + enum tier_class tier, const char *wwn, const char *serial, + uint64_t lba_start, uint64_t num_blocks, enum tier_band_state state, bool is_md) +{ + struct tier_band *band, *existing; + struct spdk_bdev *base_bdev; + uint64_t phys_offset; + int rc; + + /* R8: the band table is FROZEN at register (blockcnt + per-reactor base + * channels are fixed there). add_band already refuses a post-register add; + * assemble_band did the same TAILQ insert without the guard, so a runtime + * bdev_tier_assemble_band (SPDK_RPC_RUNTIME) could splice into t->bands while + * reactors walk it lock-free — a torn `next` pointer. Assembly is always + * create → assemble → register (CSI contract); enforce it. */ + if (t->registered) { + SPDK_ERRLOG("tier '%s': assemble_band after register is not allowed\n", t->bdev.name); + return -EBUSY; + } + /* M6/T-3/W7: the RPC decodes band_id/state as raw u32 — bound BOTH before + * they index base_ch[] or route I/O (state=5 would route like ACTIVE). */ + if (band_id >= TIER_MAX_BANDS) { + SPDK_ERRLOG("tier: assemble band_id %u out of range (max %d)\n", + band_id, TIER_MAX_BANDS - 1); + return -EINVAL; + } + if (state > TIER_BAND_RETIRED) { + SPDK_ERRLOG("tier: assemble band %u invalid state %d\n", band_id, state); + return -EINVAL; + } + if (num_blocks == 0) { + return -EINVAL; + } + /* Reject ranges whose composite end wraps u64 — otherwise the overlap and + * capacity checks below (lba_start + num_blocks, phys_offset + num_blocks) + * wrap to a small value and an oversized band slips past every guard. */ + if (lba_start > UINT64_MAX - num_blocks) { + SPDK_ERRLOG("tier: assemble band %u range [%" PRIu64 ", +%" PRIu64 ") overflows\n", + band_id, lba_start, num_blocks); + return -EINVAL; + } + /* M6: the data region starts after the mirrored md region; a band placed + * inside [0, md) would shadow the mirrored L2P range. */ + if (lba_start < t->md_num_blocks) { + SPDK_ERRLOG("tier: assemble band %u lba_start %" PRIu64 " inside md region\n", + band_id, lba_start); + return -EINVAL; + } + if (is_md && t->md_num_blocks == 0) { + SPDK_ERRLOG("tier: assemble band %u is_md on a composite without md region\n", band_id); + return -EINVAL; + } + if (is_md && t->md_mirror_a != UINT32_MAX && t->md_mirror_b != UINT32_MAX) { + SPDK_ERRLOG("tier: assemble band %u — both md mirror slots already assigned\n", band_id); + return -EINVAL; + } + if (vbdev_tier_band_by_id(t, band_id) != NULL) { + return -EEXIST; + } + /* M6: no two bands may overlap in the composite address space (a retired + * slot keeps its range as an unreclaimable hole, so it counts too). */ + TAILQ_FOREACH(existing, &t->bands, link) { + if (lba_start < existing->lba_start + existing->num_blocks && + existing->lba_start < lba_start + num_blocks) { + SPDK_ERRLOG("tier: assemble band %u [%" PRIu64 ", +%" PRIu64 + ") overlaps band %u\n", band_id, lba_start, num_blocks, + existing->band_id); + return -EEXIST; + } + /* Same duplicate-disk guard as add_band. */ + if (wwn != NULL && wwn[0] != '\0' && + strncmp(existing->wwn, wwn, sizeof(existing->wwn)) == 0) { + SPDK_ERRLOG("tier: assemble band wwn '%s' already present\n", wwn); + return -EEXIST; + } + } + band = calloc(1, sizeof(*band)); + if (band == NULL) { + return -ENOMEM; + } + band->t = t; + rc = spdk_bdev_open_ext(base_bdev_name, true, tier_base_event_cb, band, &band->desc); + if (rc != 0) { + SPDK_ERRLOG("tier: assemble cannot open '%s' rc=%d\n", base_bdev_name, rc); + free(band); + return rc; + } + base_bdev = spdk_bdev_desc_get_bdev(band->desc); + if (t->blocklen == 0) { + t->blocklen = base_bdev->blocklen; + /* R7: reject a blocklen that cannot host the fixed superblock slot (see + * vbdev_tier_add_band) — else the first SB persist fails and the composite + * is unrecoverable. */ + if (TIER_SB_SLOT_BYTES % t->blocklen != 0) { + SPDK_ERRLOG("tier: assemble band '%s' blocklen %u cannot host the superblock " + "slot (%d not a multiple)\n", base_bdev_name, t->blocklen, + TIER_SB_SLOT_BYTES); + spdk_bdev_close(band->desc); + free(band); + return -EINVAL; + } + } else if (base_bdev->blocklen != t->blocklen) { + SPDK_ERRLOG("tier: assemble band '%s' blocklen %u != composite %u\n", + base_bdev_name, base_bdev->blocklen, t->blocklen); + spdk_bdev_close(band->desc); + free(band); + return -EINVAL; + } + if (t->sb_blocks == 0) { + t->sb_blocks = spdk_divide_round_up(TIER_SB_RESERVE_BYTES, t->blocklen); + } + /* New: the stored geometry must FIT the real disk (the F1 register guard only + * checks alignment) — otherwise the band's tail returns -EIO at runtime. */ + phys_offset = is_md ? (t->sb_blocks + t->md_num_blocks) : t->sb_blocks; + if (phys_offset >= base_bdev->blockcnt || + num_blocks > base_bdev->blockcnt - phys_offset) { + SPDK_ERRLOG("tier: assemble band %u geometry (phys_off=%" PRIu64 " num=%" PRIu64 + ") exceeds disk '%s' capacity %" PRIu64 "\n", band_id, phys_offset, + num_blocks, base_bdev_name, base_bdev->blockcnt); + spdk_bdev_close(band->desc); + free(band); + return -ENOSPC; + } + rc = spdk_bdev_module_claim_bdev(base_bdev, band->desc, &tier_if); + if (rc != 0) { + SPDK_ERRLOG("tier: assemble cannot claim '%s' rc=%d\n", base_bdev_name, rc); + spdk_bdev_close(band->desc); + free(band); + return rc; + } + band->band_id = band_id; + band->tier = tier; + band->state = state; + snprintf(band->base_bdev_name, sizeof(band->base_bdev_name), "%s", base_bdev_name); + if (wwn) { + snprintf(band->wwn, sizeof(band->wwn), "%s", wwn); + } + if (serial) { + snprintf(band->serial, sizeof(band->serial), "%s", serial); + } + band->lba_start = lba_start; + band->num_blocks = num_blocks; + /* md-hosting bands carry the mirrored md region at base-physical [sb_blocks, sb_blocks+md); their + * data tail starts after it. Plain bands start at sb_blocks. (Matches vbdev_tier_add_band.) */ + band->phys_offset = is_md ? (t->sb_blocks + t->md_num_blocks) : t->sb_blocks; + if (is_md) { + if (t->md_mirror_a == UINT32_MAX) { + t->md_mirror_a = band_id; + } else if (t->md_mirror_b == UINT32_MAX) { + t->md_mirror_b = band_id; + } + } + if (band_id >= t->next_band_id) { + t->next_band_id = band_id + 1; + } + if (lba_start + num_blocks > t->total_num_blocks) { + t->total_num_blocks = lba_start + num_blocks; + } + TAILQ_INSERT_TAIL(&t->bands, band, link); + t->num_bands++; + SPDK_NOTICELOG("tier '%s': assembled band %u ('%s', tier=%d, state=%d) lba_start=%" PRIu64 + " num_blocks=%" PRIu64 " is_md=%d\n", t->bdev.name, band_id, base_bdev_name, + tier, state, lba_start, num_blocks, is_md); + return 0; +} + +struct tier_retire_ctx { + struct vbdev_tier *t; + struct tier_band *band; + void (*cb)(void *cb_arg, int rc); + void *cb_arg; + int persist_rc; +}; + +static void +tier_retire_drained(void *cb_arg, int rc) +{ + struct tier_retire_ctx *ctx = cb_arg; + + (void)rc; + SPDK_NOTICELOG("tier '%s': retired band %u (persist rc=%d)\n", + ctx->t->bdev.name, ctx->band->band_id, ctx->persist_rc); + if (ctx->cb) { + ctx->cb(ctx->cb_arg, ctx->persist_rc); + } + free(ctx); +} + +static void +tier_retire_persisted(void *cb_arg, int rc) +{ + struct tier_retire_ctx *ctx = cb_arg; + + ctx->persist_rc = rc; + /* T-4: drain the band's per-reactor channels BEFORE closing its desc (the + * old direct close raced live channels — UAF at the deferred put). On a + * persist failure we still drain+close, but report the error so the CSI + * retries the (idempotent) retire until the SB is durable (MJ6). */ + if (tier_band_drain_and_close(ctx->t, ctx->band, tier_retire_drained, ctx) != 0) { + if (ctx->cb) { + ctx->cb(ctx->cb_arg, rc != 0 ? rc : -ENOMEM); + } + free(ctx); + } +} + +int +vbdev_tier_retire_band(struct vbdev_tier *t, uint32_t band_id, + void (*cb)(void *cb_arg, int rc), void *cb_arg) +{ + struct tier_band *band = vbdev_tier_band_by_id(t, band_id); + struct tier_retire_ctx *ctx; + + if (band == NULL) { + return -ENODEV; + } + /* T-7: an md-mirror band holds one of the two L2P copies; retiring it would + * destroy the blobstore-metadata redundancy with no rebuild path. */ + if (band_id == t->md_mirror_a || band_id == t->md_mirror_b) { + SPDK_ERRLOG("tier '%s': refusing to retire md-mirror band %u\n", + t->bdev.name, band_id); + return -EBUSY; + } + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return -ENOMEM; + } + ctx->t = t; + ctx->band = band; + ctx->cb = cb; + ctx->cb_arg = cb_arg; + + /* The CSI brain guarantees the band was evacuated (clusters relocated) before + * retiring. We keep the slot and its LBA range as an unreclaimable hole. + * Re-running the flow on an already-RETIRED band is the idempotent retry + * path: it re-persists (in case the first persist failed) and re-closes. */ + band->state = TIER_BAND_RETIRED; + /* Persist to the SURVIVING bands BEFORE closing the retired one's desc (so + * the seq bump durably records the retirement), then drain+close, then + * complete (MJ6: the caller acks only a durable retirement). */ + if (t->registered) { + if (tier_sb_write_all(t, tier_retire_persisted, ctx) != 0) { + free(ctx); + return -ENOMEM; + } + } else { + tier_retire_persisted(ctx, 0); + } + return 0; +} + +int +vbdev_tier_delete(struct vbdev_tier *t) +{ + /* T-4b/R9: an in-flight async op — an SB fan-out, a relocate copy, an md + * resync, or the register-time seq rehydrate — holds this composite's base-band + * descriptors and/or pointers into `t`. Unregistering now would free `t` and its + * bands under the op (UAF + close-with-I/O). Defer the teardown: the SB fan-out + * completes it via vbdev_tier_sb_fanout_idle, the others via tier_async_op_end. + * sb_write_queued is included: the coalesced follow-up will hold descriptors. */ + if (t->sb_write_inflight || t->sb_write_queued || t->async_inflight > 0) { + t->delete_pending = true; + return 0; + } + if (t->registered) { + /* registered: unregister triggers destruct (frees bands + node) */ + spdk_bdev_unregister(&t->bdev, NULL, NULL); + } else { + /* never registered: free directly */ + TAILQ_REMOVE(&g_tier_nodes, t, link); + _tier_device_unregister_cb(t); + } + return 0; +} + +/* R9: run a teardown deferred behind async work, but only once EVERYTHING that + * pins `t` has drained (no SB fan-out in flight/queued, no relocate/resync/ + * rehydrate). Returns true if it ran the teardown — the caller must not touch `t`. */ +static bool +tier_run_deferred_delete(struct vbdev_tier *t) +{ + if (t->delete_pending && t->async_inflight == 0 && + !t->sb_write_inflight && !t->sb_write_queued) { + t->delete_pending = false; + vbdev_tier_delete(t); + return true; + } + return false; +} + +/* R9: bracket a composite async op (relocate / resync / register seq-rehydrate) + * whose context holds pointers into `t`. Call _begin before launching the async + * chain and _end at its terminal completion. _end may run a delete deferred behind + * the op — after it returns, the caller must not touch `t`. */ +static void +tier_async_op_begin(struct vbdev_tier *t) +{ + t->async_inflight++; +} + +static void +tier_async_op_end(struct vbdev_tier *t) +{ + if (t->async_inflight > 0) { + t->async_inflight--; + } + tier_run_deferred_delete(t); +} + +/* T-4b: run any teardown deferred behind an SB fan-out, now that it has drained. + * Called from tier_sb_fanout_complete. Returns true if a deferred delete consumed + * the composite (caller must not touch `t`). */ +bool +vbdev_tier_sb_fanout_idle(struct vbdev_tier *t) +{ + struct tier_band *b, *tmp; + + /* A delete is deferred behind the fan-out. If a follow-up fan-out is queued it + * still OWNS callbacks coalesced behind this fan-out — e.g. an md resync's + * persist (tier_md_resync_persisted), whose terminal handler releases the + * resync's async_inflight ref (R9). DROPPING the follow-up would strand those + * cbs, leak the ref, and the deferred delete would NEVER run (composite + descs + * leaked, delete/resync RPCs hung). So DON'T drop it: return false and let the + * caller run the follow-up, which serves the cbs; the served op's terminal then + * releases its ref and (eventually) runs the deferred delete. Only when no + * follow-up remains do we try to tear down here. */ + if (t->delete_pending) { + if (t->sb_write_queued) { + return false; /* caller runs the queued follow-up (serves the cbs) */ + } + tier_run_deferred_delete(t); /* runs iff async_inflight == 0 */ + return true; + } + /* A base hot-remove that landed during the fan-out deferred the degraded + * band's channel-drain+close. The fan-out has drained, so the desc no longer + * has an in-flight SB write — close it now (the band is DEGRADED, hence + * excluded from any follow-up fan-out). */ + TAILQ_FOREACH_SAFE(b, &t->bands, link, tmp) { + if (!b->close_pending) { + continue; + } + b->close_pending = false; + if (tier_band_drain_and_close(t, b, NULL, NULL) != 0) { + SPDK_ERRLOG("tier: deferred drain of band %u failed (out of memory); " + "desc left open\n", b->band_id); + } + } + return false; +} + +/* Fire-and-forget superblock persistence completion (logs failures). */ +static void +tier_sb_persist_cb(void *cb_arg, int rc) +{ + if (rc != 0) { + SPDK_ERRLOG("tier: superblock persist failed rc=%d\n", rc); + } +} + +/* R2: at register the CSI does not thread the on-disk `seq` back down — it reads + * seq only to arbitrate the authoritative SB (control-plane side), never through + * create/assemble/register — so the FORK owns seq monotonicity across a restart. A + * fresh composite starts t->seq = 0; without rehydration, register's first persist + * writes seq 1, which a pre-restart SB sitting at a high seq out-votes FOREVER + * (tier_sb_select is highest-seq-wins) — the composite then reassembles to the + * STALE geometry, silently undoing every retire/relocate persisted at the high seq. + * Fix: re-read every band's on-disk SB, seed t->seq to the highest seq found, THEN + * persist (its seq is max+1, so it wins). Best-effort: if no read can be launched, + * persist at the current seq. */ +struct tier_register_seed_ctx { + struct vbdev_tier *t; + int remaining; + uint64_t max_seq; +}; + +static void +tier_register_seed_finish(struct tier_register_seed_ctx *ctx) +{ + struct vbdev_tier *t = ctx->t; + + if (ctx->max_seq > t->seq) { + t->seq = ctx->max_seq; /* the persist below reserves seq max+1 (monotone) */ + } + free(ctx); + /* Launch the persist BEFORE releasing the rehydrate ref: tier_sb_write_all sets + * sb_write_inflight synchronously, so a bdev_tier_delete deferred behind the + * rehydrate is then held by the fan-out (run by vbdev_tier_sb_fanout_idle), + * never freed out from under us here. */ + if (tier_sb_write_all(t, tier_sb_persist_cb, NULL) != 0) { + SPDK_ERRLOG("tier '%s': initial superblock persist could not be launched\n", + t->bdev.name); + } + tier_async_op_end(t); /* R9 */ +} + +static void +tier_register_seed_read_cb(void *cb_arg, const struct tier_superblock *sb, int rc) +{ + struct tier_register_seed_ctx *ctx = cb_arg; + + if (rc == 0 && sb != NULL && sb->seq > ctx->max_seq) { + ctx->max_seq = sb->seq; + } + if (--ctx->remaining == 0) { + tier_register_seed_finish(ctx); + } +} + +static void +tier_register_seed_seq_and_persist(struct vbdev_tier *t) +{ + struct tier_register_seed_ctx *ctx; + struct tier_band *b; + + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + /* Fallback: persist at the current seq (may lose cross-restart monotonicity). */ + if (tier_sb_write_all(t, tier_sb_persist_cb, NULL) != 0) { + SPDK_ERRLOG("tier '%s': initial superblock persist could not be launched\n", + t->bdev.name); + } + return; + } + ctx->t = t; + ctx->remaining = 1; /* hold a ref while launching the per-band reads */ + tier_async_op_begin(t); /* R9: defer any delete until the rehydrate + persist drains */ + TAILQ_FOREACH(b, &t->bands, link) { + if (b->desc == NULL) { + continue; /* DEGRADED/absent leg — nothing to read */ + } + ctx->remaining++; + if (tier_sb_read_desc(b->desc, t->blocklen, tier_register_seed_read_cb, ctx) != 0) { + ctx->remaining--; /* this read did not launch */ + } + } + if (--ctx->remaining == 0) { + tier_register_seed_finish(ctx); + } +} + +/* Register the composite bdev once its bands are configured (called by RPC). */ +int +vbdev_tier_register(struct vbdev_tier *t) +{ + int rc; + + /* W1: a re-register would fail spdk_bdev_register with -EEXIST and the error + * path would then spdk_io_device_unregister() the io_device of the LIVE bdev + * (demolition in service). Refuse up front. */ + if (t->registered) { + return -EEXIST; + } + if (t->num_bands == 0 || t->blocklen == 0) { + return -EINVAL; + } + + /* F1 guard: every band/region boundary MUST be cluster-aligned, else a blobstore cluster can + * straddle a boundary and its I/O fails -EIO (silent corruption). Refuse to register otherwise — + * turn a latent corruption into an explicit provisioning error. */ + if (t->cluster_blocks > 1) { + struct tier_band *vb; + if (t->md_num_blocks % t->cluster_blocks != 0) { + SPDK_ERRLOG("tier '%s': md_num_blocks %" PRIu64 " not aligned to cluster %" PRIu64 "\n", + t->bdev.name, t->md_num_blocks, t->cluster_blocks); + return -EINVAL; + } + TAILQ_FOREACH(vb, &t->bands, link) { + if (vb->state == TIER_BAND_RETIRED) { + continue; + } + if (vb->lba_start % t->cluster_blocks != 0 || vb->num_blocks % t->cluster_blocks != 0) { + SPDK_ERRLOG("tier '%s': band %u geometry (lba_start=%" PRIu64 " num_blocks=%" + PRIu64 ") not cluster-aligned (%" PRIu64 ")\n", t->bdev.name, + vb->band_id, vb->lba_start, vb->num_blocks, t->cluster_blocks); + return -EINVAL; + } + } + } + + t->bdev.blocklen = t->blocklen; + t->bdev.blockcnt = t->total_num_blocks; + + spdk_io_device_register(t, tier_ch_create_cb, tier_ch_destroy_cb, + sizeof(struct tier_io_channel), t->bdev.name); + + rc = spdk_bdev_register(&t->bdev); + if (rc != 0) { + SPDK_ERRLOG("tier: bdev_register('%s') failed rc=%d\n", t->bdev.name, rc); + spdk_io_device_unregister(t, NULL); + return rc; + } + t->registered = true; + SPDK_NOTICELOG("tier '%s' registered: %u bands, %" PRIu64 " blocks of %u bytes (sb_blocks=%u)\n", + t->bdev.name, t->num_bands, t->bdev.blockcnt, t->bdev.blocklen, t->sb_blocks); + + /* R2: rehydrate t->seq from the on-disk SBs, THEN persist the superblock to every + * band (INV-T1). This makes the generation monotone across a restart (the CSI + * replays create+assemble with t->seq=0). Async + best-effort; failures logged. */ + tier_register_seed_seq_and_persist(t); + return 0; +} + +/* -------------------------------------------------------------------------- + * relocate-quiesce co-design (M2b) — only the registering module may quiesce. + * -------------------------------------------------------------------------- */ + +/* M2b: direct base-bdev copy between bands (bypasses the composite/quiesce). + * C5 (SPEC-73): after writing, the destination is read back and CRC32c-compared with the source so a + * silent media/write corruption is detected at relocate time (not later via the upper-layer redundancy). + * The copy is disk-to-disk so accel memory-copy offload does not apply; the integrity check does. */ +struct tier_copy_ctx { + struct vbdev_tier *t; /* R9: for the async-op lifecycle ref */ + struct tier_band *src_band; + struct tier_band *dst_band; + struct spdk_io_channel *src_ch; + struct spdk_io_channel *dst_ch; + void *buf; /* source data (also the write buffer) */ + void *vbuf; /* read-back verify buffer */ + uint64_t dst_phys; + uint64_t num_blocks; + uint32_t blocklen; + uint32_t src_crc; /* CRC32c of buf, computed after the source read */ + bool verify; /* PF4: run the C5 read-back+CRC (per disk class) */ + tier_relocate_cb cb_fn; + void *cb_arg; +}; + +static void +tier_copy_finish(struct tier_copy_ctx *c, int rc) +{ + struct vbdev_tier *t = c->t; + + if (c->src_ch) { + spdk_put_io_channel(c->src_ch); + } + if (c->dst_ch) { + spdk_put_io_channel(c->dst_ch); + } + /* C2: release the desc pins AFTER the channels — a hot-remove close + * deferred behind this engine (close_deferred) may run at the unpin. */ + tier_band_desc_unpin(c->src_band); + tier_band_desc_unpin(c->dst_band); + if (c->buf) { + spdk_dma_free(c->buf); + } + if (c->vbuf) { + spdk_dma_free(c->vbuf); + } + c->cb_fn(c->cb_arg, rc); + free(c); + /* R9: release the composite async ref LAST — this may run a bdev_tier_delete + * that was deferred behind this relocate, so `t` must not be touched after. */ + tier_async_op_end(t); +} + +/* A band's desc is NULLed by tier_band_drain_and_close on hot-remove. Each async + * copy step re-validates the destination before submitting the next base I/O — a + * mid-copy hot-remove would otherwise dereference a NULL desc. (With the C2 + * desc_pins the desc can no longer be closed UNDER the engine; the re-check + * remains as the early-abort on a hot-removed/degraded band.) */ +static inline bool +tier_copy_dst_alive(const struct tier_copy_ctx *c) +{ + return c->dst_band->desc != NULL && c->dst_band->state == TIER_BAND_ACTIVE; +} + +/* H5 (Q2): write-then-flush durability step, shared semantics with the SB + * persist path (54d02b2) and the CBT rebuild finish: a base WITHOUT FLUSH + * support has no volatile write cache to drain — the completed write IS + * durable, so short-circuit to the callback instead of submitting a flush + * that spdk_bdev_flush_blocks fails synchronously with -ENOTSUP (which broke + * every relocate and md-resync on bdev_uring bases). The callback must accept + * bdev_io == NULL (nothing to free on the short-circuit paths). */ +static void +tier_flush_or_durable(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, + uint64_t offset_blocks, uint64_t num_blocks, + spdk_bdev_io_completion_cb cb, void *cb_arg) +{ + int rc; + + if (!spdk_bdev_io_type_supported(spdk_bdev_desc_get_bdev(desc), + SPDK_BDEV_IO_TYPE_FLUSH)) { + cb(NULL, true, cb_arg); + return; + } + rc = spdk_bdev_flush_blocks(desc, ch, offset_blocks, num_blocks, cb, cb_arg); + if (rc != 0) { + cb(NULL, false, cb_arg); + } +} + +/* C5: destination read-back complete — CRC32c-compare with the source. */ +static void +tier_copy_verify_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct tier_copy_ctx *c = cb_arg; + uint32_t dst_crc; + + spdk_bdev_free_io(bdev_io); + if (!success) { + tier_copy_finish(c, -EIO); + return; + } + dst_crc = spdk_crc32c_update(c->vbuf, c->num_blocks * (uint64_t)c->blocklen, ~0u); + if (dst_crc != c->src_crc) { + SPDK_ERRLOG("tier: relocate CRC mismatch (src=%08x dst=%08x) — write corruption\n", + c->src_crc, dst_crc); + tier_copy_finish(c, -EIO); + return; + } + tier_copy_finish(c, 0); +} + +/* R3/F-6: destination flushed — durability is now satisfied on EVERY path. If the + * C5 verify is off (PF4), the relocate is done; otherwise read the destination + * back from MEDIA (the flush guaranteed it landed, not just the write cache) and + * CRC-compare with the source. */ +static void +tier_copy_flush_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct tier_copy_ctx *c = cb_arg; + int rc; + + if (bdev_io != NULL) { /* H5: NULL on the no-FLUSH-support short-circuit */ + spdk_bdev_free_io(bdev_io); + } + if (!success) { + tier_copy_finish(c, -EIO); + return; + } + /* PF4: the C5 read-back+verify is optional per disk class — on media the + * control-plane trusts, skip the extra read (the durability flush already ran). + * The blob-freeze (C1) already makes the move correct; verify only detects a + * SILENT media/write corruption at relocate time. */ + if (!c->verify) { + tier_copy_finish(c, 0); + return; + } + c->vbuf = spdk_dma_malloc(c->num_blocks * (uint64_t)c->blocklen, c->blocklen, NULL); + if (c->vbuf == NULL) { + tier_copy_finish(c, -ENOMEM); + return; + } + if (!tier_copy_dst_alive(c)) { + tier_copy_finish(c, -EIO); /* destination hot-removed mid-copy */ + return; + } + rc = spdk_bdev_read_blocks(c->dst_band->desc, c->dst_ch, c->vbuf, c->dst_phys, + c->num_blocks, tier_copy_verify_done, c); + if (rc != 0) { + tier_copy_finish(c, rc); + } +} + +static void +tier_copy_write_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct tier_copy_ctx *c = cb_arg; + + spdk_bdev_free_io(bdev_io); + if (!success) { + tier_copy_finish(c, -EIO); + return; + } + if (!tier_copy_dst_alive(c)) { + tier_copy_finish(c, -EIO); /* destination hot-removed mid-copy */ + return; + } + /* R3: FLUSH the destination on EVERY path (durability, F-6), not only under + * `verify`. The caller swaps the L2P and frees the SOURCE cluster the moment + * this relocate ACKs; if the moved data is still in the destination's volatile + * write cache, a power cut before the cache drains LOSES the cluster. Durability + * (flush-before-commit) is unconditional; the C5 read-back+CRC (which also needs + * this flush so the read hits media) stays opt-in per disk class (PF4). + * H5: on a base without FLUSH support the write is already durable. */ + tier_flush_or_durable(c->dst_band->desc, c->dst_ch, c->dst_phys, c->num_blocks, + tier_copy_flush_done, c); +} + +static void +tier_copy_read_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct tier_copy_ctx *c = cb_arg; + int rc; + + spdk_bdev_free_io(bdev_io); + if (!success) { + tier_copy_finish(c, -EIO); + return; + } + /* C5: snapshot the source CRC now. C1-DRAIN: the caller holds an + * lvol-bdev quiesce that DRAINED in-flight host writes before this copy + * started (the blob freeze alone did not), so the source is stable. */ + if (c->verify) { + c->src_crc = spdk_crc32c_update(c->buf, c->num_blocks * (uint64_t)c->blocklen, ~0u); + } + if (!tier_copy_dst_alive(c)) { + tier_copy_finish(c, -EIO); /* destination hot-removed mid-copy */ + return; + } + rc = spdk_bdev_write_blocks(c->dst_band->desc, c->dst_ch, c->buf, c->dst_phys, + c->num_blocks, tier_copy_write_done, c); + if (rc != 0) { + tier_copy_finish(c, rc); + } +} + +/* F11 / C1 (fixed) + C1-DRAIN: the caller (vbdev_lvol_tier_rpc.c) runs this ENTIRE copy under an + * LVOL-BDEV quiesce (spdk_bdev_quiesce — drains outstanding host I/O, then holds new I/O ABOVE the + * blob→LBA translation) plus an inner blob-level freeze, NOT a composite-level quiesce. The + * distinction is what closed the C1 lost-write: a composite quiesce holds host writes BELOW the + * translation, so a held write replays to the OLD lba after the L2P swap (ACKed write lands on a + * freed cluster). Holding ABOVE the translation makes held writes re-translate through the updated + * L2P on release. The DRAIN half matters just as much: the blob freeze alone only gated NEW + * submissions — a write already in flight to the bs_dev could land on old_lba AFTER this copy read + * it (audit C1). The window stalls the blob's I/O for drain+read+flush+readback+commit (~3× one + * cluster, 1 MiB grain) — accepted tradeoff; if latency proves unacceptable, switch to + * copy-outside-freeze + re-read-CRC-under-freeze. This copy path reads the base bdevs DIRECTLY, so + * it is not itself held by the quiesce or the freeze. */ +int +vbdev_tier_relocate_copy(struct vbdev_tier *t, uint64_t src_lba, uint64_t dst_lba, + uint64_t num_blocks, bool verify, tier_relocate_cb cb_fn, void *cb_arg) +{ + struct tier_copy_ctx *c; + struct tier_band *sb, *db; + uint64_t src_off, dst_off; + int rc; + + sb = vbdev_tier_band_of_lba(t, src_lba, &src_off); + db = vbdev_tier_band_of_lba(t, dst_lba, &dst_off); + if (sb == NULL || db == NULL || sb->state != TIER_BAND_ACTIVE || + db->state != TIER_BAND_ACTIVE || sb->desc == NULL || db->desc == NULL) { + return -EIO; + } + /* m2: the copy must stay inside both bands (a straddling range would read or + * write a NEIGHBOUR band's blocks through the wrong phys mapping). */ + if (src_off + num_blocks > sb->num_blocks || dst_off + num_blocks > db->num_blocks) { + SPDK_ERRLOG("tier: relocate copy range straddles a band boundary\n"); + return -EINVAL; + } + + c = calloc(1, sizeof(*c)); + if (c == NULL) { + return -ENOMEM; + } + c->t = t; + tier_async_op_begin(t); /* R9: defer any bdev_tier_delete until this copy drains */ + c->src_band = sb; + c->dst_band = db; + /* C2: pin both descs — a hot-remove drain landing mid-copy defers the + * close to tier_copy_finish's unpin instead of closing under our I/O. */ + tier_band_desc_pin(sb); + tier_band_desc_pin(db); + c->num_blocks = num_blocks; + c->blocklen = t->blocklen; + c->dst_phys = db->phys_offset + dst_off; + c->verify = verify; + c->cb_fn = cb_fn; + c->cb_arg = cb_arg; + c->buf = spdk_dma_malloc(num_blocks * (uint64_t)t->blocklen, t->blocklen, NULL); + c->src_ch = spdk_bdev_get_io_channel(sb->desc); + c->dst_ch = spdk_bdev_get_io_channel(db->desc); + if (c->buf == NULL || c->src_ch == NULL || c->dst_ch == NULL) { + tier_copy_finish(c, -ENOMEM); + return 0; + } + + rc = spdk_bdev_read_blocks(sb->desc, c->src_ch, c->buf, sb->phys_offset + src_off, + num_blocks, tier_copy_read_done, c); + if (rc != 0) { + tier_copy_finish(c, rc); + } + return 0; +} + +/* -------------------------------------------------------------------------- + * C3: md-mirror resync — rebuild a replacement md leg from the healthy one. + * + * The target band is typically a replacement disk assembled DEGRADED into an + * md slot (assemble_band is_md=true). The copy runs under a QUIESCE of the + * composite md range: unlike the relocate path (C1), the md region is + * IDENTITY-mapped (no L2P swap), so held writes replay to the same LBA — and + * they replay AFTER the target is activated, reaching both legs. The direct + * base-bdev copy below is not held by the composite quiesce. + * Stall bound: one full md-region copy (size the md region accordingly). + * -------------------------------------------------------------------------- */ + +struct tier_md_resync_ctx { + struct vbdev_tier *t; + struct tier_band *src; + struct tier_band *dst; + struct spdk_io_channel *src_ch; + struct spdk_io_channel *dst_ch; + void *buf; + uint64_t chunk_blocks; + uint64_t off; /* md-region blocks copied so far */ + uint64_t io_blocks; /* size of the in-flight chunk */ + int rc; + bool quiesced; + void (*cb)(void *cb_arg, int rc); + void *cb_arg; +}; + +static void tier_md_resync_next(struct tier_md_resync_ctx *c); + +/* C4: mirrors tier_copy_dst_alive for the resync engine. The healthy source + * leg must stay ACTIVE; the target is DEGRADED by design (activated at the + * end), so only its desc matters. A hot-remove of either leg mid-resync + * NULLed band->desc here before this check existed → NULL-deref crash of the + * whole target at the next chunk. (With the C2 desc_pins the descs can no + * longer be closed under the engine; this remains the early abort.) */ +static inline bool +tier_md_resync_legs_alive(const struct tier_md_resync_ctx *c) +{ + return c->src->desc != NULL && c->src->state == TIER_BAND_ACTIVE && + c->dst->desc != NULL && c->dst->state != TIER_BAND_RETIRED; +} + +static void +tier_md_resync_unquiesced(void *cb_arg, int status) +{ + struct tier_md_resync_ctx *c = cb_arg; + struct vbdev_tier *t = c->t; + + (void)status; /* best-effort; the resync outcome is c->rc */ + if (c->src_ch) { + spdk_put_io_channel(c->src_ch); + } + if (c->dst_ch) { + spdk_put_io_channel(c->dst_ch); + } + /* C2: release the desc pins AFTER the channels (deferred close may run). */ + tier_band_desc_unpin(c->src); + tier_band_desc_unpin(c->dst); + if (c->buf) { + spdk_dma_free(c->buf); + } + c->cb(c->cb_arg, c->rc); + free(c); + /* R9: release the composite async ref LAST — may run a deferred bdev_tier_delete + * (do not touch `t` afterward). */ + tier_async_op_end(t); +} + +static void +tier_md_resync_finish(struct tier_md_resync_ctx *c, int rc) +{ + c->rc = rc; + if (c->quiesced) { + if (spdk_bdev_unquiesce_range(&c->t->bdev, &tier_if, 0, c->t->md_num_blocks, + tier_md_resync_unquiesced, c) == 0) { + return; + } + SPDK_ERRLOG("tier '%s': md resync unquiesce dispatch failed\n", c->t->bdev.name); + } + tier_md_resync_unquiesced(c, 0); +} + +static void +tier_md_resync_persisted(void *cb_arg, int rc) +{ + struct tier_md_resync_ctx *c = cb_arg; + + if (rc != 0) { + /* Not durable: revert so the CSI retries (the copy itself is redoable). */ + c->dst->state = TIER_BAND_DEGRADED; + } + tier_md_resync_finish(c, rc); +} + +/* Open the resynced leg's base channel on every EXISTING tier io_channel (they + * were created before this band was assembled/degraded, so base_ch[id] is NULL + * there — activating without this would fail every md write leg to it). A + * per-reactor open failure is PROPAGATED (the leg stays DEGRADED) — never + * activate a leg only some reactors can reach. */ +static void +tier_md_resync_ch_open_iter(struct spdk_io_channel_iter *i) +{ + struct tier_md_resync_ctx *c = spdk_io_channel_iter_get_ctx(i); + struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); + struct tier_io_channel *tch = spdk_io_channel_get_ctx(_ch); + uint32_t id = c->dst->band_id; + int rc = 0; + + if (tch->base_ch[id] == NULL && c->dst->desc != NULL) { + tch->base_ch[id] = spdk_bdev_get_io_channel(c->dst->desc); + if (tch->base_ch[id] == NULL) { + rc = -ENOMEM; + } + } + spdk_for_each_channel_continue(i, rc); +} + +static void +tier_md_resync_ch_open_done(struct spdk_io_channel_iter *i, int status) +{ + struct tier_md_resync_ctx *c = spdk_io_channel_iter_get_ctx(i); + + if (status != 0) { + tier_md_resync_finish(c, status); + return; + } + /* Copy durable + channels reachable: activate the leg, persist, unquiesce. */ + c->dst->state = TIER_BAND_ACTIVE; + if (tier_sb_write_all(c->t, tier_md_resync_persisted, c) != 0) { + c->dst->state = TIER_BAND_DEGRADED; + tier_md_resync_finish(c, -ENOMEM); + } +} + +static void +tier_md_resync_flush_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct tier_md_resync_ctx *c = cb_arg; + + if (bdev_io != NULL) { /* H5: NULL on the no-FLUSH-support short-circuit */ + spdk_bdev_free_io(bdev_io); + } + if (!success) { + tier_md_resync_finish(c, -EIO); + return; + } + spdk_for_each_channel(c->t, tier_md_resync_ch_open_iter, c, + tier_md_resync_ch_open_done); +} + +static void +tier_md_resync_write_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct tier_md_resync_ctx *c = cb_arg; + + spdk_bdev_free_io(bdev_io); + if (!success) { + tier_md_resync_finish(c, -EIO); + return; + } + c->off += c->io_blocks; + tier_md_resync_next(c); +} + +static void +tier_md_resync_read_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct tier_md_resync_ctx *c = cb_arg; + int rc; + + spdk_bdev_free_io(bdev_io); + if (!success) { + tier_md_resync_finish(c, -EIO); + return; + } + if (!tier_md_resync_legs_alive(c)) { + tier_md_resync_finish(c, -ENODEV); /* C4: leg hot-removed mid-resync */ + return; + } + rc = spdk_bdev_write_blocks(c->dst->desc, c->dst_ch, c->buf, + c->t->sb_blocks + c->off, c->io_blocks, + tier_md_resync_write_done, c); + if (rc != 0) { + tier_md_resync_finish(c, rc); + } +} + +static void +tier_md_resync_next(struct tier_md_resync_ctx *c) +{ + int rc; + + if (!tier_md_resync_legs_alive(c)) { + tier_md_resync_finish(c, -ENODEV); /* C4: leg hot-removed mid-resync */ + return; + } + if (c->off >= c->t->md_num_blocks) { + /* H5: no-FLUSH bases short-circuit to the callback. */ + tier_flush_or_durable(c->dst->desc, c->dst_ch, c->t->sb_blocks, + c->t->md_num_blocks, tier_md_resync_flush_done, c); + return; + } + c->io_blocks = spdk_min(c->chunk_blocks, c->t->md_num_blocks - c->off); + rc = spdk_bdev_read_blocks(c->src->desc, c->src_ch, c->buf, + c->t->sb_blocks + c->off, c->io_blocks, + tier_md_resync_read_done, c); + if (rc != 0) { + tier_md_resync_finish(c, rc); + } +} + +static void +tier_md_resync_quiesced(void *cb_arg, int status) +{ + struct tier_md_resync_ctx *c = cb_arg; + + if (status != 0) { + tier_md_resync_finish(c, status); + return; + } + c->quiesced = true; + tier_md_resync_next(c); +} + +int +vbdev_tier_resync_md(struct vbdev_tier *t, uint32_t target_band_id, + void (*cb)(void *cb_arg, int rc), void *cb_arg) +{ + struct tier_md_resync_ctx *c; + struct tier_band *dst = vbdev_tier_band_by_id(t, target_band_id); + struct tier_band *src; + uint64_t chunk_blocks; + int rc; + + if (!t->registered || t->md_num_blocks == 0) { + return -EINVAL; + } + if (dst == NULL || dst->desc == NULL) { + return -ENODEV; + } + if (target_band_id != t->md_mirror_a && target_band_id != t->md_mirror_b) { + return -EINVAL; /* only md legs carry the mirrored region */ + } + if (dst->state != TIER_BAND_DEGRADED) { + return -EINVAL; /* resync only rebuilds a degraded/replacement leg */ + } + src = tier_md_other_leg(t, dst); + if (src == NULL || src->state != TIER_BAND_ACTIVE || src->desc == NULL) { + return -EIO; /* no healthy leg to copy from */ + } + + c = calloc(1, sizeof(*c)); + if (c == NULL) { + return -ENOMEM; + } + chunk_blocks = spdk_max(1, (1024u * 1024u) / t->blocklen); /* 1 MiB chunks */ + c->t = t; + tier_async_op_begin(t); /* R9: defer any bdev_tier_delete until this resync drains */ + c->src = src; + c->dst = dst; + /* C2: pin both descs — a hot-remove drain landing mid-resync defers the + * close to the engine's terminal unpin instead of closing under our I/O. */ + tier_band_desc_pin(src); + tier_band_desc_pin(dst); + c->chunk_blocks = chunk_blocks; + c->cb = cb; + c->cb_arg = cb_arg; + c->buf = spdk_dma_malloc(chunk_blocks * (uint64_t)t->blocklen, t->blocklen, NULL); + c->src_ch = spdk_bdev_get_io_channel(src->desc); + c->dst_ch = spdk_bdev_get_io_channel(dst->desc); + if (c->buf == NULL || c->src_ch == NULL || c->dst_ch == NULL) { + tier_md_resync_finish(c, -ENOMEM); + return 0; + } + rc = spdk_bdev_quiesce_range(&t->bdev, &tier_if, 0, t->md_num_blocks, + tier_md_resync_quiesced, c); + if (rc != 0) { + tier_md_resync_finish(c, rc); + } + return 0; +} + +/* -------------------------------------------------------------------------- + * module init / finish + * -------------------------------------------------------------------------- */ + +/* PR2: per-process boot id, minted once at module init. The CSI compares it + * across polls to detect a target restart — which invalidates ALL volatile + * state (standing pauses, in-flight relocations, cbt epochs). Exposed by + * evariops_get_capabilities (vbdev_tier_rpc.c). */ +char g_tier_boot_id[SPDK_UUID_STRING_LEN]; + +static int +vbdev_tier_init(void) +{ + struct spdk_uuid u; + + spdk_uuid_generate(&u); + spdk_uuid_fmt_lower(g_tier_boot_id, sizeof(g_tier_boot_id), &u); + return 0; +} + +static void +vbdev_tier_finish(void) +{ +} + +static int +vbdev_tier_get_ctx_size(void) +{ + return sizeof(struct tier_bdev_io); +} + +static int +vbdev_tier_config_json(struct spdk_json_write_ctx *w) +{ + /* CSI replays create/add_band from CRD; nothing to emit. */ + return 0; +} + +SPDK_LOG_REGISTER_COMPONENT(vbdev_tier) diff --git a/module/bdev/tier/vbdev_tier.h b/module/bdev/tier/vbdev_tier.h new file mode 100644 index 0000000..b151ae2 --- /dev/null +++ b/module/bdev/tier/vbdev_tier.h @@ -0,0 +1,377 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 Evariops. All rights reserved. + * + * bdev_tier — composite tier-mapped vbdev (SPEC-73A/D, M1). + * + * One vbdev_tier per node aggregates ALL local disks into a single linear + * address space, split into BANDS (one band == one physical base bdev), + * ordered fast -> slow [Ultra | Premium | Standard | Economy]. + * + * Layout (SPEC-73A §4, §5B, D1): + * LBA 0 .......................................... LBA (size-1) + * [ metadata region (md) | data region (concat of bands) ] + * [ MIRRORED RAID1 on 2 bands | band0 | band1 | band2 | ... ] + * + * - The blobstore consumes the whole thing via spdk_bdev_create_bs_dev(). + * Its super-block + masks + extent-pages (the L2P) live at LOW LBA, so they + * fall in the MIRRORED md region => a single disk loss is non-fatal (D1). + * - The data region is a pure CONCAT (sum of band sizes), routed by address + * arithmetic (no L2P table => no double indirection). lowest-first + * allocation therefore yields write-to-fast for free (SPEC-73A §3). + * - Per-band failure isolation (C-FAIL-1): a degraded band returns an I/O + * ERROR on its own LBA range only; the vbdev NEVER reports is_degraded + * globally (that would force a whole-chunk rebuild — B3 / §10A). + */ + +#ifndef SPDK_VBDEV_TIER_H +#define SPDK_VBDEV_TIER_H + +#include "spdk/stdinc.h" +#include "spdk/assert.h" +#include "spdk/bdev.h" +#include "spdk/bdev_module.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Performance tier — totally ordered, fast -> slow. + * Mirrors SpdkOperator.CustomResources.PerformanceTier (C#). The CSI brain + * resolves a concrete band; SPDK only stores/reports the class. */ +enum tier_class { + TIER_ULTRA = 0, + TIER_PREMIUM = 1, + TIER_STANDARD = 2, + TIER_ECONOMY = 3, + TIER_CLASS_MAX +}; + +/* Lifecycle state of a band (SPEC-73A §5B.4). */ +enum tier_band_state { + TIER_BAND_ACTIVE = 0, + TIER_BAND_DEGRADED = 1, /* disk failed: I/O on its range returns -EIO (C-FAIL-1) */ + TIER_BAND_RETIRED = 2, /* evacuated + removed; slot kept, range unreclaimable */ +}; + +#define TIER_WWN_LEN 64 +#define TIER_SERIAL_LEN 64 +#define TIER_BDEV_NAME_LEN 64 +#define TIER_MAX_BANDS 64 /* per node; a node won't exceed this many disks */ + +/* ---- On-disk superblock v2 (INV-T1: at native SPDK level, à la bdev_raid_sb) ---- + * See docs/FORMAT-tier-superblock.md for the authoritative layout description. + * + * One copy is written into a RESERVED region at the start of EACH base bdev. Each + * copy self-describes the WHOLE composite. There is NO examine path: assembly is + * driven by the CSI agent (SPEC-73 A2), which reads every disk's SB via + * bdev_tier_read_sb, picks the highest-seq copy, and replays + * create + assemble_band at the stored geometry. Swap/replacement detection + * (live wwn vs the slot's stored wwn) is done by the CSI during that replay; + * in-module the only wwn guard is the duplicate-wwn rejection at add/assemble. + * The reserved region is per-disk (NOT inside the mirrored md range). + * + * v2 (clean break — v1 disks are NOT readable; the project pre-dates any public + * deployment, redeploys are wipe+reinstall by design, so no migration path): + * - F-5: the 256 KiB reserve holds TWO 128 KiB SLOTS (A at 0, B at 128 KiB). + * Generation seq N is written to slot N%2, so a torn write destroys at most + * one slot; readers validate both and take the highest-seq valid one. + * - F-3: cluster_blocks widened to u64. + * - F-2: generation_uuid (fencing: identifies the composite INSTANCE — a + * re-created composite mints a new uuid, so stale disks from a previous + * life cannot be cross-assembled), created_epoch_sec (informative wall + * clock), plus 96 B of header reserve and 32 B per band descriptor. + * Format: little-endian only (F-4); layout locked by the static asserts below (F-1). */ +#define TIER_SB_MAGIC 0x5449455253423032ULL /* "TIERSB02" */ +#define TIER_SB_VERSION 2u +#define TIER_SB_RESERVE_BYTES (256 * 1024) /* reserved per base bdev for the sb */ +#define TIER_SB_SLOT_BYTES (128 * 1024) /* F-5: two A/B slots inside the reserve */ +#define TIER_SB_GEN_UUID_LEN 16 + +/* On-disk band descriptor (192 B, stable layout). */ +struct tier_sb_band { + uint32_t band_id; + uint32_t tier; /* enum tier_class */ + uint32_t state; /* enum tier_band_state */ + uint32_t reserved0; + uint64_t lba_start; /* position in the composite address space */ + uint64_t num_blocks; + char wwn[TIER_WWN_LEN]; + char serial[TIER_SERIAL_LEN]; + uint8_t reserved[32]; /* F-2 */ +}; + +/* On-disk superblock (identical content on every band; 256 B header + bands). */ +struct tier_superblock { + uint64_t magic; + uint32_t version; + uint32_t crc; /* CRC32c over the whole struct with crc field = 0 */ + uint64_t seq; /* monotone; on conflict, highest seq wins */ + uint64_t created_epoch_sec; /* wall clock at serialization (informative) */ + uint8_t generation_uuid[TIER_SB_GEN_UUID_LEN]; /* composite instance (fencing, F-2) */ + char composite_name[TIER_BDEV_NAME_LEN]; + uint64_t md_num_blocks; /* size of the mirrored md region (composite blocks) */ + uint64_t cluster_blocks; /* blobstore cluster size in blocks (grain, F1; u64 since v2) */ + uint32_t md_mirror_a; /* band slot ids holding the md RAID1 pair */ + uint32_t md_mirror_b; + uint32_t num_bands; + uint32_t this_band_id; /* which band slot this copy physically sits on */ + uint32_t blocklen; /* common block size */ + uint32_t reserved0; + uint8_t reserved[104]; /* F-2 (pads the header to exactly 256 B) */ + struct tier_sb_band bands[TIER_MAX_BANDS]; +}; + +/* F-1: lock the on-disk ABI — any layout drift is a compile error. */ +SPDK_STATIC_ASSERT(sizeof(struct tier_sb_band) == 192, "tier_sb_band on-disk ABI changed"); +SPDK_STATIC_ASSERT(offsetof(struct tier_superblock, bands) == 256, + "tier_superblock header on-disk ABI changed"); +SPDK_STATIC_ASSERT(sizeof(struct tier_superblock) == 12544, "tier_superblock on-disk ABI changed"); + +/* F-5: generation seq N lives in slot N%2 — alternating slots survive torn writes. */ +static inline uint32_t +tier_sb_slot_for_seq(uint64_t seq) +{ + return (uint32_t)(seq & 1); +} + +/* + * One band == one physical base bdev. bandId is a STABLE monotone slot, + * never reused (a retired disk keeps its slot) — same model as a raid + * base_bdev slot (SPEC-73A §5B.4). + */ +struct tier_band { + uint32_t band_id; + enum tier_class tier; + enum tier_band_state state; + + char base_bdev_name[TIER_BDEV_NAME_LEN]; + char wwn[TIER_WWN_LEN]; /* disk identity — detect a swapped disk in a slot */ + char serial[TIER_SERIAL_LEN]; + + /* Position in the composite linear address space (in blocks). */ + uint64_t lba_start; /* composite start of this band's contribution */ + uint64_t num_blocks; /* usable blocks contributed by this band */ + uint64_t phys_offset; /* base-bdev physical block where lba_start maps + * (>= sb_blocks; mirror band A adds md_num_blocks) */ + + /* Open handle to the underlying disk (NULL while retired or hot-removed). */ + struct spdk_bdev_desc *desc; + + /* Lifecycle (T-4b): a hot-remove that lands while an SB fan-out is in flight + * defers this band's channel-drain+close (closing the desc under an in-flight + * SB write on it would violate the channel-before-desc contract). Resolved by + * vbdev_tier_sb_fanout_idle() once the fan-out drains. */ + bool close_pending; + + /* C2: drain state. A band drain must not put a reactor's base channel while + * host legs are in flight on it (bdev_channel_destroy asserts + * io_outstanding == 0 — abort in debug, channel UAF in release), and must + * not close the desc while a relocate/resync engine still submits on it. + * drain_refs counts the drain fan-out itself (+1) plus every reactor that + * deferred its channel put to its last leg completion; the final release + * closes the desc on the app thread. desc_pins (app-thread only, engines + * run entirely on the app thread) counts engines using this band's desc; a + * close requested while pinned is deferred to the engine's terminal + * (close_deferred). */ + bool draining; + uint32_t drain_refs; /* atomic (released from reactor threads) */ + bool close_deferred; + uint32_t desc_pins; /* app-thread only */ + void (*drain_cb)(void *cb_arg, int rc); + void *drain_cb_arg; + + /* Back-pointer to the composite (needed by the hot-remove event callback, + * which only receives the band as event_ctx). */ + struct vbdev_tier *t; + + TAILQ_ENTRY(tier_band) link; +}; + +/* + * The composite vbdev. One per node. + */ +struct vbdev_tier { + struct spdk_bdev bdev; /* the bdev we register */ + + TAILQ_HEAD(, tier_band) bands; /* ordered fast -> slow, by lba_start */ + uint32_t num_bands; + uint32_t next_band_id; /* monotone slot allocator */ + + /* Mirrored metadata region [0, md_num_blocks) — RAID1 across two bands (D1). + * md_mirror_a / md_mirror_b are band_ids; both hold an identical copy of the + * low LBA range so blobstore metadata survives a single disk loss. */ + uint64_t md_num_blocks; /* size of the mirrored md region, in blocks */ + uint32_t md_mirror_a; + uint32_t md_mirror_b; + + uint32_t blocklen; /* common block size of all bands (must match) */ + uint32_t sb_blocks; /* reserved superblock blocks at the start of EACH base bdev */ + uint8_t gen_uuid[TIER_SB_GEN_UUID_LEN]; /* composite instance uuid (minted at + * create, stored in every SB copy — F-2 fencing) */ + uint64_t cluster_blocks; /* blobstore cluster size in blocks; ALL band/md boundaries are + * aligned to this so no cluster ever straddles a band/region + * boundary (F1) — a straddling cluster would fail I/O (-EIO). */ + uint64_t seq; /* current superblock generation (monotone). M5(a): RESERVED + * (incremented) at write_all entry, so no two write_all + * generations can ever share a seq — gaps are harmless + * ("highest seq wins"), duplicates are fatal. */ + uint64_t total_num_blocks; /* md region + Σ data bands (excludes per-disk sb reserve) */ + bool registered; + + /* H3: the module thread (captured at create — the app/RPC thread). ALL + * composite-global SB-persist state (seq, sb_write_inflight/queued, + * sb_pending_cbs, delete/close_pending) is owned by this thread; reactor- + * side events (md-leg degrade in _tier_leg_complete) funnel their persist + * through spdk_thread_send_msg instead of mutating it in place. */ + struct spdk_thread *thread; + + /* M5(a): serialize tier_sb_write_all — one fan-out in flight at a time; + * concurrent requests queue their callbacks and are coalesced into ONE + * follow-up fan-out that persists the latest state. */ + bool sb_write_inflight; + bool sb_write_queued; + /* T-4b / R9: a bdev_tier_delete that arrives while ANY async op holds this + * composite is deferred here (tearing down now would free `t` and its bands + * under the in-flight op). Honored once every async op drains — by + * vbdev_tier_sb_fanout_idle() for an SB fan-out, or by the last relocate/resync/ + * seq-rehydrate completion. */ + bool delete_pending; + /* R9: count of in-flight composite async ops whose context holds pointers into + * `t` (bands/descs) but which do NOT run through `t`'s io_device (so the SPDK + * io_device refcount does not defer the teardown for them): relocate copies, md + * resyncs, and the register-time seq rehydrate reads. bdev_tier_delete defers + * while this is > 0; the last op to finish runs the deferred teardown. (Retire / + * hot-remove drains use spdk_for_each_channel on `t`'s io_device and ARE covered + * by the io_device refcount, so they are NOT counted here.) */ + uint32_t async_inflight; + TAILQ_HEAD(, tier_sb_pending_cb) sb_pending_cbs; + + TAILQ_ENTRY(vbdev_tier) link; +}; + +/* Queued completion callback for a serialized tier_sb_write_all request. */ +struct tier_sb_pending_cb { + void (*cb)(void *cb_arg, int rc); + void *cb_arg; + TAILQ_ENTRY(tier_sb_pending_cb) link; +}; + +/* + * Per-IO-channel context: one base channel per band (+ the md mirror channels). + */ +struct tier_io_channel { + struct spdk_io_channel *base_ch[TIER_MAX_BANDS]; /* indexed by band slot */ + + /* C2: legs in flight per band on THIS reactor (single-thread access: legs + * complete on their submitting thread — no atomics needed). A band drain + * finding inflight > 0 defers the channel put to the last completion via + * drain_deferred. */ + uint32_t inflight[TIER_MAX_BANDS]; + bool drain_deferred[TIER_MAX_BANDS]; +}; + +/* ---- Internal API (consumed by vbdev_tier.c, vbdev_tier_rpc.c, and the + * relocate/quiesce co-design in M2b) ------------------------------------ */ + +/* Resolve a composite LBA to the owning band + offset within that band. + * Returns NULL if the LBA falls outside any active band's range. The md region + * [0, md_num_blocks) is special-cased by the caller (mirrored). */ +struct tier_band *vbdev_tier_band_of_lba(struct vbdev_tier *t, uint64_t lba, + uint64_t *band_offset); + +/* Resolve by stable slot id. */ +struct tier_band *vbdev_tier_band_by_id(struct vbdev_tier *t, uint32_t band_id); + +/* True when [offset, offset+num_blocks) lies entirely in the mirrored md region. */ +static inline bool +vbdev_tier_is_md_range(const struct vbdev_tier *t, uint64_t offset, uint64_t num_blocks) +{ + return offset < t->md_num_blocks && + (offset + num_blocks) <= t->md_num_blocks; +} + +/* Round an LBA/length down/up to the composite cluster grain (F1: keep every band/region + * boundary cluster-aligned so no blobstore cluster straddles a boundary). cluster_blocks==0 + * (legacy) ⇒ no alignment. */ +static inline uint64_t +tier_align_down(const struct vbdev_tier *t, uint64_t v) +{ + return (t->cluster_blocks > 1) ? (v - (v % t->cluster_blocks)) : v; +} +static inline uint64_t +tier_align_up(const struct vbdev_tier *t, uint64_t v) +{ + return (t->cluster_blocks > 1) ? tier_align_down(t, v + t->cluster_blocks - 1) : v; +} + +/* Lifecycle (RPC-driven, SPEC-73A §9.1 / C-MUT-2). */ +struct vbdev_tier *vbdev_tier_create(const char *name, uint64_t md_num_blocks, uint64_t cluster_blocks); +struct vbdev_tier *vbdev_tier_get_by_name(const char *name); +int vbdev_tier_add_band(struct vbdev_tier *t, const char *base_bdev_name, + enum tier_class tier, const char *wwn, const char *serial, + uint32_t *out_band_id); +/* SPEC-73 A2: place a band at explicit stored geometry (superblock-authoritative reassembly). */ +int vbdev_tier_assemble_band(struct vbdev_tier *t, const char *base_bdev_name, uint32_t band_id, + enum tier_class tier, const char *wwn, const char *serial, + uint64_t lba_start, uint64_t num_blocks, enum tier_band_state state, bool is_md); +/* MJ6: async — cb fires AFTER the retirement is persisted to the surviving bands' + * superblocks (rc != 0 ⇒ NOT durable, caller must retry; in-memory state is + * already RETIRED). T-7: retiring an md-mirror band is refused (-EBUSY). */ +int vbdev_tier_retire_band(struct vbdev_tier *t, uint32_t band_id, + void (*cb)(void *cb_arg, int rc), void *cb_arg); +/* C3: resync the mirrored md region onto a DEGRADED md leg (typically a + * replacement disk assembled DEGRADED into an md slot), then activate it and + * persist. Runs under a quiesce of the composite md range (identity-mapped, so + * held writes replay correctly to BOTH legs after activation). Async; cb gets + * rc != 0 on failure (leg left DEGRADED — retry). */ +int vbdev_tier_resync_md(struct vbdev_tier *t, uint32_t target_band_id, + void (*cb)(void *cb_arg, int rc), void *cb_arg); +/* Register the composite bdev once its bands are configured. */ +int vbdev_tier_register(struct vbdev_tier *t); +/* Tear down + unregister (cleanup). If an SB fan-out is in flight the teardown is + * deferred (T-4b) — see delete_pending. */ +int vbdev_tier_delete(struct vbdev_tier *t); +/* T-4b: called by tier_sb_write_all's fan-out completion once the fan-out has + * drained. Runs any teardown deferred behind the fan-out (a pending delete, or a + * hot-removed band's channel-drain+close). Returns true if the composite was + * consumed by a deferred delete — the caller must not touch `t` afterward. */ +bool vbdev_tier_sb_fanout_idle(struct vbdev_tier *t); + +/* Superblock (vbdev_tier_sb.c) — native-level persistence (INV-T1). */ +void tier_sb_serialize(struct vbdev_tier *t, struct tier_band *self, uint64_t seq, + uint64_t created_epoch_sec, struct tier_superblock *sb); +bool tier_sb_valid(const struct tier_superblock *sb); /* magic + crc check (LE-only, F-4) */ +/* F-5: pick the best (valid, highest-seq) slot inside a full reserve buffer + * (slot A at 0, slot B at TIER_SB_SLOT_BYTES). NULL if neither is valid. + * Pure — unit-tested host-side. */ +const struct tier_superblock *tier_sb_select(const void *reserve_buf, size_t reserve_len); +/* Async-write the (serialized) superblock to every ACTIVE band (M5(b): DEGRADED + * bands are excluded — their stale copy is out-voted by seq at reassembly), then + * FLUSH the written slot (F-6). Generation seq N goes to slot N%2 (F-5), so a + * torn write can only destroy one slot. Serialized (M5(a)): a call while a + * fan-out is in flight queues cb and coalesces into one follow-up fan-out of the + * LATEST state. cb fires once, rc != 0 if any band failed. cb may be NULL. */ +int tier_sb_write_all(struct vbdev_tier *t, void (*cb)(void *cb_arg, int rc), void *cb_arg); +/* Async-read BOTH superblock slots from a base bdev desc; cb receives the best + * valid slot per tier_sb_select (NULL + rc on failure). The sb pointer is only + * valid for the duration of the callback. */ +int tier_sb_read_desc(struct spdk_bdev_desc *desc, uint32_t blocklen, + void (*cb)(void *cb_arg, const struct tier_superblock *sb, int rc), void *cb_arg); + +/* M2b: copy num_blocks from src composite-LBA to dst composite-LBA by resolving + * each to its band + physical offset and doing a direct base-bdev read+write + * (bypasses the composite, so it is NOT held by the caller's blob-level freeze). + * C1: the caller must hold spdk_blob_freeze_io on the owning blob for the whole + * copy+commit — a composite-level quiesce is NOT a valid barrier here (it holds + * writes below the blob→LBA translation and replays them to the OLD lba). */ +typedef void (*tier_relocate_cb)(void *cb_arg, int status); +/* verify (PF4): run the C5 read-back + CRC32c check after the write (detects a + * silent media/write corruption at relocate time). Optional per disk class. */ +int vbdev_tier_relocate_copy(struct vbdev_tier *t, uint64_t src_lba, uint64_t dst_lba, + uint64_t num_blocks, bool verify, tier_relocate_cb cb_fn, void *cb_arg); + +#ifdef __cplusplus +} +#endif + +#endif /* SPDK_VBDEV_TIER_H */ diff --git a/module/bdev/tier/vbdev_tier_rpc.c b/module/bdev/tier/vbdev_tier_rpc.c new file mode 100644 index 0000000..e8a0d01 --- /dev/null +++ b/module/bdev/tier/vbdev_tier_rpc.c @@ -0,0 +1,634 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 Evariops. All rights reserved. + * + * bdev_tier JSON-RPC surface (SPEC-73A §9.1 / 73B C-OBS-2, C-MUT-2). + * + * The CSI control-plane (source of truth in CRD) replays, on agent startup: + * bdev_tier_create -> bdev_tier_add_band (xN) -> bdev_tier_register + * reproducing the identical composite layout. Runtime ops: retire_band, + * get_bands, delete. + */ + +#include "vbdev_tier.h" + +#include "spdk/rpc.h" +#include "spdk/util.h" +#include "spdk/string.h" +#include "spdk/log.h" +#include "spdk/uuid.h" + +/* ---- bdev_tier_create {name, md_num_blocks} ---------------------------------- */ + +struct rpc_tier_create { + char *name; + uint64_t md_num_blocks; + uint64_t cluster_blocks; /* F1: boundary alignment grain (blobstore cluster size in blocks) */ +}; + +static const struct spdk_json_object_decoder rpc_tier_create_decoders[] = { + {"name", offsetof(struct rpc_tier_create, name), spdk_json_decode_string}, + {"md_num_blocks", offsetof(struct rpc_tier_create, md_num_blocks), spdk_json_decode_uint64}, + {"cluster_blocks", offsetof(struct rpc_tier_create, cluster_blocks), spdk_json_decode_uint64, true}, +}; + +static void +rpc_bdev_tier_create(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_tier_create req = {}; + struct vbdev_tier *t; + + if (spdk_json_decode_object(params, rpc_tier_create_decoders, + SPDK_COUNTOF(rpc_tier_create_decoders), &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "invalid parameters"); + free(req.name); /* R15: decode may have strdup'd name before a later field failed */ + return; + } + if (vbdev_tier_get_by_name(req.name) != NULL) { + spdk_jsonrpc_send_error_response_fmt(request, -EEXIST, "tier '%s' already exists", req.name); + free(req.name); + return; + } + /* A composite without an md region silently disables the mirrored-L2P design + * (D1): vbdev_tier_is_md_range() would never match. Refuse it. */ + if (req.md_num_blocks == 0) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "md_num_blocks must be > 0"); + free(req.name); + return; + } + /* R13: cluster_blocks is the F1 alignment grain (blobstore cluster size in + * blocks). 0 is the ONLY legacy value the control-plane replays verbatim for a + * pre-F1 composite (alignment intentionally dormant); >= 2 is a real grain. + * 1 is neither: it silently no-ops tier_align_* AND the register alignment guard + * (t->cluster_blocks > 1), so an unaligned composite registers and later fails + * -EIO on a straddling blobstore cluster. Fresh provisioning always passes the + * real grain (>= 2, in practice >= 256). Reject the ambiguous 1. */ + if (req.cluster_blocks == 1) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "cluster_blocks must be 0 (legacy, no alignment) or >= 2"); + free(req.name); + return; + } + /* R17: reject an md_num_blocks so large the cluster-align round-up + * (tier_align_up: v + cluster_blocks - 1) would overflow u64 and wrap the md + * region to a tiny value — silently disabling the mirrored L2P. A real md region + * is a few GiB; anything near UINT64_MAX is a caller error. */ + if (req.cluster_blocks > 1 && req.md_num_blocks > UINT64_MAX - (req.cluster_blocks - 1)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "md_num_blocks too large (cluster-align overflow)"); + free(req.name); + return; + } + /* F-3: cluster_blocks is stored as u64 on disk since superblock v2 — no u32 cap. */ + t = vbdev_tier_create(req.name, req.md_num_blocks, req.cluster_blocks); + free(req.name); + if (t == NULL) { + spdk_jsonrpc_send_error_response(request, -ENOMEM, "could not create tier"); + return; + } + spdk_jsonrpc_send_bool_response(request, true); +} +SPDK_RPC_REGISTER("bdev_tier_create", rpc_bdev_tier_create, SPDK_RPC_RUNTIME) + +/* ---- bdev_tier_add_band {name, base_bdev_name, tier, wwn?, serial?} ---------- */ + +struct rpc_tier_add_band { + char *name; + char *base_bdev_name; + uint32_t tier; + char *wwn; + char *serial; +}; + +static const struct spdk_json_object_decoder rpc_tier_add_band_decoders[] = { + {"name", offsetof(struct rpc_tier_add_band, name), spdk_json_decode_string}, + {"base_bdev_name", offsetof(struct rpc_tier_add_band, base_bdev_name), spdk_json_decode_string}, + {"tier", offsetof(struct rpc_tier_add_band, tier), spdk_json_decode_uint32}, + {"wwn", offsetof(struct rpc_tier_add_band, wwn), spdk_json_decode_string, true}, + {"serial", offsetof(struct rpc_tier_add_band, serial), spdk_json_decode_string, true}, +}; + +static void +rpc_bdev_tier_add_band(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_tier_add_band req = {}; + struct vbdev_tier *t; + uint32_t band_id = 0; + int rc; + + if (spdk_json_decode_object(params, rpc_tier_add_band_decoders, + SPDK_COUNTOF(rpc_tier_add_band_decoders), &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "invalid parameters"); + goto cleanup; + } + if (req.tier >= TIER_CLASS_MAX) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "invalid tier"); + goto cleanup; + } + t = vbdev_tier_get_by_name(req.name); + if (t == NULL) { + spdk_jsonrpc_send_error_response_fmt(request, -ENODEV, "tier '%s' not found", req.name); + goto cleanup; + } + rc = vbdev_tier_add_band(t, req.base_bdev_name, (enum tier_class)req.tier, req.wwn, req.serial, + &band_id); + if (rc != 0) { + spdk_jsonrpc_send_error_response_fmt(request, rc, "add_band failed: %s", spdk_strerror(-rc)); + goto cleanup; + } + struct spdk_json_write_ctx *w = spdk_jsonrpc_begin_result(request); + spdk_json_write_object_begin(w); + spdk_json_write_named_uint32(w, "band_id", band_id); + spdk_json_write_object_end(w); + spdk_jsonrpc_end_result(request, w); + +cleanup: + free(req.name); + free(req.base_bdev_name); + free(req.wwn); + free(req.serial); +} +SPDK_RPC_REGISTER("bdev_tier_add_band", rpc_bdev_tier_add_band, SPDK_RPC_RUNTIME) + +/* ---- bdev_tier_register {name} ----------------------------------------------- */ + +struct rpc_tier_name { + char *name; +}; + +static const struct spdk_json_object_decoder rpc_tier_name_decoders[] = { + {"name", offsetof(struct rpc_tier_name, name), spdk_json_decode_string}, +}; + +static void +rpc_bdev_tier_register(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_tier_name req = {}; + struct vbdev_tier *t; + int rc; + + if (spdk_json_decode_object(params, rpc_tier_name_decoders, + SPDK_COUNTOF(rpc_tier_name_decoders), &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "invalid parameters"); + free(req.name); /* R15 */ + return; + } + t = vbdev_tier_get_by_name(req.name); + free(req.name); + if (t == NULL) { + spdk_jsonrpc_send_error_response(request, -ENODEV, "tier not found"); + return; + } + rc = vbdev_tier_register(t); + if (rc != 0) { + spdk_jsonrpc_send_error_response_fmt(request, rc, "register failed: %s", spdk_strerror(-rc)); + return; + } + spdk_jsonrpc_send_bool_response(request, true); +} +SPDK_RPC_REGISTER("bdev_tier_register", rpc_bdev_tier_register, SPDK_RPC_RUNTIME) + +/* ---- bdev_tier_delete {name} ------------------------------------------------- */ + +static void +rpc_bdev_tier_delete(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_tier_name req = {}; + struct vbdev_tier *t; + + if (spdk_json_decode_object(params, rpc_tier_name_decoders, + SPDK_COUNTOF(rpc_tier_name_decoders), &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "invalid parameters"); + free(req.name); /* R15 */ + return; + } + /* SEC1: audit this destructive op (tears down the whole composite). */ + spdk_jsonrpc_request_audit(request, "bdev_tier_delete", req.name); + t = vbdev_tier_get_by_name(req.name); + free(req.name); + if (t == NULL) { + spdk_jsonrpc_send_error_response(request, -ENODEV, "tier not found"); + return; + } + vbdev_tier_delete(t); + spdk_jsonrpc_send_bool_response(request, true); +} +SPDK_RPC_REGISTER("bdev_tier_delete", rpc_bdev_tier_delete, SPDK_RPC_RUNTIME) + +/* ---- bdev_tier_retire_band {name, band_id} ---------------------------------- */ + +struct rpc_tier_retire { + char *name; + uint32_t band_id; +}; + +static const struct spdk_json_object_decoder rpc_tier_retire_decoders[] = { + {"name", offsetof(struct rpc_tier_retire, name), spdk_json_decode_string}, + {"band_id", offsetof(struct rpc_tier_retire, band_id), spdk_json_decode_uint32}, +}; + +/* MJ6: the RPC acks only once the retirement is durably persisted to the + * surviving bands' superblocks (rc != 0 ⇒ retry the idempotent retire). */ +static void +rpc_tier_retire_done(void *cb_arg, int rc) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (rc != 0) { + spdk_jsonrpc_send_error_response_fmt(request, rc, + "retire_band not durable (retry): %s", + spdk_strerror(-rc)); + return; + } + spdk_jsonrpc_send_bool_response(request, true); +} + +static void +rpc_bdev_tier_retire_band(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_tier_retire req = {}; + struct vbdev_tier *t; + int rc; + + if (spdk_json_decode_object(params, rpc_tier_retire_decoders, + SPDK_COUNTOF(rpc_tier_retire_decoders), &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "invalid parameters"); + free(req.name); /* R15 */ + return; + } + /* SEC1: audit this destructive op (evacuates + removes a band). */ + { + char detail[128]; + + snprintf(detail, sizeof(detail), "name=%s band_id=%u", req.name, req.band_id); + spdk_jsonrpc_request_audit(request, "bdev_tier_retire_band", detail); + } + t = vbdev_tier_get_by_name(req.name); + free(req.name); + if (t == NULL) { + spdk_jsonrpc_send_error_response(request, -ENODEV, "tier not found"); + return; + } + rc = vbdev_tier_retire_band(t, req.band_id, rpc_tier_retire_done, request); + if (rc != 0) { + spdk_jsonrpc_send_error_response_fmt(request, rc, "retire_band failed: %s", + spdk_strerror(-rc)); + return; + } + /* Response sent from rpc_tier_retire_done after persistence. */ +} +SPDK_RPC_REGISTER("bdev_tier_retire_band", rpc_bdev_tier_retire_band, SPDK_RPC_RUNTIME) + +/* ---- bdev_tier_get_bands {name} -> [{band_id, tier, state, lba_start, ...}] -- */ + +static void +rpc_bdev_tier_get_bands(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_tier_name req = {}; + struct vbdev_tier *t; + struct tier_band *b; + struct spdk_json_write_ctx *w; + + if (spdk_json_decode_object(params, rpc_tier_name_decoders, + SPDK_COUNTOF(rpc_tier_name_decoders), &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "invalid parameters"); + free(req.name); /* R15 */ + return; + } + t = vbdev_tier_get_by_name(req.name); + free(req.name); + if (t == NULL) { + spdk_jsonrpc_send_error_response(request, -ENODEV, "tier not found"); + return; + } + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_object_begin(w); + spdk_json_write_named_array_begin(w, "bands"); + TAILQ_FOREACH(b, &t->bands, link) { + /* Geometry + state only. Fill accounting (used/capacity) is logical and + * lives in the blobstore; the CSI derives it from get_cluster_placement + * (C-OBS-1), so the composite does not emit a duplicate/always-zero copy. */ + spdk_json_write_object_begin(w); + spdk_json_write_named_uint32(w, "band_id", b->band_id); + spdk_json_write_named_uint32(w, "tier", b->tier); + spdk_json_write_named_uint32(w, "state", b->state); + spdk_json_write_named_string(w, "base_bdev_name", b->base_bdev_name); + spdk_json_write_named_string(w, "wwn", b->wwn); + spdk_json_write_named_string(w, "serial", b->serial); + spdk_json_write_named_uint64(w, "lba_start", b->lba_start); + spdk_json_write_named_uint64(w, "num_blocks", b->num_blocks); + spdk_json_write_object_end(w); + } + spdk_json_write_array_end(w); + spdk_json_write_object_end(w); + spdk_jsonrpc_end_result(request, w); +} +SPDK_RPC_REGISTER("bdev_tier_get_bands", rpc_bdev_tier_get_bands, SPDK_RPC_RUNTIME) + +/* ---- SPEC-73 A2: bdev_tier_assemble_band {name, base_bdev_name, band_id, tier, wwn?, serial?, + * lba_start, num_blocks, state, is_md} — place a band at explicit stored geometry ------------ */ + +struct rpc_tier_assemble { + char *name; + char *base_bdev_name; + uint32_t band_id; + uint32_t tier; + char *wwn; + char *serial; + uint64_t lba_start; + uint64_t num_blocks; + uint32_t state; + bool is_md; +}; + +static const struct spdk_json_object_decoder rpc_tier_assemble_decoders[] = { + {"name", offsetof(struct rpc_tier_assemble, name), spdk_json_decode_string}, + {"base_bdev_name", offsetof(struct rpc_tier_assemble, base_bdev_name), spdk_json_decode_string}, + {"band_id", offsetof(struct rpc_tier_assemble, band_id), spdk_json_decode_uint32}, + {"tier", offsetof(struct rpc_tier_assemble, tier), spdk_json_decode_uint32}, + {"wwn", offsetof(struct rpc_tier_assemble, wwn), spdk_json_decode_string, true}, + {"serial", offsetof(struct rpc_tier_assemble, serial), spdk_json_decode_string, true}, + {"lba_start", offsetof(struct rpc_tier_assemble, lba_start), spdk_json_decode_uint64}, + {"num_blocks", offsetof(struct rpc_tier_assemble, num_blocks), spdk_json_decode_uint64}, + {"state", offsetof(struct rpc_tier_assemble, state), spdk_json_decode_uint32, true}, + {"is_md", offsetof(struct rpc_tier_assemble, is_md), spdk_json_decode_bool, true}, +}; + +static void +rpc_bdev_tier_assemble_band(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_tier_assemble req = {}; + struct vbdev_tier *t; + int rc; + + if (spdk_json_decode_object(params, rpc_tier_assemble_decoders, + SPDK_COUNTOF(rpc_tier_assemble_decoders), &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "invalid parameters"); + goto cleanup; + } + if (req.tier >= TIER_CLASS_MAX) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "invalid tier"); + goto cleanup; + } + t = vbdev_tier_get_by_name(req.name); + if (t == NULL) { + spdk_jsonrpc_send_error_response_fmt(request, -ENODEV, "tier '%s' not found", req.name); + goto cleanup; + } + rc = vbdev_tier_assemble_band(t, req.base_bdev_name, req.band_id, (enum tier_class)req.tier, + req.wwn, req.serial, req.lba_start, req.num_blocks, + (enum tier_band_state)req.state, req.is_md); + if (rc != 0) { + spdk_jsonrpc_send_error_response_fmt(request, rc, "assemble_band failed: %s", spdk_strerror(-rc)); + goto cleanup; + } + spdk_jsonrpc_send_bool_response(request, true); + +cleanup: + free(req.name); + free(req.base_bdev_name); + free(req.wwn); + free(req.serial); +} +SPDK_RPC_REGISTER("bdev_tier_assemble_band", rpc_bdev_tier_assemble_band, SPDK_RPC_RUNTIME) + +/* ---- C3: bdev_tier_resync_md {name, band_id} — rebuild a DEGRADED md leg ---- */ + +static void +rpc_tier_resync_md_done(void *cb_arg, int rc) +{ + struct spdk_jsonrpc_request *request = cb_arg; + + if (rc != 0) { + spdk_jsonrpc_send_error_response_fmt(request, rc, "resync_md failed: %s", + spdk_strerror(-rc)); + return; + } + spdk_jsonrpc_send_bool_response(request, true); +} + +static void +rpc_bdev_tier_resync_md(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_tier_retire req = {}; + struct vbdev_tier *t; + int rc; + + if (spdk_json_decode_object(params, rpc_tier_retire_decoders, + SPDK_COUNTOF(rpc_tier_retire_decoders), &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "invalid parameters"); + free(req.name); /* R15 */ + return; + } + /* SEC1: audit this op (rewrites redundancy state of the mirrored md region). */ + { + char detail[128]; + + snprintf(detail, sizeof(detail), "name=%s target_band_id=%u", req.name, req.band_id); + spdk_jsonrpc_request_audit(request, "bdev_tier_resync_md", detail); + } + t = vbdev_tier_get_by_name(req.name); + free(req.name); + if (t == NULL) { + spdk_jsonrpc_send_error_response(request, -ENODEV, "tier not found"); + return; + } + rc = vbdev_tier_resync_md(t, req.band_id, rpc_tier_resync_md_done, request); + if (rc != 0) { + spdk_jsonrpc_send_error_response_fmt(request, rc, "resync_md failed: %s", + spdk_strerror(-rc)); + return; + } + /* Response sent from rpc_tier_resync_md_done. */ +} +SPDK_RPC_REGISTER("bdev_tier_resync_md", rpc_bdev_tier_resync_md, SPDK_RPC_RUNTIME) + +/* ---- SPEC-73 A2: bdev_tier_read_sb {name=base_bdev} -> the on-disk superblock (swap detection) ---- */ + +struct rpc_read_sb_ctx { + struct spdk_jsonrpc_request *request; + struct spdk_bdev_desc *desc; +}; + +static void +rpc_read_sb_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) +{ + /* m7: the desc lives only for the duration of one async SB read and is closed + * in rpc_read_sb_done. A REMOVE during that window just delays the base + * bdev's unregister until the read completes (bounded); log it. */ + if (type == SPDK_BDEV_EVENT_REMOVE) { + SPDK_WARNLOG("tier read_sb: '%s' removed mid-read; desc closes at read completion\n", + bdev->name); + } + (void)ctx; +} + +static void +rpc_read_sb_done(void *cb_arg, const struct tier_superblock *sb, int rc) +{ + struct rpc_read_sb_ctx *c = cb_arg; + struct spdk_json_write_ctx *w; + uint32_t i; + + w = spdk_jsonrpc_begin_result(c->request); + spdk_json_write_object_begin(w); + if (rc != 0 || sb == NULL) { + spdk_json_write_named_bool(w, "valid", false); + } else { + char uuid_str[SPDK_UUID_STRING_LEN]; + + spdk_json_write_named_bool(w, "valid", true); + spdk_json_write_named_string(w, "composite_name", sb->composite_name); + spdk_json_write_named_uint64(w, "seq", sb->seq); + spdk_json_write_named_uint32(w, "version", sb->version); + spdk_json_write_named_uint64(w, "created_epoch_sec", sb->created_epoch_sec); + /* F-2: expose the generation uuid so the CSI can fence stale disks. */ + spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), + (const struct spdk_uuid *)sb->generation_uuid); + spdk_json_write_named_string(w, "generation_uuid", uuid_str); + spdk_json_write_named_uint64(w, "md_num_blocks", sb->md_num_blocks); + spdk_json_write_named_uint64(w, "cluster_blocks", sb->cluster_blocks); + spdk_json_write_named_uint32(w, "md_mirror_a", sb->md_mirror_a); + spdk_json_write_named_uint32(w, "md_mirror_b", sb->md_mirror_b); + spdk_json_write_named_uint32(w, "num_bands", sb->num_bands); + spdk_json_write_named_uint32(w, "this_band_id", sb->this_band_id); + spdk_json_write_named_array_begin(w, "bands"); + for (i = 0; i < sb->num_bands && i < TIER_MAX_BANDS; i++) { + spdk_json_write_object_begin(w); + spdk_json_write_named_uint32(w, "band_id", sb->bands[i].band_id); + spdk_json_write_named_uint32(w, "tier", sb->bands[i].tier); + spdk_json_write_named_uint32(w, "state", sb->bands[i].state); + spdk_json_write_named_uint64(w, "lba_start", sb->bands[i].lba_start); + spdk_json_write_named_uint64(w, "num_blocks", sb->bands[i].num_blocks); + spdk_json_write_named_string(w, "wwn", sb->bands[i].wwn); + spdk_json_write_named_string(w, "serial", sb->bands[i].serial); + spdk_json_write_object_end(w); + } + spdk_json_write_array_end(w); + } + spdk_json_write_object_end(w); + spdk_jsonrpc_end_result(c->request, w); + + spdk_bdev_close(c->desc); + free(c); +} + +static void +rpc_bdev_tier_read_sb(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) +{ + struct rpc_tier_name req = {}; + struct rpc_read_sb_ctx *c; + struct spdk_bdev_desc *desc = NULL; + uint32_t blocklen; + int rc; + + if (spdk_json_decode_object(params, rpc_tier_name_decoders, + SPDK_COUNTOF(rpc_tier_name_decoders), &req)) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "invalid parameters"); + free(req.name); /* R15 */ + return; + } + rc = spdk_bdev_open_ext(req.name, false, rpc_read_sb_event_cb, NULL, &desc); + free(req.name); + if (rc != 0) { + spdk_jsonrpc_send_error_response_fmt(request, rc, "open failed: %s", spdk_strerror(-rc)); + return; + } + blocklen = spdk_bdev_get_block_size(spdk_bdev_desc_get_bdev(desc)); + c = calloc(1, sizeof(*c)); + if (c == NULL) { + spdk_bdev_close(desc); + spdk_jsonrpc_send_error_response(request, -ENOMEM, "oom"); + return; + } + c->request = request; + c->desc = desc; + rc = tier_sb_read_desc(desc, blocklen, rpc_read_sb_done, c); + if (rc != 0) { + spdk_bdev_close(desc); + free(c); + spdk_jsonrpc_send_error_response_fmt(request, rc, "read_sb failed: %s", spdk_strerror(-rc)); + } +} +SPDK_RPC_REGISTER("bdev_tier_read_sb", rpc_bdev_tier_read_sb, SPDK_RPC_RUNTIME) + +/* ---- PR2: evariops_get_capabilities — boot_id + schema versions + methods --- */ + +extern char g_tier_boot_id[]; /* minted at module init (vbdev_tier.c) */ + +/* Candidate list of the Evariops-fork RPC surface the control-plane probes for. + * Deferred #3: this is NOT emitted verbatim — each entry is filtered through the + * LIVE RPC registry (spdk_rpc_get_method_state_mask) so `methods[]` reflects what + * is actually registered in this binary, never a stale hand-maintained list. This + * kills the false POSITIVE the CSI cannot tolerate (a method left here but not + * built in → the CSI gates a path "present" that then fails -32601 at runtime). + * The list scopes WHICH methods to report as fork-capabilities; the registry is + * the source of truth for WHETHER each is present. */ +static const char *g_evariops_methods[] = { + "bdev_tier_create", "bdev_tier_add_band", "bdev_tier_assemble_band", + "bdev_tier_register", "bdev_tier_delete", "bdev_tier_retire_band", + "bdev_tier_resync_md", "bdev_tier_get_bands", "bdev_tier_read_sb", + "bdev_lvol_get_cluster_placement", "bdev_lvol_relocate_cluster", + "bdev_lvol_relocate_clusters", "bdev_lvol_remap_cluster", + "bdev_lvol_remap_clusters", + "bdev_lvol_get_allocated_ranges", + "bdev_raid_rebuild_ranges", "vbdev_nexus_enable_heat", + "vbdev_nexus_disable_heat", "vbdev_nexus_get_heat", + "nvmf_subsystem_pause", "nvmf_subsystem_resume", + "bdev_cbt_create", "bdev_cbt_delete", "bdev_cbt_epoch_open", + "bdev_cbt_epoch_freeze", "bdev_cbt_epoch_close", "bdev_cbt_epoch_invalidate", + "bdev_cbt_epoch_get_dirty_ranges", "bdev_cbt_partial_rebuild", + "bdev_cbt_start_rebuild", "bdev_cbt_get_rebuild_status", + "bdev_cbt_cancel_rebuild", "bdev_cbt_reset", + "evariops_get_capabilities", +}; + +static void +rpc_evariops_get_capabilities(struct spdk_jsonrpc_request *request, + const struct spdk_json_val *params) +{ + struct spdk_json_write_ctx *w; + size_t i; + + /* No params. A spurious body is tolerated (forward-compat with a probe that + * sends options), but an explicit non-object is rejected. */ + if (params != NULL && params->type != SPDK_JSON_VAL_OBJECT_BEGIN) { + spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, + "no parameters expected"); + return; + } + + w = spdk_jsonrpc_begin_result(request); + spdk_json_write_object_begin(w); + /* boot_id: a fresh value across polls means the target restarted — the CSI + * must then treat all volatile state (pauses, relocations, cbt epochs) as lost. */ + spdk_json_write_named_string(w, "boot_id", g_tier_boot_id); + spdk_json_write_named_uint32(w, "tier_sb_version", TIER_SB_VERSION); + spdk_json_write_named_uint32(w, "capabilities_schema", 1); + spdk_json_write_named_bool(w, "single_reactor_assumed", true); /* D5 (see RPC-CONTRACT) */ + spdk_json_write_named_array_begin(w, "methods"); + for (i = 0; i < SPDK_COUNTOF(g_evariops_methods); i++) { + uint32_t state_mask; + + /* Deferred #3: emit only methods present in the LIVE registry. A method + * built in but forgotten from the candidate list is a false NEGATIVE — + * harmless: the CSI falls back to its per-call -32601 probe. Fully removing + * false negatives would need a public registry iterator, which upstream does + * not expose; filtering the candidate list closes the dangerous direction. */ + if (spdk_rpc_get_method_state_mask(g_evariops_methods[i], &state_mask) == 0) { + spdk_json_write_string(w, g_evariops_methods[i]); + } + } + spdk_json_write_array_end(w); + spdk_json_write_object_end(w); + spdk_jsonrpc_end_result(request, w); +} +SPDK_RPC_REGISTER("evariops_get_capabilities", rpc_evariops_get_capabilities, SPDK_RPC_RUNTIME) diff --git a/module/bdev/tier/vbdev_tier_sb.c b/module/bdev/tier/vbdev_tier_sb.c new file mode 100644 index 0000000..8e0aa8c --- /dev/null +++ b/module/bdev/tier/vbdev_tier_sb.c @@ -0,0 +1,437 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2026 Evariops. All rights reserved. + * + * bdev_tier on-disk superblock (SPEC-73A INV-T1, à la bdev_raid_sb). + * One copy per band, in the reserved region [0, sb_blocks) of each base bdev + * (inside the RAID1-mirrored md range). Self-describes the whole composite so + * any present band can drive self-assembly + wwn validation. + */ + +#include "vbdev_tier.h" + +#include "spdk/env.h" +#include "spdk/crc32.h" +#include "spdk/string.h" +#include "spdk/log.h" +#include "spdk/util.h" + +/* ---- serialize / validate -------------------------------------------------- */ + +void +tier_sb_serialize(struct vbdev_tier *t, struct tier_band *self, uint64_t seq, + uint64_t created_epoch_sec, struct tier_superblock *sb) +{ + struct tier_band *b; + uint32_t i = 0; + + memset(sb, 0, sizeof(*sb)); + sb->magic = TIER_SB_MAGIC; + sb->version = TIER_SB_VERSION; + sb->seq = seq; /* M5(a): generation RESERVED at write_all entry (t->seq already bumped) */ + sb->created_epoch_sec = created_epoch_sec; + memcpy(sb->generation_uuid, t->gen_uuid, sizeof(sb->generation_uuid)); + snprintf(sb->composite_name, sizeof(sb->composite_name), "%s", t->bdev.name); + sb->md_num_blocks = t->md_num_blocks; + sb->md_mirror_a = t->md_mirror_a; + sb->md_mirror_b = t->md_mirror_b; + sb->num_bands = t->num_bands; + sb->this_band_id = self ? self->band_id : UINT32_MAX; + sb->blocklen = t->blocklen; + sb->cluster_blocks = t->cluster_blocks; /* F1 grain; u64 on disk since v2 (F-3) */ + + TAILQ_FOREACH(b, &t->bands, link) { + if (i >= TIER_MAX_BANDS) { + break; + } + sb->bands[i].band_id = b->band_id; + sb->bands[i].tier = b->tier; + sb->bands[i].state = b->state; + sb->bands[i].lba_start = b->lba_start; + sb->bands[i].num_blocks = b->num_blocks; + snprintf(sb->bands[i].wwn, sizeof(sb->bands[i].wwn), "%s", b->wwn); + snprintf(sb->bands[i].serial, sizeof(sb->bands[i].serial), "%s", b->serial); + i++; + } + + sb->crc = 0; + sb->crc = spdk_crc32c_update(sb, sizeof(*sb), ~0u); +} + +bool +tier_sb_valid(const struct tier_superblock *sb) +{ + struct tier_superblock tmp; + uint32_t crc; + + /* F-4: the format is declared little-endian only (amd64/arm64 targets). A + * byte-swapped magic means the SB was written by a big-endian host — refuse + * loudly instead of failing the CRC silently. */ + if (sb->magic == __builtin_bswap64(TIER_SB_MAGIC)) { + SPDK_ERRLOG("tier sb: byte-swapped magic (big-endian writer?) — refusing\n"); + return false; + } + if (sb->magic != TIER_SB_MAGIC || sb->version != TIER_SB_VERSION) { + return false; + } + tmp = *sb; + tmp.crc = 0; + crc = spdk_crc32c_update(&tmp, sizeof(tmp), ~0u); + return crc == sb->crc; +} + +/* F-5: pick the best (valid, highest-seq) of the two slots in a reserve buffer. */ +const struct tier_superblock * +tier_sb_select(const void *reserve_buf, size_t reserve_len) +{ + const struct tier_superblock *a, *b; + bool a_ok, b_ok; + + if (reserve_buf == NULL || reserve_len < TIER_SB_RESERVE_BYTES) { + return NULL; + } + a = (const struct tier_superblock *)reserve_buf; + b = (const struct tier_superblock *)((const uint8_t *)reserve_buf + TIER_SB_SLOT_BYTES); + a_ok = tier_sb_valid(a); + b_ok = tier_sb_valid(b); + if (a_ok && b_ok) { + return (b->seq > a->seq) ? b : a; + } + if (a_ok) { + return a; + } + if (b_ok) { + return b; + } + return NULL; +} + +/* ---- async write to all bands ---------------------------------------------- + * + * M5(a): serialized. One fan-out in flight per composite; requests arriving + * meanwhile queue their callback on t->sb_pending_cbs and are coalesced into + * ONE follow-up fan-out of the latest in-memory state. The generation is + * RESERVED (t->seq++) at fan-out start: a partially-failed fan-out can never + * share a seq with a later, different table ("highest seq wins" stays sound — + * gaps are harmless, duplicates are fatal). + * M5(b): only ACTIVE bands are written. The old `!= RETIRED` filter included + * DEGRADED bands whose write always fails → status stuck at -EIO → a + * retirement was never observed as persisted once any band degraded. + * F-6: each copy is FLUSHed after the write — a "committed" seq sitting in a + * volatile write cache is not a commit. */ + +struct tier_sb_write_ctx { + struct vbdev_tier *t; + int remaining; + int status; + TAILQ_HEAD(, tier_sb_pending_cb) cbs; /* callbacks served by THIS fan-out */ +}; + +struct tier_sb_band_write { + struct tier_sb_write_ctx *parent; + struct spdk_bdev_desc *desc; + struct spdk_io_channel *ch; + void *buf; + uint64_t slot_off_blocks; /* F-5: this generation's slot */ + uint32_t slot_blocks; +}; + +static void tier_sb_write_start(struct vbdev_tier *t); + +static void +tier_sb_fanout_complete(struct tier_sb_write_ctx *ctx) +{ + struct vbdev_tier *t = ctx->t; + struct tier_sb_pending_cb *p; + int status = ctx->status; + + /* Run the callbacks queued behind this fan-out. A callback may request a + * teardown (bdev_tier_delete) — that is DEFERRED (delete_pending), never run + * synchronously here, so `t` is guaranteed to survive this loop. */ + while ((p = TAILQ_FIRST(&ctx->cbs)) != NULL) { + TAILQ_REMOVE(&ctx->cbs, p, link); + p->cb(p->cb_arg, status); + free(p); + } + free(ctx); + t->sb_write_inflight = false; + + /* T-4b: honor teardown deferred behind this fan-out (a pending delete, or a + * hot-removed band's drain+close). If a deferred delete consumed the + * composite, `t` is gone/unregistering — do not touch it. */ + if (vbdev_tier_sb_fanout_idle(t)) { + return; + } + if (t->sb_write_queued) { + t->sb_write_queued = false; + tier_sb_write_start(t); + } +} + +static void +tier_sb_band_io_done(struct tier_sb_band_write *bw, bool success) +{ + struct tier_sb_write_ctx *ctx = bw->parent; + + spdk_put_io_channel(bw->ch); + spdk_dma_free(bw->buf); + free(bw); + if (!success) { + ctx->status = -EIO; + } + if (--ctx->remaining == 0) { + tier_sb_fanout_complete(ctx); + } +} + +static void +tier_sb_flush_band_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + spdk_bdev_free_io(bdev_io); + tier_sb_band_io_done(cb_arg, success); +} + +static void +tier_sb_write_band_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct tier_sb_band_write *bw = cb_arg; + int rc; + + spdk_bdev_free_io(bdev_io); + if (!success) { + tier_sb_band_io_done(bw, false); + return; + } + /* F-6: flush before calling this copy durable. Same contract as the CBT + * rebuild path: a base bdev without FLUSH support (bdev_uring) has no + * volatile cache to drain — treat the completed write as durable instead + * of failing the persist. This EIO'd EVERY superblock persist on uring + * bases, so no tier SB was ever written on such nodes. */ + if (!spdk_bdev_io_type_supported(spdk_bdev_desc_get_bdev(bw->desc), + SPDK_BDEV_IO_TYPE_FLUSH)) { + tier_sb_band_io_done(bw, true); + return; + } + rc = spdk_bdev_flush_blocks(bw->desc, bw->ch, bw->slot_off_blocks, bw->slot_blocks, + tier_sb_flush_band_done, bw); + if (rc != 0) { + tier_sb_band_io_done(bw, false); + } +} + +static void +tier_sb_write_start(struct vbdev_tier *t) +{ + struct tier_sb_write_ctx *ctx; + struct tier_band *b; + size_t bufsz = TIER_SB_SLOT_BYTES; /* F-5: one slot per generation */ + uint32_t slot_blocks = TIER_SB_SLOT_BYTES / t->blocklen; + uint64_t target_seq, slot_off_blocks, now_sec; + + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + struct tier_sb_pending_cb *p; + + t->sb_write_inflight = false; + while ((p = TAILQ_FIRST(&t->sb_pending_cbs)) != NULL) { + TAILQ_REMOVE(&t->sb_pending_cbs, p, link); + p->cb(p->cb_arg, -ENOMEM); + free(p); + } + /* M2: this IS a fan-out termination. A bdev_tier_delete deferred + * behind the queued follow-up (delete_pending), or a hot-removed + * band's close_pending, was waiting on vbdev_tier_sb_fanout_idle — + * returning without it stranded the composite (and its claimed base + * descs) forever when the drained callbacks held no async_inflight + * reference. Do not touch `t` if the teardown consumed it. */ + vbdev_tier_sb_fanout_idle(t); + return; + } + t->sb_write_inflight = true; + ctx->t = t; + ctx->status = 0; + TAILQ_INIT(&ctx->cbs); + /* Take ownership of the queued callbacks (no TAILQ_CONCAT on glibc/SPDK). */ + { + struct tier_sb_pending_cb *p; + + while ((p = TAILQ_FIRST(&t->sb_pending_cbs)) != NULL) { + TAILQ_REMOVE(&t->sb_pending_cbs, p, link); + TAILQ_INSERT_TAIL(&ctx->cbs, p, link); + } + } + ctx->remaining = 1; /* hold a ref while we launch, released at the end */ + + /* M5(a): reserve the generation up front (see block comment above). */ + t->seq++; + target_seq = t->seq; + /* F-5: generation N lands in slot N%2 — a torn write only hurts one slot. */ + slot_off_blocks = (uint64_t)tier_sb_slot_for_seq(target_seq) * slot_blocks; + now_sec = (uint64_t)time(NULL); + + int launched = 0; + + TAILQ_FOREACH(b, &t->bands, link) { + struct tier_sb_band_write *bw; + int rc; + + /* M5(b): ACTIVE bands only. */ + if (b->state != TIER_BAND_ACTIVE || b->desc == NULL) { + continue; + } + bw = calloc(1, sizeof(*bw)); + if (bw == NULL) { + ctx->status = -ENOMEM; + continue; + } + bw->parent = ctx; + bw->desc = b->desc; + bw->slot_off_blocks = slot_off_blocks; + bw->slot_blocks = slot_blocks; + bw->buf = spdk_dma_zmalloc(bufsz, t->blocklen, NULL); + if (bw->buf == NULL) { + ctx->status = -ENOMEM; + free(bw); + continue; + } + tier_sb_serialize(t, b, target_seq, now_sec, (struct tier_superblock *)bw->buf); + bw->ch = spdk_bdev_get_io_channel(b->desc); + if (bw->ch == NULL) { + ctx->status = -ENOMEM; + spdk_dma_free(bw->buf); + free(bw); + continue; + } + ctx->remaining++; + rc = spdk_bdev_write_blocks(b->desc, bw->ch, bw->buf, slot_off_blocks, slot_blocks, + tier_sb_write_band_done, bw); + if (rc != 0) { + ctx->remaining--; + ctx->status = rc; + spdk_put_io_channel(bw->ch); + spdk_dma_free(bw->buf); + free(bw); + continue; + } + launched++; + } + + /* M3: zero copies written must NOT complete rc == 0 — the callers' + * contract (rc != 0 ⇒ NOT durable, MJ6) treats 0 as "the state is on + * disk". With every band skipped (all DEGRADED/RETIRED/desc-less), a + * retire acked "durable" would silently resurrect after reboot from the + * old highest-seq superblocks. */ + if (launched == 0 && ctx->status == 0) { + SPDK_ERRLOG("tier '%s': superblock persist wrote ZERO copies (no " + "ACTIVE band with an open desc) — reporting not durable\n", + t->bdev.name); + ctx->status = -ENODEV; + } + + /* Release the holding ref; if no band write was launched, complete now. */ + if (--ctx->remaining == 0) { + tier_sb_fanout_complete(ctx); + } +} + +int +tier_sb_write_all(struct vbdev_tier *t, void (*cb)(void *cb_arg, int rc), void *cb_arg) +{ + if (t->sb_blocks == 0 || t->blocklen == 0 || + TIER_SB_SLOT_BYTES % t->blocklen != 0 || + sizeof(struct tier_superblock) > TIER_SB_SLOT_BYTES) { + return -EINVAL; + } + if (cb != NULL) { + struct tier_sb_pending_cb *p = calloc(1, sizeof(*p)); + + if (p == NULL) { + return -ENOMEM; + } + p->cb = cb; + p->cb_arg = cb_arg; + TAILQ_INSERT_TAIL(&t->sb_pending_cbs, p, link); + } + if (t->sb_write_inflight) { + /* M5(a): coalesce — one follow-up fan-out will persist the latest state. */ + t->sb_write_queued = true; + return 0; + } + tier_sb_write_start(t); + return 0; +} + +/* ---- async read from one base bdev desc ------------------------------------ */ + +struct tier_sb_read_ctx { + struct spdk_bdev_desc *desc; + struct spdk_io_channel *ch; + void *buf; + uint32_t sb_blocks; + uint32_t blocklen; + void (*cb)(void *cb_arg, const struct tier_superblock *sb, int rc); + void *cb_arg; +}; + +static void +tier_sb_read_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + struct tier_sb_read_ctx *rc_ctx = cb_arg; + const struct tier_superblock *sb = NULL; + int rc = 0; + + spdk_bdev_free_io(bdev_io); + + if (!success) { + rc = -EIO; + } else { + /* F-5: both slots were read — take the valid one with the highest seq. */ + sb = tier_sb_select(rc_ctx->buf, (size_t)rc_ctx->sb_blocks * rc_ctx->blocklen); + if (sb == NULL) { + rc = -EILSEQ; /* no / invalid superblock on this disk */ + } + } + + rc_ctx->cb(rc_ctx->cb_arg, sb, rc); + + spdk_put_io_channel(rc_ctx->ch); + spdk_dma_free(rc_ctx->buf); + free(rc_ctx); +} + +int +tier_sb_read_desc(struct spdk_bdev_desc *desc, uint32_t blocklen, + void (*cb)(void *cb_arg, const struct tier_superblock *sb, int rc), void *cb_arg) +{ + struct tier_sb_read_ctx *ctx; + uint32_t sb_blocks = spdk_divide_round_up(TIER_SB_RESERVE_BYTES, blocklen); + size_t bufsz = (size_t)sb_blocks * blocklen; + int rc; + + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) { + return -ENOMEM; + } + ctx->desc = desc; + ctx->sb_blocks = sb_blocks; + ctx->blocklen = blocklen; + ctx->cb = cb; + ctx->cb_arg = cb_arg; + ctx->buf = spdk_dma_zmalloc(bufsz, blocklen, NULL); + if (ctx->buf == NULL) { + free(ctx); + return -ENOMEM; + } + ctx->ch = spdk_bdev_get_io_channel(desc); + if (ctx->ch == NULL) { + spdk_dma_free(ctx->buf); + free(ctx); + return -ENOMEM; + } + rc = spdk_bdev_read_blocks(desc, ctx->ch, ctx->buf, 0, sb_blocks, tier_sb_read_done, ctx); + if (rc != 0) { + spdk_put_io_channel(ctx->ch); + spdk_dma_free(ctx->buf); + free(ctx); + } + return rc; +} diff --git a/patches/0001-raid-add-skip_rebuild-parameter.patch b/patches/0001-raid-add-skip_rebuild-parameter.patch index fa77e30..378f012 100644 --- a/patches/0001-raid-add-skip_rebuild-parameter.patch +++ b/patches/0001-raid-add-skip_rebuild-parameter.patch @@ -1,31 +1,24 @@ -From c004e1653bd1fe24a4f585bc416b80f350223f35 Mon Sep 17 00:00:00 2001 -From: Evariops -Date: Sat, 23 May 2026 12:57:10 +0200 -Subject: [PATCH] raid: add skip_rebuild parameter to bdev_raid_add_base_bdev +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 10:58:25 +0200 +Subject: [PATCH 01/12] raid: add skip_rebuild parameter to + bdev_raid_add_base_bdev -When a base bdev is re-added after a CBT-driven partial rebuild, -the full surface rebuild is unnecessary. This adds an optional -skip_rebuild boolean to the bdev_raid_add_base_bdev RPC. - -When true, the RAID module: -1. Marks the base bdev ONLINE immediately (no full rebuild) -2. Quiesces the RAID bdev to safely update IO channels -3. Opens base_channel[slot] in all existing IO channels -4. Unquiesces to resume IO -5. Writes superblock if enabled (persists across reboot) - -Signed-off-by: Evariops +Evariops 0001. Promotes a CBT-resynced member without a full rebuild. +Includes: CBT-6 channel-promotion error propagation + unwind (never ONLINE +with a NULL channel on a reactor), P-5 runtime slot check (was an +NDEBUG-erased assert), membership rollback on quiesce failure. --- - module/bdev/raid/bdev_raid.c | 139 ++++++++++++++++++++++++++++++- + module/bdev/raid/bdev_raid.c | 217 ++++++++++++++++++++++++++++++- module/bdev/raid/bdev_raid.h | 4 + module/bdev/raid/bdev_raid_rpc.c | 9 +- - 3 files changed, 149 insertions(+), 3 deletions(-) + 3 files changed, 227 insertions(+), 3 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c -index 7ab1607..94ada40 100644 +index 7ab1607..cc515bc 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c -@@ -3201,10 +3201,121 @@ raid_bdev_ch_sync(struct spdk_io_channel_iter *i) +@@ -3201,10 +3201,199 @@ raid_bdev_ch_sync(struct spdk_io_channel_iter *i) spdk_for_each_channel_continue(i, 0); } @@ -35,6 +28,7 @@ index 7ab1607..94ada40 100644 + struct raid_base_bdev_info *base_info; + raid_base_bdev_cb cb_fn; + void *cb_ctx; ++ int err; /* first failure; drives the unwind path */ +}; + +static void @@ -63,6 +57,9 @@ index 7ab1607..94ada40 100644 + struct raid_bdev_sb_base_bdev *sb_base_bdev; + uint8_t i; + ++ /* CBT-7 (documented): the SB flips to CONFIGURED only after the ++ * unquiesce. A crash between the two costs a full rebuild at reboot — ++ * conservative and safe (never CONFIGURED without live channels). */ + for (i = 0; i < sb->base_bdevs_size; i++) { + sb_base_bdev = &sb->base_bdevs[i]; + if (sb_base_bdev->slot == raid_bdev_base_bdev_slot(base_info)) { @@ -82,6 +79,56 @@ index 7ab1607..94ada40 100644 + free(sr_ctx); +} + ++/* ── CBT-6 unwind: a reactor failed to open the member's channel. Close every ++ * channel we opened, unquiesce, revert the membership (the online remove path ++ * also rolls back num_base_bdevs_operational) and report the error — the ++ * member must NOT come up ONLINE with a NULL channel on any reactor (silent ++ * divergence: writes skip it while the SB says CONFIGURED). ── */ ++ ++static void ++raid_bdev_skip_rebuild_fail_unquiesced(void *ctx, int status) ++{ ++ struct skip_rebuild_ctx *sr_ctx = ctx; ++ struct raid_base_bdev_info *base_info = sr_ctx->base_info; ++ ++ (void)status; ++ _raid_bdev_remove_base_bdev(base_info, NULL, NULL); ++ if (sr_ctx->cb_fn) { ++ sr_ctx->cb_fn(sr_ctx->cb_ctx, sr_ctx->err); ++ } ++ free(sr_ctx); ++} ++ ++static void ++raid_bdev_skip_rebuild_unwind_ch_iter(struct spdk_io_channel_iter *i) ++{ ++ struct skip_rebuild_ctx *sr_ctx = spdk_io_channel_iter_get_ctx(i); ++ struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); ++ struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch); ++ uint8_t slot = raid_bdev_base_bdev_slot(sr_ctx->base_info); ++ ++ if (raid_ch->base_channel[slot] != NULL) { ++ spdk_put_io_channel(raid_ch->base_channel[slot]); ++ raid_ch->base_channel[slot] = NULL; ++ } ++ spdk_for_each_channel_continue(i, 0); ++} ++ ++static void ++raid_bdev_skip_rebuild_unwind_ch_done(struct spdk_io_channel_iter *i, int status) ++{ ++ struct skip_rebuild_ctx *sr_ctx = spdk_io_channel_iter_get_ctx(i); ++ struct raid_bdev *raid_bdev = sr_ctx->base_info->raid_bdev; ++ int rc; ++ ++ (void)status; ++ rc = spdk_bdev_unquiesce(&raid_bdev->bdev, &g_raid_if, ++ raid_bdev_skip_rebuild_fail_unquiesced, sr_ctx); ++ if (rc != 0) { ++ raid_bdev_skip_rebuild_fail_unquiesced(sr_ctx, rc); ++ } ++} ++ +static void +raid_bdev_skip_rebuild_ch_done(struct spdk_io_channel_iter *i, int status) +{ @@ -89,6 +136,17 @@ index 7ab1607..94ada40 100644 + struct raid_bdev *raid_bdev = sr_ctx->base_info->raid_bdev; + int rc; + ++ if (status != 0) { ++ /* CBT-6: at least one reactor could not open the channel — unwind ++ * instead of bringing the member up half-wired. */ ++ SPDK_ERRLOG("skip_rebuild: channel promotion failed (%s) — unwinding\n", ++ spdk_strerror(-status)); ++ sr_ctx->err = status; ++ spdk_for_each_channel(raid_bdev, raid_bdev_skip_rebuild_unwind_ch_iter, ++ sr_ctx, raid_bdev_skip_rebuild_unwind_ch_done); ++ return; ++ } ++ + rc = spdk_bdev_unquiesce(&raid_bdev->bdev, &g_raid_if, + raid_bdev_skip_rebuild_unquiesced, sr_ctx); + if (rc != 0) { @@ -105,10 +163,20 @@ index 7ab1607..94ada40 100644 + struct raid_base_bdev_info *base_info = sr_ctx->base_info; + uint8_t slot = raid_bdev_base_bdev_slot(base_info); + -+ assert(raid_ch->base_channel[slot] == NULL); ++ if (raid_ch->base_channel[slot] != NULL) { ++ /* P-5: was an assert — NDEBUG builds dropped it and silently overwrote ++ * a live channel (leak + double channel). A populated slot is already ++ * what promotion wants: reuse it. */ ++ spdk_for_each_channel_continue(i, 0); ++ return; ++ } + raid_ch->base_channel[slot] = spdk_bdev_get_io_channel(base_info->desc); + if (raid_ch->base_channel[slot] == NULL) { + SPDK_ERRLOG("skip_rebuild: failed to get io channel for slot %u\n", slot); ++ /* CBT-6: propagate — hardcoding 0 here brought the member ONLINE with a ++ * NULL channel on this reactor. */ ++ spdk_for_each_channel_continue(i, -ENOMEM); ++ return; + } + + spdk_for_each_channel_continue(i, 0); @@ -124,6 +192,9 @@ index 7ab1607..94ada40 100644 + if (status != 0) { + SPDK_ERRLOG("Failed to quiesce for skip_rebuild: %s\n", + spdk_strerror(-status)); ++ /* Revert the membership taken in configure_base_bdev_cont — ++ * num_base_bdevs_operational rolls back via the online remove path. */ ++ _raid_bdev_remove_base_bdev(base_info, NULL, NULL); + if (sr_ctx->cb_fn) { + sr_ctx->cb_fn(sr_ctx->cb_ctx, status); + } @@ -147,7 +218,7 @@ index 7ab1607..94ada40 100644 raid_base_bdev_cb configure_cb; int rc; -@@ -3243,7 +3354,30 @@ raid_bdev_configure_base_bdev_cont(struct raid_base_bdev_info *base_info) +@@ -3243,7 +3432,30 @@ raid_bdev_configure_base_bdev_cont(struct raid_base_bdev_info *base_info) } } else if (base_info->is_process_target) { raid_bdev->num_base_bdevs_operational++; @@ -179,7 +250,7 @@ index 7ab1607..94ada40 100644 if (rc != 0) { SPDK_ERRLOG("Failed to start rebuild: %s\n", spdk_strerror(-rc)); _raid_bdev_remove_base_bdev(base_info, NULL, NULL); -@@ -3488,6 +3622,7 @@ out: +@@ -3488,6 +3700,7 @@ out: int raid_bdev_add_base_bdev(struct raid_bdev *raid_bdev, const char *name, @@ -187,7 +258,7 @@ index 7ab1607..94ada40 100644 raid_base_bdev_cb cb_fn, void *cb_ctx) { struct raid_base_bdev_info *base_info = NULL, *iter; -@@ -3543,6 +3678,8 @@ raid_bdev_add_base_bdev(struct raid_bdev *raid_bdev, const char *name, +@@ -3543,6 +3756,8 @@ raid_bdev_add_base_bdev(struct raid_bdev *raid_bdev, const char *name, return -ENOMEM; } diff --git a/patches/0002-bdev-lvol-add-get-allocated-ranges-rpc.patch b/patches/0002-bdev-lvol-add-get-allocated-ranges-rpc.patch index 0a64830..ae1f4cf 100644 --- a/patches/0002-bdev-lvol-add-get-allocated-ranges-rpc.patch +++ b/patches/0002-bdev-lvol-add-get-allocated-ranges-rpc.patch @@ -1,20 +1,9 @@ -From af7a521923ec587b18151e12c393b9c24f8ff675 Mon Sep 17 00:00:00 2001 -From: Evariops -Date: Tue, 9 Jun 2026 22:05:00 +0200 -Subject: [PATCH] bdev/lvol: add bdev_lvol_get_allocated_ranges RPC +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 10:32:48 +0200 +Subject: [PATCH 02/12] lvol: add bdev_lvol_get_allocated_ranges RPC (Evariops + 0002) -Adds a read-only JSON-RPC that reports the allocated cluster extents of -a single lvol blob layer, walking the in-memory cluster map via the -exported spdk_blob_get_next_allocated_io_unit / -spdk_blob_get_next_unallocated_io_unit APIs. Emits merged, ascending, -cluster-aligned extents with max_ranges-based pagination. - -This is the native changed-block-tracking surface for thin snapshots, -consumed by spdk-csi for CSI SnapshotMetadata (KEP-3314) incremental -backups. New file plus a one-line Makefile hunk to minimise upstream -conflict surface. - -Signed-off-by: Evariops --- module/bdev/lvol/Makefile | 2 +- module/bdev/lvol/vbdev_lvol_ranges_rpc.c | 102 +++++++++++++++++++++++ diff --git a/patches/0003-nvmf-add-subsystem-pause-resume-rpc.patch b/patches/0003-nvmf-add-subsystem-pause-resume-rpc.patch index 9e76104..ea9e6ef 100644 --- a/patches/0003-nvmf-add-subsystem-pause-resume-rpc.patch +++ b/patches/0003-nvmf-add-subsystem-pause-resume-rpc.patch @@ -1,44 +1,17 @@ -From b261bd5574138a0a82ce7c1eeff6cc5be1af48c9 Mon Sep 17 00:00:00 2001 -From: Evariops -Date: Sun, 14 Jun 2026 01:30:31 +0200 -Subject: [PATCH] nvmf: add nvmf_subsystem_pause / nvmf_subsystem_resume RPC +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 10:58:26 +0200 +Subject: [PATCH 03/12] nvmf: add nvmf_subsystem_pause / nvmf_subsystem_resume + RPC -Adds two SPDK_RPC_RUNTIME JSON-RPCs that turn the public -spdk_nvmf_subsystem_pause/resume C API into a STANDING, drain-certified -I/O barrier that holds a subsystem Paused across RPC calls -- the -primitive spdk-csi needs for cross-volume crash-consistent group -snapshots (SPEC-66 H10 / fork docs/SPEC-subsystem-pause-resume.md). - -Both RPCs respond only from the state-change completion callback, so a -200 OK certifies the drain (pause) / the resume. A process-RAM registry -of standing pauses provides: - - - idempotent re-pause (refreshes a mandatory server-side TTL), - - TTL auto-resume so a leaked pause self-heals, - - a PSTATE_RESUMING guard so overlapping resumes (TTL vs explicit, or - explicit vs explicit) can never double-free a registry entry, - - a barrier-integrity token: pause returns an opaque ":" - token, resume reports barrier_intact so the controller can drop a - snapshot whose freeze was broken by a TTL, a target restart, or a - third-party resume. - -The TTL ceiling is 60 s and must stay below the host I/O timeout -(nvme_core.io_timeout); a pause that outlasts it makes the host abort -despite the target queuing without error. - -The registry is lock-free and assumes a single reactor (-m 0x1, the -production SPDK_CPU_MASK); the first pause logs an error if more than one -reactor is active. Guard g_paused before widening the CPU mask. - -New file plus a one-line lib/nvmf/Makefile hunk to minimise upstream -conflict surface. Additive and consumer-invisible behind a capability -probe (rpc_get_methods). - -Signed-off-by: Evariops +Evariops 0003. Standing, drain-certified, TTL-guarded I/O barrier. +Includes: registry entry preallocated before the pause dispatch (a Paused +subsystem can no longer end up without entry+TTL on OOM), UUID-based +per-process nonce (P-6), loud TTL clamp (S-8 follow-up). --- lib/nvmf/Makefile | 2 +- - lib/nvmf/nvmf_pause_rpc.c | 416 ++++++++++++++++++++++++++++++++++++++ - 2 files changed, 417 insertions(+), 1 deletion(-) + lib/nvmf/nvmf_pause_rpc.c | 467 ++++++++++++++++++++++++++++++++++++++ + 2 files changed, 468 insertions(+), 1 deletion(-) create mode 100644 lib/nvmf/nvmf_pause_rpc.c diff --git a/lib/nvmf/Makefile b/lib/nvmf/Makefile @@ -56,10 +29,10 @@ index 3136f7a..0af5836 100644 C_SRCS-$(CONFIG_HAVE_EVP_MAC) += auth.c diff --git a/lib/nvmf/nvmf_pause_rpc.c b/lib/nvmf/nvmf_pause_rpc.c new file mode 100644 -index 0000000..21e6951 +index 0000000..a162553 --- /dev/null +++ b/lib/nvmf/nvmf_pause_rpc.c -@@ -0,0 +1,416 @@ +@@ -0,0 +1,467 @@ +/* lib/nvmf/nvmf_pause_rpc.c — SPDX-License-Identifier: BSD-3-Clause + * Evariops fork — companion of spdk-csi SPEC-66 (H10). + * nvmf_subsystem_pause / nvmf_subsystem_resume: a STANDING, drain-certified, @@ -73,7 +46,8 @@ index 0000000..21e6951 +#include "spdk/util.h" +#include "spdk/log.h" +#include "spdk/queue.h" -+#include "spdk/env.h" /* spdk_env_get_core_count, spdk_get_ticks */ ++#include "spdk/env.h" /* spdk_env_get_core_count */ ++#include "spdk/uuid.h" /* P-6: random per-process nonce */ + +#define PAUSE_TTL_DEFAULT_MS 30000u +#define PAUSE_TTL_MAX_MS 60000u /* 60 s ceiling; keep below host io_timeout (R2) */ @@ -87,6 +61,9 @@ index 0000000..21e6951 + +struct paused_subsystem { + struct spdk_nvmf_subsystem *subsystem; ++ struct spdk_nvmf_tgt *tgt; /* re-resolve `subsystem` by NQN before the TTL ++ * resume dereferences it — a subsystem destroyed ++ * while paused would otherwise be a use-after-free */ + char nqn[SPDK_NVMF_NQN_MAX_LEN + 1]; + uint32_t nsid; + uint32_t ttl_ms; /* remembered to re-arm the TTL on a failed resume */ @@ -110,7 +87,14 @@ index 0000000..21e6951 +{ + static bool warned; + if (g_instance == 0) { -+ g_instance = (spdk_get_ticks() << 1) | 1; /* nonzero per-process nonce */ ++ /* P-6: random nonce (the old ticks<<1 discarded the high bit and could ++ * collide across quick restarts). |1 keeps it provably nonzero. */ ++ struct spdk_uuid u; ++ uint64_t hi; ++ ++ spdk_uuid_generate(&u); ++ memcpy(&hi, &u, sizeof(hi)); ++ g_instance = hi | 1; + } + /* RISK R4: the lock-free registry assumes a single reactor (-m 0x1). */ + if (!warned && spdk_env_get_core_count() > 1) { @@ -193,6 +177,16 @@ index 0000000..21e6951 +ttl_expired(void *arg) +{ + struct paused_subsystem *p = arg; ++ /* The registry holds only a raw subsystem pointer; the subsystem may have been ++ * stopped/destroyed since the pause was armed. Re-resolve by NQN and confirm ++ * identity before touching it — a vanished or replaced subsystem must be dropped, ++ * never resumed (that would dereference freed memory). */ ++ if (spdk_nvmf_tgt_find_subsystem(p->tgt, p->nqn) != p->subsystem) { ++ SPDK_WARNLOG("nvmf pause: subsystem %s gone before TTL; dropping stale pause\n", ++ p->nqn); ++ paused_entry_free(p); ++ return SPDK_POLLER_BUSY; ++ } + /* one-shot: paused_begin_resume unregisters this poller (or, on the rare + * dispatch failure, re-arms a fresh one). */ + if (paused_begin_resume(p, auto_resume_done, p)) { @@ -208,7 +202,13 @@ index 0000000..21e6951 + char *tgt_name; + uint32_t nsid; /* 0 => resolve to first namespace */ + uint32_t ttl_ms; ++ struct spdk_nvmf_tgt *tgt; + struct spdk_nvmf_subsystem *subsystem; ++ /* Registry entry PREALLOCATED before the pause dispatch: a post-drain alloc ++ * failure used to leave the subsystem Paused with no registry entry and no ++ * TTL (the exact leaked-pause state the design refuses), undone only by a ++ * best-effort resume that could itself fail silently. */ ++ struct paused_subsystem *prealloc; +}; + +static const struct spdk_json_object_decoder rpc_pause_decoders[] = { @@ -221,6 +221,7 @@ index 0000000..21e6951 +static void +rpc_pause_ctx_free(struct rpc_pause_ctx *ctx) +{ ++ free(ctx->prealloc); /* NULL once consumed by pause_done */ + free(ctx->nqn); free(ctx->tgt_name); free(ctx); +} + @@ -240,16 +241,13 @@ index 0000000..21e6951 + /* Drain certified: in-flight I/O complete, new I/O now queued target-side. */ + p = paused_find(ss); + if (p == NULL) { -+ p = calloc(1, sizeof(*p)); -+ if (p == NULL) { -+ /* cannot arm TTL → refuse to leave a pause we can't auto-heal */ -+ spdk_nvmf_subsystem_resume(ss, NULL, NULL); /* best-effort undo */ -+ spdk_jsonrpc_send_error_response(ctx->request, -+ SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "out of memory"); -+ rpc_pause_ctx_free(ctx); -+ return; -+ } ++ /* Entry was preallocated before dispatch — no OOM window exists here ++ * (a Paused subsystem always gets its registry entry + TTL). */ ++ p = ctx->prealloc; ++ ctx->prealloc = NULL; ++ assert(p != NULL); + p->subsystem = ss; ++ p->tgt = ctx->tgt; + p->nsid = ctx->nsid; + p->epoch = ++g_epoch; /* new barrier → new token */ + snprintf(p->nqn, sizeof(p->nqn), "%s", spdk_nvmf_subsystem_get_nqn(ss)); @@ -304,11 +302,18 @@ index 0000000..21e6951 + return; + } + if (ctx->ttl_ms == 0) { ctx->ttl_ms = PAUSE_TTL_DEFAULT_MS; } -+ if (ctx->ttl_ms > PAUSE_TTL_MAX_MS) { ctx->ttl_ms = PAUSE_TTL_MAX_MS; } ++ if (ctx->ttl_ms > PAUSE_TTL_MAX_MS) { ++ /* S-8 follow-up: clamp loudly — the caller must know its barrier is ++ * shorter than requested. */ ++ SPDK_WARNLOG("nvmf pause: ttl_ms %u clamped to %u (host io_timeout ceiling)\n", ++ ctx->ttl_ms, PAUSE_TTL_MAX_MS); ++ ctx->ttl_ms = PAUSE_TTL_MAX_MS; ++ } + + tgt = spdk_nvmf_get_tgt(ctx->tgt_name); /* NULL name => the single default tgt */ + if (!tgt) { spdk_jsonrpc_send_error_response(request, -ENODEV, "target not found"); + rpc_pause_ctx_free(ctx); return; } ++ ctx->tgt = tgt; + ss = spdk_nvmf_tgt_find_subsystem(tgt, ctx->nqn); + if (!ss) { spdk_jsonrpc_send_error_response(request, -ENOENT, "subsystem not found"); + rpc_pause_ctx_free(ctx); return; } @@ -327,6 +332,25 @@ index 0000000..21e6951 + return; + } + ++ /* SEC1: audit this drain barrier (pauses host I/O to a subsystem). */ ++ { ++ char detail[320]; ++ ++ snprintf(detail, sizeof(detail), "nqn=%s nsid=%u ttl_ms=%u", ++ ctx->nqn ? ctx->nqn : "(default)", ctx->nsid, ctx->ttl_ms); ++ spdk_jsonrpc_request_audit(request, "nvmf_subsystem_pause", detail); ++ } ++ ++ /* Preallocate the registry entry (see struct comment): fail BEFORE pausing, ++ * never after — a drained subsystem must always get its entry + TTL. */ ++ ctx->prealloc = calloc(1, sizeof(*ctx->prealloc)); ++ if (ctx->prealloc == NULL) { ++ spdk_jsonrpc_send_error_response(request, ++ SPDK_JSONRPC_ERROR_INTERNAL_ERROR, "out of memory"); ++ rpc_pause_ctx_free(ctx); ++ return; ++ } ++ + /* idempotent re-pause: already standing-paused → refresh TTL + ack. + * But if a resume is in flight, the subsystem is leaving Paused: refuse with + * -EAGAIN rather than ack a false barrier. */ diff --git a/patches/0004-blob-relocate-primitives.patch b/patches/0004-blob-relocate-primitives.patch new file mode 100644 index 0000000..a5a24ae --- /dev/null +++ b/patches/0004-blob-relocate-primitives.patch @@ -0,0 +1,523 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 10:58:26 +0200 +Subject: [PATCH 04/12] blob: relocate primitives + public blob I/O freeze + +Evariops 0004. claim/commit/release relocate primitives (invariants A/B) and +spdk_blob_freeze_io/spdk_blob_unfreeze_io - the blob-level barrier that +closes the C1 lost-write (held I/O re-translates through the updated L2P on +release). m8: assert the bit-pool claim invariant. +--- + include/spdk/bit_pool.h | 16 ++ + include/spdk/blob.h | 69 +++++++++ + lib/blob/blobstore.c | 330 ++++++++++++++++++++++++++++++++++++++++ + lib/blob/spdk_blob.map | 7 + + lib/util/bit_array.c | 6 + + lib/util/spdk_util.map | 1 + + 6 files changed, 429 insertions(+) + +diff --git a/include/spdk/bit_pool.h b/include/spdk/bit_pool.h +index 316c66a..81297f1 100644 +--- a/include/spdk/bit_pool.h ++++ b/include/spdk/bit_pool.h +@@ -94,6 +94,22 @@ int spdk_bit_pool_resize(struct spdk_bit_pool **pool, uint32_t num_bits); + */ + bool spdk_bit_pool_is_allocated(const struct spdk_bit_pool *pool, uint32_t bit_index); + ++/** ++ * Find the index of the first free (clear) bit at or after start_bit_index, ++ * scanning word-wise (fully-allocated 64-bit words are skipped in one step). ++ * Read-only: does NOT allocate — a windowed/resumable caller claims the returned ++ * index itself via spdk_bit_pool_set_bit_allocated so the pool's free-count ++ * invariant is maintained. Unlike spdk_bit_pool_allocate_bit (global lowest free ++ * bit), this scans from an arbitrary start so a caller can bound it to a window. ++ * ++ * \param pool Bit pool to search. ++ * \param start_bit_index Index to start searching from. ++ * ++ * \return index of the first free bit >= start_bit_index, or UINT32_MAX if none. ++ */ ++uint32_t spdk_bit_pool_find_first_free_from(const struct spdk_bit_pool *pool, ++ uint32_t start_bit_index); ++ + /** + * Allocate a bit from the bit pool. + * +diff --git a/include/spdk/blob.h b/include/spdk/blob.h +index fe98215..9cebfc8 100644 +--- a/include/spdk/blob.h ++++ b/include/spdk/blob.h +@@ -544,6 +544,75 @@ uint64_t spdk_blob_get_num_allocated_clusters(struct spdk_blob *blob); + */ + uint64_t spdk_blob_get_next_allocated_io_unit(struct spdk_blob *blob, uint64_t offset); + ++/** ++ * SPEC-73 M2a: physical LBA of a cluster on the bs_dev (bdev_tier composite), ++ * so the control-plane can map clusters to bands/tiers. ++ * \return physical LBA, or 0 if unallocated / out of range. ++ */ ++uint64_t spdk_blob_get_cluster_lba(struct spdk_blob *blob, uint64_t cluster_num); ++ ++/** ++ * SPEC-73 M2b: claim a free cluster whose physical LBA falls within ++ * [lba_start, lba_start+lba_count) (a band's range). The destination of a ++ * relocate. Returns the new physical LBA via new_lba. Caller must copy the data ++ * to new_lba then call spdk_blob_relocate_commit(). ++ * \return 0 on success, -ENOSPC if the band has no free cluster. ++ */ ++int spdk_blob_claim_cluster_in_range(struct spdk_blob *blob, uint64_t lba_start, ++ uint64_t lba_count, uint64_t *new_lba); ++ ++/** ++ * SPEC-73 (deferred #4): resumable variant of spdk_blob_claim_cluster_in_range for ++ * a campaign that claims MANY clusters into the same band window (batch relocate/ ++ * remap). cursor is an in/out CLUSTER-index hint: the scan starts at ++ * max(window_start, *cursor) and, on success, *cursor is set to claimed+1 so the ++ * next call skips the just-claimed prefix — turning an N-claim campaign into one ++ * band from O(N^2) into O(bandsize + N). cursor == NULL behaves exactly like the ++ * legacy entry point (start at the window start). PERF ONLY: the cursor is a hint; ++ * the scan always honors [window_start, window_end) and never returns an ++ * already-allocated or out-of-window cluster, so a stale cursor is always safe. ++ * \return 0 on success, -ENOSPC if no free cluster at/after the hint in the window. ++ */ ++int spdk_blob_claim_cluster_in_range_from(struct spdk_blob *blob, uint64_t lba_start, ++ uint64_t lba_count, uint32_t *cursor, uint64_t *new_lba); ++ ++/** ++ * SPEC-73 M2b: atomically re-point a cluster to new_lba (swap in-place, no count ++ * change), persist the extent page, and release the old cluster ONLY after the ++ * extent is durable (crash-safe; native replay reclaims any orphan). Runs on the ++ * metadata thread. The caller must have copied the data to new_lba first. ++ * ++ * release_old (R11): true for a normal relocate (old cluster on a healthy band, ++ * return it to the pool). false for a REMAP (old cluster on a DEGRADED/dead band) — ++ * the old cluster is QUARANTINED (kept marked-allocated) so the thin allocator ++ * never re-serves that bit into the dead band (which would -EIO). Refuses -EBUSY on ++ * a snapshot blob (F2 made intrinsic, deferred #5). ++ */ ++int spdk_blob_relocate_commit(struct spdk_blob *blob, uint64_t cluster_num, uint64_t new_lba, ++ bool release_old, spdk_blob_op_complete cb_fn, void *cb_arg); ++ ++/** ++ * SPEC-73 M2b: release a cluster claimed via spdk_blob_claim_cluster_in_range that ++ * was not committed (relocate abort), avoiding a leak until reboot. ++ */ ++void spdk_blob_release_cluster_lba(struct spdk_blob *blob, uint64_t lba); ++ ++/** ++ * SPEC-73 C1: hold (freeze) all I/O submitted to this blob. Queued I/O resumes ++ * on unfreeze and RE-EXECUTES the blob→LBA translation, so a cluster relocated ++ * meanwhile is reached at its NEW physical location (the property a bdev-level ++ * quiesce cannot give — it replays the pre-translation LBA). Refcounted; may be ++ * called from any thread (internally dispatched to the blobstore md thread); ++ * cb_fn fires on the calling thread. ++ * \return 0 if dispatched, -ENOMEM otherwise. ++ */ ++int spdk_blob_freeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg); ++ ++/** ++ * SPEC-73 C1: release one freeze reference and replay the held I/O. ++ */ ++int spdk_blob_unfreeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg); ++ + /** + * Get next unallocated io_unit + * +diff --git a/lib/blob/blobstore.c b/lib/blob/blobstore.c +index 7ae791c..d6d632c 100644 +--- a/lib/blob/blobstore.c ++++ b/lib/blob/blobstore.c +@@ -6246,6 +6246,20 @@ spdk_blob_get_next_unallocated_io_unit(struct spdk_blob *blob, uint64_t offset) + return blob_find_io_unit(blob, offset, false); + } + ++/* SPEC-73 M2a: physical LBA of a cluster on the underlying bs_dev (the bdev_tier ++ * composite). Returns 0 if the cluster is unallocated or out of range. Lets the ++ * control-plane map each cluster to a band/tier (LBA -> band) for placement. */ ++uint64_t ++spdk_blob_get_cluster_lba(struct spdk_blob *blob, uint64_t cluster_num) ++{ ++ assert(blob != NULL); ++ ++ if (cluster_num >= blob->active.num_clusters) { ++ return 0; ++ } ++ return blob->active.clusters[cluster_num]; ++} ++ + /* START spdk_bs_create_blob */ + + static void +@@ -9030,6 +9044,322 @@ blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, + spdk_thread_send_msg(blob->bs->md_thread, blob_insert_cluster_msg, ctx); + } + ++/* ===== SPEC-73 M2b: cluster relocation primitives ===================== ++ * The relocate orchestration lives in the data-plane (bdev_tier control): ++ * 1. old_lba = spdk_blob_get_cluster_lba(blob, cn) ++ * 2. spdk_blob_claim_cluster_in_range(blob, band_lba_start, band_lba_count, &new_lba) ++ * 3. copy old_lba -> new_lba (accel) ; quiesce old range ; reconcile (CRC) ++ * 4. spdk_blob_relocate_commit(blob, cn, new_lba, cb) [swap + persist + release old] ++ * 5. unquiesce ++ * Crash-safety (invariants A/B): data is written to new BEFORE the extent is ++ * re-pointed (step 3 precedes step 4); old is released ONLY after the extent is ++ * durable (below). A crash in any window leaves an orphan auto-reclaimed by the ++ * native replay (used_clusters rebuilt from extents). */ ++ ++/* Deferred #4: word-wise + resume-cursor cluster claim. The old body probed the ++ * used_clusters bit-pool one BIT at a time from c_start on every call, so a ++ * campaign filling one band re-scanned its allocated prefix each time -> O(N^2). ++ * Now: (1) spdk_bit_pool_find_first_free_from skips fully-allocated 64-bit words ++ * in one step, and (2) an optional in/out `cursor` lets a campaign resume past the ++ * last claimed bit instead of restarting at c_start -> O(bandsize + N). Semantics ++ * are unchanged: first free cluster at/after the start hint within the window, ++ * under used_lock, -ENOSPC when none. The cursor is a HINT only — the live scan ++ * still bounds to [c_start, c_end) and set_bit_allocated re-checks the bit, so a ++ * stale/over-shot cursor can never yield an allocated or out-of-window cluster. */ ++static int ++blob_claim_cluster_in_range(struct spdk_blob *blob, uint64_t lba_start, uint64_t lba_count, ++ uint32_t *cursor, uint64_t *new_lba) ++{ ++ struct spdk_blob_store *bs = blob->bs; ++ uint32_t c_start, c_end, cap, start, idx; ++ int rc = -ENOSPC; ++ ++ assert(new_lba != NULL); ++ ++ c_start = (uint32_t)bs_lba_to_cluster(bs, lba_start); ++ c_end = (uint32_t)bs_lba_to_cluster(bs, lba_start + lba_count); ++ cap = spdk_bit_pool_capacity(bs->used_clusters); ++ if (c_end > cap) { ++ c_end = cap; ++ } ++ ++ /* Resume hint: start at the cursor if it points past c_start (never before, ++ * so the window's lower bound is always honored). */ ++ start = c_start; ++ if (cursor != NULL && *cursor > start) { ++ start = *cursor; ++ } ++ ++ spdk_spin_lock(&bs->used_lock); ++ if (start < c_end) { ++ idx = spdk_bit_pool_find_first_free_from(bs->used_clusters, start); ++ if (idx != UINT32_MAX && idx < c_end) { ++ int claim_rc = spdk_bit_pool_set_bit_allocated(bs->used_clusters, idx); ++ ++ /* m8: cannot fail — idx < capacity and the bit is free, both under ++ * used_lock. Assert the invariant instead of ignoring it. */ ++ assert(claim_rc == 0); ++ (void)claim_rc; ++ bs->num_free_clusters--; ++ *new_lba = bs_cluster_to_lba(bs, idx); ++ if (cursor != NULL) { ++ *cursor = idx + 1; /* next candidate; skip the claimed prefix */ ++ } ++ rc = 0; ++ } ++ } ++ spdk_spin_unlock(&bs->used_lock); ++ return rc; ++} ++ ++int ++spdk_blob_claim_cluster_in_range(struct spdk_blob *blob, uint64_t lba_start, ++ uint64_t lba_count, uint64_t *new_lba) ++{ ++ return blob_claim_cluster_in_range(blob, lba_start, lba_count, NULL, new_lba); ++} ++ ++int ++spdk_blob_claim_cluster_in_range_from(struct spdk_blob *blob, uint64_t lba_start, ++ uint64_t lba_count, uint32_t *cursor, uint64_t *new_lba) ++{ ++ return blob_claim_cluster_in_range(blob, lba_start, lba_count, cursor, new_lba); ++} ++ ++/* SPEC-73 M2b: release a cluster claimed by spdk_blob_claim_cluster_in_range that ++ * was NOT committed (relocate abort), so it is not leaked until the next reboot. */ ++void ++spdk_blob_release_cluster_lba(struct spdk_blob *blob, uint64_t lba) ++{ ++ struct spdk_blob_store *bs = blob->bs; ++ uint32_t cluster_num; ++ ++ if (lba == 0) { ++ return; ++ } ++ cluster_num = (uint32_t)bs_lba_to_cluster(bs, lba); ++ spdk_spin_lock(&bs->used_lock); ++ if (spdk_bit_pool_is_allocated(bs->used_clusters, cluster_num)) { ++ bs_release_cluster(bs, cluster_num); ++ } ++ spdk_spin_unlock(&bs->used_lock); ++} ++ ++struct blob_relocate_ctx { ++ struct spdk_blob *blob; ++ uint64_t cluster_num; ++ uint64_t new_lba; ++ uint64_t old_lba; ++ struct spdk_blob_md_page *page; ++ struct spdk_thread *orig_thread; ++ spdk_blob_op_complete cb_fn; ++ void *cb_arg; ++ bool release_old; /* R11: false = quarantine old (remap) */ ++ int rc; ++}; ++ ++static void ++blob_relocate_done(void *arg) ++{ ++ struct blob_relocate_ctx *ctx = arg; ++ ++ ctx->cb_fn(ctx->cb_arg, ctx->rc); ++ free(ctx); ++} ++ ++static void ++blob_relocate_extent_written(void *cb_arg, int bserrno) ++{ ++ struct blob_relocate_ctx *ctx = cb_arg; ++ struct spdk_blob_store *bs = ctx->blob->bs; ++ ++ if (bserrno == 0) { ++ if (ctx->old_lba != 0 && ctx->release_old) { ++ /* Invariant B: release old ONLY after the extent is durable. */ ++ spdk_spin_lock(&bs->used_lock); ++ bs_release_cluster(bs, (uint32_t)bs_lba_to_cluster(bs, ctx->old_lba)); ++ spdk_spin_unlock(&bs->used_lock); ++ } ++ /* R11 (remap): release_old == false QUARANTINES the old cluster instead of ++ * returning it to the pool. A remap re-homes a cluster whose OLD copy sits on ++ * a DEGRADED/dead band; releasing that bit would let the thin allocator ++ * re-serve it to a normal host write → routed into the dead band → -EIO on an ++ * otherwise-healthy volume. Leaving it marked-allocated is a bounded, in-RAM ++ * leak for THIS instance only: a reboot rebuilds used_clusters from the blob ++ * extents (which now point at new_lba), and the control-plane reassembles the ++ * dead band DEGRADED/RETIRED so its range is not re-served. */ ++ } else { ++ /* Persist failed: roll the in-memory swap back to old_lba (commit_msg ++ * already overwrote active.clusters[cluster_num] with new_lba). ++ * H4 — CALLER CONTRACT: the new cluster must be QUARANTINED, not ++ * released. An error completion is ambiguous — the extent-page write ++ * may still have reached media (timeout/abort/reset), leaving the ++ * DURABLE extent pointing at new_lba. Releasing it would let a later ++ * thin allocation durably commit the same cluster in another blob → ++ * two owners after restart, with no double-reference detection at ++ * replay. Keeping the in-RAM claim is a bounded single-boot leak: ++ * replay either reclaims new_lba as an orphan (extent never landed) ++ * or the extent legitimately owns it (its content was fully copied ++ * before the commit was dispatched). */ ++ ctx->blob->active.clusters[ctx->cluster_num] = ctx->old_lba; ++ } ++ ctx->rc = bserrno; ++ if (ctx->page != NULL) { ++ spdk_free(ctx->page); ++ ctx->page = NULL; ++ } ++ spdk_thread_send_msg(ctx->orig_thread, blob_relocate_done, ctx); ++} ++ ++static void ++blob_relocate_commit_msg(void *arg) ++{ ++ struct blob_relocate_ctx *ctx = arg; ++ struct spdk_blob *blob = ctx->blob; ++ uint32_t *extent_page; ++ ++ ctx->old_lba = blob->active.clusters[ctx->cluster_num]; ++ ++ if (blob->use_extent_table == false) { ++ blob->active.clusters[ctx->cluster_num] = ctx->new_lba; ++ blob->state = SPDK_BLOB_STATE_DIRTY; ++ blob_sync_md(blob, blob_relocate_extent_written, ctx); ++ return; ++ } ++ ++ extent_page = bs_cluster_to_extent_page(blob, ctx->cluster_num); ++ if (extent_page == NULL || *extent_page == 0) { ++ /* Relocating an allocated cluster must have an existing extent page. */ ++ ctx->rc = -EINVAL; ++ spdk_free(ctx->page); ++ ctx->page = NULL; ++ spdk_thread_send_msg(ctx->orig_thread, blob_relocate_done, ctx); ++ return; ++ } ++ /* Invariant A holds (caller copied data to new before this). Swap in-place ++ * (overwrite map entry; cluster count unchanged) then persist the extent. */ ++ blob->active.clusters[ctx->cluster_num] = ctx->new_lba; ++ blob_write_extent_page(blob, *extent_page, ctx->cluster_num, ctx->page, ++ blob_relocate_extent_written, ctx); ++} ++ ++int ++spdk_blob_relocate_commit(struct spdk_blob *blob, uint64_t cluster_num, uint64_t new_lba, ++ bool release_old, spdk_blob_op_complete cb_fn, void *cb_arg) ++{ ++ struct blob_relocate_ctx *ctx; ++ ++ /* Deferred #5 (F2 made intrinsic): refuse to relocate/remap a snapshot's cluster ++ * from ANY caller, not only the RPC handler. A snapshot's clusters are shared ++ * copy-on-write with its clones; re-pointing one would corrupt the clone that ++ * reads it. Same -EBUSY the RPC layer already returns (errno contract preserved). */ ++ if (spdk_blob_is_snapshot(blob)) { ++ return -EBUSY; ++ } ++ if (cluster_num >= blob->active.num_clusters || blob->active.clusters[cluster_num] == 0) { ++ return -ENOENT; ++ } ++ ++ ctx = calloc(1, sizeof(*ctx)); ++ if (ctx == NULL) { ++ return -ENOMEM; ++ } ++ ctx->blob = blob; ++ ctx->cluster_num = cluster_num; ++ ctx->new_lba = new_lba; ++ ctx->release_old = release_old; ++ ctx->orig_thread = spdk_get_thread(); ++ ctx->cb_fn = cb_fn; ++ ctx->cb_arg = cb_arg; ++ ctx->page = spdk_zmalloc(blob->bs->md_page_size, 0, NULL, SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_DMA); ++ if (ctx->page == NULL) { ++ free(ctx); ++ return -ENOMEM; ++ } ++ ++ spdk_thread_send_msg(blob->bs->md_thread, blob_relocate_commit_msg, ctx); ++ return 0; ++} ++ ++/* ===== SPEC-73 C1: public blob-level I/O freeze ======================== ++ * Exposes the internal blob_freeze_io/blob_unfreeze_io (the primitive the ++ * snapshot/resize/delete paths already rely on) so the relocate orchestration ++ * can hold host I/O ABOVE the blob→LBA translation. This is what closes the ++ * C1 lost-write: I/O held by a freeze re-executes the translation on release ++ * (landing on the post-swap cluster), whereas I/O held by a composite-level ++ * quiesce replays a stale pre-swap LBA. Dispatches to the md thread (the ++ * internal primitives assert it); the callback returns on the caller thread. */ ++ ++struct blob_freeze_pub_ctx { ++ struct spdk_blob *blob; ++ bool freeze; ++ struct spdk_thread *orig_thread; ++ spdk_blob_op_complete cb_fn; ++ void *cb_arg; ++ int rc; ++}; ++ ++static void ++blob_freeze_pub_done_msg(void *arg) ++{ ++ struct blob_freeze_pub_ctx *ctx = arg; ++ ++ ctx->cb_fn(ctx->cb_arg, ctx->rc); ++ free(ctx); ++} ++ ++static void ++blob_freeze_pub_done(void *cb_arg, int bserrno) ++{ ++ struct blob_freeze_pub_ctx *ctx = cb_arg; ++ ++ ctx->rc = bserrno; ++ spdk_thread_send_msg(ctx->orig_thread, blob_freeze_pub_done_msg, ctx); ++} ++ ++static void ++blob_freeze_pub_msg(void *arg) ++{ ++ struct blob_freeze_pub_ctx *ctx = arg; ++ ++ if (ctx->freeze) { ++ blob_freeze_io(ctx->blob, blob_freeze_pub_done, ctx); ++ } else { ++ blob_unfreeze_io(ctx->blob, blob_freeze_pub_done, ctx); ++ } ++} ++ ++static int ++blob_freeze_pub(struct spdk_blob *blob, bool freeze, ++ spdk_blob_op_complete cb_fn, void *cb_arg) ++{ ++ struct blob_freeze_pub_ctx *ctx; ++ ++ ctx = calloc(1, sizeof(*ctx)); ++ if (ctx == NULL) { ++ return -ENOMEM; ++ } ++ ctx->blob = blob; ++ ctx->freeze = freeze; ++ ctx->orig_thread = spdk_get_thread(); ++ ctx->cb_fn = cb_fn; ++ ctx->cb_arg = cb_arg; ++ spdk_thread_send_msg(blob->bs->md_thread, blob_freeze_pub_msg, ctx); ++ return 0; ++} ++ ++int ++spdk_blob_freeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) ++{ ++ return blob_freeze_pub(blob, true, cb_fn, cb_arg); ++} ++ ++int ++spdk_blob_unfreeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) ++{ ++ return blob_freeze_pub(blob, false, cb_fn, cb_arg); ++} ++ + static void + blob_free_cluster_msg(void *arg) + { +diff --git a/lib/blob/spdk_blob.map b/lib/blob/spdk_blob.map +index d4d85c2..5a03b76 100644 +--- a/lib/blob/spdk_blob.map ++++ b/lib/blob/spdk_blob.map +@@ -24,6 +24,13 @@ + spdk_blob_get_num_allocated_clusters; + spdk_blob_get_next_allocated_io_unit; + spdk_blob_get_next_unallocated_io_unit; ++ spdk_blob_get_cluster_lba; ++ spdk_blob_claim_cluster_in_range; ++ spdk_blob_claim_cluster_in_range_from; ++ spdk_blob_relocate_commit; ++ spdk_blob_release_cluster_lba; ++ spdk_blob_freeze_io; ++ spdk_blob_unfreeze_io; + spdk_blob_opts_init; + spdk_bs_create_blob_ext; + spdk_bs_create_blob; +diff --git a/lib/util/bit_array.c b/lib/util/bit_array.c +index ff51705..067b25a 100644 +--- a/lib/util/bit_array.c ++++ b/lib/util/bit_array.c +@@ -431,6 +431,12 @@ spdk_bit_pool_is_allocated(const struct spdk_bit_pool *pool, uint32_t bit_index) + return spdk_bit_array_get(pool->array, bit_index); + } + ++uint32_t ++spdk_bit_pool_find_first_free_from(const struct spdk_bit_pool *pool, uint32_t start_bit_index) ++{ ++ return spdk_bit_array_find_first_clear(pool->array, start_bit_index); ++} ++ + uint32_t + spdk_bit_pool_allocate_bit(struct spdk_bit_pool *pool) + { +diff --git a/lib/util/spdk_util.map b/lib/util/spdk_util.map +index bf2df25..40ee55b 100644 +--- a/lib/util/spdk_util.map ++++ b/lib/util/spdk_util.map +@@ -30,6 +30,7 @@ + spdk_bit_pool_free; + spdk_bit_pool_resize; + spdk_bit_pool_is_allocated; ++ spdk_bit_pool_find_first_free_from; + spdk_bit_pool_allocate_bit; + spdk_bit_pool_set_bit_allocated; + spdk_bit_pool_free_bit; +-- +2.50.1 (Apple Git-155) + diff --git a/patches/0005-lvol-get-cluster-placement-rpc.patch b/patches/0005-lvol-get-cluster-placement-rpc.patch new file mode 100644 index 0000000..8af6507 --- /dev/null +++ b/patches/0005-lvol-get-cluster-placement-rpc.patch @@ -0,0 +1,1475 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 15:26:58 +0200 +Subject: [PATCH 05/12] 0005-lvol-get-cluster-placement-rpc.patch + +--- + module/bdev/lvol/Makefile | 3 +- + module/bdev/lvol/vbdev_lvol_tier_rpc.c | 1441 ++++++++++++++++++++++++ + 2 files changed, 1443 insertions(+), 1 deletion(-) + create mode 100644 module/bdev/lvol/vbdev_lvol_tier_rpc.c + +diff --git a/module/bdev/lvol/Makefile b/module/bdev/lvol/Makefile +index aba6620..a8ed387 100644 +--- a/module/bdev/lvol/Makefile ++++ b/module/bdev/lvol/Makefile +@@ -9,7 +9,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk + SO_VER := 7 + SO_MINOR := 0 + +-C_SRCS = vbdev_lvol.c vbdev_lvol_rpc.c vbdev_lvol_ranges_rpc.c ++CFLAGS += -I$(SPDK_ROOT_DIR)/module/bdev/tier ++C_SRCS = vbdev_lvol.c vbdev_lvol_rpc.c vbdev_lvol_ranges_rpc.c vbdev_lvol_tier_rpc.c + LIBNAME = bdev_lvol + + SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map +diff --git a/module/bdev/lvol/vbdev_lvol_tier_rpc.c b/module/bdev/lvol/vbdev_lvol_tier_rpc.c +new file mode 100644 +index 0000000..6bf3ca1 +--- /dev/null ++++ b/module/bdev/lvol/vbdev_lvol_tier_rpc.c +@@ -0,0 +1,1441 @@ ++/* SPDX-License-Identifier: BSD-3-Clause ++ * Copyright (C) 2026 Evariops. ++ * All rights reserved. ++ * ++ * SPEC-73 M2a/M2b: bdev_lvol_get_cluster_placement + bdev_lvol_relocate_cluster ++ * + bdev_lvol_remap_cluster (F6). ++ * ++ * Placement (M2a): per allocated cluster -> physical LBA on the bs_dev (the ++ * bdev_tier composite); the control-plane maps LBA->band/tier from its band ++ * table. Read-only. Both the emission AND the scan are bounded per call (m9). ++ * ++ * Relocate (M2b): move one cluster to a destination band, fully in the ++ * data-plane (INV-T2). C1: the barrier is an LVOL-BDEV quiesce ++ * (spdk_bdev_quiesce — DRAINS outstanding host I/O, then holds new I/O ++ * ABOVE the blob→LBA translation) plus an inner blob-level freeze ++ * (spdk_blob_freeze_io) — NOT a composite-level quiesce. A composite ++ * quiesce holds writes BELOW the translation and replays them to the OLD ++ * lba after the L2P swap (ACKed write lands on a freed cluster = lost ++ * write). The blob freeze alone gated only NEW submissions: a write already ++ * submitted to the bs_dev before the freeze round could land on old_lba ++ * AFTER the copy read it — the C1-DRAIN hole; the bdev quiesce closes it by ++ * waiting for outstanding I/O (LBA-range lock semantics) before the copy. ++ * Held I/O re-translates through the updated L2P on release. Order: ++ * open blob ref -> claim dst -> quiesce lvol bdev (drain+hold) -> freeze ++ * blob -> copy (base-direct, not held) -> ++ * commit (md-thread swap+persist+release old) -> ++ * unfreeze -> close ref. ++ * Crash-safety: invariants A/B inside spdk_blob_relocate_commit. ++ * ++ * F2: a SNAPSHOT blob's clusters may be shared (clones read through back_bs_dev), ++ * so relocate/remap REFUSE snapshot blobs (owned-only, the v3 invariant). ++ * F4: at most one relocate/remap in flight per blob (self-safe primitive). ++ * N-2: the blob is pinned with its own open-ref for the whole async chain, so ++ * a concurrent lvol close/delete cannot free it under the callbacks. ++ * P-2: the lvol must live on the SAME composite as tier_name (a mismatched ++ * pair would freeze/copy on the wrong device). ++ * N-7/W6 (remap): the source band must be DEGRADED — remap intentionally ++ * swaps the L2P to an UNWRITTEN cluster (the data is rebuilt afterwards by ++ * bdev_raid_rebuild_ranges); doing that to a HEALTHY band serves garbage. ++ */ ++ ++#include "spdk/rpc.h" ++#include "spdk/util.h" ++#include "spdk/queue.h" ++#include "spdk/string.h" ++#include "spdk/blob.h" ++#include "spdk/bdev.h" ++#include "spdk/bdev_module.h" /* C1-DRAIN: spdk_bdev_quiesce/unquiesce */ ++#include "spdk/log.h" ++#include "vbdev_lvol.h" ++#include "vbdev_tier.h" ++ ++#define RPC_PLACEMENT_DEFAULT_MAX 4096u ++#define RPC_PLACEMENT_HARD_MAX 1048576u ++/* m9: max cluster-table entries EXAMINED per call — bounds the synchronous scan ++ * on the reactor even for sparse multi-million-cluster blobs. */ ++#define RPC_PLACEMENT_SCAN_BUDGET 262144u ++ ++/* ===================== F4: per-blob relocate serialization ===================== */ ++/* Two in-flight relocate/remap ops on the SAME blob can issue two concurrent extent-page device ++ * writes (the commit re-points an extent page); a reordered completion loses one update -> an ++ * orphaned/stale cluster -> corruption. The saga fence serializes per VOLUME, but make the ++ * primitive self-safe too: at most one relocate/remap in flight per blob. The JSON-RPC handlers ++ * run on a single reactor thread, so no lock is needed. */ ++struct relocate_inflight { ++ struct spdk_blob *blob; ++ TAILQ_ENTRY(relocate_inflight) link; ++}; ++static TAILQ_HEAD(, relocate_inflight) g_relocate_inflight = TAILQ_HEAD_INITIALIZER(g_relocate_inflight); ++ ++static bool ++relocate_blob_acquire(struct spdk_blob *blob) ++{ ++ struct relocate_inflight *e; ++ ++ TAILQ_FOREACH(e, &g_relocate_inflight, link) { ++ if (e->blob == blob) { ++ return false; /* already in flight on this blob */ ++ } ++ } ++ e = calloc(1, sizeof(*e)); ++ if (e == NULL) { ++ return false; ++ } ++ e->blob = blob; ++ TAILQ_INSERT_TAIL(&g_relocate_inflight, e, link); ++ return true; ++} ++ ++static void ++relocate_blob_release(struct spdk_blob *blob) ++{ ++ struct relocate_inflight *e; ++ ++ TAILQ_FOREACH(e, &g_relocate_inflight, link) { ++ if (e->blob == blob) { ++ TAILQ_REMOVE(&g_relocate_inflight, e, link); ++ free(e); ++ return; ++ } ++ } ++} ++ ++/* P-2: true when the lvol's blobstore sits on THIS composite bdev. */ ++static bool ++lvol_lives_on_tier(struct spdk_lvol *lvol, struct vbdev_tier *tier) ++{ ++ struct lvol_store_bdev *lvs_bdev = vbdev_get_lvs_bdev_by_lvs(lvol->lvol_store); ++ ++ return lvs_bdev != NULL && lvs_bdev->bdev == &tier->bdev; ++} ++ ++/* ===================== M2a: get_cluster_placement ===================== */ ++ ++struct rpc_lvol_placement { ++ char *name; ++ uint64_t start_cluster; ++ uint32_t max_clusters; ++}; ++ ++static const struct spdk_json_object_decoder rpc_lvol_placement_decoders[] = { ++ {"name", offsetof(struct rpc_lvol_placement, name), spdk_json_decode_string}, ++ {"start_cluster", offsetof(struct rpc_lvol_placement, start_cluster), spdk_json_decode_uint64, true}, ++ {"max_clusters", offsetof(struct rpc_lvol_placement, max_clusters), spdk_json_decode_uint32, true}, ++}; ++ ++static void ++rpc_bdev_lvol_get_cluster_placement(struct spdk_jsonrpc_request *request, ++ const struct spdk_json_val *params) ++{ ++ struct rpc_lvol_placement req = { .start_cluster = 0, .max_clusters = RPC_PLACEMENT_DEFAULT_MAX }; ++ struct spdk_json_write_ctx *w; ++ struct spdk_bdev *bdev; ++ struct spdk_lvol *lvol; ++ struct spdk_blob *blob; ++ uint64_t cluster_size, num_clusters, c, emitted = 0, scanned = 0, next_cursor = 0, lba; ++ ++ if (spdk_json_decode_object(params, rpc_lvol_placement_decoders, ++ SPDK_COUNTOF(rpc_lvol_placement_decoders), &req)) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, ++ "spdk_json_decode_object failed"); ++ goto cleanup; ++ } ++ if (req.max_clusters == 0 || req.max_clusters > RPC_PLACEMENT_HARD_MAX) { ++ req.max_clusters = RPC_PLACEMENT_DEFAULT_MAX; ++ } ++ ++ bdev = spdk_bdev_get_by_name(req.name); ++ if (bdev == NULL || (lvol = vbdev_lvol_get_from_bdev(bdev)) == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, spdk_strerror(ENODEV)); ++ goto cleanup; ++ } ++ ++ blob = lvol->blob; ++ cluster_size = spdk_bs_get_cluster_size(lvol->lvol_store->blobstore); ++ num_clusters = spdk_blob_get_num_clusters(blob); ++ ++ w = spdk_jsonrpc_begin_result(request); ++ spdk_json_write_object_begin(w); ++ spdk_json_write_named_string(w, "name", req.name); ++ spdk_json_write_named_uint64(w, "cluster_size_bytes", cluster_size); ++ spdk_json_write_named_uint64(w, "num_clusters", num_clusters); ++ spdk_json_write_named_array_begin(w, "clusters"); ++ ++ for (c = req.start_cluster; c < num_clusters; c++) { ++ /* m9: bound the SCAN, not only the emission — a sparse blob used to ++ * walk millions of unallocated entries on the reactor in one call. */ ++ if (scanned++ >= RPC_PLACEMENT_SCAN_BUDGET || emitted >= req.max_clusters) { ++ next_cursor = c; ++ break; ++ } ++ lba = spdk_blob_get_cluster_lba(blob, c); ++ if (lba == 0) { ++ continue; /* unallocated */ ++ } ++ spdk_json_write_object_begin(w); ++ spdk_json_write_named_uint64(w, "index", c); ++ spdk_json_write_named_uint64(w, "lba", lba); ++ spdk_json_write_object_end(w); ++ emitted++; ++ } ++ ++ spdk_json_write_array_end(w); ++ spdk_json_write_named_uint64(w, "next_cursor", next_cursor); ++ spdk_json_write_object_end(w); ++ spdk_jsonrpc_end_result(request, w); ++cleanup: ++ free(req.name); ++} ++SPDK_RPC_REGISTER("bdev_lvol_get_cluster_placement", rpc_bdev_lvol_get_cluster_placement, ++ SPDK_RPC_RUNTIME) ++ ++/* ===================== M2b: relocate_cluster / remap_cluster shared decode ===== */ ++ ++struct rpc_lvol_relocate { ++ char *name; ++ char *tier_name; ++ uint64_t cluster_num; ++ uint64_t dst_lba_start; ++ uint64_t dst_lba_count; ++}; ++ ++static const struct spdk_json_object_decoder rpc_lvol_relocate_decoders[] = { ++ {"name", offsetof(struct rpc_lvol_relocate, name), spdk_json_decode_string}, ++ {"tier_name", offsetof(struct rpc_lvol_relocate, tier_name), spdk_json_decode_string}, ++ {"cluster_num", offsetof(struct rpc_lvol_relocate, cluster_num), spdk_json_decode_uint64}, ++ {"dst_lba_start", offsetof(struct rpc_lvol_relocate, dst_lba_start), spdk_json_decode_uint64}, ++ {"dst_lba_count", offsetof(struct rpc_lvol_relocate, dst_lba_count), spdk_json_decode_uint64}, ++}; ++ ++struct relocate_ctx { ++ struct spdk_jsonrpc_request *request; ++ struct spdk_blob *blob; ++ struct vbdev_tier *tier; ++ struct spdk_bdev *lvol_bdev; /* C1-DRAIN: quiesce target */ ++ uint64_t cluster_num; ++ uint64_t old_lba; ++ uint64_t new_lba; ++ uint64_t cluster_blocks; ++ bool quiesced; /* C1-DRAIN: lvol bdev quiesce held */ ++ bool frozen; /* C1: blob freeze held */ ++ bool ref_held; /* N-2: our own blob open-ref */ ++ bool claimed; ++ bool committed; ++ bool commit_ambiguous; /* H4: commit failed AFTER dispatch */ ++ int rc; ++}; ++ ++static void ++relocate_reply(struct relocate_ctx *c, int rc) ++{ ++ struct spdk_json_write_ctx *w; ++ ++ relocate_blob_release(c->blob); /* F4: clear the per-blob in-flight guard */ ++ if (rc == 0) { ++ w = spdk_jsonrpc_begin_result(c->request); ++ spdk_json_write_object_begin(w); ++ spdk_json_write_named_uint64(w, "from_lba", c->old_lba); ++ spdk_json_write_named_uint64(w, "to_lba", c->new_lba); ++ spdk_json_write_object_end(w); ++ spdk_jsonrpc_end_result(c->request, w); ++ } else { ++ spdk_jsonrpc_send_error_response_fmt(c->request, rc, "relocate failed: %s", ++ spdk_strerror(rc < 0 ? -rc : rc)); ++ } ++ free(c); ++} ++ ++static void ++relocate_ref_closed(void *cb_arg, int bserrno) ++{ ++ struct relocate_ctx *c = cb_arg; ++ ++ (void)bserrno; /* the relocate outcome is c->rc */ ++ relocate_reply(c, c->rc); ++} ++ ++static void ++relocate_close_ref(struct relocate_ctx *c) ++{ ++ if (c->ref_held) { ++ c->ref_held = false; ++ spdk_blob_close(c->blob, relocate_ref_closed, c); ++ return; ++ } ++ relocate_reply(c, c->rc); ++} ++ ++static void ++relocate_unquiesced(void *cb_arg, int status) ++{ ++ struct relocate_ctx *c = cb_arg; ++ ++ (void)status; /* best-effort unquiesce completion; outcome is c->rc */ ++ relocate_close_ref(c); ++} ++ ++static void ++relocate_unfrozen(void *cb_arg, int status) ++{ ++ struct relocate_ctx *c = cb_arg; ++ ++ (void)status; /* best-effort unfreeze completion; outcome is c->rc */ ++ if (c->quiesced) { ++ c->quiesced = false; ++ if (spdk_bdev_unquiesce(c->lvol_bdev, c->lvol_bdev->module, ++ relocate_unquiesced, c) == 0) { ++ return; ++ } ++ SPDK_ERRLOG("relocate: UNQUIESCE DISPATCH FAILED — lvol I/O left held!\n"); ++ } ++ relocate_close_ref(c); ++} ++ ++/* Final stage: release the dst claim on failure, lift the freeze, drop the blob ++ * ref, then reply. */ ++static void ++relocate_finish(struct relocate_ctx *c, int rc) ++{ ++ c->rc = rc; ++ if (rc != 0 && c->claimed && !c->committed) { ++ if (c->commit_ambiguous) { ++ /* H4: the commit failed AFTER dispatch — the extent-page write may ++ * still have reached media (timeout/abort/reset completions are ++ * ambiguous). If the durable extent points at new_lba, releasing it ++ * lets a later thin allocation durably commit the same cluster in ++ * another blob → two owners after restart. QUARANTINE instead: the ++ * in-RAM claim stays, so nothing reuses new_lba this boot; at the ++ * next restart replay either reclaims it as an orphan (extent never ++ * landed) or the extent legitimately owns it (its content was fully ++ * copied and verified BEFORE the commit). */ ++ SPDK_ERRLOG("relocate: ambiguous commit failure — quarantining " ++ "cluster lba %" PRIu64 " until restart (no release)\n", ++ c->new_lba); ++ c->claimed = false; ++ } else { ++ spdk_blob_release_cluster_lba(c->blob, c->new_lba); ++ c->claimed = false; ++ } ++ } ++ if (c->frozen) { ++ c->frozen = false; ++ if (spdk_blob_unfreeze_io(c->blob, relocate_unfrozen, c) == 0) { ++ return; ++ } ++ /* Unfreeze dispatch can only fail on OOM. A leaked freeze stalls this ++ * blob's I/O FOREVER — scream; operator remediation: restart target. */ ++ SPDK_ERRLOG("relocate: UNFREEZE DISPATCH FAILED — blob I/O left frozen!\n"); ++ } ++ relocate_unfrozen(c, 0); ++} ++ ++static void ++relocate_committed(void *cb_arg, int bserrno) ++{ ++ struct relocate_ctx *c = cb_arg; ++ ++ if (bserrno == 0) { ++ c->committed = true; /* commit took ownership of new + released old */ ++ } else { ++ c->commit_ambiguous = true; /* H4: dispatched — media state unknown */ ++ } ++ relocate_finish(c, bserrno); ++} ++ ++static void ++relocate_copied(void *cb_arg, int status) ++{ ++ struct relocate_ctx *c = cb_arg; ++ int rc; ++ ++ if (status != 0) { ++ relocate_finish(c, status); ++ return; ++ } ++ /* Data is on new (invariant A). Swap the map + persist + release old (relocate: ++ * old is on a healthy band → return it to the pool, R11 release_old=true). */ ++ rc = spdk_blob_relocate_commit(c->blob, c->cluster_num, c->new_lba, true, relocate_committed, c); ++ if (rc != 0) { ++ relocate_finish(c, rc); ++ } ++} ++ ++static void ++relocate_frozen(void *cb_arg, int status) ++{ ++ struct relocate_ctx *c = cb_arg; ++ int rc; ++ ++ if (status != 0) { ++ relocate_finish(c, status); ++ return; ++ } ++ c->frozen = true; ++ /* The copy reads/writes the base bdevs DIRECTLY (below the blob), so it is ++ * NOT held by the blob freeze. Single-cluster path always verifies (PF4 opt-out ++ * is exposed on the batch RPC). */ ++ rc = vbdev_tier_relocate_copy(c->tier, c->old_lba, c->new_lba, c->cluster_blocks, ++ true, relocate_copied, c); ++ if (rc != 0) { ++ relocate_finish(c, rc); ++ } ++} ++ ++/* C1-DRAIN: the lvol bdev is quiesced — every host I/O submitted BEFORE the ++ * quiesce has now COMPLETED (the LBA-range lock waits for outstanding I/O), ++ * and new I/O is held ABOVE the blob→LBA translation (re-translates through ++ * the updated L2P on release). Only now is the copy's source stable: the blob ++ * freeze alone gated new submissions but never drained writes already in ++ * flight to the bs_dev, so a pre-freeze ACKed write could land on old_lba ++ * AFTER the copy read it — lost on commit, cross-blob overwrite once old_lba ++ * was reallocated (the C1 hole the audit confirmed). The blob freeze is kept ++ * as an inner belt (blob-level consumers, no bdev detour). */ ++static void ++relocate_quiesced(void *cb_arg, int status) ++{ ++ struct relocate_ctx *c = cb_arg; ++ int rc; ++ ++ if (status != 0) { ++ relocate_finish(c, status); ++ return; ++ } ++ c->quiesced = true; ++ rc = spdk_blob_freeze_io(c->blob, relocate_frozen, c); ++ if (rc != 0) { ++ relocate_finish(c, rc); ++ } ++} ++ ++/* N-2: our pin on the blob is established — start the drained+frozen section. */ ++static void ++relocate_blob_opened(void *cb_arg, struct spdk_blob *blob, int bserrno) ++{ ++ struct relocate_ctx *c = cb_arg; ++ int rc; ++ ++ if (bserrno != 0) { ++ relocate_finish(c, bserrno); ++ return; ++ } ++ assert(blob == c->blob); ++ c->ref_held = true; ++ rc = spdk_bdev_quiesce(c->lvol_bdev, c->lvol_bdev->module, relocate_quiesced, c); ++ if (rc != 0) { ++ relocate_finish(c, rc); ++ } ++} ++ ++static void ++rpc_bdev_lvol_relocate_cluster(struct spdk_jsonrpc_request *request, ++ const struct spdk_json_val *params) ++{ ++ struct rpc_lvol_relocate req = {}; ++ struct spdk_bdev *bdev; ++ struct spdk_lvol *lvol; ++ struct spdk_blob *blob; ++ struct vbdev_tier *tier; ++ struct relocate_ctx *c; ++ uint64_t cluster_size, blocklen, old_lba, new_lba; ++ int rc; ++ ++ if (spdk_json_decode_object(params, rpc_lvol_relocate_decoders, ++ SPDK_COUNTOF(rpc_lvol_relocate_decoders), &req)) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, ++ "spdk_json_decode_object failed"); ++ goto cleanup; ++ } ++ ++ /* SEC1: audit this L2P mutation (copies a cluster's data + swaps its mapping). */ ++ { ++ char detail[256]; ++ ++ snprintf(detail, sizeof(detail), ++ "lvol=%s tier=%s cluster=%" PRIu64 " dst_lba=%" PRIu64 "+%" PRIu64, ++ req.name ? req.name : "?", req.tier_name ? req.tier_name : "?", ++ req.cluster_num, req.dst_lba_start, req.dst_lba_count); ++ spdk_jsonrpc_request_audit(request, "bdev_lvol_relocate_cluster", detail); ++ } ++ ++ bdev = spdk_bdev_get_by_name(req.name); ++ if (bdev == NULL || (lvol = vbdev_lvol_get_from_bdev(bdev)) == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "lvol not found"); ++ goto cleanup; ++ } ++ tier = vbdev_tier_get_by_name(req.tier_name); ++ if (tier == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "tier not found"); ++ goto cleanup; ++ } ++ blob = lvol->blob; ++ /* P-2: name and tier_name are independent parameters — verify the lvol's ++ * blobstore actually sits on THIS composite before freezing/copying on it. */ ++ if (!lvol_lives_on_tier(lvol, tier)) { ++ spdk_jsonrpc_send_error_response(request, -EINVAL, ++ "lvol does not live on this tier composite (P-2)"); ++ goto cleanup; ++ } ++ /* F2 (reinstates the v3 "owned-only" invariant; reverses C6): a SNAPSHOT blob's clusters may be ++ * shared with clones reading THROUGH back_bs_dev (and delete_snapshot has a transient sharing ++ * window). Relocating one would free `old` under a clone -> corruption. The S-D-snap spike (safe ++ * read-through under relocate) was never validated on multi-tier HW, so refuse snapshot blobs. The ++ * active leaf lvol's OWN allocated clusters are uniquely owned (CoW already copied), so safe. */ ++ if (spdk_blob_is_snapshot(blob)) { ++ spdk_jsonrpc_send_error_response(request, -EBUSY, ++ "cannot relocate a snapshot blob's cluster (shared with clones; F2)"); ++ goto cleanup; ++ } ++ cluster_size = spdk_bs_get_cluster_size(lvol->lvol_store->blobstore); ++ blocklen = spdk_bdev_get_block_size(bdev); ++ if (blocklen == 0) { ++ spdk_jsonrpc_send_error_response(request, -EINVAL, "bad blocklen"); ++ goto cleanup; ++ } ++ ++ old_lba = spdk_blob_get_cluster_lba(blob, req.cluster_num); ++ if (old_lba == 0) { ++ spdk_jsonrpc_send_error_response(request, -ENOENT, "cluster not allocated"); ++ goto cleanup; ++ } ++ ++ if (!relocate_blob_acquire(blob)) { /* F4: one relocate/remap per blob */ ++ spdk_jsonrpc_send_error_response(request, -EBUSY, ++ "another relocate is in flight on this blob (F4)"); ++ goto cleanup; ++ } ++ ++ rc = spdk_blob_claim_cluster_in_range(blob, req.dst_lba_start, req.dst_lba_count, &new_lba); ++ if (rc != 0) { ++ relocate_blob_release(blob); ++ spdk_jsonrpc_send_error_response_fmt(request, rc, "no free cluster in target band: %s", ++ spdk_strerror(rc < 0 ? -rc : rc)); ++ goto cleanup; ++ } ++ ++ c = calloc(1, sizeof(*c)); ++ if (c == NULL) { ++ spdk_blob_release_cluster_lba(blob, new_lba); ++ relocate_blob_release(blob); ++ spdk_jsonrpc_send_error_response(request, -ENOMEM, "oom"); ++ goto cleanup; ++ } ++ c->request = request; ++ c->blob = blob; ++ c->tier = tier; ++ c->lvol_bdev = bdev; /* C1-DRAIN */ ++ c->cluster_num = req.cluster_num; ++ c->old_lba = old_lba; ++ c->new_lba = new_lba; ++ c->cluster_blocks = cluster_size / blocklen; ++ c->claimed = true; ++ ++ /* N-2: pin the blob for the whole async chain, then quiesce (drain) + ++ * freeze + copy + commit. */ ++ spdk_bs_open_blob(lvol->lvol_store->blobstore, spdk_blob_get_id(blob), ++ relocate_blob_opened, c); ++cleanup: ++ free(req.name); ++ free(req.tier_name); ++} ++SPDK_RPC_REGISTER("bdev_lvol_relocate_cluster", rpc_bdev_lvol_relocate_cluster, SPDK_RPC_RUNTIME) ++ ++/* ===================== PF3: batch relocate — ONE freeze over N clusters ======== ++ * bdev_lvol_relocate_cluster freezes/unfreezes the blob per cluster (~3× cluster ++ * of stalled blob I/O each). PF3 amortizes that: freeze ONCE, run claim→copy→ ++ * commit for every {cluster_num, dst_lba_start, dst_lba_count} sequentially, then ++ * unfreeze ONCE. Correctness is identical (each cluster still swaps its L2P under ++ * the same held freeze); only the freeze/unfreeze count drops from N to 1. ++ * verify (PF4): forwarded to each copy — false skips the C5 read-back on trusted ++ * media. On any per-cluster error the batch STOPS at that cluster (releasing its ++ * un-committed claim) and reports how many committed — the caller retries the ++ * tail. Bounded to RPC_BATCH_MAX clusters per call. */ ++ ++#define RPC_BATCH_MAX 4096u ++ ++struct batch_item { ++ uint64_t cluster_num; ++ uint64_t dst_lba_start; ++ uint64_t dst_lba_count; ++}; ++ ++/* Deferred #4: per-band resume cursor for a batch claim. Threading a cursor into ++ * spdk_blob_claim_cluster_in_range_from turns an N-cluster campaign filling ONE ++ * band from O(N^2) into O(bandsize + N). claim_cursor_for() advances the cursor ++ * while consecutive items target the SAME dst window and RESETS it (pos = 0) when ++ * the window changes, so a stale hint from a previous window can never spuriously ++ * -ENOSPC — correctness always comes from the live windowed scan, this is a hint. */ ++struct claim_cursor { ++ uint64_t win_start; ++ uint64_t win_count; ++ uint32_t pos; ++}; ++ ++static uint32_t * ++claim_cursor_for(struct claim_cursor *cc, uint64_t win_start, uint64_t win_count) ++{ ++ if (win_start != cc->win_start || win_count != cc->win_count) { ++ cc->win_start = win_start; ++ cc->win_count = win_count; ++ cc->pos = 0; ++ } ++ return &cc->pos; ++} ++ ++struct batch_ctx { ++ struct spdk_jsonrpc_request *request; ++ struct spdk_blob *blob; ++ struct vbdev_tier *tier; ++ struct spdk_bdev *lvol_bdev; /* C1-DRAIN: quiesce target */ ++ uint64_t cluster_blocks; ++ bool verify; ++ struct batch_item *items; ++ uint32_t num_items; ++ uint32_t cur; /* index being processed */ ++ uint32_t done; /* committed count */ ++ uint64_t cur_old_lba; ++ uint64_t cur_new_lba; ++ bool cur_claimed; ++ bool quiesced; /* C1-DRAIN */ ++ bool frozen; ++ bool ref_held; ++ bool commit_ambiguous; /* H4 */ ++ struct claim_cursor cc; /* deferred #4: resume hint */ ++ int rc; /* first fatal error */ ++}; ++ ++static void batch_step(struct batch_ctx *c); ++ ++static void ++batch_reply(struct batch_ctx *c, int rc) ++{ ++ struct spdk_json_write_ctx *w; ++ ++ relocate_blob_release(c->blob); ++ /* Partial success is still a success at the RPC layer: the caller reads ++ * `relocated` and retries the remainder. A hard rc only when NOTHING moved. */ ++ if (rc == 0 || c->done > 0) { ++ w = spdk_jsonrpc_begin_result(c->request); ++ spdk_json_write_object_begin(w); ++ spdk_json_write_named_uint32(w, "relocated", c->done); ++ spdk_json_write_named_uint32(w, "requested", c->num_items); ++ spdk_json_write_named_int32(w, "error", rc); ++ spdk_json_write_object_end(w); ++ spdk_jsonrpc_end_result(c->request, w); ++ } else { ++ spdk_jsonrpc_send_error_response_fmt(c->request, rc, "relocate batch failed: %s", ++ spdk_strerror(rc < 0 ? -rc : rc)); ++ } ++ free(c->items); ++ free(c); ++} ++ ++static void ++batch_ref_closed(void *cb_arg, int bserrno) ++{ ++ struct batch_ctx *c = cb_arg; ++ ++ (void)bserrno; ++ batch_reply(c, c->rc); ++} ++ ++static void ++batch_ref_close_or_reply(struct batch_ctx *c) ++{ ++ if (c->ref_held) { ++ c->ref_held = false; ++ spdk_blob_close(c->blob, batch_ref_closed, c); ++ return; ++ } ++ batch_reply(c, c->rc); ++} ++ ++static void ++batch_unquiesced(void *cb_arg, int status) ++{ ++ struct batch_ctx *c = cb_arg; ++ ++ (void)status; ++ batch_ref_close_or_reply(c); ++} ++ ++static void ++batch_unfrozen(void *cb_arg, int status) ++{ ++ struct batch_ctx *c = cb_arg; ++ ++ (void)status; ++ if (c->quiesced) { ++ c->quiesced = false; ++ if (spdk_bdev_unquiesce(c->lvol_bdev, c->lvol_bdev->module, ++ batch_unquiesced, c) == 0) { ++ return; ++ } ++ SPDK_ERRLOG("relocate batch: UNQUIESCE DISPATCH FAILED — lvol I/O left held!\n"); ++ } ++ batch_ref_close_or_reply(c); ++} ++ ++/* Terminal: release any un-committed claim, unfreeze + unquiesce ONCE, drop ++ * ref, reply. */ ++static void ++batch_finish(struct batch_ctx *c, int rc) ++{ ++ c->rc = rc; ++ if (c->cur_claimed) { ++ if (c->commit_ambiguous) { ++ /* H4: same quarantine contract as the single-cluster path — an ++ * ambiguous commit failure must NOT return cur_new_lba to the ++ * pool while the durable extent may reference it. */ ++ SPDK_ERRLOG("relocate batch: ambiguous commit failure — quarantining " ++ "cluster lba %" PRIu64 " until restart (no release)\n", ++ c->cur_new_lba); ++ c->cur_claimed = false; ++ } else { ++ spdk_blob_release_cluster_lba(c->blob, c->cur_new_lba); ++ c->cur_claimed = false; ++ } ++ } ++ if (c->frozen) { ++ c->frozen = false; ++ if (spdk_blob_unfreeze_io(c->blob, batch_unfrozen, c) == 0) { ++ return; ++ } ++ SPDK_ERRLOG("relocate batch: UNFREEZE DISPATCH FAILED — blob I/O left frozen!\n"); ++ } ++ batch_unfrozen(c, 0); ++} ++ ++static void ++batch_committed(void *cb_arg, int bserrno) ++{ ++ struct batch_ctx *c = cb_arg; ++ ++ if (bserrno != 0) { ++ c->commit_ambiguous = true; /* H4: dispatched — media state unknown */ ++ batch_finish(c, bserrno); ++ return; ++ } ++ c->cur_claimed = false; /* commit took ownership of new + released old */ ++ c->done++; ++ c->cur++; ++ batch_step(c); /* next cluster, still under the same freeze */ ++} ++ ++static void ++batch_copied(void *cb_arg, int status) ++{ ++ struct batch_ctx *c = cb_arg; ++ int rc; ++ ++ if (status != 0) { ++ batch_finish(c, status); ++ return; ++ } ++ rc = spdk_blob_relocate_commit(c->blob, c->items[c->cur].cluster_num, c->cur_new_lba, ++ true, batch_committed, c); /* R11: batch is relocate → release old */ ++ if (rc != 0) { ++ batch_finish(c, rc); ++ } ++} ++ ++/* Process one cluster: resolve+claim+copy. Runs while the blob stays frozen. */ ++static void ++batch_step(struct batch_ctx *c) ++{ ++ struct batch_item *it; ++ int rc; ++ ++ if (c->cur >= c->num_items) { ++ batch_finish(c, 0); /* all done */ ++ return; ++ } ++ it = &c->items[c->cur]; ++ c->cur_old_lba = spdk_blob_get_cluster_lba(c->blob, it->cluster_num); ++ if (c->cur_old_lba == 0) { ++ batch_finish(c, -ENOENT); /* cluster not allocated */ ++ return; ++ } ++ rc = spdk_blob_claim_cluster_in_range_from(c->blob, it->dst_lba_start, it->dst_lba_count, ++ claim_cursor_for(&c->cc, it->dst_lba_start, it->dst_lba_count), ++ &c->cur_new_lba); ++ if (rc != 0) { ++ batch_finish(c, rc); ++ return; ++ } ++ c->cur_claimed = true; ++ rc = vbdev_tier_relocate_copy(c->tier, c->cur_old_lba, c->cur_new_lba, c->cluster_blocks, ++ c->verify, batch_copied, c); ++ if (rc != 0) { ++ batch_finish(c, rc); ++ } ++} ++ ++static void ++batch_frozen(void *cb_arg, int status) ++{ ++ struct batch_ctx *c = cb_arg; ++ ++ if (status != 0) { ++ batch_finish(c, status); ++ return; ++ } ++ c->frozen = true; ++ batch_step(c); ++} ++ ++/* C1-DRAIN: see relocate_quiesced — outstanding host I/O has drained, new I/O ++ * is held above the L2P translation for the whole batch. */ ++static void ++batch_quiesced(void *cb_arg, int status) ++{ ++ struct batch_ctx *c = cb_arg; ++ int rc; ++ ++ if (status != 0) { ++ batch_finish(c, status); ++ return; ++ } ++ c->quiesced = true; ++ rc = spdk_blob_freeze_io(c->blob, batch_frozen, c); ++ if (rc != 0) { ++ batch_finish(c, rc); ++ } ++} ++ ++static void ++batch_blob_opened(void *cb_arg, struct spdk_blob *blob, int bserrno) ++{ ++ struct batch_ctx *c = cb_arg; ++ int rc; ++ ++ if (bserrno != 0) { ++ batch_finish(c, bserrno); ++ return; ++ } ++ assert(blob == c->blob); ++ c->ref_held = true; ++ rc = spdk_bdev_quiesce(c->lvol_bdev, c->lvol_bdev->module, batch_quiesced, c); ++ if (rc != 0) { ++ batch_finish(c, rc); ++ } ++} ++ ++struct rpc_lvol_relocate_batch { ++ char *name; ++ char *tier_name; ++ bool verify; ++ struct batch_item *items; ++ size_t num_items; ++}; ++ ++static int ++decode_batch_item(const struct spdk_json_val *val, void *out) ++{ ++ struct batch_item *it = out; ++ const struct spdk_json_object_decoder d[] = { ++ {"cluster_num", offsetof(struct batch_item, cluster_num), spdk_json_decode_uint64}, ++ {"dst_lba_start", offsetof(struct batch_item, dst_lba_start), spdk_json_decode_uint64}, ++ {"dst_lba_count", offsetof(struct batch_item, dst_lba_count), spdk_json_decode_uint64}, ++ }; ++ ++ return spdk_json_decode_object(val, d, SPDK_COUNTOF(d), it) ? -1 : 0; ++} ++ ++static int ++decode_batch_items(const struct spdk_json_val *val, void *out) ++{ ++ struct rpc_lvol_relocate_batch *req = SPDK_CONTAINEROF(out, struct rpc_lvol_relocate_batch, items); ++ ++ /* R16: a duplicate "clusters" JSON key calls this decoder twice; without the ++ * guard the second calloc overwrites req->items and leaks the first array ++ * (~RPC_BATCH_MAX entries). Allocate once, mirroring decode_repair_ranges. */ ++ if (req->items == NULL) { ++ req->items = calloc(RPC_BATCH_MAX, sizeof(struct batch_item)); ++ if (req->items == NULL) { ++ return -1; ++ } ++ } ++ return spdk_json_decode_array(val, decode_batch_item, req->items, RPC_BATCH_MAX, ++ &req->num_items, sizeof(struct batch_item)); ++} ++ ++static const struct spdk_json_object_decoder rpc_lvol_relocate_batch_decoders[] = { ++ {"name", offsetof(struct rpc_lvol_relocate_batch, name), spdk_json_decode_string}, ++ {"tier_name", offsetof(struct rpc_lvol_relocate_batch, tier_name), spdk_json_decode_string}, ++ {"verify", offsetof(struct rpc_lvol_relocate_batch, verify), spdk_json_decode_bool, true}, ++ {"clusters", offsetof(struct rpc_lvol_relocate_batch, items), decode_batch_items}, ++}; ++ ++static void ++rpc_bdev_lvol_relocate_clusters(struct spdk_jsonrpc_request *request, ++ const struct spdk_json_val *params) ++{ ++ struct rpc_lvol_relocate_batch req = { .verify = true }; ++ struct spdk_bdev *bdev; ++ struct spdk_lvol *lvol; ++ struct spdk_blob *blob; ++ struct vbdev_tier *tier; ++ struct batch_ctx *c; ++ uint64_t cluster_size, blocklen; ++ ++ if (spdk_json_decode_object(params, rpc_lvol_relocate_batch_decoders, ++ SPDK_COUNTOF(rpc_lvol_relocate_batch_decoders), &req)) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, ++ "spdk_json_decode_object failed"); ++ goto cleanup; ++ } ++ if (req.num_items == 0) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "no clusters"); ++ goto cleanup; ++ } ++ /* SEC1: audit this batch L2P mutation (copies + remaps N clusters). */ ++ { ++ char detail[192]; ++ ++ snprintf(detail, sizeof(detail), "lvol=%s tier=%s clusters=%zu verify=%d", ++ req.name ? req.name : "?", req.tier_name ? req.tier_name : "?", ++ req.num_items, req.verify ? 1 : 0); ++ spdk_jsonrpc_request_audit(request, "bdev_lvol_relocate_clusters", detail); ++ } ++ bdev = spdk_bdev_get_by_name(req.name); ++ if (bdev == NULL || (lvol = vbdev_lvol_get_from_bdev(bdev)) == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "lvol not found"); ++ goto cleanup; ++ } ++ tier = vbdev_tier_get_by_name(req.tier_name); ++ if (tier == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "tier not found"); ++ goto cleanup; ++ } ++ blob = lvol->blob; ++ if (!lvol_lives_on_tier(lvol, tier)) { /* P-2 */ ++ spdk_jsonrpc_send_error_response(request, -EINVAL, ++ "lvol does not live on this tier composite (P-2)"); ++ goto cleanup; ++ } ++ if (spdk_blob_is_snapshot(blob)) { /* F2 */ ++ spdk_jsonrpc_send_error_response(request, -EBUSY, ++ "cannot relocate a snapshot blob's clusters (F2)"); ++ goto cleanup; ++ } ++ blocklen = spdk_bdev_get_block_size(bdev); ++ if (blocklen == 0) { ++ spdk_jsonrpc_send_error_response(request, -EINVAL, "bad blocklen"); ++ goto cleanup; ++ } ++ cluster_size = spdk_bs_get_cluster_size(lvol->lvol_store->blobstore); ++ if (!relocate_blob_acquire(blob)) { /* F4: one relocate/remap per blob */ ++ spdk_jsonrpc_send_error_response(request, -EBUSY, ++ "another relocate is in flight on this blob (F4)"); ++ goto cleanup; ++ } ++ c = calloc(1, sizeof(*c)); ++ if (c == NULL) { ++ relocate_blob_release(blob); ++ spdk_jsonrpc_send_error_response(request, -ENOMEM, "oom"); ++ goto cleanup; ++ } ++ c->request = request; ++ c->blob = blob; ++ c->tier = tier; ++ c->lvol_bdev = bdev; /* C1-DRAIN */ ++ c->cluster_blocks = cluster_size / blocklen; ++ c->verify = req.verify; ++ c->items = req.items; /* ownership transferred to the ctx */ ++ req.items = NULL; ++ c->num_items = (uint32_t)req.num_items; ++ ++ /* N-2: pin the blob, quiesce (drain) + freeze ONCE, then run the whole ++ * batch under that held window. */ ++ spdk_bs_open_blob(lvol->lvol_store->blobstore, spdk_blob_get_id(blob), ++ batch_blob_opened, c); ++cleanup: ++ free(req.name); ++ free(req.tier_name); ++ free(req.items); ++} ++SPDK_RPC_REGISTER("bdev_lvol_relocate_clusters", rpc_bdev_lvol_relocate_clusters, SPDK_RPC_RUNTIME) ++ ++/* ===================== F6: remap_cluster (re-home a LOST cluster, no copy) ===== */ ++ ++/* Per-item remap pre-checks, shared by the single and the batch remap RPCs so ++ * both apply IDENTICAL guards (the idempotency contract — re-issuing a partially ++ * committed batch cleanly -EINVALs the already-moved items — depends on this). ++ * On success (0): *old_lba and *new_lba are set and a claim is HELD on *new_lba; ++ * the caller commits it (the commit consumes the claim) or releases it on abort. ++ * On failure (-errno): NO claim is held (released internally if a later check ++ * failed after the claim). Does NOT take the F4 in-flight guard nor the P-2/F2 ++ * blob-level checks — those are batch-wide and stay in the callers. Guards, in ++ * order: cluster allocated (ENOENT) -> source band DEGRADED (N-7/W6, EINVAL) -> ++ * claim a dst cluster (ENOSPC/…) -> destination band ACTIVE (EINVAL). ++ * cursor (deferred #4): in/out resume hint for the word-wise claim, NULL for the ++ * single form (no campaign to resume). */ ++static int ++remap_one_precheck(struct vbdev_tier *tier, struct spdk_blob *blob, ++ const struct batch_item *it, uint32_t *cursor, ++ uint64_t *old_lba, uint64_t *new_lba) ++{ ++ struct tier_band *src_band, *dst_band; ++ uint64_t band_off, o, n; ++ int rc; ++ ++ /* F6: re-home a cluster whose band DIED (unreadable) onto a healthy band WITHOUT copying — the ++ * data is reconstructed afterwards by bdev_raid_rebuild_ranges writing through the nexus. We swap ++ * the L2P to a freshly-claimed cluster on the target band and quarantine the old (dead) cluster. ++ * Invariant A is intentionally relaxed (no data on new yet): the cluster was ALREADY lost, the ++ * nexus serves reads from k+m redundancy meanwhile, and the subsequent range rebuild fills new. */ ++ o = spdk_blob_get_cluster_lba(blob, it->cluster_num); ++ if (o == 0) { ++ return -ENOENT; /* cluster not allocated */ ++ } ++ /* N-7/W6: relaxing invariant A is only legitimate when the data is ALREADY ++ * lost — the source band must be DEGRADED. Remapping a healthy cluster would ++ * serve uninitialized garbage to every future read. */ ++ src_band = vbdev_tier_band_of_lba(tier, o, &band_off); ++ if (src_band == NULL || src_band->state != TIER_BAND_DEGRADED) { ++ return -EINVAL; /* source band not DEGRADED */ ++ } ++ rc = spdk_blob_claim_cluster_in_range_from(blob, it->dst_lba_start, it->dst_lba_count, ++ cursor, &n); ++ if (rc != 0) { ++ return rc; /* no free cluster in target band */ ++ } ++ /* The destination must be a HEALTHY band (the whole point is to re-home). */ ++ dst_band = vbdev_tier_band_of_lba(tier, n, &band_off); ++ if (dst_band == NULL || dst_band->state != TIER_BAND_ACTIVE) { ++ spdk_blob_release_cluster_lba(blob, n); ++ return -EINVAL; /* destination band not ACTIVE */ ++ } ++ *old_lba = o; ++ *new_lba = n; ++ return 0; ++} ++ ++struct remap_ctx { ++ struct spdk_jsonrpc_request *request; ++ struct spdk_blob *blob; ++ uint64_t cluster_num; ++ uint64_t old_lba; ++ uint64_t new_lba; ++ bool claimed; ++ bool ref_held; /* N-2 */ ++ int rc; ++}; ++ ++static void ++remap_reply(struct remap_ctx *c, int rc) ++{ ++ struct spdk_json_write_ctx *w; ++ ++ if (rc != 0 && c->claimed) { ++ spdk_blob_release_cluster_lba(c->blob, c->new_lba); ++ } ++ relocate_blob_release(c->blob); ++ if (rc == 0) { ++ w = spdk_jsonrpc_begin_result(c->request); ++ spdk_json_write_object_begin(w); ++ spdk_json_write_named_uint64(w, "from_lba", c->old_lba); ++ spdk_json_write_named_uint64(w, "to_lba", c->new_lba); ++ spdk_json_write_object_end(w); ++ spdk_jsonrpc_end_result(c->request, w); ++ } else { ++ spdk_jsonrpc_send_error_response_fmt(c->request, rc, "remap failed: %s", ++ spdk_strerror(rc < 0 ? -rc : rc)); ++ } ++ free(c); ++} ++ ++static void ++remap_ref_closed(void *cb_arg, int bserrno) ++{ ++ struct remap_ctx *c = cb_arg; ++ ++ (void)bserrno; ++ remap_reply(c, c->rc); ++} ++ ++static void ++remap_finish(struct remap_ctx *c, int rc) ++{ ++ c->rc = rc; ++ if (c->ref_held) { ++ c->ref_held = false; ++ spdk_blob_close(c->blob, remap_ref_closed, c); ++ return; ++ } ++ remap_reply(c, rc); ++} ++ ++static void ++remap_committed(void *cb_arg, int bserrno) ++{ ++ struct remap_ctx *c = cb_arg; ++ ++ if (bserrno == 0) { ++ c->claimed = false; /* commit took ownership of new + released old */ ++ } ++ remap_finish(c, bserrno); ++} ++ ++static void ++remap_blob_opened(void *cb_arg, struct spdk_blob *blob, int bserrno) ++{ ++ struct remap_ctx *c = cb_arg; ++ int rc; ++ ++ if (bserrno != 0) { ++ remap_finish(c, bserrno); ++ return; ++ } ++ assert(blob == c->blob); ++ c->ref_held = true; ++ /* R11: remap re-homes a cluster whose OLD copy is on a DEGRADED band → QUARANTINE ++ * the old cluster (release_old=false), never return it to the thin pool. */ ++ rc = spdk_blob_relocate_commit(c->blob, c->cluster_num, c->new_lba, false, remap_committed, c); ++ if (rc != 0) { ++ remap_finish(c, rc); ++ } ++} ++ ++static void ++rpc_bdev_lvol_remap_cluster(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) ++{ ++ struct rpc_lvol_relocate req = {}; ++ struct spdk_bdev *bdev; ++ struct spdk_lvol *lvol; ++ struct spdk_blob *blob; ++ struct vbdev_tier *tier; ++ struct batch_item item; ++ struct remap_ctx *c; ++ uint64_t old_lba, new_lba; ++ int rc; ++ ++ if (spdk_json_decode_object(params, rpc_lvol_relocate_decoders, ++ SPDK_COUNTOF(rpc_lvol_relocate_decoders), &req)) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, ++ "spdk_json_decode_object failed"); ++ goto cleanup; ++ } ++ /* SEC1: audit this L2P remap (rewrites a cluster's mapping to a new band). */ ++ { ++ char detail[256]; ++ ++ snprintf(detail, sizeof(detail), ++ "lvol=%s tier=%s cluster=%" PRIu64 " dst_lba=%" PRIu64 "+%" PRIu64, ++ req.name ? req.name : "?", req.tier_name ? req.tier_name : "?", ++ req.cluster_num, req.dst_lba_start, req.dst_lba_count); ++ spdk_jsonrpc_request_audit(request, "bdev_lvol_remap_cluster", detail); ++ } ++ bdev = spdk_bdev_get_by_name(req.name); ++ if (bdev == NULL || (lvol = vbdev_lvol_get_from_bdev(bdev)) == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "lvol not found"); ++ goto cleanup; ++ } ++ tier = vbdev_tier_get_by_name(req.tier_name); ++ if (tier == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "tier not found"); ++ goto cleanup; ++ } ++ blob = lvol->blob; ++ if (!lvol_lives_on_tier(lvol, tier)) { /* P-2 */ ++ spdk_jsonrpc_send_error_response(request, -EINVAL, ++ "lvol does not live on this tier composite (P-2)"); ++ goto cleanup; ++ } ++ if (spdk_blob_is_snapshot(blob)) { /* F2 */ ++ spdk_jsonrpc_send_error_response(request, -EBUSY, "cannot remap a snapshot blob's cluster (F2)"); ++ goto cleanup; ++ } ++ /* F4: one relocate/remap in flight per blob. Taken before the per-item ++ * pre-checks (which claim a dst cluster) and released on any error path. */ ++ if (!relocate_blob_acquire(blob)) { ++ spdk_jsonrpc_send_error_response(request, -EBUSY, ++ "another relocate is in flight on this blob (F4)"); ++ goto cleanup; ++ } ++ /* Shared per-item guards (cluster allocated, source DEGRADED, claim dst, ++ * dst ACTIVE): on success a claim is held on new_lba; on failure none is. */ ++ item = (struct batch_item){ .cluster_num = req.cluster_num, ++ .dst_lba_start = req.dst_lba_start, ++ .dst_lba_count = req.dst_lba_count }; ++ rc = remap_one_precheck(tier, blob, &item, NULL, &old_lba, &new_lba); ++ if (rc != 0) { ++ relocate_blob_release(blob); ++ spdk_jsonrpc_send_error_response_fmt(request, rc, "remap failed: %s", ++ spdk_strerror(rc < 0 ? -rc : rc)); ++ goto cleanup; ++ } ++ c = calloc(1, sizeof(*c)); ++ if (c == NULL) { ++ spdk_blob_release_cluster_lba(blob, new_lba); ++ relocate_blob_release(blob); ++ spdk_jsonrpc_send_error_response(request, -ENOMEM, "oom"); ++ goto cleanup; ++ } ++ c->request = request; ++ c->blob = blob; ++ c->cluster_num = req.cluster_num; ++ c->old_lba = old_lba; ++ c->new_lba = new_lba; ++ c->claimed = true; ++ /* N-2: pin the blob across the async commit. */ ++ spdk_bs_open_blob(lvol->lvol_store->blobstore, spdk_blob_get_id(blob), ++ remap_blob_opened, c); ++cleanup: ++ free(req.name); ++ free(req.tier_name); ++} ++SPDK_RPC_REGISTER("bdev_lvol_remap_cluster", rpc_bdev_lvol_remap_cluster, SPDK_RPC_RUNTIME) ++ ++/* ===================== F6 batch: remap_clusters (re-home N LOST clusters) ====== ++ * The no-copy analogue of bdev_lvol_relocate_clusters: collapse N per-cluster ++ * bdev_lvol_remap_cluster round-trips (a full band drain is hundreds–thousands of ++ * clusters) into ONE RPC before a single bdev_raid_rebuild_ranges. Because remap ++ * is no-copy on a DEAD source band, it takes NO freeze (there is no in-flight to ++ * drain) — so unlike the relocate batch there is nothing to amortize and no ++ * foreground I/O stall: this is pure orchestration-overhead elimination. ++ * Per item it runs the SAME remap_one_precheck guards as the single form, then ++ * spdk_blob_relocate_commit(release_old=false) (R11 quarantine). Processed ++ * SEQUENTIALLY; the FIRST per-item error STOPS the batch (releasing that item's ++ * un-committed claim) and replies {remapped, requested, error} — partial success ++ * is a 200 (caller retries the tail from `remapped`). Bounded to RPC_BATCH_MAX. ++ * Crash-safety per item is identical to the single remap (invariant B); the ++ * control-plane journals the remap set and re-drives tail + rebuild at restart. */ ++ ++struct remap_batch_ctx { ++ struct spdk_jsonrpc_request *request; ++ struct spdk_blob *blob; ++ struct vbdev_tier *tier; ++ struct batch_item *items; ++ uint32_t num_items; ++ uint32_t cur; /* index being processed */ ++ uint32_t done; /* committed count */ ++ uint64_t cur_old_lba; ++ uint64_t cur_new_lba; ++ bool cur_claimed; ++ bool ref_held; /* N-2 */ ++ struct claim_cursor cc; /* deferred #4: resume hint */ ++ int rc; /* first fatal error */ ++}; ++ ++static void remap_batch_step(struct remap_batch_ctx *c); ++ ++static void ++remap_batch_reply(struct remap_batch_ctx *c, int rc) ++{ ++ struct spdk_json_write_ctx *w; ++ ++ relocate_blob_release(c->blob); ++ /* Partial success is still a success at the RPC layer: the caller reads ++ * `remapped` and retries the remainder. A hard rc only when NOTHING moved. */ ++ if (rc == 0 || c->done > 0) { ++ w = spdk_jsonrpc_begin_result(c->request); ++ spdk_json_write_object_begin(w); ++ spdk_json_write_named_uint32(w, "remapped", c->done); ++ spdk_json_write_named_uint32(w, "requested", c->num_items); ++ spdk_json_write_named_int32(w, "error", rc); ++ spdk_json_write_object_end(w); ++ spdk_jsonrpc_end_result(c->request, w); ++ } else { ++ spdk_jsonrpc_send_error_response_fmt(c->request, rc, "remap batch failed: %s", ++ spdk_strerror(rc < 0 ? -rc : rc)); ++ } ++ free(c->items); ++ free(c); ++} ++ ++static void ++remap_batch_ref_closed(void *cb_arg, int bserrno) ++{ ++ struct remap_batch_ctx *c = cb_arg; ++ ++ (void)bserrno; ++ remap_batch_reply(c, c->rc); ++} ++ ++/* Terminal: release any un-committed claim, drop the blob ref, reply. There is NO ++ * unfreeze (the batch never freezes — the source band is dead). */ ++static void ++remap_batch_finish(struct remap_batch_ctx *c, int rc) ++{ ++ c->rc = rc; ++ if (c->cur_claimed) { ++ spdk_blob_release_cluster_lba(c->blob, c->cur_new_lba); ++ c->cur_claimed = false; ++ } ++ if (c->ref_held) { ++ c->ref_held = false; ++ spdk_blob_close(c->blob, remap_batch_ref_closed, c); ++ return; ++ } ++ remap_batch_reply(c, rc); ++} ++ ++static void ++remap_batch_committed(void *cb_arg, int bserrno) ++{ ++ struct remap_batch_ctx *c = cb_arg; ++ ++ if (bserrno != 0) { ++ remap_batch_finish(c, bserrno); ++ return; ++ } ++ c->cur_claimed = false; /* commit took ownership of new + quarantined old */ ++ c->done++; ++ c->cur++; ++ remap_batch_step(c); /* next cluster */ ++} ++ ++/* Process one cluster: shared pre-check (claim) + async commit. */ ++static void ++remap_batch_step(struct remap_batch_ctx *c) ++{ ++ struct batch_item *it; ++ int rc; ++ ++ if (c->cur >= c->num_items) { ++ remap_batch_finish(c, 0); /* all done */ ++ return; ++ } ++ it = &c->items[c->cur]; ++ rc = remap_one_precheck(c->tier, c->blob, it, ++ claim_cursor_for(&c->cc, it->dst_lba_start, it->dst_lba_count), ++ &c->cur_old_lba, &c->cur_new_lba); ++ if (rc != 0) { ++ remap_batch_finish(c, rc); /* stop at first per-item error */ ++ return; ++ } ++ c->cur_claimed = true; ++ /* R11: remap → the OLD cluster is on a DEGRADED band → quarantine it ++ * (release_old=false), never return it to the thin pool. */ ++ rc = spdk_blob_relocate_commit(c->blob, it->cluster_num, c->cur_new_lba, false, ++ remap_batch_committed, c); ++ if (rc != 0) { ++ remap_batch_finish(c, rc); ++ } ++} ++ ++static void ++remap_batch_blob_opened(void *cb_arg, struct spdk_blob *blob, int bserrno) ++{ ++ struct remap_batch_ctx *c = cb_arg; ++ ++ if (bserrno != 0) { ++ remap_batch_finish(c, bserrno); ++ return; ++ } ++ assert(blob == c->blob); ++ c->ref_held = true; ++ remap_batch_step(c); ++} ++ ++struct rpc_lvol_remap_batch { ++ char *name; ++ char *tier_name; ++ struct batch_item *items; ++ size_t num_items; ++}; ++ ++static int ++decode_remap_batch_items(const struct spdk_json_val *val, void *out) ++{ ++ struct rpc_lvol_remap_batch *req = SPDK_CONTAINEROF(out, struct rpc_lvol_remap_batch, items); ++ ++ /* R16: a duplicate "clusters" JSON key calls this decoder twice; without the ++ * guard the second calloc overwrites req->items and leaks the first array. ++ * Allocate once. (Same shape as decode_batch_items, distinct container type.) */ ++ if (req->items == NULL) { ++ req->items = calloc(RPC_BATCH_MAX, sizeof(struct batch_item)); ++ if (req->items == NULL) { ++ return -1; ++ } ++ } ++ return spdk_json_decode_array(val, decode_batch_item, req->items, RPC_BATCH_MAX, ++ &req->num_items, sizeof(struct batch_item)); ++} ++ ++static const struct spdk_json_object_decoder rpc_lvol_remap_batch_decoders[] = { ++ {"name", offsetof(struct rpc_lvol_remap_batch, name), spdk_json_decode_string}, ++ {"tier_name", offsetof(struct rpc_lvol_remap_batch, tier_name), spdk_json_decode_string}, ++ {"clusters", offsetof(struct rpc_lvol_remap_batch, items), decode_remap_batch_items}, ++}; ++ ++static void ++rpc_bdev_lvol_remap_clusters(struct spdk_jsonrpc_request *request, ++ const struct spdk_json_val *params) ++{ ++ struct rpc_lvol_remap_batch req = {}; ++ struct spdk_bdev *bdev; ++ struct spdk_lvol *lvol; ++ struct spdk_blob *blob; ++ struct vbdev_tier *tier; ++ struct remap_batch_ctx *c; ++ ++ if (spdk_json_decode_object(params, rpc_lvol_remap_batch_decoders, ++ SPDK_COUNTOF(rpc_lvol_remap_batch_decoders), &req)) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, ++ "spdk_json_decode_object failed"); ++ goto cleanup; ++ } ++ if (req.num_items == 0) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "no clusters"); ++ goto cleanup; ++ } ++ /* SEC1: audit this batch L2P remap (re-homes N lost clusters). */ ++ { ++ char detail[192]; ++ ++ snprintf(detail, sizeof(detail), "lvol=%s tier=%s clusters=%zu", ++ req.name ? req.name : "?", req.tier_name ? req.tier_name : "?", ++ req.num_items); ++ spdk_jsonrpc_request_audit(request, "bdev_lvol_remap_clusters", detail); ++ } ++ bdev = spdk_bdev_get_by_name(req.name); ++ if (bdev == NULL || (lvol = vbdev_lvol_get_from_bdev(bdev)) == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "lvol not found"); ++ goto cleanup; ++ } ++ tier = vbdev_tier_get_by_name(req.tier_name); ++ if (tier == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "tier not found"); ++ goto cleanup; ++ } ++ blob = lvol->blob; ++ if (!lvol_lives_on_tier(lvol, tier)) { /* P-2 */ ++ spdk_jsonrpc_send_error_response(request, -EINVAL, ++ "lvol does not live on this tier composite (P-2)"); ++ goto cleanup; ++ } ++ if (spdk_blob_is_snapshot(blob)) { /* F2 */ ++ spdk_jsonrpc_send_error_response(request, -EBUSY, ++ "cannot remap a snapshot blob's clusters (F2)"); ++ goto cleanup; ++ } ++ if (!relocate_blob_acquire(blob)) { /* F4: one relocate/remap per blob */ ++ spdk_jsonrpc_send_error_response(request, -EBUSY, ++ "another relocate is in flight on this blob (F4)"); ++ goto cleanup; ++ } ++ c = calloc(1, sizeof(*c)); ++ if (c == NULL) { ++ relocate_blob_release(blob); ++ spdk_jsonrpc_send_error_response(request, -ENOMEM, "oom"); ++ goto cleanup; ++ } ++ c->request = request; ++ c->blob = blob; ++ c->tier = tier; ++ c->items = req.items; /* ownership transferred to the ctx */ ++ req.items = NULL; ++ c->num_items = (uint32_t)req.num_items; ++ ++ /* N-2: pin the blob for the whole async chain (no freeze — the source is dead). */ ++ spdk_bs_open_blob(lvol->lvol_store->blobstore, spdk_blob_get_id(blob), ++ remap_batch_blob_opened, c); ++cleanup: ++ free(req.name); ++ free(req.tier_name); ++ free(req.items); ++} ++SPDK_RPC_REGISTER("bdev_lvol_remap_clusters", rpc_bdev_lvol_remap_clusters, SPDK_RPC_RUNTIME) +-- +2.50.1 (Apple Git-155) + diff --git a/patches/0006-raid5f-degraded-read-fallback.patch b/patches/0006-raid5f-degraded-read-fallback.patch new file mode 100644 index 0000000..303ffa1 --- /dev/null +++ b/patches/0006-raid5f-degraded-read-fallback.patch @@ -0,0 +1,93 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 10:32:48 +0200 +Subject: [PATCH 06/12] raid5f: degraded-read fallback + double-fault guard + (Evariops 0006) + +--- + module/bdev/raid/raid5f.c | 55 +++++++++++++++++++++++++++++++++++++-- + 1 file changed, 53 insertions(+), 2 deletions(-) + +diff --git a/module/bdev/raid/raid5f.c b/module/bdev/raid/raid5f.c +index 0a2274f..1d02ec8 100644 +--- a/module/bdev/raid/raid5f.c ++++ b/module/bdev/raid/raid5f.c +@@ -649,15 +649,48 @@ raid5f_submit_write_request(struct raid_bdev_io *raid_io, uint64_t stripe_index) + return 0; + } + ++/* SPEC-73 M3: forward declarations so the read-completion path can fall back to ++ * a degraded reconstruct read (both defined later in this file). */ ++static int raid5f_submit_reconstruct_read(struct raid_bdev_io *raid_io, uint64_t stripe_index, ++ uint8_t chunk_idx, uint64_t chunk_offset, stripe_req_xor_cb cb); ++static void raid5f_stripe_request_reconstruct_xor_done(struct stripe_request *stripe_req, int status); ++ + static void + raid5f_chunk_read_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) + { + struct raid_bdev_io *raid_io = cb_arg; ++ struct raid_bdev *raid_bdev; ++ struct raid5f_info *r5f_info; ++ uint64_t stripe_index, stripe_offset, chunk_offset; ++ uint8_t chunk_data_idx, p_idx, chunk_idx; ++ int ret; + + spdk_bdev_free_io(bdev_io); + +- raid_bdev_io_complete(raid_io, success ? SPDK_BDEV_IO_STATUS_SUCCESS : +- SPDK_BDEV_IO_STATUS_FAILED); ++ if (spdk_likely(success)) { ++ raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_SUCCESS); ++ return; ++ } ++ ++ /* A PRESENT member returned a read error (e.g. a degraded band of the ++ * bdev_tier composite). Fall back to a degraded read: reconstruct this chunk ++ * from the survivors + parity instead of failing. The direct read path did ++ * not touch base_bdev_io_submitted, so the reconstruct path starts clean. */ ++ raid_bdev = raid_io->raid_bdev; ++ r5f_info = raid_bdev->module_private; ++ stripe_index = raid_io->offset_blocks / r5f_info->stripe_blocks; ++ stripe_offset = raid_io->offset_blocks % r5f_info->stripe_blocks; ++ chunk_data_idx = stripe_offset >> raid_bdev->strip_size_shift; ++ p_idx = raid5f_stripe_parity_chunk_index(raid_bdev, stripe_index); ++ chunk_idx = chunk_data_idx < p_idx ? chunk_data_idx : chunk_data_idx + 1; ++ chunk_offset = stripe_offset - ((uint64_t)chunk_data_idx << raid_bdev->strip_size_shift); ++ ++ ret = raid5f_submit_reconstruct_read(raid_io, stripe_index, chunk_idx, chunk_offset, ++ raid5f_stripe_request_reconstruct_xor_done); ++ if (ret != 0) { ++ raid_bdev_io_complete(raid_io, ret == -ENOMEM ? SPDK_BDEV_IO_STATUS_NOMEM : ++ SPDK_BDEV_IO_STATUS_FAILED); ++ } + } + + static void raid5f_submit_rw_request(struct raid_bdev_io *raid_io); +@@ -709,6 +742,24 @@ raid5f_submit_reconstruct_read(struct raid_bdev_io *raid_io, uint64_t stripe_ind + + assert(cb != NULL); + ++ /* SPEC-73 M3 double-fault guard: a reconstruct read consumes EVERY other member ++ * (raid5 tolerates a single fault). If a second member is unavailable (its base ++ * bdev was removed → NULL channel), reconstruction is impossible — fail cleanly with ++ * -EIO instead of submitting a read to a missing base bdev (crash / silent garbage). ++ * A present-but-erroring member (e.g. a degraded bdev_tier band) keeps a valid channel ++ * and is handled by the read-completion path, not here. */ ++ { ++ uint8_t i; ++ for (i = 0; i < raid_bdev->num_base_bdevs; i++) { ++ if (i == chunk_idx) { ++ continue; ++ } ++ if (raid_bdev_channel_get_base_channel(raid_io->raid_ch, i) == NULL) { ++ return -EIO; ++ } ++ } ++ } ++ + stripe_req = TAILQ_FIRST(&r5ch->free_stripe_requests.reconstruct); + if (!stripe_req) { + return -ENOMEM; +-- +2.50.1 (Apple Git-155) + diff --git a/patches/0007-raid-nexus-heat.patch b/patches/0007-raid-nexus-heat.patch new file mode 100644 index 0000000..5ea6889 --- /dev/null +++ b/patches/0007-raid-nexus-heat.patch @@ -0,0 +1,410 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 10:58:26 +0200 +Subject: [PATCH 07/12] raid: nexus logical heat instrumentation + +Evariops 0007. M7: release-publish / acquire-load of the heat pointer +(arm64-safe), per-region TSC recency instead of a shared tick counter. +N-8/P-4: disable_heat reclaims the sketch after a for_each_channel grace +period (the old "reclaimed at teardown" claim was false - it leaked). +--- + module/bdev/raid/Makefile | 2 +- + module/bdev/raid/bdev_raid.c | 17 ++ + module/bdev/raid/bdev_raid.h | 9 + + module/bdev/raid/bdev_raid_heat.c | 311 ++++++++++++++++++++++++++++++ + 4 files changed, 338 insertions(+), 1 deletion(-) + create mode 100644 module/bdev/raid/bdev_raid_heat.c + +diff --git a/module/bdev/raid/Makefile b/module/bdev/raid/Makefile +index 3dff369..cb46945 100644 +--- a/module/bdev/raid/Makefile ++++ b/module/bdev/raid/Makefile +@@ -10,7 +10,7 @@ SO_VER := 7 + SO_MINOR := 0 + + CFLAGS += -I$(SPDK_ROOT_DIR)/lib/bdev/ +-C_SRCS = bdev_raid.c bdev_raid_rpc.c bdev_raid_sb.c raid0.c raid1.c concat.c ++C_SRCS = bdev_raid.c bdev_raid_rpc.c bdev_raid_sb.c bdev_raid_heat.c raid0.c raid1.c concat.c + + ifeq ($(CONFIG_RAID5F),y) + C_SRCS += raid5f.c +diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c +index cc515bc..636eda1 100644 +--- a/module/bdev/raid/bdev_raid.c ++++ b/module/bdev/raid/bdev_raid.c +@@ -393,6 +393,9 @@ raid_bdev_cleanup(struct raid_bdev *raid_bdev) + static void + raid_bdev_free(struct raid_bdev *raid_bdev) + { ++ /* SPEC-73 M5: reclaim the logical heat sketch at nexus teardown (no I/O in flight here). */ ++ raid_tier_heat_destroy(raid_bdev->tier_heat); ++ raid_bdev->tier_heat = NULL; + raid_bdev_free_superblock(raid_bdev); + free(raid_bdev->base_bdev_info); + free(raid_bdev->bdev.name); +@@ -968,6 +971,20 @@ static void + raid_bdev_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) + { + struct raid_bdev_io *raid_io = (struct raid_bdev_io *)bdev_io->driver_ctx; ++ struct raid_bdev *raid_bdev = (struct raid_bdev *)bdev_io->bdev->ctxt; ++ void *heat; ++ ++ /* SPEC-73 M5: record logical heat on the hot path (lossy by design). ++ * M7: ACQUIRE pairs with the RELEASE publish in enable_heat — on arm64 the ++ * regions array must be visible before the pointer. No branch hint: heat is ++ * always-on in the nominal SPEC-73 deployment (the old spdk_unlikely was ++ * inverted for that state). */ ++ heat = __atomic_load_n(&raid_bdev->tier_heat, __ATOMIC_ACQUIRE); ++ if (heat != NULL && ++ (bdev_io->type == SPDK_BDEV_IO_TYPE_READ || bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE)) { ++ raid_tier_heat_record(heat, bdev_io->u.bdev.offset_blocks, ++ bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE); ++ } + + raid_bdev_io_init(raid_io, spdk_io_channel_get_ctx(ch), bdev_io->type, + bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks, +diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h +index d0484ca..e7d7ac5 100644 +--- a/module/bdev/raid/bdev_raid.h ++++ b/module/bdev/raid/bdev_raid.h +@@ -237,6 +237,9 @@ struct raid_bdev { + /* Private data for the raid module */ + void *module_private; + ++ /* SPEC-73 M5: logical heat sketch (NULL unless enabled at the nexus). */ ++ void *tier_heat; ++ + /* Superblock */ + bool superblock_enabled; + struct raid_bdev_superblock *sb; +@@ -570,4 +573,10 @@ struct spdk_raid_bdev_opts { + void raid_bdev_get_opts(struct spdk_raid_bdev_opts *opts); + int raid_bdev_set_opts(const struct spdk_raid_bdev_opts *opts); + ++/* SPEC-73 M5: logical heat instrumentation (bdev_raid_heat.c). Record an I/O on ++ * the hot path (cheap, lossy by design — INV-52); enable/destroy/query. */ ++void raid_tier_heat_record(void *heat, uint64_t offset_blocks, bool is_write); ++void *raid_tier_heat_create(uint64_t num_blocks, uint64_t region_blocks); ++void raid_tier_heat_destroy(void *heat); ++ + #endif /* SPDK_BDEV_RAID_INTERNAL_H */ +diff --git a/module/bdev/raid/bdev_raid_heat.c b/module/bdev/raid/bdev_raid_heat.c +new file mode 100644 +index 0000000..027a55d +--- /dev/null ++++ b/module/bdev/raid/bdev_raid_heat.c +@@ -0,0 +1,311 @@ ++/* SPDX-License-Identifier: BSD-3-Clause ++ * Copyright (C) 2026 Evariops. ++ * All rights reserved. ++ * ++ * SPEC-73 M5 (option A): logical heat instrumentation at the nexus (raid bdev). ++ * A dense per-region counter array, updated on the hot path by ++ * raid_bdev_submit_request (lossy by design, INV-52). The control-plane reads ++ * it asynchronously to rank regions for tiering — never on the I/O path (INV-T2). ++ */ ++ ++#include "bdev_raid.h" ++ ++#include "spdk/rpc.h" ++#include "spdk/util.h" ++#include "spdk/string.h" ++#include "spdk/likely.h" ++#include "spdk/log.h" ++#include "spdk/env.h" ++ ++struct raid_region_heat { ++ uint64_t reads; ++ uint64_t writes; ++ uint64_t last_tick; ++ /* Pad each entry to a full cacheline: adjacent regions touched by different ++ * reactors must not share a line, or every I/O false-shares with its neighbour. */ ++ uint8_t pad[64 - 3 * sizeof(uint64_t)]; ++}; ++ ++struct raid_tier_heat { ++ uint64_t region_blocks; /* power of two (see create) */ ++ uint32_t region_shift; /* log2(region_blocks): hot path shifts, never divides */ ++ uint64_t num_regions; ++ struct raid_region_heat *regions; ++}; ++ ++void * ++raid_tier_heat_create(uint64_t num_blocks, uint64_t region_blocks) ++{ ++ struct raid_tier_heat *h; ++ ++ if (region_blocks == 0) { ++ return NULL; ++ } ++ h = calloc(1, sizeof(*h)); ++ if (h == NULL) { ++ return NULL; ++ } ++ /* Round the region size DOWN to a power of two so raid_tier_heat_record maps ++ * offset->region with a shift instead of a per-I/O 64-bit divide. region_blocks ++ * is >= 1 here (guarded above), so the builtin is well-defined. */ ++ h->region_shift = 63u - (uint32_t)__builtin_clzll(region_blocks); ++ h->region_blocks = (uint64_t)1 << h->region_shift; ++ h->num_regions = spdk_divide_round_up(num_blocks, h->region_blocks); ++ h->regions = calloc(h->num_regions, sizeof(*h->regions)); ++ if (h->regions == NULL) { ++ free(h); ++ return NULL; ++ } ++ return h; ++} ++ ++void ++raid_tier_heat_destroy(void *heat) ++{ ++ struct raid_tier_heat *h = heat; ++ ++ if (h == NULL) { ++ return; ++ } ++ free(h->regions); ++ free(h); ++} ++ ++/* Hot path: cheap, racy increment (approximate counts tolerated — INV-52; the ++ * counters are per-region so concurrent reactors only contend when hitting the ++ * SAME 64 MiB region). M7: recency uses the per-CPU TSC — the old shared ++ * `++h->tick` was a guaranteed cacheline ping-pong across every reactor on ++ * every I/O and a torn read/write on 32-bit stores. */ ++void ++raid_tier_heat_record(void *heat, uint64_t offset_blocks, bool is_write) ++{ ++ struct raid_tier_heat *h = heat; ++ uint64_t region; ++ ++ if (spdk_unlikely(h == NULL)) { ++ return; ++ } ++ region = offset_blocks >> h->region_shift; ++ if (spdk_unlikely(region >= h->num_regions)) { ++ return; ++ } ++ if (is_write) { ++ h->regions[region].writes++; ++ } else { ++ h->regions[region].reads++; ++ } ++ h->regions[region].last_tick = spdk_get_ticks(); ++} ++ ++/* ---- RPC: vbdev_nexus_enable_heat {name, region_mib} ----------------------- */ ++ ++struct rpc_heat_enable { ++ char *name; ++ uint32_t region_mib; ++}; ++ ++static const struct spdk_json_object_decoder rpc_heat_enable_decoders[] = { ++ {"name", offsetof(struct rpc_heat_enable, name), spdk_json_decode_string}, ++ {"region_mib", offsetof(struct rpc_heat_enable, region_mib), spdk_json_decode_uint32, true}, ++}; ++ ++static void ++rpc_vbdev_nexus_enable_heat(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) ++{ ++ struct rpc_heat_enable req = { .region_mib = 64 }; ++ struct raid_bdev *raid_bdev; ++ uint64_t region_blocks; ++ ++ if (spdk_json_decode_object(params, rpc_heat_enable_decoders, ++ SPDK_COUNTOF(rpc_heat_enable_decoders), &req)) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "decode failed"); ++ goto cleanup; ++ } ++ if (req.region_mib == 0) { ++ req.region_mib = 64; ++ } ++ raid_bdev = raid_bdev_find_by_name(req.name); ++ if (raid_bdev == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "nexus not found"); ++ goto cleanup; ++ } ++ if (raid_bdev->bdev.blocklen == 0) { ++ /* A nexus that exists but is not yet configured has blocklen 0 — dividing ++ * by it below would fault. Refuse until the nexus is online. */ ++ spdk_jsonrpc_send_error_response(request, -EINVAL, "nexus not ready (blocklen unset)"); ++ goto cleanup; ++ } ++ if (__atomic_load_n(&raid_bdev->tier_heat, __ATOMIC_RELAXED) == NULL) { ++ void *h; ++ ++ region_blocks = ((uint64_t)req.region_mib << 20) / raid_bdev->bdev.blocklen; ++ if (region_blocks == 0) { ++ region_blocks = 1; ++ } ++ h = raid_tier_heat_create(raid_bdev->bdev.blockcnt, region_blocks); ++ if (h == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENOMEM, "heat alloc failed"); ++ goto cleanup; ++ } ++ /* M7: RELEASE publish — pairs with the ACQUIRE load in the submit hot ++ * path so the regions array is visible before the pointer (arm64). */ ++ __atomic_store_n(&raid_bdev->tier_heat, h, __ATOMIC_RELEASE); ++ } ++ spdk_jsonrpc_send_bool_response(request, true); ++cleanup: ++ free(req.name); ++} ++SPDK_RPC_REGISTER("vbdev_nexus_enable_heat", rpc_vbdev_nexus_enable_heat, SPDK_RPC_RUNTIME) ++ ++/* ---- RPC: vbdev_nexus_disable_heat {name} --------------------------------- */ ++ ++struct rpc_heat_name { char *name; }; ++static const struct spdk_json_object_decoder rpc_heat_name_decoders[] = { ++ {"name", offsetof(struct rpc_heat_name, name), spdk_json_decode_string}, ++}; ++ ++/* N-8/P-4: disable must actually reclaim the sketch — "reclaimed at teardown" ++ * was false (no reference was kept), so enable/disable/enable leaked the first ++ * array forever. Reclaim happens after a for_each_channel GRACE PERIOD: SPDK ++ * reactors are run-to-completion, so once every reactor has processed one ++ * message, any record() that loaded the old pointer has finished. */ ++struct heat_disable_ctx { ++ struct spdk_jsonrpc_request *request; ++ void *old; ++}; ++ ++static void ++heat_disable_grace_iter(struct spdk_io_channel_iter *i) ++{ ++ spdk_for_each_channel_continue(i, 0); ++} ++ ++static void ++heat_disable_grace_done(struct spdk_io_channel_iter *i, int status) ++{ ++ struct heat_disable_ctx *ctx = spdk_io_channel_iter_get_ctx(i); ++ ++ (void)status; ++ raid_tier_heat_destroy(ctx->old); ++ spdk_jsonrpc_send_bool_response(ctx->request, true); ++ free(ctx); ++} ++ ++static void ++rpc_vbdev_nexus_disable_heat(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) ++{ ++ struct rpc_heat_name req = {}; ++ struct raid_bdev *raid_bdev; ++ void *old = NULL; ++ ++ if (spdk_json_decode_object(params, rpc_heat_name_decoders, ++ SPDK_COUNTOF(rpc_heat_name_decoders), &req)) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "decode failed"); ++ goto cleanup; ++ } ++ raid_bdev = raid_bdev_find_by_name(req.name); ++ if (raid_bdev != NULL) { ++ old = __atomic_exchange_n(&raid_bdev->tier_heat, NULL, __ATOMIC_ACQ_REL); ++ } ++ if (old == NULL) { ++ spdk_jsonrpc_send_bool_response(request, true); ++ goto cleanup; ++ } ++ if (raid_bdev->state == RAID_BDEV_STATE_ONLINE) { ++ struct heat_disable_ctx *ctx = calloc(1, sizeof(*ctx)); ++ ++ if (ctx == NULL) { ++ /* Cannot run the grace sweep: leave the array orphaned (leak, ++ * never a use-after-free) — teardown still reclaims it if the ++ * nexus is later freed. */ ++ SPDK_ERRLOG("disable_heat: oom — heat array orphaned until teardown\n"); ++ spdk_jsonrpc_send_bool_response(request, true); ++ goto cleanup; ++ } ++ ctx->request = request; ++ ctx->old = old; ++ spdk_for_each_channel(raid_bdev, heat_disable_grace_iter, ctx, ++ heat_disable_grace_done); ++ free(req.name); ++ return; ++ } ++ /* Not ONLINE: no channels exist, no reactor can be inside record(). */ ++ raid_tier_heat_destroy(old); ++ spdk_jsonrpc_send_bool_response(request, true); ++cleanup: ++ free(req.name); ++} ++SPDK_RPC_REGISTER("vbdev_nexus_disable_heat", rpc_vbdev_nexus_disable_heat, SPDK_RPC_RUNTIME) ++ ++/* ---- RPC: vbdev_nexus_get_heat {name, start_region, max_regions} ----------- */ ++ ++struct rpc_heat_get { ++ char *name; ++ uint64_t start_region; ++ uint32_t max_regions; ++}; ++ ++static const struct spdk_json_object_decoder rpc_heat_get_decoders[] = { ++ {"name", offsetof(struct rpc_heat_get, name), spdk_json_decode_string}, ++ {"start_region", offsetof(struct rpc_heat_get, start_region), spdk_json_decode_uint64, true}, ++ {"max_regions", offsetof(struct rpc_heat_get, max_regions), spdk_json_decode_uint32, true}, ++}; ++ ++#define HEAT_GET_DEFAULT_MAX 8192u ++ ++static void ++rpc_vbdev_nexus_get_heat(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) ++{ ++ struct rpc_heat_get req = { .start_region = 0, .max_regions = HEAT_GET_DEFAULT_MAX }; ++ struct raid_bdev *raid_bdev; ++ struct raid_tier_heat *h; ++ struct spdk_json_write_ctx *w; ++ uint64_t r, emitted = 0, next_cursor = 0; ++ ++ if (spdk_json_decode_object(params, rpc_heat_get_decoders, ++ SPDK_COUNTOF(rpc_heat_get_decoders), &req)) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "decode failed"); ++ goto cleanup; ++ } ++ if (req.max_regions == 0) { ++ req.max_regions = HEAT_GET_DEFAULT_MAX; ++ } ++ raid_bdev = raid_bdev_find_by_name(req.name); ++ h = raid_bdev != NULL ? ++ __atomic_load_n(&raid_bdev->tier_heat, __ATOMIC_ACQUIRE) : NULL; ++ if (h == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "nexus/heat not found"); ++ goto cleanup; ++ } ++ ++ w = spdk_jsonrpc_begin_result(request); ++ spdk_json_write_object_begin(w); ++ spdk_json_write_named_uint64(w, "region_blocks", h->region_blocks); ++ spdk_json_write_named_uint64(w, "num_regions", h->num_regions); ++ /* M7: recency reference is the TSC now (last_tick is per-region TSC). */ ++ spdk_json_write_named_uint64(w, "tick", spdk_get_ticks()); ++ spdk_json_write_named_array_begin(w, "regions"); ++ for (r = req.start_region; r < h->num_regions; r++) { ++ if (h->regions[r].reads == 0 && h->regions[r].writes == 0) { ++ continue; /* cold/untouched */ ++ } ++ if (emitted >= req.max_regions) { ++ next_cursor = r; ++ break; ++ } ++ spdk_json_write_object_begin(w); ++ spdk_json_write_named_uint64(w, "region", r); ++ spdk_json_write_named_uint64(w, "reads", h->regions[r].reads); ++ spdk_json_write_named_uint64(w, "writes", h->regions[r].writes); ++ spdk_json_write_named_uint64(w, "last_tick", h->regions[r].last_tick); ++ spdk_json_write_object_end(w); ++ emitted++; ++ } ++ spdk_json_write_array_end(w); ++ spdk_json_write_named_uint64(w, "next_cursor", next_cursor); ++ spdk_json_write_object_end(w); ++ spdk_jsonrpc_end_result(request, w); ++cleanup: ++ free(req.name); ++} ++SPDK_RPC_REGISTER("vbdev_nexus_get_heat", rpc_vbdev_nexus_get_heat, SPDK_RPC_RUNTIME) +-- +2.50.1 (Apple Git-155) + diff --git a/patches/0008-raid-rebuild-ranges.patch b/patches/0008-raid-rebuild-ranges.patch new file mode 100644 index 0000000..bb55cfe --- /dev/null +++ b/patches/0008-raid-rebuild-ranges.patch @@ -0,0 +1,548 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 10:58:26 +0200 +Subject: [PATCH 08/12] raid: bdev_raid_rebuild_ranges + read-reconstruct-writeback repair + +Evariops 0008. C2: every read/write-back chunk runs under a channel-owned +LBA range lock (spdk_bdev_lock_lba_range, newly exported from lib/bdev - +the primitive COMPARE_AND_WRITE emulation already uses); host writes to the +chunk are held and replayed after the write-back. P-3: stripe alignment +validated for parity raids, ranges heap-allocated. N-6: REMOVE aborts. +--- + include/spdk/bdev_module.h | 17 ++ + lib/bdev/bdev.c | 25 ++ + lib/bdev/spdk_bdev.map | 2 + + module/bdev/raid/Makefile | 2 +- + module/bdev/raid/bdev_raid.c | 11 + + module/bdev/raid/bdev_raid_repair.c | 406 ++++++++++++++++++++++++++++ + 6 files changed, 462 insertions(+), 1 deletion(-) + create mode 100644 module/bdev/raid/bdev_raid_repair.c + +diff --git a/include/spdk/bdev_module.h b/include/spdk/bdev_module.h +index fce200b..79b43f0 100644 +--- a/include/spdk/bdev_module.h ++++ b/include/spdk/bdev_module.h +@@ -1954,6 +1954,23 @@ int spdk_bdev_unquiesce_range(struct spdk_bdev *bdev, struct spdk_bdev_module *m + uint64_t offset, uint64_t length, + spdk_bdev_quiesce_cb cb_fn, void *cb_arg); + ++/* ── Evariops SPEC-73 C2: channel-owned LBA range lock ───────────────────── ++ * Holds NEW I/O overlapping [offset, offset+length) (queued, replayed at ++ * unlock) and waits for in-flight I/O to drain before granting the lock. The ++ * OWNING channel is exempt: an I/O submitted on `ch` whose cb_arg equals the ++ * lock's `cb_arg` passes through — the pattern a read-modify-write repair ++ * needs. This is the same internal primitive COMPARE_AND_WRITE emulation uses. ++ * Both calls must be made from the thread owning `ch`. */ ++struct lba_range; ++typedef void (*spdk_bdev_lba_range_cb)(struct lba_range *range, void *ctx, int status); ++ ++int spdk_bdev_lock_lba_range(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, ++ uint64_t offset, uint64_t length, ++ spdk_bdev_lba_range_cb cb_fn, void *cb_arg); ++int spdk_bdev_unlock_lba_range(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, ++ uint64_t offset, uint64_t length, ++ spdk_bdev_lba_range_cb cb_fn, void *cb_arg); ++ + /* + * Macro used to register module for later initialization. + */ +diff --git a/lib/bdev/bdev.c b/lib/bdev/bdev.c +index 98e01ca..80f0cf7 100644 +--- a/lib/bdev/bdev.c ++++ b/lib/bdev/bdev.c +@@ -11189,6 +11189,31 @@ spdk_bdev_unquiesce_range(struct spdk_bdev *bdev, struct spdk_bdev_module *modul + return _spdk_bdev_quiesce(bdev, module, offset, length, cb_fn, cb_arg, true); + } + ++/* Evariops SPEC-73 C2: expose the channel-owned LBA range lock (the primitive ++ * COMPARE_AND_WRITE emulation already uses). Unlike quiesce, the OWNER channel ++ * may keep submitting I/O inside the locked range when the I/O's cb_arg equals ++ * the lock's cb_arg (see bdev_io_range_is_locked) — exactly what a ++ * read-reconstruct-writeback repair needs: host I/O to the range is held and ++ * replayed at unlock, while the repair's own I/O flows. */ ++int ++spdk_bdev_lock_lba_range(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, ++ uint64_t offset, uint64_t length, ++ spdk_bdev_lba_range_cb cb_fn, void *cb_arg) ++{ ++ /* No cast: spdk_bdev_lba_range_cb and the internal lock_range_cb have identical ++ * signatures, so this compiles as compatible types. Casting instead would hide a ++ * future divergence of the internal typedef behind a silent runtime mismatch. */ ++ return bdev_lock_lba_range(desc, ch, offset, length, cb_fn, cb_arg); ++} ++ ++int ++spdk_bdev_unlock_lba_range(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, ++ uint64_t offset, uint64_t length, ++ spdk_bdev_lba_range_cb cb_fn, void *cb_arg) ++{ ++ return bdev_unlock_lba_range(desc, ch, offset, length, cb_fn, cb_arg); ++} ++ + int + spdk_bdev_get_memory_domains(struct spdk_bdev *bdev, struct spdk_memory_domain **domains, + int array_size) +diff --git a/lib/bdev/spdk_bdev.map b/lib/bdev/spdk_bdev.map +index 57c7889..889d36c 100644 +--- a/lib/bdev/spdk_bdev.map ++++ b/lib/bdev/spdk_bdev.map +@@ -201,6 +201,8 @@ + spdk_bdev_quiesce; + spdk_bdev_unquiesce; + spdk_bdev_quiesce_range; ++ spdk_bdev_lock_lba_range; ++ spdk_bdev_unlock_lba_range; + spdk_bdev_unquiesce_range; + spdk_bdev_io_hide_metadata; + +diff --git a/module/bdev/raid/Makefile b/module/bdev/raid/Makefile +index cb46945..e118002 100644 +--- a/module/bdev/raid/Makefile ++++ b/module/bdev/raid/Makefile +@@ -10,7 +10,7 @@ SO_VER := 7 + SO_MINOR := 0 + + CFLAGS += -I$(SPDK_ROOT_DIR)/lib/bdev/ +-C_SRCS = bdev_raid.c bdev_raid_rpc.c bdev_raid_sb.c bdev_raid_heat.c raid0.c raid1.c concat.c ++C_SRCS = bdev_raid.c bdev_raid_rpc.c bdev_raid_sb.c bdev_raid_heat.c bdev_raid_repair.c raid0.c raid1.c concat.c + + ifeq ($(CONFIG_RAID5F),y) + C_SRCS += raid5f.c +diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c +index 636eda1..53b88bc 100644 +--- a/module/bdev/raid/bdev_raid.c ++++ b/module/bdev/raid/bdev_raid.c +@@ -1122,6 +1122,17 @@ raid_bdev_write_info_json(struct raid_bdev *raid_bdev, struct spdk_json_write_ct + spdk_json_write_named_uint32(w, "num_base_bdevs_discovered", raid_bdev->num_base_bdevs_discovered); + spdk_json_write_named_uint32(w, "num_base_bdevs_operational", + raid_bdev->num_base_bdevs_operational); ++ /* SPEC-73 C3: expose the full-stripe size (in blocks) that bdev_raid_rebuild_ranges ++ * requires its ranges to be aligned to, so the control-plane reads it instead of ++ * re-deriving `k` (data chunks) per raid level. This is EXACTLY the value the repair ++ * validates: strip_size × min_base_bdevs_operational (for raid5f, min == num-1 == the ++ * data-chunk count `k`; for a striped non-parity raid, min == num == k). Emitted only ++ * for striped raids — raid1/concat (strip_size == 0) have no full-stripe constraint. */ ++ if (raid_bdev->strip_size > 0) { ++ spdk_json_write_named_uint64(w, "full_stripe_blocks", ++ (uint64_t)raid_bdev->strip_size * ++ raid_bdev->min_base_bdevs_operational); ++ } + if (raid_bdev->process) { + struct raid_bdev_process *process = raid_bdev->process; + uint64_t offset = process->window_offset; +diff --git a/module/bdev/raid/bdev_raid_repair.c b/module/bdev/raid/bdev_raid_repair.c +new file mode 100644 +index 0000000..f0aff59 +--- /dev/null ++++ b/module/bdev/raid/bdev_raid_repair.c +@@ -0,0 +1,406 @@ ++/* SPDX-License-Identifier: BSD-3-Clause ++ * Copyright (C) 2026 Evariops. ++ * All rights reserved. ++ * ++ * SPEC-73 M4: bdev_raid_rebuild_ranges — repair lost ranges of a degraded member by ++ * READ-RECONSTRUCT-WRITEBACK through the raid bdev. ++ * ++ * A degraded bdev_tier band means a chunk lost some clusters' data (the disk failed). Repair: ++ * for each stripe-aligned range, read it from the raid bdev — M3 (0006) reconstructs the ++ * degraded chunk from survivors+parity for raid5f; raid1 reads a healthy mirror — then write ++ * the buffer back, which recomputes parity (raid5f) / rewrites all mirrors (raid1), repairing ++ * the chunk's data on its relocated/replacement band. INV-T2: data never leaves SPDK. ++ * ++ * C2 (atomicity vs host I/O): each read/write-back pair runs under a CHANNEL-OWNED LBA RANGE ++ * LOCK (spdk_bdev_lock_lba_range) on exactly the chunk being repaired. Host writes overlapping ++ * the chunk are HELD at submission and replayed after the unlock (so they land AFTER our ++ * write-back — host data wins); our own repair I/O is exempt (same channel + same cb_arg as the ++ * lock). Without the lock, a host write racing between our read and our write-back was silently ++ * overwritten with the stale buffer. Host reads are not held (a plain lock only holds writes). ++ * ++ * P-3: ranges must be stripe-aligned for parity raids (validated at the RPC — the old code ++ * trusted the caller and issued a partial trailing write that raid5f rejects -EIO mid-repair). ++ * raid1/concat (strip_size == 0) have no full-stripe constraint: a fixed chunk is used. ++ * ++ * N-6: a REMOVE of the raid bdev mid-repair aborts the repair at the next chunk boundary ++ * (in-flight I/O completes with an error) instead of holding the desc open indefinitely. ++ * ++ * Sequential (QD=1) — this is a background repair, paced by the control-plane op budget. ++ */ ++ ++#include "bdev_raid.h" ++ ++#include "spdk/rpc.h" ++#include "spdk/util.h" ++#include "spdk/string.h" ++#include "spdk/log.h" ++#include "spdk/env.h" ++ ++#define RAID_REPAIR_MAX_RANGES 4096u ++/* Chunk for non-parity raids (raid1/concat, strip_size == 0): 2048 blocks ++ * (1 MiB @ 512B). Parity raids always use the full stripe. */ ++#define RAID_REPAIR_NOSTRIPE_CHUNK_BLOCKS 2048u ++ ++struct raid_repair_range { ++ uint64_t start_lba; ++ uint64_t num_blocks; ++}; ++ ++struct raid_repair_ctx { ++ struct spdk_jsonrpc_request *request; ++ struct spdk_bdev_desc *desc; ++ struct spdk_io_channel *ch; ++ struct raid_repair_range *ranges; ++ uint32_t num_ranges; ++ uint32_t cur; /* current range index */ ++ uint64_t cur_off; /* blocks completed within the current range */ ++ uint64_t io_blocks; /* size of the in-flight I/O */ ++ uint64_t io_lba; /* start of the in-flight (locked) chunk */ ++ uint64_t stripe_blocks; /* chunk size (full stripe for parity raids) */ ++ bool locked; /* C2: chunk range lock held */ ++ bool removed; /* N-6: REMOVE seen — abort at next boundary */ ++ int rc; ++ void *buf; ++ uint64_t ranges_done; ++}; ++ ++static void raid_repair_next(struct raid_repair_ctx *c); ++ ++static void ++raid_repair_complete(struct raid_repair_ctx *c, int rc) ++{ ++ struct spdk_json_write_ctx *w; ++ ++ if (c->ch != NULL) { ++ spdk_put_io_channel(c->ch); ++ } ++ if (c->desc != NULL) { ++ spdk_bdev_close(c->desc); ++ } ++ if (c->buf != NULL) { ++ spdk_dma_free(c->buf); ++ } ++ if (rc == 0) { ++ w = spdk_jsonrpc_begin_result(c->request); ++ spdk_json_write_object_begin(w); ++ spdk_json_write_named_uint64(w, "ranges_repaired", c->ranges_done); ++ spdk_json_write_object_end(w); ++ spdk_jsonrpc_end_result(c->request, w); ++ } else { ++ spdk_jsonrpc_send_error_response_fmt(c->request, rc, "rebuild_ranges failed: %s", ++ spdk_strerror(rc < 0 ? -rc : rc)); ++ } ++ free(c->ranges); ++ free(c); ++} ++ ++static void ++raid_repair_finish_unlocked(struct lba_range *range, void *cb_arg, int status) ++{ ++ struct raid_repair_ctx *c = cb_arg; ++ ++ (void)range; ++ (void)status; ++ raid_repair_complete(c, c->rc); ++} ++ ++/* Terminal path: release a held chunk lock first (host I/O queued behind it ++ * would hang forever otherwise), then reply. */ ++static void ++raid_repair_finish(struct raid_repair_ctx *c, int rc) ++{ ++ c->rc = rc; ++ if (c->locked) { ++ c->locked = false; ++ if (spdk_bdev_unlock_lba_range(c->desc, c->ch, c->io_lba, c->io_blocks, ++ raid_repair_finish_unlocked, c) == 0) { ++ return; ++ } ++ SPDK_ERRLOG("raid repair: terminal unlock dispatch failed — host I/O on " ++ "[%" PRIu64 ", +%" PRIu64 ") left held!\n", c->io_lba, c->io_blocks); ++ } ++ raid_repair_complete(c, rc); ++} ++ ++static void ++raid_repair_chunk_unlocked(struct lba_range *range, void *cb_arg, int status) ++{ ++ struct raid_repair_ctx *c = cb_arg; ++ ++ (void)range; ++ if (status != 0) { ++ SPDK_ERRLOG("raid repair: chunk unlock failed: %s\n", spdk_strerror(-status)); ++ raid_repair_complete(c, status); ++ return; ++ } ++ raid_repair_next(c); ++} ++ ++static void ++raid_repair_write_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) ++{ ++ struct raid_repair_ctx *c = cb_arg; ++ ++ spdk_bdev_free_io(bdev_io); ++ if (!success) { ++ raid_repair_finish(c, -EIO); ++ return; ++ } ++ c->cur_off += c->io_blocks; ++ /* C2: chunk repaired — release the lock (held host writes replay NOW, after ++ * our write-back, so the freshest data wins), then move on. */ ++ c->locked = false; ++ if (spdk_bdev_unlock_lba_range(c->desc, c->ch, c->io_lba, c->io_blocks, ++ raid_repair_chunk_unlocked, c) != 0) { ++ SPDK_ERRLOG("raid repair: unlock dispatch failed — aborting\n"); ++ raid_repair_complete(c, -EIO); ++ } ++} ++ ++static void ++raid_repair_read_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) ++{ ++ struct raid_repair_ctx *c = cb_arg; ++ int rc; ++ ++ spdk_bdev_free_io(bdev_io); ++ if (!success) { ++ /* M3 should have reconstructed a degraded read; a hard failure here means the stripe ++ * is unrecoverable (>1 fault) — fail cleanly, the data is still protected elsewhere. ++ * Name the offending chunk so operators can pinpoint the unrecoverable stripe (which ++ * range/LBA lost redundancy) instead of seeing an opaque -EIO from the RPC. */ ++ SPDK_ERRLOG("raid repair: unrecoverable read at chunk [%" PRIu64 ", +%" PRIu64 ++ ") (range %u/%u) — stripe has >1 fault, aborting\n", ++ c->io_lba, c->io_blocks, c->cur, c->num_ranges); ++ raid_repair_finish(c, -EIO); ++ return; ++ } ++ rc = spdk_bdev_write_blocks(c->desc, c->ch, c->buf, c->io_lba, c->io_blocks, ++ raid_repair_write_done, c); ++ if (rc != 0) { ++ raid_repair_finish(c, rc); ++ } ++} ++ ++/* C2: the chunk range is locked (in-flight host I/O drained, new host writes ++ * held) — do the read-reconstruct-writeback. Our own I/O passes the lock ++ * because it is submitted on the locking channel with cb_arg == locked_ctx. */ ++static void ++raid_repair_chunk_locked(struct lba_range *range, void *cb_arg, int status) ++{ ++ struct raid_repair_ctx *c = cb_arg; ++ int rc; ++ ++ (void)range; ++ if (status != 0) { ++ raid_repair_complete(c, status); ++ return; ++ } ++ c->locked = true; ++ rc = spdk_bdev_read_blocks(c->desc, c->ch, c->buf, c->io_lba, c->io_blocks, ++ raid_repair_read_done, c); ++ if (rc != 0) { ++ raid_repair_finish(c, rc); ++ } ++} ++ ++static void ++raid_repair_next(struct raid_repair_ctx *c) ++{ ++ uint64_t rem; ++ int rc; ++ ++ if (c->removed) { ++ raid_repair_finish(c, -ENODEV); /* N-6: abort on REMOVE */ ++ return; ++ } ++ /* Advance past finished ranges. */ ++ while (c->cur < c->num_ranges && c->cur_off >= c->ranges[c->cur].num_blocks) { ++ c->cur++; ++ c->cur_off = 0; ++ c->ranges_done++; ++ } ++ if (c->cur >= c->num_ranges) { ++ raid_repair_finish(c, 0); ++ return; ++ } ++ c->io_lba = c->ranges[c->cur].start_lba + c->cur_off; ++ rem = c->ranges[c->cur].num_blocks - c->cur_off; ++ c->io_blocks = spdk_min(rem, c->stripe_blocks); ++ rc = spdk_bdev_lock_lba_range(c->desc, c->ch, c->io_lba, c->io_blocks, ++ raid_repair_chunk_locked, c); ++ if (rc != 0) { ++ raid_repair_finish(c, rc); ++ } ++} ++ ++/* ---- RPC: bdev_raid_rebuild_ranges {name, ranges:[{start_lba,num_blocks}]} -------------- */ ++ ++struct rpc_raid_repair { ++ char *name; ++ size_t num_ranges; ++ /* P-3(c): heap-allocated in the handler (4096×16B on the handler stack ++ * before). */ ++ struct raid_repair_range *ranges; ++}; ++ ++static int ++decode_repair_range(const struct spdk_json_val *val, void *out) ++{ ++ const struct spdk_json_object_decoder decoders[] = { ++ {"start_lba", offsetof(struct raid_repair_range, start_lba), spdk_json_decode_uint64}, ++ {"num_blocks", offsetof(struct raid_repair_range, num_blocks), spdk_json_decode_uint64}, ++ }; ++ return spdk_json_decode_object(val, decoders, SPDK_COUNTOF(decoders), ++ (struct raid_repair_range *)out) ? -1 : 0; ++} ++ ++static int ++decode_repair_ranges(const struct spdk_json_val *val, void *out) ++{ ++ struct rpc_raid_repair *req = SPDK_CONTAINEROF(out, struct rpc_raid_repair, ranges); ++ ++ if (req->ranges == NULL) { ++ req->ranges = calloc(RAID_REPAIR_MAX_RANGES, sizeof(struct raid_repair_range)); ++ if (req->ranges == NULL) { ++ return -ENOMEM; ++ } ++ } ++ return spdk_json_decode_array(val, decode_repair_range, req->ranges, ++ RAID_REPAIR_MAX_RANGES, &req->num_ranges, ++ sizeof(struct raid_repair_range)); ++} ++ ++static const struct spdk_json_object_decoder rpc_raid_repair_decoders[] = { ++ {"name", offsetof(struct rpc_raid_repair, name), spdk_json_decode_string}, ++ {"ranges", offsetof(struct rpc_raid_repair, ranges), decode_repair_ranges}, ++}; ++ ++/* N-6: remember the REMOVE so the repair aborts at the next chunk boundary ++ * (the desc is closed by the normal finish path). */ ++static void ++raid_repair_base_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) ++{ ++ struct raid_repair_ctx *c = ctx; ++ ++ if (type == SPDK_BDEV_EVENT_REMOVE && c != NULL) { ++ SPDK_WARNLOG("raid repair: '%s' removed — aborting at next chunk\n", bdev->name); ++ c->removed = true; ++ } ++} ++ ++static void ++rpc_bdev_raid_rebuild_ranges(struct spdk_jsonrpc_request *request, const struct spdk_json_val *params) ++{ ++ struct rpc_raid_repair req = {}; ++ struct raid_repair_ctx *c = NULL; ++ struct raid_bdev *raid_bdev = NULL, *it; ++ uint64_t stripe_blocks; ++ uint32_t i; ++ int rc; ++ ++ if (spdk_json_decode_object(params, rpc_raid_repair_decoders, ++ SPDK_COUNTOF(rpc_raid_repair_decoders), &req)) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "decode failed"); ++ goto cleanup; ++ } ++ if (req.num_ranges == 0) { ++ spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "no ranges"); ++ goto cleanup; ++ } ++ ++ TAILQ_FOREACH(it, &g_raid_bdev_list, global_link) { ++ if (it->bdev.name != NULL && strcmp(it->bdev.name, req.name) == 0) { ++ raid_bdev = it; ++ break; ++ } ++ } ++ if (raid_bdev == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENODEV, "raid bdev not found"); ++ goto cleanup; ++ } ++ ++ if (raid_bdev->strip_size > 0) { ++ /* Full-stripe size = strip_size × number of DATA chunks. For raid5f ++ * min_base_bdevs_operational == num_base_bdevs - 1 == data chunks; for ++ * striped non-parity raids min == num == data chunks. */ ++ stripe_blocks = (uint64_t)raid_bdev->strip_size * ++ raid_bdev->min_base_bdevs_operational; ++ if (stripe_blocks == 0) { ++ spdk_jsonrpc_send_error_response(request, -EINVAL, ++ "raid has strip_size but zero data chunks"); ++ goto cleanup; ++ } ++ /* P-3(a): validate alignment up front — a partial trailing write would ++ * fail -EIO in the middle of the repair (raid5f accepts only full ++ * stripes) after having already rewritten earlier stripes. */ ++ for (i = 0; i < req.num_ranges; i++) { ++ if (req.ranges[i].num_blocks == 0 || ++ req.ranges[i].start_lba % stripe_blocks != 0 || ++ req.ranges[i].num_blocks % stripe_blocks != 0) { ++ spdk_jsonrpc_send_error_response_fmt(request, ++ SPDK_JSONRPC_ERROR_INVALID_PARAMS, ++ "range %u [%" PRIu64 ", +%" PRIu64 ") not aligned to the " ++ "full stripe (%" PRIu64 " blocks)", i, ++ req.ranges[i].start_lba, req.ranges[i].num_blocks, ++ stripe_blocks); ++ goto cleanup; ++ } ++ } ++ } else { ++ /* raid1/concat: no full-stripe constraint — fixed repair chunk. */ ++ stripe_blocks = RAID_REPAIR_NOSTRIPE_CHUNK_BLOCKS; ++ for (i = 0; i < req.num_ranges; i++) { ++ if (req.ranges[i].num_blocks == 0) { ++ spdk_jsonrpc_send_error_response(request, ++ SPDK_JSONRPC_ERROR_INVALID_PARAMS, "zero-length range"); ++ goto cleanup; ++ } ++ } ++ } ++ ++ c = calloc(1, sizeof(*c)); ++ if (c == NULL) { ++ spdk_jsonrpc_send_error_response(request, -ENOMEM, "oom"); ++ goto cleanup; ++ } ++ c->request = request; ++ c->num_ranges = (uint32_t)req.num_ranges; ++ c->stripe_blocks = stripe_blocks; ++ /* Hand the heap ranges array over to the async context. */ ++ c->ranges = req.ranges; ++ req.ranges = NULL; ++ ++ rc = spdk_bdev_open_ext(req.name, true, raid_repair_base_event_cb, c, &c->desc); ++ if (rc != 0) { ++ raid_repair_complete(c, rc); ++ c = NULL; ++ goto cleanup; ++ } ++ c->ch = spdk_bdev_get_io_channel(c->desc); ++ c->buf = spdk_dma_malloc(stripe_blocks * (uint64_t)raid_bdev->bdev.blocklen, ++ raid_bdev->bdev.blocklen, NULL); ++ if (c->ch == NULL || c->buf == NULL) { ++ raid_repair_complete(c, -ENOMEM); ++ c = NULL; ++ goto cleanup; ++ } ++ ++ /* SEC1: audit this data-mutating repair (reads+rewrites full stripes). */ ++ { ++ char detail[128]; ++ ++ snprintf(detail, sizeof(detail), "name=%s ranges=%u chunk_blocks=%" PRIu64, ++ req.name, c->num_ranges, stripe_blocks); ++ spdk_jsonrpc_request_audit(request, "bdev_raid_rebuild_ranges", detail); ++ } ++ SPDK_NOTICELOG("raid '%s': rebuild_ranges over %u range(s), chunk=%" PRIu64 ++ " blocks (per-chunk LBA lock)\n", req.name, c->num_ranges, stripe_blocks); ++ raid_repair_next(c); ++ c = NULL; /* ownership transferred to the async chain */ ++ ++cleanup: ++ free(req.ranges); ++ free(req.name); ++} ++SPDK_RPC_REGISTER("bdev_raid_rebuild_ranges", rpc_bdev_raid_rebuild_ranges, SPDK_RPC_RUNTIME) +-- +2.50.1 (Apple Git-155) + diff --git a/patches/0009-lvol-raid-enospc-capacity-exceeded.patch b/patches/0009-lvol-raid-enospc-capacity-exceeded.patch new file mode 100644 index 0000000..c62a59b --- /dev/null +++ b/patches/0009-lvol-raid-enospc-capacity-exceeded.patch @@ -0,0 +1,142 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 10:55:35 +0200 +Subject: [PATCH 09/12] lvol/raid: surface thin-pool ENOSPC as NVMe + CAPACITY_EXCEEDED (Evariops 0009, C-1) + +lvol maps blob -ENOSPC to NVMe CAPACITY_EXCEEDED (survives the NVMe-oF hop); +raid1 does NOT fail a member on CAPACITY_EXCEEDED (thin exhaustion is not a +device fault) and the raid_io completes CAPACITY_EXCEEDED to the consumer +even if another leg succeeded - no more silently-diverged, silently-degraded +mirror on asymmetric ENOSPC. +--- + module/bdev/lvol/vbdev_lvol.c | 9 +++++++++ + module/bdev/raid/bdev_raid.c | 10 ++++++++++ + module/bdev/raid/bdev_raid.h | 6 ++++++ + module/bdev/raid/raid1.c | 23 +++++++++++++++++++---- + 4 files changed, 44 insertions(+), 4 deletions(-) + +diff --git a/module/bdev/lvol/vbdev_lvol.c b/module/bdev/lvol/vbdev_lvol.c +index 4c74027..713f905 100644 +--- a/module/bdev/lvol/vbdev_lvol.c ++++ b/module/bdev/lvol/vbdev_lvol.c +@@ -6,6 +6,7 @@ + + #include "spdk/rpc.h" + #include "spdk/bdev_module.h" ++#include "spdk/nvme_spec.h" /* Evariops C-1 */ + #include "spdk/log.h" + #include "spdk/string.h" + #include "spdk/uuid.h" +@@ -868,6 +869,14 @@ lvol_op_comp(void *cb_arg, int bserrno) + if (bserrno != 0) { + if (bserrno == -ENOMEM) { + status = SPDK_BDEV_IO_STATUS_NOMEM; ++ } else if (bserrno == -ENOSPC) { ++ /* Evariops C-1: surface thin-pool exhaustion as NVMe ++ * CAPACITY_EXCEEDED so it survives the NVMe-oF hop and upper ++ * layers (raid1) can distinguish it from a media fault (the ++ * generic FAILED mapping erased that information). */ ++ spdk_bdev_io_complete_nvme_status(bdev_io, 0, SPDK_NVME_SCT_GENERIC, ++ SPDK_NVME_SC_CAPACITY_EXCEEDED); ++ return; + } else { + status = SPDK_BDEV_IO_STATUS_FAILED; + } +diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c +index 53b88bc..ad5143d 100644 +--- a/module/bdev/raid/bdev_raid.c ++++ b/module/bdev/raid/bdev_raid.c +@@ -5,6 +5,7 @@ + */ + + #include "bdev_raid.h" ++#include "spdk/nvme_spec.h" /* Evariops C-1 */ + #include "spdk/env.h" + #include "spdk/thread.h" + #include "spdk/log.h" +@@ -681,6 +682,14 @@ raid_bdev_io_complete(struct raid_bdev_io *raid_io, enum spdk_bdev_io_status sta + status = SPDK_BDEV_IO_STATUS_FAILED; + } + } ++ /* Evariops C-1: a thin-exhausted member surfaces to the consumer as ++ * CAPACITY_EXCEEDED (the FS sees ENOSPC), overriding the any-leg-SUCCESS ++ * outcome — one replica silently missed this write. */ ++ if (spdk_unlikely(raid_io->enospc)) { ++ spdk_bdev_io_complete_nvme_status(bdev_io, 0, SPDK_NVME_SCT_GENERIC, ++ SPDK_NVME_SC_CAPACITY_EXCEEDED); ++ return; ++ } + spdk_bdev_io_complete(bdev_io, status); + } + } +@@ -951,6 +960,7 @@ raid_bdev_io_init(struct raid_bdev_io *raid_io, struct raid_bdev_io_channel *rai + raid_io->base_bdev_io_remaining = 0; + raid_io->base_bdev_io_submitted = 0; + raid_io->completion_cb = NULL; ++ raid_io->enospc = false; /* Evariops C-1 */ + raid_io->split.offset = RAID_OFFSET_BLOCKS_INVALID; + + raid_bdev_io_set_default_status(raid_io, SPDK_BDEV_IO_STATUS_SUCCESS); +diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h +index e7d7ac5..7ee701a 100644 +--- a/module/bdev/raid/bdev_raid.h ++++ b/module/bdev/raid/bdev_raid.h +@@ -150,6 +150,12 @@ struct raid_bdev_io { + /* This will be the raid_io completion status unless any base io's status is different. */ + enum spdk_bdev_io_status base_bdev_io_status_default; + ++ /* Evariops C-1: a member completed a write with NVMe CAPACITY_EXCEEDED ++ * (thin-pool exhaustion). Sticky: the raid_io completes CAPACITY_EXCEEDED ++ * to the consumer even if another leg succeeded (the any-leg-SUCCESS rule ++ * of complete_part must not ACK a write one replica silently missed). */ ++ bool enospc; ++ + /* Private data for the raid module */ + void *module_private; + +diff --git a/module/bdev/raid/raid1.c b/module/bdev/raid/raid1.c +index 4b24829..af633de 100644 +--- a/module/bdev/raid/raid1.c ++++ b/module/bdev/raid/raid1.c +@@ -7,6 +7,7 @@ + + #include "spdk/likely.h" + #include "spdk/log.h" ++#include "spdk/nvme_spec.h" /* Evariops C-1: CAPACITY_EXCEEDED discrimination */ + + struct raid1_info { + /* The parent raid bdev */ +@@ -54,11 +55,25 @@ raid1_write_bdev_io_completion(struct spdk_bdev_io *bdev_io, bool success, void + struct raid_bdev_io *raid_io = cb_arg; + + if (!success) { +- struct raid_base_bdev_info *base_info; ++ uint32_t cdw0; ++ int sct, sc; ++ ++ spdk_bdev_io_get_nvme_status(bdev_io, &cdw0, &sct, &sc); ++ if (sct == SPDK_NVME_SCT_GENERIC && sc == SPDK_NVME_SC_CAPACITY_EXCEEDED) { ++ /* Evariops C-1: thin-pool exhaustion is NOT a device fault. The old ++ * behavior failed the member (silent redundancy loss) while ++ * complete_part still ACKed the write off the surviving leg ++ * (silent divergence). Instead: keep the member, mark the raid_io ++ * so it completes CAPACITY_EXCEEDED to the consumer (the FS sees ++ * ENOSPC; a later successful rewrite reconverges both legs). */ ++ raid_io->enospc = true; ++ } else { ++ struct raid_base_bdev_info *base_info; + +- base_info = raid_bdev_channel_get_base_info(raid_io->raid_ch, bdev_io->bdev); +- if (base_info) { +- raid_bdev_fail_base_bdev(base_info); ++ base_info = raid_bdev_channel_get_base_info(raid_io->raid_ch, bdev_io->bdev); ++ if (base_info) { ++ raid_bdev_fail_base_bdev(base_info); ++ } + } + } + +-- +2.50.1 (Apple Git-155) + diff --git a/patches/0010-rpc-socket-chmod.patch b/patches/0010-rpc-socket-chmod.patch new file mode 100644 index 0000000..00aec6d --- /dev/null +++ b/patches/0010-rpc-socket-chmod.patch @@ -0,0 +1,72 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 10:56:39 +0200 +Subject: [PATCH 10/12] rpc: chmod 0600 the unix socket after bind (Evariops + 0010, S-2) + +--- + lib/rpc/rpc.c | 23 +++++++++++++++++++++++ + 1 file changed, 23 insertions(+) + +diff --git a/lib/rpc/rpc.c b/lib/rpc/rpc.c +index ead7c66..7af1f9c 100644 +--- a/lib/rpc/rpc.c ++++ b/lib/rpc/rpc.c +@@ -5,6 +5,7 @@ + */ + + #include ++#include /* Evariops R14: umask() / mode_t for atomic owner-only socket */ + + #include "spdk/stdinc.h" + +@@ -147,6 +148,7 @@ struct spdk_rpc_server * + spdk_rpc_server_listen(const char *listen_addr) + { + struct spdk_rpc_server *server; ++ mode_t saved_umask; + int rc; + + server = calloc(1, sizeof(struct spdk_rpc_server)); +@@ -194,10 +196,18 @@ spdk_rpc_server_listen(const char *listen_addr) + */ + unlink(server->listen_addr_unix.sun_path); + ++ /* Evariops R14 (TOCTOU): bind() (inside spdk_jsonrpc_server_listen) creates the ++ * socket file with the process umask — often 0755/0775 in a shared /var/tmp — and ++ * the socket is CONNECTABLE the instant it is bound+listening. A chmod done AFTER ++ * leaves a window in which a local unprivileged process can connect() and issue ++ * destructive RPCs. Force umask 077 across the bind so the socket is created ++ * owner-only (0600 = 0666 & ~077) ATOMICALLY; restore the umask right after. */ ++ saved_umask = umask(077); + server->jsonrpc_server = spdk_jsonrpc_server_listen(AF_UNIX, 0, + (struct sockaddr *) & server->listen_addr_unix, + sizeof(server->listen_addr_unix), + jsonrpc_handler); ++ umask(saved_umask); + if (server->jsonrpc_server == NULL) { + SPDK_ERRLOG("spdk_jsonrpc_server_listen() failed\n"); + close(server->lock_fd); +@@ -205,6 +215,19 @@ spdk_rpc_server_listen(const char *listen_addr) + goto ret; + } + ++ /* Evariops S-2 / R14 (belt-and-braces): the umask above already created the ++ * socket 0600; re-assert it in case the platform's bind() ignores umask for ++ * AF_UNIX. This chmod is no longer the primary control (it closed a TOCTOU). */ ++ if (chmod(server->listen_addr_unix.sun_path, 0600) != 0) { ++ SPDK_ERRLOG("Cannot chmod 0600 RPC socket %s: %s\n", ++ server->listen_addr_unix.sun_path, spdk_strerror(errno)); ++ spdk_jsonrpc_server_shutdown(server->jsonrpc_server); ++ close(server->lock_fd); ++ unlink(server->lock_path); ++ unlink(server->listen_addr_unix.sun_path); ++ goto ret; ++ } ++ + return server; + + ret: +-- +2.50.1 (Apple Git-155) + diff --git a/patches/0011-jsonrpc-peer-audit.patch b/patches/0011-jsonrpc-peer-audit.patch new file mode 100644 index 0000000..f4c7092 --- /dev/null +++ b/patches/0011-jsonrpc-peer-audit.patch @@ -0,0 +1,169 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sat, 4 Jul 2026 19:16:48 +0200 +Subject: [PATCH 11/12] jsonrpc: SO_PEERCRED peer-credential audit hook + (Evariops 0011, SEC1) + +Capture the RPC client's Unix-socket credentials (SO_PEERCRED) at accept and +expose spdk_jsonrpc_request_get_peer_ucred() + spdk_jsonrpc_request_audit(), so +destructive mutation handlers (bdev_tier_delete/retire, relocate/remap, pause, +rebuild_ranges) log pid/uid/gid + operation. Applied with the series; earlier +patches (0003/0005/0008) call the helper defined here. +--- + include/spdk/jsonrpc.h | 27 ++++++++++++++++++++++ + lib/jsonrpc/jsonrpc_internal.h | 7 ++++++ + lib/jsonrpc/jsonrpc_server.c | 39 ++++++++++++++++++++++++++++++++ + lib/jsonrpc/jsonrpc_server_tcp.c | 17 ++++++++++++++ + lib/jsonrpc/spdk_jsonrpc.map | 2 ++ + 5 files changed, 92 insertions(+) + +diff --git a/include/spdk/jsonrpc.h b/include/spdk/jsonrpc.h +index 760039d..7a157ad 100644 +--- a/include/spdk/jsonrpc.h ++++ b/include/spdk/jsonrpc.h +@@ -124,6 +124,33 @@ void spdk_jsonrpc_server_shutdown(struct spdk_jsonrpc_server *server); + */ + struct spdk_jsonrpc_server_conn *spdk_jsonrpc_get_conn(struct spdk_jsonrpc_request *request); + ++/** ++ * SPEC-73 SEC1: peer credentials of the client that issued \c request, captured ++ * with SO_PEERCRED at connection accept. Only meaningful over a Unix-domain ++ * socket on Linux; a TCP transport or a non-Linux build has none. ++ * ++ * \param request JSON-RPC request ++ * \param pid filled with the peer PID (may be NULL) ++ * \param uid filled with the peer UID (may be NULL) ++ * \param gid filled with the peer GID (may be NULL) ++ * \return 0 on success; -ENOTSUP if peer credentials are unavailable ++ */ ++int spdk_jsonrpc_request_get_peer_ucred(struct spdk_jsonrpc_request *request, ++ pid_t *pid, uid_t *uid, gid_t *gid); ++ ++/** ++ * SPEC-73 SEC1: emit an audit NOTICELOG line for a destructive RPC, prefixed ++ * with the caller's peer credentials (pid/uid/gid, or "unknown"). Intended to be ++ * called at the top of every mutation handler (delete, retire, relocate/remap, ++ * pause, rebuild). \c detail is an optional pre-formatted parameter string. ++ * ++ * \param request JSON-RPC request being served ++ * \param method RPC method name (for the log line) ++ * \param detail optional extra context (may be NULL) ++ */ ++void spdk_jsonrpc_request_audit(struct spdk_jsonrpc_request *request, ++ const char *method, const char *detail); ++ + /** + * Add callback called when connection is closed. Pair of \c cb and \c ctx must be unique or error is returned. + * Registered callback is called only once and there is no need to call \c spdk_jsonrpc_conn_del_close_cb +diff --git a/lib/jsonrpc/jsonrpc_internal.h b/lib/jsonrpc/jsonrpc_internal.h +index d13ec7a..81a17a4 100644 +--- a/lib/jsonrpc/jsonrpc_internal.h ++++ b/lib/jsonrpc/jsonrpc_internal.h +@@ -49,6 +49,13 @@ struct spdk_jsonrpc_server_conn { + struct spdk_jsonrpc_server *server; + int sockfd; + bool closed; ++ /* SPEC-73 SEC1: peer credentials captured at accept (SO_PEERCRED on a Unix ++ * socket, Linux only). peer_cred_valid is false for a TCP transport or a ++ * platform without SO_PEERCRED. */ ++ bool peer_cred_valid; ++ pid_t peer_pid; ++ uid_t peer_uid; ++ gid_t peer_gid; + size_t recv_len; + uint8_t recv_buf[SPDK_JSONRPC_RECV_BUF_SIZE]; + uint32_t outstanding_requests; +diff --git a/lib/jsonrpc/jsonrpc_server.c b/lib/jsonrpc/jsonrpc_server.c +index c122613..158ed7f 100644 +--- a/lib/jsonrpc/jsonrpc_server.c ++++ b/lib/jsonrpc/jsonrpc_server.c +@@ -278,6 +278,45 @@ spdk_jsonrpc_get_conn(struct spdk_jsonrpc_request *request) + return request->conn; + } + ++int ++spdk_jsonrpc_request_get_peer_ucred(struct spdk_jsonrpc_request *request, ++ pid_t *pid, uid_t *uid, gid_t *gid) ++{ ++ struct spdk_jsonrpc_server_conn *conn = request ? request->conn : NULL; ++ ++ if (conn == NULL || !conn->peer_cred_valid) { ++ return -ENOTSUP; ++ } ++ if (pid != NULL) { ++ *pid = conn->peer_pid; ++ } ++ if (uid != NULL) { ++ *uid = conn->peer_uid; ++ } ++ if (gid != NULL) { ++ *gid = conn->peer_gid; ++ } ++ return 0; ++} ++ ++void ++spdk_jsonrpc_request_audit(struct spdk_jsonrpc_request *request, const char *method, ++ const char *detail) ++{ ++ pid_t pid = 0; ++ uid_t uid = 0; ++ gid_t gid = 0; ++ ++ if (spdk_jsonrpc_request_get_peer_ucred(request, &pid, &uid, &gid) == 0) { ++ SPDK_NOTICELOG("audit rpc=%s peer=pid:%d,uid:%u,gid:%u%s%s\n", ++ method, (int)pid, (unsigned)uid, (unsigned)gid, ++ detail != NULL ? " " : "", detail != NULL ? detail : ""); ++ } else { ++ SPDK_NOTICELOG("audit rpc=%s peer=unknown%s%s\n", method, ++ detail != NULL ? " " : "", detail != NULL ? detail : ""); ++ } ++} ++ + /* Never return NULL */ + static struct spdk_json_write_ctx * + begin_response(struct spdk_jsonrpc_request *request) +diff --git a/lib/jsonrpc/jsonrpc_server_tcp.c b/lib/jsonrpc/jsonrpc_server_tcp.c +index e30f37f..4312469 100644 +--- a/lib/jsonrpc/jsonrpc_server_tcp.c ++++ b/lib/jsonrpc/jsonrpc_server_tcp.c +@@ -192,6 +192,23 @@ jsonrpc_server_accept(struct spdk_jsonrpc_server *server) + conn->server = server; + conn->sockfd = rc; + conn->closed = false; ++ /* SPEC-73 SEC1: capture the client's credentials for the audit trail. Works ++ * for a Unix-domain socket (the default SPDK RPC transport); a TCP peer or a ++ * non-Linux build simply has none (peer_cred_valid stays false). */ ++ conn->peer_cred_valid = false; ++#ifdef __linux__ ++ { ++ struct ucred cred; ++ socklen_t clen = sizeof(cred); ++ ++ if (getsockopt(conn->sockfd, SOL_SOCKET, SO_PEERCRED, &cred, &clen) == 0) { ++ conn->peer_cred_valid = true; ++ conn->peer_pid = cred.pid; ++ conn->peer_uid = cred.uid; ++ conn->peer_gid = cred.gid; ++ } ++ } ++#endif + conn->recv_len = 0; + conn->outstanding_requests = 0; + STAILQ_INIT(&conn->send_queue); +diff --git a/lib/jsonrpc/spdk_jsonrpc.map b/lib/jsonrpc/spdk_jsonrpc.map +index a62f4a4..5555990 100644 +--- a/lib/jsonrpc/spdk_jsonrpc.map ++++ b/lib/jsonrpc/spdk_jsonrpc.map +@@ -6,6 +6,8 @@ + spdk_jsonrpc_server_poll; + spdk_jsonrpc_server_shutdown; + spdk_jsonrpc_get_conn; ++ spdk_jsonrpc_request_get_peer_ucred; ++ spdk_jsonrpc_request_audit; + spdk_jsonrpc_conn_add_close_cb; + spdk_jsonrpc_conn_del_close_cb; + spdk_jsonrpc_begin_result; +-- +2.50.1 (Apple Git-155) + diff --git a/patches/0012-lvol-shutdown-unload-observability.patch b/patches/0012-lvol-shutdown-unload-observability.patch new file mode 100644 index 0000000..4f3d3c0 --- /dev/null +++ b/patches/0012-lvol-shutdown-unload-observability.patch @@ -0,0 +1,70 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Evariops +Date: Sun, 5 Jul 2026 18:20:00 +0200 +Subject: [PATCH 12/12] lvol: NOTICE-level shutdown unload observability + (Evariops 0012) + +A lvolstore whose shutdown unload fails (or never runs because lvols +were still open at fini_start and the unregister-path unload never +fired) leaves the blobstore superblock dirty: the next load performs a +full metadata recovery - tens of seconds on SATA, which inflates every +rollout's node-ready time and the CBT dirty window with it. Observed on +turing: ~70-95s boots on rolled nodes vs ~6s on clean loads, with the +existing INFOLOG outcome invisible in production logs. + +Log all three outcomes (clean unload via fini_start, clean unload via +unregister path, FAILED unload with errno) and the fini_start skip at +NOTICE so field logs pinpoint which path ran. +--- + module/bdev/lvol/vbdev_lvol.c | 19 +++++++++++++++++-- + 1 file changed, 17 insertions(+), 2 deletions(-) + +diff --git a/module/bdev/lvol/vbdev_lvol.c b/module/bdev/lvol/vbdev_lvol.c +index 713f905..f309354 100644 +--- a/module/bdev/lvol/vbdev_lvol.c ++++ b/module/bdev/lvol/vbdev_lvol.c +@@ -586,8 +586,14 @@ _vbdev_lvol_unregister_unload_lvs(void *cb_arg, int lvserrno) + struct lvol_bdev *lvol_bdev = cb_arg; + struct lvol_store_bdev *lvs_bdev = lvol_bdev->lvs_bdev; + ++ /* Evariops 0012: a failed shutdown unload leaves the blobstore superblock ++ * dirty — the next load replays md for tens of seconds on SATA. Surface the ++ * outcome at NOTICE so field logs pinpoint the errno and the path taken. */ + if (lvserrno != 0) { +- SPDK_INFOLOG(vbdev_lvol, "Lvol store removed with error: %d.\n", lvserrno); ++ SPDK_NOTICELOG("shutdown: lvolstore unload FAILED err=%d " ++ "(superblock left dirty; next load will recover)\n", lvserrno); ++ } else if (g_shutdown_started) { ++ SPDK_NOTICELOG("shutdown: lvolstore unloaded clean (unregister path)\n"); + } + + TAILQ_REMOVE(&g_spdk_lvol_pairs, lvs_bdev, lvol_stores); +@@ -1504,8 +1510,12 @@ vbdev_lvs_fini_start_unload_cb(void *cb_arg, int lvserrno) + struct lvol_store_bdev *lvs_bdev = cb_arg; + struct lvol_store_bdev *next_lvs_bdev = vbdev_lvol_store_next(lvs_bdev); + ++ /* Evariops 0012: see _vbdev_lvol_unregister_unload_lvs — NOTICE both outcomes. */ + if (lvserrno != 0) { +- SPDK_INFOLOG(vbdev_lvol, "Lvol store removed with error: %d.\n", lvserrno); ++ SPDK_NOTICELOG("shutdown: lvolstore unload FAILED err=%d " ++ "(superblock left dirty; next load will recover)\n", lvserrno); ++ } else { ++ SPDK_NOTICELOG("shutdown: lvolstore unloaded clean (fini_start path)\n"); + } + + TAILQ_REMOVE(&g_spdk_lvol_pairs, lvs_bdev, lvol_stores); +@@ -1526,6 +1536,11 @@ vbdev_lvs_fini_start_iter(struct lvol_store_bdev *lvs_bdev) + spdk_lvs_unload(lvs, vbdev_lvs_fini_start_unload_cb, lvs_bdev); + return; + } ++ /* Evariops 0012: an lvstore skipped here relies on the unregister path ++ * to unload later; if THAT never fires, the superblock stays dirty and ++ * the next boot pays a full md recovery. Make the skip visible. */ ++ SPDK_NOTICELOG("shutdown: lvolstore NOT unloaded at fini_start " ++ "(lvols still open) — deferring to unregister path\n"); + lvs_bdev = vbdev_lvol_store_next(lvs_bdev); + } + +-- +2.50.1 (Apple Git-155) + diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 0000000..a87e779 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,97 @@ +# Evariops SPDK patches + +Out-of-tree patches applied on top of upstream SPDK during the container build +(`images/spdk/Dockerfile`). They add the primitives the SPEC-73 tiering +data-plane and the SPEC-66 group-snapshot barrier need, and harden a few +upstream paths. The two out-of-tree bdev **modules** (`module/bdev/cbt`, +`module/bdev/tier`) are copied in whole, not patched. + +## Application order (U-5/U-6) + +Patches are applied in **lexicographic order of filename** (`0001` … `0012`) — +the Dockerfile globs `patches/*.patch` and `git apply`s each. The numeric prefix +IS the contract; do not rely on any other ordering. Order matters: + +| # | Patch | Touches | Depends on | +|--:|:------|:--------|:-----------| +| 0001 | raid skip_rebuild | bdev_raid | — | +| 0002 | lvol get_allocated_ranges | bdev_lvol | — | +| 0003 | nvmf pause/resume | lib/nvmf | 0011 (audit hook) | +| 0004 | blob relocate primitives + freeze_io | lib/blob | — | +| 0005 | lvol placement/relocate/remap RPCs | bdev_lvol, module/bdev/tier | 0004, 0011, tier module | +| 0006 | raid5f degraded-read | bdev_raid | — | +| 0007 | raid nexus heat | bdev_raid | — | +| 0008 | raid rebuild_ranges + full_stripe_blocks (C3) | bdev_raid, lib/bdev (lock_lba_range) | 0006, 0011 | +| 0009 | ENOSPC → CAPACITY_EXCEEDED | bdev_lvol, bdev_raid | — | +| 0010 | rpc socket chmod 0600 | lib/rpc | — | +| 0011 | jsonrpc SO_PEERCRED audit hook (SEC1) | lib/jsonrpc | — | +| 0012 | lvol shutdown-unload observability | bdev_lvol | — | + +0005 `#include`s `vbdev_tier.h` and adds `-I module/bdev/tier` to the lvol module +CFLAGS via its own Makefile hunk; the Dockerfile injects the module dirs before +applying patches (copy-before-apply ordering matters). + +**0011 is a shared substrate, not a leaf.** It adds `spdk_jsonrpc_request_audit()` ++ `spdk_jsonrpc_request_get_peer_ucred()` to `lib/jsonrpc`, which the destructive +handlers in 0003 (pause), 0005 (relocate/remap), and 0008 (rebuild_ranges) call — +so those numerically-earlier patches reference a symbol added by 0011. This is +sound because the series is applied **as a whole** before anything is compiled +(the Dockerfile `git apply`s all of `patches/*.patch`, then builds once); the +number is a *filename apply order*, not an incremental-compile order. It sits last +to avoid renumbering the existing `Evariops 000X` labels baked into every commit. + +## Makefile / build wiring (NOT patches) + +The module registration is done by four `sed -i` edits in the Dockerfile, not by +patches, because they are one-line list insertions that would conflict on every +upstream bump. They are **fragile** (a `sed` that finds no anchor silently does +nothing, unlike `git apply` which errors) — if a build ever links without +`bdev_tier`/`bdev_cbt`, check those `sed` anchors first: + +- `module/bdev/Makefile`: `DIRS-y += … cbt tier` +- `mk/spdk.modules.mk`: `BLOCKDEV_MODULES_LIST += bdev_cbt` / `bdev_tier` +- `mk/spdk.lib_deps.mk`: `DEPDIRS-bdev_cbt`/`bdev_tier` + `DEPDIRS-bdev_lvol … bdev_tier` + +## Upstream pin (W4) + +The upstream commit is pinned in `images/spdk/Dockerfile` +(`ARG SPDK_COMMIT_SHA`) and verified after clone — a moved tag fails the build. +`scripts/patches.sh` reads the same ARG, so tooling and build never disagree. + +## Tooling — `scripts/patches.sh` + +Never hand-edit hunk offsets (the audit's U-3 finding: a `fix: correct patch +hunk line count` commit is proof of manual editing gone wrong). Instead: + +```sh +# Apply the series into a local SPDK checkout that is at the pinned commit. +scripts/patches.sh apply /path/to/spdk + +# Dry-run the whole series in order; stops at the first that fails. +scripts/patches.sh check /path/to/spdk + +# Zero-network-state check: clone the pinned upstream to a temp dir and check. +scripts/patches.sh verify + +# Regenerate patches/*.patch from a worktree with one commit per patch, +# in order, on top of the pinned base commit. +scripts/patches.sh regen /path/to/spdk-worktree +``` + +### To change a patch + +1. Apply the series into a clean SPDK checkout at the pinned commit, committing + one patch per commit (in order). +2. Edit the source, `git commit --amend` (or a fixup) into the owning commit. +3. `scripts/patches.sh regen ` to rewrite `patches/*.patch`. +4. `scripts/patches.sh verify` to confirm the whole series still applies. + +`git format-patch --zero-commit` is used so the `From ` line is a constant +(all-zero) instead of a per-regeneration hash — this keeps the patch files stable +under review. The `From: 000…` header line is therefore expected, not a bug. + +## Upstreaming (UP4) + +Candidates, easiest first: 0006 (degraded-read, small/general), 0003 +(pause/resume), 0002 (allocated_ranges). 0004 (blob relocate) needs an RFC. Each +merged upstream removes rebase surface here. diff --git a/scripts/patches.sh b/scripts/patches.sh new file mode 100755 index 0000000..1ff6f3a --- /dev/null +++ b/scripts/patches.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Evariops. +# +# patches.sh — tooling for the Evariops SPDK patch series (UP2/U-3). +# +# The audit flagged manual hunk editing (`git apply` of by-hand-edited .patch +# files) as a real risk: `sed`/hand edits silently corrupt offsets and only +# surface at build time. This script makes the round trip mechanical: +# +# patches.sh check verify the whole series applies in order +# (non-destructive: checks against a scratch +# index, never touches the caller's tree). +# patches.sh apply git-apply the series into an SPDK checkout. +# patches.sh regen regenerate patches/*.patch from a +# worktree whose commits (one per patch, +# in order) sit on top of the pinned base. +# patches.sh verify [] clone the pinned upstream into a temp dir +# and run `check` against it (no local SPDK +# needed; requires network + git). +# +# The pinned upstream is read from images/spdk/Dockerfile (ARG SPDK_COMMIT_SHA), +# so the script and the container build can never disagree (W4). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PATCH_DIR="$REPO_ROOT/patches" +DOCKERFILE="$REPO_ROOT/images/spdk/Dockerfile" + +pinned_sha() { sed -n 's/^ARG SPDK_COMMIT_SHA=\([0-9a-f]\{7,\}\).*/\1/p' "$DOCKERFILE"; } +pinned_tag() { sed -n 's/^ARG SPDK_VERSION=\(v[^ ]*\).*/\1/p' "$DOCKERFILE"; } + +# NUL-delimited list so a patch path with spaces/globs is never word-split. +patches() { find "$PATCH_DIR" -maxdepth 1 -name '[0-9]*.patch' -print0 | sort -z; } +patches_count() { patches | tr -cd '\0' | wc -c | tr -d ' '; } + +# Non-destructive: `git apply --check` verifies the WHOLE series applies in order +# without touching the caller's tree. --check alone stops at the first patch, so +# each subsequent patch is checked against a scratch index (git apply --cached +# into a temporary index copied from HEAD), leaving the working tree untouched. +cmd_check() { + local src="${1:?usage: patches.sh check }" + local tmpidx err p rc=0 + tmpidx="$(mktemp)" + err="$(mktemp)" + # Seed the scratch index from HEAD so --cached checks stack in order. + GIT_INDEX_FILE="$tmpidx" git -C "$src" read-tree HEAD + while IFS= read -r -d '' p; do + if GIT_INDEX_FILE="$tmpidx" git -C "$src" apply --cached --check "$p" 2>"$err"; then + GIT_INDEX_FILE="$tmpidx" git -C "$src" apply --cached "$p" + echo "OK $(basename "$p")" + else + echo "FAIL $(basename "$p")" + sed 's/^/ /' "$err" + rc=1 + break + fi + done < <(patches) + rm -f "$tmpidx" "$err" + [ "$rc" -eq 0 ] && echo "All $(patches_count) patches apply cleanly." + return "$rc" +} + +cmd_apply() { + local src="${1:?usage: patches.sh apply }" + local p + while IFS= read -r -d '' p; do + echo "Applying $(basename "$p")" + git -C "$src" apply "$p" + done < <(patches) +} + +cmd_regen() { + local wt="${1:?usage: patches.sh regen }" + local base + base="$(pinned_sha)" + local n + n="$(patches_count)" + rm -f "$PATCH_DIR"/[0-9]*.patch + git -C "$wt" format-patch --zero-commit "$base..HEAD" -o "$PATCH_DIR" >/dev/null + echo "Regenerated $(patches_count) patches from $wt ($base..HEAD); was $n." +} + +cmd_verify() { + local src="${1:-}" + if [[ -z "$src" ]]; then + src="$(mktemp -d)/spdk" + echo "Cloning $(pinned_tag) into $src ..." + git clone --branch "$(pinned_tag)" --depth 1 "https://github.com/spdk/spdk.git" "$src" >/dev/null 2>&1 + local actual + actual="$(git -C "$src" rev-parse --short=7 HEAD)" + if [[ "$actual" != "$(pinned_sha)" ]]; then + echo "FATAL: $(pinned_tag) resolves to $actual, expected $(pinned_sha) (tag moved?)" >&2 + return 1 + fi + fi + cmd_check "$src" +} + +case "${1:-}" in + check) shift; cmd_check "$@";; + apply) shift; cmd_apply "$@";; + regen) shift; cmd_regen "$@";; + verify) shift; cmd_verify "$@";; + *) sed -n '2,30p' "${BASH_SOURCE[0]}"; exit 1;; +esac