From d37f19acdcbd8ddd408b000aa5a773b53e0bd534 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:32:08 +0800 Subject: [PATCH 01/10] docs(plans): ext4 metadata volume design for snapshot-prepare fsync cost (ABX-496) --- internal-docs/plans/ext4-metadata-volume.md | 213 ++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 internal-docs/plans/ext4-metadata-volume.md diff --git a/internal-docs/plans/ext4-metadata-volume.md b/internal-docs/plans/ext4-metadata-volume.md new file mode 100644 index 000000000..9e51c8563 --- /dev/null +++ b/internal-docs/plans/ext4-metadata-volume.md @@ -0,0 +1,213 @@ +# ext4 Metadata Volume — snapshot-prepare fsync fix (ABX-496) + +Status: **Planned** (design locked, awaiting implementation) +Owner: ABX-496 (docker build performance vs Colima) +Companion fix already shipped separately: `rcu_expedited` for runc-create (arcbox#496). + +## 1. Problem and evidence + +Container start on ArcBox spends ~182 ms in containerd snapshot-prepare vs ~20 ms +on Colima. Profiling (static strace with `-y` fd resolution, dockerd debug +lifecycle timeline, `dd conv=fsync` microbench) pinned the cost: + +- A single `fsync` on the guest data volume costs **9.5 ms on ArcBox (btrfs)** + vs **1.0 ms on Colima (ext4)** on the same virtio-blk stack. The cost is the + btrfs commit path (`write_all_supers`: superblock write + FLUSH barriers), + not COW extent allocation — `chattr +C` (NOCOW) was measured at 9.5 ms vs + 10.5 ms, i.e. no help. +- ~90 % of the fsyncs during a container start hit small boltdb metadata + files: containerd `io.containerd.metadata.v1.bolt/meta.db` (dominant), + overlayfs snapshotter `metadata.db`, dockerd `network/files/local-kv.db` + and `image/*.db`. Content-store blobs and snapshot dirs are a small + minority of fsyncs. + +The fsync-hot files are small, incompressible, and rewritten in place; the +bulk data (extracted layers, blobs, volumes) is large and benefits from btrfs +zstd compression. They have opposite filesystem needs — so we split them. + +## 2. Locked design + +Add a second, small disk image formatted **ext4 (journaled, fast_commit)** +that carries only the fsync-hot metadata directories, bind-mounted over their +current btrfs locations. Everything else stays on btrfs unchanged. + +| Decision | Choice | Rationale / rejected alternatives | +|---|---|---| +| Storage for ext4 | Second virtio-blk disk image | Loop file on btrfs rejected: loop FLUSH re-enters the btrfs commit path, keeping the 9.5 ms cost. GPT-partitioning docker.img rejected: destructive migration of existing whole-disk btrfs images. | +| Image name | `-meta.img` derived from the data image (`docker.img` → `docker-meta.img`, `docker-rosetta.img` → `docker-rosetta-meta.img`) | Follows the existing per-VM data-image parameterization (`vm_lifecycle/mod.rs::for_machine`). | +| Virtual size | 2 GiB, sparse (`ensure_sparse_block_image`), no resize path | Holds only bolt DBs + small configs; 2 GiB is ~100× headroom. No resize2fs in guest; fixed size avoids that dependency. | +| Guest device | `/dev/vdc` (third `BlockDeviceConfig` in the Vec; both backends attach in Vec order) + cmdline override key `arcbox.docker_metadata_device=` mirroring the existing data-device pattern | HV additionally exposes serial `arcbox-blk-docker-meta.img` via GET_ID, but ordering is already the load-bearing contract for vdb. No HVC fast-path analogue — metadata I/O is tiny. | +| Formatter | `mkfs.ext4` (static e2fsprogs) added to the guest rootfs via boot-assets, invoked guest-side like `mkfs.btrfs` | `arcbox-ext4` crate rejected: its formatter sets no `HAS_JOURNAL` (registry source, `formatter.rs:1107-1111`), and journal-less ext4 has no crash replay — VM force-stop is a routine event and these DBs are the system of record. Host-side formatting impossible (no mke2fs on macOS). | +| mkfs options | `mkfs.ext4 -F -O fast_commit -E lazy_itable_init=0,lazy_journal_init=0 -L arcbox-meta` | fast_commit reduces exactly our fsync pattern (small metadata commits); kernel is 6.x with `CONFIG_EXT4_FS=y` on both arches. Lazy init off: one-time cost at first format, no background trickle. | +| Mount options | `noatime` (defaults otherwise: `data=ordered`, barriers on) | `discard` skipped — bolt files are stable in size, few deletes. | +| Recovery | Ship static `e2fsck` alongside; on mount failure run `e2fsck -y` once, retry mount once, then fail hard | Journal replay is in-kernel; e2fsck covers the residual corruption class. | + +### Directories moved to ext4 (the profiled hot set, nothing more) + +| ext4 volume path | Bind target | Kind | +|---|---|---| +| `containerd-bolt/` | `/var/lib/containerd/io.containerd.metadata.v1.bolt` | dir bind | +| `snapshotter-metadata.db` | `/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/metadata.db` | **file** bind (the dir also holds `snapshots/`, which stays on btrfs) | +| `docker-network/` | `/var/lib/docker/network` | dir bind | +| `docker-image/` | `/var/lib/docker/image` | dir bind | +| `docker-buildkit/` | `/var/lib/docker/buildkit` | dir bind | + +The file bind is safe for boltdb: bolt opens, mmaps, writes in place and +fsyncs — it never renames or recreates its DB file. Both sides of the bind +are pre-created (empty files) before containerd starts; bolt initializes a +0-byte file as a fresh DB. + +EXDEV audit: none of the five targets receives cross-boundary renames. Bolt +never renames; libnetwork and docker's image store use same-directory +tmp+rename atomic writes; buildkit DBs are bolt. Content-store ingest→blobs +and snapshotter tmp→snapshots renames stay entirely within btrfs subtrees. +Any regression here fails loudly (EXDEV errors the operation) and is covered +by the validation suite. + +### Explicitly excluded (with reasons) + +- `/var/lib/docker/containers` — config fsyncs are a minor cost; the dir also + holds json-file logs, which are unbounded and would risk filling the 2 GiB + volume. +- `/var/lib/docker/volumes/metadata.db`, `builder/`, `trust/` — cold paths, + not in the profile. +- Distro machines (`data.img`, machine-init.sh) — out of scope; System VM + and Rosetta VM only (both go through `create_default_machine`). + +### NOCOW removal rides along + +`btrfs.rs::disable_cow_on_metadata_dirs` is removed in the same change. Its +premise (NOCOW reduces bolt fsync amplification) is disproven by measurement +(9.5 vs 10.5 ms), the dirs it targeted either move to ext4 or are cold, and +its inherited NOCOW flag on `io.containerd.snapshotter.v1.overlayfs` silently +disables zstd compression for every extracted layer file — the one thing +btrfs is being kept for. Existing installs keep the on-disk flag on old dirs +(only re-flagging stops); new snapshot dirs regain compression. + +## 3. Guest changes (`guest/arcbox-agent`) + +New module `agent/linux/metadata_volume.rs`, called from +`try_start_bundled_runtime` (`runtime.rs`) immediately after +`ensure_data_mount()` — i.e. after btrfs targets exist, before the NFS export +and before containerd/dockerd start (bolt files guaranteed closed): + +1. Resolve device: cmdline override → `/dev/vdc`; wait up to 5 s for the node + (same loop as `btrfs.rs`). +2. Probe ext4 superblock magic (0xEF53 at offset 0x438). If absent → + `mkfs.ext4` with the locked options. +3. Mount at `/run/arcbox/metadata` (`noatime`). On failure: `e2fsck -y`, + retry once, else hard-fail. +4. Per-mapping migrate-then-bind, idempotent and crash-safe per entry: + 1. If the ext4-side final entry is absent: if the btrfs-side source + exists at its canonical path, copy it (recursive, pure Rust) to + `.partial` on the volume, fsync, then rename to final — + an interrupted copy leaves only a `.partial` that is discarded and + redone next boot, never a half-populated final that would get bound. + If no source exists, create the entry empty (fresh install). + 2. If the ext4-side final exists AND the btrfs-side source is still at + its canonical path, rename the source to `.pre-ext4` (covers a + crash between step 1 and this rename). The rename — rather than + leaving the source in place — is load-bearing: it is what prevents a + later blank metadata volume (user deleted the image; host recreated + it) from silently re-migrating stale pre-upgrade metadata against a + data store that has moved on. The renamed copy doubles as a manual + recovery artifact. + 3. Ensure an empty mountpoint stub exists at the canonical path + (dir, or 0-byte file for the metadata.db mapping), then bind. + +Failure policy (version-skew safe): + +| Condition | Behavior | +|---|---| +| Device node absent (older daemon without the third disk) | warn + continue on btrfs — perf-only degradation | +| mkfs binary absent AND device blank (older rootfs) | warn + continue — nothing was ever migrated, no split-brain possible | +| Device present with ext4 but mount/fsck fails | **hard fail** runtime start — booting dockerd against the stale shadowed btrfs DBs would fork state | + +Constants: `DOCKER_METADATA_BLOCK_DEVICE = "/dev/vdc"` +(`arcbox-constants/src/devices.rs`) and the cmdline key +(`arcbox-constants/src/cmdline.rs`), mirroring the data-device pair. + +The Kubernetes path (`kubernetes.rs`) is untouched — k3s state lives under +`/var/lib/rancher` and none of the five targets. + +### User migration matrix + +| Scenario | Outcome | +|---|---| +| Fresh install | Format + empty entries; no migration. | +| Upgrade with populated docker.img | Atomic per-mapping migration (above); docker state fully preserved; sources renamed to `*.pre-ext4` on btrfs. | +| metadata.img deleted or corrupted by the user after migration | Host recreates a blank image; guest formats; sources are renamed away, so the result is a clean **empty** docker state (images re-pull; orphaned blobs/snapshots leak on btrfs but state is consistent) — never a resurrection of stale pre-upgrade metadata. | +| Downgrade to a pre-feature release | **Unsupported (one-way door).** Old binaries mount btrfs only and see the empty mountpoint stubs → empty docker state, consistent. Manual recovery: rename the `*.pre-ext4` dirs back, restoring the state as of the moment of upgrade. Document this in the release notes. | +| Backups / moving data dirs by hand | `docker.img` and `docker-meta.img` are a paired set; copying one without the other loses metadata or orphans data. Documented in `docs/data-directories.md`; `resource_cleanup` already treats them as a unit. | + +## 4. Host changes (`app/`) + +- `vm_lifecycle/mod.rs`: `DOCKER_METADATA_IMAGE_SIZE_BYTES` (2 GiB); + metadata filename derived next to `data_image_filename`. +- `vm_lifecycle/boot.rs::create_default_machine`: + `ensure_sparse_block_image` + third `block_devices.push` (rw); update the + vda/vdb doc comment to cover vdc. +- `arcbox-daemon/src/startup/resource_cleanup.rs::DISK_IMAGE_NAMES`: add + `docker-meta.img`, `docker-rosetta-meta.img` (the pair is a unit — deleting + one without the other orphans state). +- `arcbox-core/src/config.rs`: `docker_meta_img_path()` accessor; + `arcbox-cli/commands/disk.rs`: include it in usage reporting. +- `docs/data-directories.md`: add the `data/docker-meta.img` row. + +No dockerd/containerd config changes — the split is invisible to both. + +## 5. boot-assets changes (sibling repo) + +- `src/build/scripts/build-rootfs-binaries.sh`: static e2fsprogs build from + the upstream source tarball with `LDFLAGS=-static`, mirroring the + btrfs-progs recipe (e2fsprogs vendors its own libuuid/libblkid); stage + `mke2fs` (installed as `/sbin/mkfs.ext4`) and `/sbin/e2fsck`. +- `src/build/rootfs.rs`: extend `CORE_STATIC_BINARIES` + `copy_executable` + lines; the static-check loop picks them up. +- Tag `v0.6.11` → release publishes → arcbox bumps `assets.lock` + (`[boot] version` + `manifest_sha256`) in the same PR that ships the + daemon/agent changes (the lock is compile-time embedded; daemon rebuild + required). + +## 6. NFS / `~/ArcBox` interplay + +None required. The docker export is a **non-recursive** read-only bind +(`nfs.rs::bind_readonly`, plain `MS_BIND`), so the new submounts do not +surface over NFS; `~/ArcBox/network|image|buildkit` show the (empty, +shadowed) underlying @docker dirs. These are internal dirs with no browsing +use case. No new exports, no fsid allocation. + +## 7. Validation and acceptance + +1. Unit: migrate-then-bind decision logic and superblock probe + (pure functions, `cargo test -p arcbox-agent` host-side where possible; + musl cross-compile for the rest). Must cover every crash window of the + migration algorithm: interruption during copy (`.partial` present), + between ext4 rename and btrfs source rename, and re-entry after each — + each replay must converge on the same final state. +2. e2e: `docker_build` suite D1–D10 green (VZ), `boot_assets` green on VZ + **and** HV (`ARCBOX_VM_BACKEND=hv`) — covers full Docker lifecycle over + the new mounts; any EXDEV or bind mistake fails these loudly. +3. Migration: boot a data dir populated by current master (images + + containers present) with the new build; `docker images` / `docker ps -a` + intact; bolt DBs physically on the ext4 volume (`findmnt`, file sizes). +4. Version-skew: new agent + two-disk-less daemon boots with the warn path; + fresh install formats and mounts cleanly. +5. Perf acceptance (the point of it all), measured via dockerd debug + lifecycle timeline as in the ABX-496 campaign: + - guest `dd conv=fsync` on the metadata mount ≈ 1 ms (vs 9.5 ms); + - snapshot-prepare 182 ms → **< 60 ms**; + - combined with the shipped rcu_expedited fix, `docker run` container + start approaches Colima parity; re-run the bench table in + `internal-docs/plans/docker-build-e2e-matrix.md` and record the delta + on ABX-496. + +## 8. Rollout sequence + +1. **PR 1 (boot-assets)**: e2fsprogs in rootfs; tag `v0.6.11`. Inert for + existing arcbox releases. +2. **PR 2 (arcbox)**: constants + host attach/cleanup/CLI + guest + `metadata_volume.rs` + NOCOW removal + `assets.lock` bump + doc updates, + as separate atomic commits. +3. Bench + record results on ABX-496; close the snapshot-prepare half of the + issue. From 152adc41fd4c79809e12ddd7660415a2537737b4 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:49:31 +0800 Subject: [PATCH 02/10] feat(core): attach ext4 metadata volume as third block device (ABX-496) The default (and Rosetta) machine gains a 2 GiB sparse docker-meta.img attached as vdc. Existing machines are recreated via a new block_devices drift arm so upgrades pick up the disk; image files are untouched by recreation. arcbox disk usage reports the paired image. --- app/arcbox-cli/src/commands/disk.rs | 16 ++++++++ app/arcbox-core/src/config.rs | 9 +++++ app/arcbox-core/src/vm_lifecycle/boot.rs | 21 +++++++++-- app/arcbox-core/src/vm_lifecycle/mod.rs | 6 +++ app/arcbox-core/src/vm_lifecycle/tests.rs | 37 +++++++++++++++++-- app/arcbox-core/src/vm_lifecycle/types.rs | 19 ++++++++++ .../src/startup/resource_cleanup.rs | 7 +++- common/arcbox-constants/src/cmdline.rs | 3 ++ common/arcbox-constants/src/devices.rs | 6 +++ docs/data-directories.md | 1 + 10 files changed, 118 insertions(+), 7 deletions(-) diff --git a/app/arcbox-cli/src/commands/disk.rs b/app/arcbox-cli/src/commands/disk.rs index 5d7b1579d..ee1e2ec44 100644 --- a/app/arcbox-cli/src/commands/disk.rs +++ b/app/arcbox-cli/src/commands/disk.rs @@ -103,6 +103,22 @@ async fn execute_usage() -> Result<()> { ); println!(" Reclaimable: {:.1} GiB", usage.reclaimable_gib()); + // The metadata image is small but part of the data set; report it so + // "what is on disk" is complete. + let meta_path = config.docker_meta_img_path(); + if meta_path.exists() { + let meta = read_disk_usage(&meta_path)?; + println!(); + println!("Docker metadata disk:"); + println!(" Path: {}", meta_path.display()); + println!(" Logical: {:.1} GiB", meta.logical_gib()); + println!( + " Physical: {:.1} GiB ({:.1}%)", + meta.physical_gib(), + meta.usage_pct() + ); + } + Ok(()) } diff --git a/app/arcbox-core/src/config.rs b/app/arcbox-core/src/config.rs index 2ad951d2f..850a077c6 100644 --- a/app/arcbox-core/src/config.rs +++ b/app/arcbox-core/src/config.rs @@ -179,6 +179,15 @@ impl Config { pub fn docker_img_path(&self) -> PathBuf { self.data_subdir().join("docker.img") } + + /// Returns the path to the Docker metadata image (`data/docker-meta.img`). + /// + /// Paired with [`Self::docker_img_path`]: the ext4 volume carrying the + /// fsync-hot boltdb metadata (see internal-docs/plans/ext4-metadata-volume.md). + #[must_use] + pub fn docker_meta_img_path(&self) -> PathBuf { + self.data_subdir().join("docker-meta.img") + } } /// Default VM configuration. diff --git a/app/arcbox-core/src/vm_lifecycle/boot.rs b/app/arcbox-core/src/vm_lifecycle/boot.rs index d55d4ee12..f4cbc5291 100644 --- a/app/arcbox-core/src/vm_lifecycle/boot.rs +++ b/app/arcbox-core/src/vm_lifecycle/boot.rs @@ -21,8 +21,8 @@ use arcbox_constants::cmdline::{ use arcbox_error::CommonError; use super::actor::{Completion, InternalEvent, LifecycleShared}; -use super::types::{DesiredBoot, machine_drift_reason}; -use super::{DOCKER_DATA_IMAGE_SIZE_BYTES, RecoveryAction}; +use super::types::{DesiredBoot, machine_drift_reason, metadata_image_filename}; +use super::{DOCKER_DATA_IMAGE_SIZE_BYTES, DOCKER_METADATA_IMAGE_SIZE_BYTES, RecoveryAction}; impl LifecycleShared { /// Boots the VM end-to-end (create if needed, start with retries, wait for @@ -342,7 +342,8 @@ impl LifecycleShared { /// /// Block devices: /// - vda: rootfs.erofs (read-only) - /// - vdb: docker-data.img (read-write) + /// - vdb: docker-data.img (read-write, btrfs bulk data) + /// - vdc: docker-meta.img (read-write, ext4 metadata volume) async fn create_default_machine(&self) -> Result<()> { let boot = self.resolve_desired_boot().await?; let rootfs_path = boot.rootfs_image.to_string_lossy().to_string(); @@ -368,6 +369,20 @@ impl LifecycleShared { read_only: false, }); + // Attach the ext4 metadata volume (vdc): the fsync-hot boltdb + // metadata lives there while bulk data stays on the btrfs data disk. + // The two images are a paired set — see + // internal-docs/plans/ext4-metadata-volume.md. + let metadata_image = self + .data_dir + .join(arcbox_constants::paths::host::DATA) + .join(metadata_image_filename(&self.data_image_filename)); + ensure_sparse_block_image(&metadata_image, DOCKER_METADATA_IMAGE_SIZE_BYTES)?; + block_devices.push(crate::vm::BlockDeviceConfig { + path: metadata_image.to_string_lossy().to_string(), + read_only: false, + }); + let config = MachineConfig { name: self.machine_name.clone(), cpus: self.config.default_vm.cpus, diff --git a/app/arcbox-core/src/vm_lifecycle/mod.rs b/app/arcbox-core/src/vm_lifecycle/mod.rs index 1ff5a1615..3f7133d0c 100644 --- a/app/arcbox-core/src/vm_lifecycle/mod.rs +++ b/app/arcbox-core/src/vm_lifecycle/mod.rs @@ -84,6 +84,12 @@ const DOCKER_DATA_IMAGE_NAME: &str = "docker.img"; /// only consumes actual disk space for written blocks. 8 TiB matches OrbStack /// and prevents users from hitting artificial limits. const DOCKER_DATA_IMAGE_SIZE_BYTES: u64 = 8 * 1024 * 1024 * 1024 * 1024; +/// Persistent guest metadata image size (2 GiB sparse file). +/// +/// The ext4 volume holds only the fsync-hot boltdb metadata directories — +/// bulk data stays on the btrfs data image — so 2 GiB is ~100x headroom. +/// See internal-docs/plans/ext4-metadata-volume.md. +const DOCKER_METADATA_IMAGE_SIZE_BYTES: u64 = 2 * 1024 * 1024 * 1024; pub(crate) use boot::ensure_sparse_block_image; pub use health::HealthMonitor; diff --git a/app/arcbox-core/src/vm_lifecycle/tests.rs b/app/arcbox-core/src/vm_lifecycle/tests.rs index d1a355d2c..9e7f1835b 100644 --- a/app/arcbox-core/src/vm_lifecycle/tests.rs +++ b/app/arcbox-core/src/vm_lifecycle/tests.rs @@ -1,5 +1,5 @@ use super::boot::{agent_timeout_error, ensure_earlycon, ensure_sparse_block_image}; -use super::types::{DesiredBoot, machine_drift_reason}; +use super::types::{DesiredBoot, machine_drift_reason, metadata_image_filename}; use super::*; use crate::machine::MachineState; use arcbox_constants::cmdline::HV_EARLYCON_DIRECTIVE; @@ -152,7 +152,20 @@ fn sample_machine(cpus: u32, memory_mb: u64, kernel: &str, cmdline: &str) -> Mac disk_gb: 50, kernel: Some(kernel.to_string()), cmdline: Some(cmdline.to_string()), - block_devices: Vec::new(), + block_devices: vec![ + crate::vm::BlockDeviceConfig { + path: "/rootfs.erofs".to_string(), + read_only: true, + }, + crate::vm::BlockDeviceConfig { + path: "/data/docker.img".to_string(), + read_only: false, + }, + crate::vm::BlockDeviceConfig { + path: "/data/docker-meta.img".to_string(), + read_only: false, + }, + ], distro: None, distro_version: None, disk_path: None, @@ -197,7 +210,25 @@ fn machine_drift_detects_each_overridable_field() { // The cmdline gap that previously slipped through (e.g. arm64.nosve // added/removed without bumping the boot-asset version). - let mut m = current; + let mut m = current.clone(); m.cmdline = Some("console=hvc0 earlycon arm64.nosve".to_string()); assert_eq!(machine_drift_reason(&m, &want, &boot), Some("cmdline")); + + // A machine persisted before the ext4 metadata volume (two disks) must + // be recreated so the guest receives vdc. + let mut m = current; + m.block_devices.pop(); + assert_eq!( + machine_drift_reason(&m, &want, &boot), + Some("block_devices") + ); +} + +#[test] +fn metadata_image_filename_pairs_with_data_image() { + assert_eq!(metadata_image_filename("docker.img"), "docker-meta.img"); + assert_eq!( + metadata_image_filename("docker-rosetta.img"), + "docker-rosetta-meta.img" + ); } diff --git a/app/arcbox-core/src/vm_lifecycle/types.rs b/app/arcbox-core/src/vm_lifecycle/types.rs index 55b7aba68..05558c5ed 100644 --- a/app/arcbox-core/src/vm_lifecycle/types.rs +++ b/app/arcbox-core/src/vm_lifecycle/types.rs @@ -188,7 +188,26 @@ pub(super) fn machine_drift_reason( Some("kernel") } else if persisted.cmdline.as_deref() != Some(boot.cmdline.as_str()) { Some("cmdline") + } else if persisted.block_devices.len() != DEFAULT_MACHINE_DISK_COUNT { + // A machine persisted before the ext4 metadata volume carries only + // two disks; recreating rewrites the machine record (image files are + // untouched) so the guest receives vdc and can migrate. + Some("block_devices") } else { None } } + +/// Number of block devices `create_default_machine` attaches: EROFS rootfs +/// (vda), btrfs data image (vdb), ext4 metadata image (vdc). +pub(super) const DEFAULT_MACHINE_DISK_COUNT: usize = 3; + +/// Derives the metadata-volume image filename paired with a data image: +/// `docker.img` → `docker-meta.img`, `docker-rosetta.img` → +/// `docker-rosetta-meta.img`. +pub(super) fn metadata_image_filename(data_image_filename: &str) -> String { + let stem = data_image_filename + .strip_suffix(".img") + .unwrap_or(data_image_filename); + format!("{stem}-meta.img") +} diff --git a/app/arcbox-daemon/src/startup/resource_cleanup.rs b/app/arcbox-daemon/src/startup/resource_cleanup.rs index 96e75de78..3dde37f32 100644 --- a/app/arcbox-daemon/src/startup/resource_cleanup.rs +++ b/app/arcbox-daemon/src/startup/resource_cleanup.rs @@ -4,7 +4,12 @@ use std::path::{Path, PathBuf}; use arcbox_core::persistence::MachinePersistence; -const DISK_IMAGE_NAMES: [&str; 2] = ["docker.img", "docker-rosetta.img"]; +const DISK_IMAGE_NAMES: [&str; 4] = [ + "docker.img", + "docker-meta.img", + "docker-rosetta.img", + "docker-rosetta-meta.img", +]; /// Why startup must scan for stale disk-image holders. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/common/arcbox-constants/src/cmdline.rs b/common/arcbox-constants/src/cmdline.rs index 2646337ec..922f527d5 100644 --- a/common/arcbox-constants/src/cmdline.rs +++ b/common/arcbox-constants/src/cmdline.rs @@ -4,6 +4,9 @@ pub const GUEST_DOCKER_VSOCK_PORT_KEY: &str = "arcbox.guest_docker_vsock_port="; /// Kernel cmdline key for guest Docker data block-device path. pub const DOCKER_DATA_DEVICE_KEY: &str = "arcbox.docker_data_device="; +/// Kernel cmdline key for guest Docker metadata block-device path. +pub const DOCKER_METADATA_DEVICE_KEY: &str = "arcbox.docker_metadata_device="; + /// Kernel cmdline key carrying the host path of the interactive debug-console /// Unix socket (custom-HV backend). /// diff --git a/common/arcbox-constants/src/devices.rs b/common/arcbox-constants/src/devices.rs index 06fa9fde1..7c19ee0a6 100644 --- a/common/arcbox-constants/src/devices.rs +++ b/common/arcbox-constants/src/devices.rs @@ -3,3 +3,9 @@ pub const ROOT_BLOCK_DEVICE: &str = "/dev/vda"; /// Default Docker data block device path in guest. pub const DOCKER_DATA_BLOCK_DEVICE: &str = "/dev/vdb"; + +/// Default Docker metadata block device path in guest. +/// +/// The ext4 volume carrying the fsync-hot boltdb metadata directories; paired +/// with the btrfs data device (see internal-docs/plans/ext4-metadata-volume.md). +pub const DOCKER_METADATA_BLOCK_DEVICE: &str = "/dev/vdc"; diff --git a/docs/data-directories.md b/docs/data-directories.md index 18eda7997..dfa4fe141 100644 --- a/docs/data-directories.md +++ b/docs/data-directories.md @@ -57,6 +57,7 @@ Defined in `app/arcbox-core/src/config.rs`. | `data/machines/` | Virtual machine data | daemon | | `data/volumes/` | Named volumes | daemon | | `data/docker.img` | Docker persistent disk image (Btrfs) | daemon | +| `data/docker-meta.img` | Docker metadata disk image (ext4, fsync-hot boltdb state); paired with `docker.img` — back up or move the two together | daemon | ### 1.4 `boot/` — Boot Asset Cache From 6c7931ad408a66f8f375ec51e304edf17102c91a Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:57:22 +0800 Subject: [PATCH 03/10] feat(agent): crash-safe migration state machine for the ext4 metadata volume (ABX-496) Copy via .partial + atomic rename, retire the btrfs-side source to *.pre-ext4, and leave an empty mountpoint stub. Every crash window re-converges; a blank volume never resurrects retired state. Pure std::fs so the tests run on any host. --- guest/arcbox-agent/src/lib.rs | 1 + guest/arcbox-agent/src/metadata_migrate.rs | 383 +++++++++++++++++++++ 2 files changed, 384 insertions(+) create mode 100644 guest/arcbox-agent/src/metadata_migrate.rs diff --git a/guest/arcbox-agent/src/lib.rs b/guest/arcbox-agent/src/lib.rs index 2f450b6f3..f57d6264b 100644 --- a/guest/arcbox-agent/src/lib.rs +++ b/guest/arcbox-agent/src/lib.rs @@ -13,6 +13,7 @@ pub mod dns_server; #[cfg(target_os = "linux")] pub mod error; pub mod memory_pressure; +pub mod metadata_migrate; pub mod rootfs_builder; #[cfg(target_os = "linux")] pub mod sandbox; diff --git a/guest/arcbox-agent/src/metadata_migrate.rs b/guest/arcbox-agent/src/metadata_migrate.rs new file mode 100644 index 000000000..f4c683538 --- /dev/null +++ b/guest/arcbox-agent/src/metadata_migrate.rs @@ -0,0 +1,383 @@ +//! Crash-safe migrate-then-retire state machine for the ext4 metadata volume. +//! +//! The Linux agent (`agent/linux/metadata_volume.rs`) moves the fsync-hot +//! boltdb metadata directories from the btrfs data volume onto a small ext4 +//! volume and bind-mounts them back over their canonical paths. This module +//! owns the on-disk state machine — copy, retire, mountpoint stub — and is +//! kept free of mount syscalls so every crash window stays unit-testable on +//! any host. Design: internal-docs/plans/ext4-metadata-volume.md. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +/// What a migration entry is on disk. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntryKind { + /// A directory bind-mounted as a whole. + Dir, + /// A single file (boltdb) bind-mounted over its canonical path. + File, +} + +/// How [`prepare_entry`] left the volume side. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Prepared { + /// Existing btrfs-side content was copied onto the volume. + Migrated, + /// No source content existed; the volume entry was created empty. + Fresh, + /// The volume entry already existed (nothing copied). + Existing, +} + +/// Suffix of the retired btrfs-side source after a successful migration. +/// +/// Renaming the source — rather than leaving it shadowed under the bind — is +/// what prevents a later blank metadata volume (deleted image, recreated by +/// the host) from silently re-migrating stale pre-upgrade state against a +/// data store that has moved on. The renamed copy doubles as a manual +/// recovery artifact for the (unsupported) downgrade path. +pub const RETIRED_SUFFIX: &str = ".pre-ext4"; + +/// Suffix of an in-progress copy on the volume; never bound, discarded and +/// redone on re-entry. +const PARTIAL_SUFFIX: &str = ".partial"; + +/// Prepares one metadata entry on the volume and retires its btrfs-side +/// source. Idempotent and crash-safe: +/// +/// 1. If `/` is absent: discard any stale `.partial`, +/// then either copy the source into `.partial` and atomically rename it +/// into place, or create the entry empty when no source content exists. +/// 2. If the final entry exists while the source still has content at its +/// canonical path, rename the source to `*.pre-ext4` (also covers a crash +/// between the copy and this retire). +/// 3. Ensure an empty mountpoint stub exists at the canonical path. +/// +/// Every interruption re-converges: a torn copy is redone from scratch, a +/// completed copy is never redone, and the retire is retried until it lands. +pub fn prepare_entry( + volume_root: &Path, + target: &Path, + name: &str, + kind: EntryKind, +) -> io::Result { + let final_path = volume_root.join(name); + let partial = volume_root.join(format!("{name}{PARTIAL_SUFFIX}")); + + let prepared = if final_path.symlink_metadata().is_ok() { + Prepared::Existing + } else { + remove_existing(&partial)?; + if has_content(target, kind) { + copy_entry(target, &partial, kind)?; + fs::rename(&partial, &final_path)?; + // Persist the rename itself; best-effort (a lost rename is + // redone on the next boot). + if let Ok(dir) = fs::File::open(volume_root) { + let _ = dir.sync_all(); + } + Prepared::Migrated + } else { + create_entry(&final_path, kind)?; + Prepared::Fresh + } + }; + + if has_content(target, kind) { + fs::rename(target, free_retired_path(target)?)?; + } + ensure_stub(target, kind)?; + Ok(prepared) +} + +/// Whether the canonical source still carries data worth migrating/retiring: +/// a non-empty directory or a non-empty file. +fn has_content(path: &Path, kind: EntryKind) -> bool { + match kind { + EntryKind::Dir => fs::read_dir(path).is_ok_and(|mut dir| dir.next().is_some()), + EntryKind::File => fs::metadata(path).is_ok_and(|meta| meta.len() > 0), + } +} + +fn remove_existing(path: &Path) -> io::Result<()> { + match path.symlink_metadata() { + Ok(meta) if meta.is_dir() => fs::remove_dir_all(path), + Ok(_) => fs::remove_file(path), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +fn create_entry(path: &Path, kind: EntryKind) -> io::Result<()> { + match kind { + EntryKind::Dir => fs::create_dir_all(path), + EntryKind::File => { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map(|_| ()) + } + } +} + +/// Creates the (empty) mountpoint at the canonical path without touching +/// existing content — an existing empty file/dir is reused as the stub. +fn ensure_stub(target: &Path, kind: EntryKind) -> io::Result<()> { + create_entry(target, kind) +} + +fn copy_entry(src: &Path, dst: &Path, kind: EntryKind) -> io::Result<()> { + match kind { + EntryKind::Dir => copy_dir_synced(src, dst), + EntryKind::File => { + if let Some(parent) = dst.parent() { + fs::create_dir_all(parent)?; + } + copy_file_synced(src, dst) + } + } +} + +fn copy_dir_synced(src: &Path, dst: &Path) -> io::Result<()> { + fs::create_dir_all(dst)?; + fs::set_permissions(dst, fs::metadata(src)?.permissions())?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let to = dst.join(entry.file_name()); + let file_type = entry.file_type()?; + if file_type.is_dir() { + copy_dir_synced(&entry.path(), &to)?; + } else if file_type.is_symlink() { + std::os::unix::fs::symlink(fs::read_link(entry.path())?, &to)?; + } else { + copy_file_synced(&entry.path(), &to)?; + } + } + Ok(()) +} + +/// `fs::copy` + fsync: the copy must be durable before the rename that +/// publishes it, or a crash could publish a hollow entry. +fn copy_file_synced(src: &Path, dst: &Path) -> io::Result<()> { + fs::copy(src, dst)?; + fs::File::open(dst)?.sync_all() +} + +/// First free `*.pre-ext4[.N]` sibling for retiring the source. A numbered +/// fallback covers state re-accumulated at the canonical path by boots that +/// ran without the metadata volume. +fn free_retired_path(target: &Path) -> io::Result { + let base = path_with_suffix(target, RETIRED_SUFFIX); + if base.symlink_metadata().is_err() { + return Ok(base); + } + for n in 1..100u32 { + let candidate = path_with_suffix(target, &format!("{RETIRED_SUFFIX}.{n}")); + if candidate.symlink_metadata().is_err() { + return Ok(candidate); + } + } + Err(io::Error::other(format!( + "no free {RETIRED_SUFFIX} backup name next to {}", + target.display() + ))) +} + +fn path_with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut os = path.as_os_str().to_owned(); + os.push(suffix); + PathBuf::from(os) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn setup() -> (tempfile::TempDir, PathBuf, PathBuf) { + let tmp = tempfile::tempdir().unwrap(); + let volume = tmp.path().join("volume"); + let data = tmp.path().join("data"); + fs::create_dir_all(&volume).unwrap(); + fs::create_dir_all(&data).unwrap(); + (tmp, volume, data) + } + + fn seed_dir(dir: &Path) { + fs::create_dir_all(dir.join("sub")).unwrap(); + fs::write(dir.join("meta.db"), b"bolt").unwrap(); + fs::write(dir.join("sub/inner.json"), b"{}").unwrap(); + } + + #[test] + fn fresh_install_creates_empty_entries() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Fresh); + assert!(volume.join("docker-network").is_dir()); + assert!(target.is_dir(), "mountpoint stub must exist"); + + let file_target = data.join("overlayfs/metadata.db"); + let out = prepare_entry( + &volume, + &file_target, + "snapshotter-metadata.db", + EntryKind::File, + ) + .unwrap(); + assert_eq!(out, Prepared::Fresh); + assert_eq!( + fs::metadata(volume.join("snapshotter-metadata.db")) + .unwrap() + .len(), + 0 + ); + assert_eq!(fs::metadata(&file_target).unwrap().len(), 0); + } + + #[test] + fn populated_source_is_migrated_and_retired() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Migrated); + let migrated = volume.join("docker-network"); + assert_eq!(fs::read(migrated.join("meta.db")).unwrap(), b"bolt"); + assert_eq!(fs::read(migrated.join("sub/inner.json")).unwrap(), b"{}"); + // Source retired, empty stub in its place. + assert!(data.join("network.pre-ext4").join("meta.db").exists()); + assert!(fs::read_dir(&target).unwrap().next().is_none()); + } + + #[test] + fn torn_copy_is_discarded_and_redone() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + // A crashed previous copy left a partial with garbage. + fs::create_dir_all(volume.join("docker-network.partial")).unwrap(); + fs::write(volume.join("docker-network.partial/meta.db"), b"torn").unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Migrated); + assert!(!volume.join("docker-network.partial").exists()); + assert_eq!( + fs::read(volume.join("docker-network/meta.db")).unwrap(), + b"bolt" + ); + } + + #[test] + fn crash_between_copy_and_retire_converges() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + // The copy landed (final exists) but the retire never ran. + fs::create_dir_all(volume.join("docker-network")).unwrap(); + fs::write(volume.join("docker-network/meta.db"), b"bolt").unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + // Nothing re-copied, but the retire and stub landed. + assert_eq!(out, Prepared::Existing); + assert!(data.join("network.pre-ext4/meta.db").exists()); + assert!(fs::read_dir(&target).unwrap().next().is_none()); + } + + #[test] + fn replay_after_full_migration_is_a_noop() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Existing); + assert!( + !data.join("network.pre-ext4.1").exists(), + "no second backup" + ); + } + + #[test] + fn blank_volume_after_retire_starts_fresh_not_stale() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + // The metadata image was deleted and recreated blank: volume side + // empty, retired backup still on btrfs. + fs::remove_dir_all(volume.join("docker-network")).unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + // The retired backup must NOT be resurrected — that state predates + // whatever the data store has since moved on to. + assert_eq!(out, Prepared::Fresh); + assert!( + fs::read_dir(volume.join("docker-network")) + .unwrap() + .next() + .is_none() + ); + } + + #[test] + fn interim_state_retires_under_numbered_backup() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + // Boots without the volume re-accumulated state at the canonical path. + fs::write(target.join("meta.db"), b"interim").unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Existing); + assert_eq!( + fs::read(data.join("network.pre-ext4.1/meta.db")).unwrap(), + b"interim" + ); + assert!(fs::read_dir(&target).unwrap().next().is_none()); + } + + #[test] + fn file_entry_migrates_and_stubs() { + let (_tmp, volume, data) = setup(); + let target = data.join("overlayfs/metadata.db"); + fs::create_dir_all(target.parent().unwrap()).unwrap(); + fs::write(&target, b"boltdb-pages").unwrap(); + // A torn file copy is discarded too. + fs::write(volume.join("snapshotter-metadata.db.partial"), b"torn").unwrap(); + + let out = + prepare_entry(&volume, &target, "snapshotter-metadata.db", EntryKind::File).unwrap(); + + assert_eq!(out, Prepared::Migrated); + assert!(!volume.join("snapshotter-metadata.db.partial").exists()); + assert_eq!( + fs::read(volume.join("snapshotter-metadata.db")).unwrap(), + b"boltdb-pages" + ); + assert_eq!( + fs::read(data.join("overlayfs/metadata.db.pre-ext4")).unwrap(), + b"boltdb-pages" + ); + assert_eq!(fs::metadata(&target).unwrap().len(), 0, "0-byte stub"); + } +} From 71872d95f0f090ae3fad0246e0af970ca12cde34 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:57:22 +0800 Subject: [PATCH 04/10] feat(agent): mount ext4 metadata volume and bind fsync-hot boltdb dirs (ABX-496) Formats /dev/vdc (journaled ext4 + fast_commit) on first use, mounts it at /run/arcbox/metadata, migrates the five profiled hot entries, and binds them before containerd/dockerd start. Device absent or mkfs missing on a blank disk degrade to the btrfs-only layout; a present but unmountable volume hard-fails (e2fsck -y retry first) so dockerd never boots against stale shadowed state. --- guest/arcbox-agent/src/agent/linux/cmdline.rs | 19 +- .../src/agent/linux/metadata_volume.rs | 253 ++++++++++++++++++ guest/arcbox-agent/src/agent/linux/mod.rs | 1 + guest/arcbox-agent/src/agent/linux/runtime.rs | 9 + guest/arcbox-agent/src/main.rs | 5 + 5 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 guest/arcbox-agent/src/agent/linux/metadata_volume.rs diff --git a/guest/arcbox-agent/src/agent/linux/cmdline.rs b/guest/arcbox-agent/src/agent/linux/cmdline.rs index 29d843465..7652ab5ad 100644 --- a/guest/arcbox-agent/src/agent/linux/cmdline.rs +++ b/guest/arcbox-agent/src/agent/linux/cmdline.rs @@ -3,9 +3,13 @@ use std::path::Path; use arcbox_constants::cmdline::{ - DOCKER_DATA_DEVICE_KEY as DOCKER_DATA_DEVICE_CMDLINE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, + DOCKER_DATA_DEVICE_KEY as DOCKER_DATA_DEVICE_CMDLINE_KEY, + DOCKER_METADATA_DEVICE_KEY as DOCKER_METADATA_DEVICE_CMDLINE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, +}; +use arcbox_constants::devices::{ + DOCKER_DATA_BLOCK_DEVICE as DOCKER_DATA_DEVICE_DEFAULT, + DOCKER_METADATA_BLOCK_DEVICE as DOCKER_METADATA_DEVICE_DEFAULT, }; -use arcbox_constants::devices::DOCKER_DATA_BLOCK_DEVICE as DOCKER_DATA_DEVICE_DEFAULT; use arcbox_constants::env::GUEST_DOCKER_VSOCK_PORT as GUEST_DOCKER_VSOCK_PORT_ENV; use arcbox_constants::ports::{DOCKER_API_VSOCK_PORT, KUBERNETES_API_VSOCK_PORT}; @@ -59,6 +63,17 @@ pub(super) fn docker_data_device() -> String { DOCKER_DATA_DEVICE_DEFAULT.to_string() } +pub(super) fn docker_metadata_device() -> String { + // Prefer explicit kernel cmdline override; no HVC fast-path analogue — + // metadata I/O is tiny, plain virtio-blk is plenty. + if let Some(v) = cmdline_value(DOCKER_METADATA_DEVICE_CMDLINE_KEY) { + if !v.trim().is_empty() { + return v; + } + } + DOCKER_METADATA_DEVICE_DEFAULT.to_string() +} + pub(super) fn kubernetes_api_vsock_port() -> u32 { KUBERNETES_API_VSOCK_PORT } diff --git a/guest/arcbox-agent/src/agent/linux/metadata_volume.rs b/guest/arcbox-agent/src/agent/linux/metadata_volume.rs new file mode 100644 index 000000000..1ab81a9dd --- /dev/null +++ b/guest/arcbox-agent/src/agent/linux/metadata_volume.rs @@ -0,0 +1,253 @@ +//! ext4 metadata volume: format, mount, migrate, and bind the fsync-hot +//! boltdb metadata onto a small journaled ext4 disk. +//! +//! Container-start profiling (ABX-496) put ~90 % of fsyncs on these boltdb +//! files, and one fsync costs ~9.5 ms on btrfs vs ~1 ms on ext4 over the +//! same virtio-blk stack — so the hot metadata moves to ext4 while bulk data +//! (layers, blobs, volumes) stays on the compressed btrfs data volume. The +//! crash-safe migration state machine lives in `crate::metadata_migrate`; +//! design and failure policy: internal-docs/plans/ext4-metadata-volume.md. + +use std::io::{Read as _, Seek as _, SeekFrom}; +use std::path::Path; + +use arcbox_constants::paths::{CONTAINERD_DATA_MOUNT_POINT, DOCKER_DATA_MOUNT_POINT}; + +use super::cmdline::docker_metadata_device; +use crate::metadata_migrate::{EntryKind, Prepared, prepare_entry}; + +/// Mount point of the raw ext4 volume (`/run` is tmpfs, writable). +const METADATA_MOUNT: &str = "/run/arcbox/metadata"; +/// Both binaries are baked into the EROFS rootfs (static e2fsprogs). +const MKFS_EXT4: &str = "/sbin/mkfs.ext4"; +const E2FSCK: &str = "/sbin/e2fsck"; + +/// ext4 superblock magic `0xEF53`, little-endian at byte 56 of the +/// superblock (which starts at byte 1024). +const EXT4_MAGIC_OFFSET: u64 = 1024 + 56; +const EXT4_MAGIC: [u8; 2] = [0x53, 0xEF]; + +/// One fsync-hot metadata location: an entry on the volume bound over its +/// canonical btrfs-side path. The set is exactly the profiled hot set — +/// everything else (containers/, volumes/, builder/, trust/) stays on btrfs. +struct Mapping { + /// Entry name inside the metadata volume. + name: &'static str, + /// Canonical path the runtime opens (bind target). + target: String, + kind: EntryKind, +} + +fn mappings() -> Vec { + vec![ + Mapping { + name: "containerd-bolt", + target: format!("{CONTAINERD_DATA_MOUNT_POINT}/io.containerd.metadata.v1.bolt"), + kind: EntryKind::Dir, + }, + // The snapshotter dir also holds snapshots/ (bulk layer data), so + // only its boltdb is bound — a file bind is safe for bolt, which + // writes in place and never renames its database file. + Mapping { + name: "snapshotter-metadata.db", + target: format!( + "{CONTAINERD_DATA_MOUNT_POINT}/io.containerd.snapshotter.v1.overlayfs/metadata.db" + ), + kind: EntryKind::File, + }, + Mapping { + name: "docker-network", + target: format!("{DOCKER_DATA_MOUNT_POINT}/network"), + kind: EntryKind::Dir, + }, + Mapping { + name: "docker-image", + target: format!("{DOCKER_DATA_MOUNT_POINT}/image"), + kind: EntryKind::Dir, + }, + Mapping { + name: "docker-buildkit", + target: format!("{DOCKER_DATA_MOUNT_POINT}/buildkit"), + kind: EntryKind::Dir, + }, + ] +} + +/// Mounts the ext4 metadata volume and binds the hot metadata dirs over +/// their btrfs-side paths. Must run after `ensure_data_mount` (targets live +/// on the data subvolumes) and before containerd/dockerd start (their boltdb +/// files must be closed while entries migrate). +/// +/// Failure policy (version-skew safe, see the plan doc): +/// - device absent (daemon without the third disk) → `Ok`, btrfs-only boot; +/// - mkfs binary absent AND device blank (older rootfs) → `Ok`, skip; +/// - device present with ext4 but mount/prepare/bind fails → `Err` — booting +/// dockerd against the stale shadowed btrfs state would fork it. +pub(super) fn ensure_metadata_mount() -> Result { + let maps = mappings(); + if maps.iter().all(|m| crate::mount::is_mounted(&m.target)) { + return Ok("metadata binds already mounted".to_string()); + } + + let device = docker_metadata_device(); + if !wait_for_device(&device) { + tracing::warn!(device, "metadata device absent; running btrfs-only layout"); + return Ok(format!("metadata device {device} absent; skipped")); + } + + let mut notes = Vec::new(); + + if !has_ext4_superblock(&device) { + if !Path::new(MKFS_EXT4).exists() { + // Older rootfs without e2fsprogs and a never-used disk: nothing + // was ever migrated, so a btrfs-only boot is consistent. + tracing::warn!("mkfs.ext4 missing and metadata device blank; skipping metadata volume"); + return Ok("metadata volume skipped (no mkfs.ext4)".to_string()); + } + notes.push(format_ext4(&device)?); + } + + mount_metadata(&device, &mut notes)?; + + for mapping in &maps { + if crate::mount::is_mounted(&mapping.target) { + continue; + } + let volume_entry = Path::new(METADATA_MOUNT).join(mapping.name); + match prepare_entry( + Path::new(METADATA_MOUNT), + Path::new(&mapping.target), + mapping.name, + mapping.kind, + ) { + Ok(Prepared::Migrated) => notes.push(format!("migrated {}", mapping.target)), + Ok(_) => {} + Err(e) => return Err(format!("prepare {} failed: {e}", mapping.target)), + } + bind(&volume_entry, &mapping.target)?; + } + + if notes.is_empty() { + Ok("metadata volume mounted".to_string()) + } else { + Ok(notes.join("; ")) + } +} + +/// Waits up to 5 s for the VirtIO block device node (same budget and +/// rationale as the data-device wait in `btrfs.rs`). +fn wait_for_device(device: &str) -> bool { + for attempt in 0..50 { + if Path::new(device).exists() { + if attempt > 0 { + tracing::info!(device, attempt, "waited for metadata device"); + } + return true; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + false +} + +fn has_ext4_superblock(device: &str) -> bool { + let Ok(mut file) = std::fs::File::open(device) else { + return false; + }; + if file.seek(SeekFrom::Start(EXT4_MAGIC_OFFSET)).is_err() { + return false; + } + let mut magic = [0_u8; 2]; + file.read_exact(&mut magic).is_ok() && magic == EXT4_MAGIC +} + +fn format_ext4(device: &str) -> Result { + // Explicit feature list so the result is deterministic regardless of + // any mke2fs.conf; fast_commit targets exactly the small-metadata-commit + // fsync pattern boltdb produces, lazy init off pays the one-time cost at + // format instead of trickling background writes into first boot. + match std::process::Command::new(MKFS_EXT4) + .args([ + "-F", + "-t", + "ext4", + "-O", + "has_journal,extent,huge_file,flex_bg,metadata_csum,64bit,dir_nlink,extra_isize,fast_commit", + "-E", + "lazy_itable_init=0,lazy_journal_init=0", + "-L", + "arcbox-meta", + device, + ]) + .status() + { + Ok(status) if status.success() => Ok(format!("formatted {device} as ext4")), + Ok(status) => Err(format!( + "mkfs.ext4 failed on {device} (exit={})", + status.code().unwrap_or(-1) + )), + Err(e) => Err(format!("failed to execute mkfs.ext4: {e}")), + } +} + +/// Mounts the volume; on failure runs `e2fsck -y` once (journal replay is +/// in-kernel — fsck covers the residual corruption class) and retries once. +fn mount_metadata(device: &str, notes: &mut Vec) -> Result<(), String> { + if crate::mount::is_mounted(METADATA_MOUNT) { + return Ok(()); + } + std::fs::create_dir_all(METADATA_MOUNT) + .map_err(|e| format!("failed to create {METADATA_MOUNT}: {e}"))?; + + if try_mount(device) { + return Ok(()); + } + + if !Path::new(E2FSCK).exists() { + return Err(format!( + "mount {device} on {METADATA_MOUNT} failed and {E2FSCK} is unavailable" + )); + } + // e2fsck exit codes 0/1/2 mean clean or corrected; >=4 is a real failure. + match std::process::Command::new(E2FSCK) + .args(["-y", device]) + .status() + { + Ok(status) if status.code().is_some_and(|c| c <= 2) => { + notes.push(format!("e2fsck repaired {device}")); + } + Ok(status) => { + return Err(format!( + "e2fsck failed on {device} (exit={})", + status.code().unwrap_or(-1) + )); + } + Err(e) => return Err(format!("failed to execute e2fsck: {e}")), + } + if try_mount(device) { + Ok(()) + } else { + Err(format!( + "mount {device} on {METADATA_MOUNT} failed even after e2fsck" + )) + } +} + +fn try_mount(device: &str) -> bool { + matches!( + std::process::Command::new("/bin/busybox") + .args(["mount", "-t", "ext4", "-o", "noatime", device, METADATA_MOUNT]) + .status(), + Ok(status) if status.success() + ) +} + +fn bind(source: &Path, target: &str) -> Result<(), String> { + nix::mount::mount( + Some(source), + target, + None::<&str>, + nix::mount::MsFlags::MS_BIND, + None::<&str>, + ) + .map_err(|e| format!("bind {} -> {target} failed: {e}", source.display())) +} diff --git a/guest/arcbox-agent/src/agent/linux/mod.rs b/guest/arcbox-agent/src/agent/linux/mod.rs index ac43e71d0..5913796a0 100644 --- a/guest/arcbox-agent/src/agent/linux/mod.rs +++ b/guest/arcbox-agent/src/agent/linux/mod.rs @@ -10,6 +10,7 @@ mod disk; mod kubernetes; mod machine_exec; mod memory_pressure; +mod metadata_volume; mod port_forward; mod probe; mod proxy; diff --git a/guest/arcbox-agent/src/agent/linux/runtime.rs b/guest/arcbox-agent/src/agent/linux/runtime.rs index eb211cc4a..8e765698e 100644 --- a/guest/arcbox-agent/src/agent/linux/runtime.rs +++ b/guest/arcbox-agent/src/agent/linux/runtime.rs @@ -774,6 +774,15 @@ async fn try_start_bundled_runtime() -> String { Err(e) => return format!("data volume setup failed: {}", e), } + // Bind the fsync-hot metadata dirs onto the ext4 volume before the + // daemons open their boltdb files. A hard error means the volume exists + // but is unusable — starting dockerd against the stale shadowed btrfs + // state would fork it, so abort instead. + match super::metadata_volume::ensure_metadata_mount() { + Ok(note) => notes.push(note), + Err(e) => return format!("metadata volume setup failed: {}", e), + } + // The docker data mount now exists; export it read-only over NFS. setup_nfs_export(&mut notes); diff --git a/guest/arcbox-agent/src/main.rs b/guest/arcbox-agent/src/main.rs index 8898fe151..456ce1741 100644 --- a/guest/arcbox-agent/src/main.rs +++ b/guest/arcbox-agent/src/main.rs @@ -23,6 +23,11 @@ mod memory_pressure; #[cfg(target_os = "linux")] mod stats; +// Same arrangement for the ext4 metadata-volume migration state machine +// (pure std::fs; the mount syscalls live in agent/linux/metadata_volume.rs). +#[cfg(target_os = "linux")] +mod metadata_migrate; + #[cfg(target_os = "linux")] mod error; From bf6ef6531324450f4bf9557bde22c48e05ede4df Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:57:54 +0800 Subject: [PATCH 05/10] perf(agent): drop btrfs NOCOW flagging of metadata dirs (ABX-496) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured no fsync benefit (9.5 ms nocow vs 10.5 ms cow — the cost is the btrfs commit path, not COW), the boltdb dirs it targeted now live on the ext4 metadata volume, and the inherited flag on the snapshotter dir silently disabled zstd compression for every extracted layer file. --- guest/arcbox-agent/src/agent/linux/btrfs.rs | 69 +-------------------- 1 file changed, 3 insertions(+), 66 deletions(-) diff --git a/guest/arcbox-agent/src/agent/linux/btrfs.rs b/guest/arcbox-agent/src/agent/linux/btrfs.rs index f66d81e3a..150102fb8 100644 --- a/guest/arcbox-agent/src/agent/linux/btrfs.rs +++ b/guest/arcbox-agent/src/agent/linux/btrfs.rs @@ -2,8 +2,9 @@ //! //! On first boot the data device is formatted as Btrfs with five subvolumes //! (`@docker`, `@containerd`, `@k3s`, `@kubelet`, `@cni`), each bind-mounted -//! to its canonical path. Metadata-heavy directories get `NOCOW` to avoid -//! Btrfs + APFS double write amplification. +//! to its canonical path. The fsync-hot boltdb metadata is NOT kept here — +//! it lives on the ext4 metadata volume (`metadata_volume.rs`); btrfs holds +//! the bulk, compression-friendly data. use std::io::{Read as _, Seek as _, SeekFrom}; use std::path::Path; @@ -247,11 +248,6 @@ pub(super) fn ensure_data_mount() -> Result { .status() { Ok(s) if s.success() => { - // Disable Btrfs COW on metadata-heavy subdirectories. - // BoltDB (containerd/dockerd) does frequent fdatasync on - // small pages. Without NOCOW, each write triggers Btrfs - // copy-on-write + APFS COW on the host = double amplification. - disable_cow_on_metadata_dirs(target); notes.push(format!("mounted {} -> {}", subvol, target)); } Ok(s) => { @@ -273,65 +269,6 @@ pub(super) fn ensure_data_mount() -> Result { } } -/// Disables Btrfs COW (sets NOCOW attribute) on metadata-heavy subdirectories. -/// -/// BoltDB and other metadata stores do frequent fdatasync on small pages. -/// Btrfs COW amplifies each write (copy 16KB metadata page + update B-tree), -/// and the host's APFS does another COW on top — double write amplification. -/// NOCOW converts these to in-place overwrites at the Btrfs layer. -fn disable_cow_on_metadata_dirs(mount_point: &str) { - // FS_IOC_SETFLAGS = _IOW('f', 2, long) - // FS_NOCOW_FL = 0x00800000 - const FS_NOCOW_FL: libc::c_long = 0x0080_0000; - - // Subdirectories that contain BoltDB or other fsync-heavy metadata. - // The NOCOW attribute is inherited by new files created in these dirs. - let metadata_subdirs = [ - "io.containerd.metadata.v1.bolt", - "io.containerd.snapshotter.v1.overlayfs", - "containerd", - "network", - "builder", - "buildkit", - "image", - "trust", - ]; - - for subdir in &metadata_subdirs { - let path = format!("{}/{}", mount_point, subdir); - let _ = std::fs::create_dir_all(&path); - - let Ok(cpath) = std::ffi::CString::new(path.as_str()) else { - continue; - }; - // SAFETY: valid path, O_RDONLY | O_DIRECTORY. - let fd = unsafe { libc::open(cpath.as_ptr(), libc::O_RDONLY | libc::O_DIRECTORY) }; - if fd < 0 { - continue; - } - - let mut flags: libc::c_long = 0; - // Get current flags, then set NOCOW. - // SAFETY: FS_IOC_GETFLAGS/SETFLAGS on a valid directory fd. - // NOTE: `libc::Ioctl` differs per target — `c_ulong` on - // Linux GNU, `c_int` on Linux musl. Using the typedef keeps - // the cast right for whichever target we cross-compile to. - unsafe { - #[allow(clippy::cast_possible_wrap)] - let get_flags = 0x8008_6601u32 as libc::Ioctl; // FS_IOC_GETFLAGS - #[allow(clippy::cast_possible_wrap)] - let set_flags = 0x4008_6602u32 as libc::Ioctl; // FS_IOC_SETFLAGS - if libc::ioctl(fd, get_flags, &mut flags) == 0 { - flags |= FS_NOCOW_FL; - if libc::ioctl(fd, set_flags, &flags) == 0 { - tracing::debug!("set NOCOW on {}", path); - } - } - libc::close(fd); - } - } -} - // BTRFS_IOC_SUBVOL_CREATE = _IOW(0x94, 14, struct btrfs_ioctl_vol_args) // struct btrfs_ioctl_vol_args { __s64 fd; char name[4088]; } total = 4096 bytes // From 5eadfbc8a5add30a41051ef707b2779e84402bc7 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 00:05:56 +0800 Subject: [PATCH 06/10] refactor(storage): declare metadata device on the cmdline, drop guest probing (ABX-496) The host injects arcbox.docker_metadata_device=/dev/vdc when it attaches the disk. Declaration is authoritative: declared-but-missing hard-fails instead of silently skipping, and undeclared boots (older daemon, bare hv_e2e probe) skip with zero probe delay instead of a 5 s timeout. An already-present default node without a declaration is still honored for bespoke e2e configs. --- app/arcbox-core/src/vm_lifecycle/boot.rs | 18 +++++++++- guest/arcbox-agent/src/agent/linux/cmdline.rs | 19 ++++------ .../src/agent/linux/metadata_volume.rs | 35 ++++++++++++++----- internal-docs/plans/ext4-metadata-volume.md | 11 +++--- 4 files changed, 56 insertions(+), 27 deletions(-) diff --git a/app/arcbox-core/src/vm_lifecycle/boot.rs b/app/arcbox-core/src/vm_lifecycle/boot.rs index f4cbc5291..50410db77 100644 --- a/app/arcbox-core/src/vm_lifecycle/boot.rs +++ b/app/arcbox-core/src/vm_lifecycle/boot.rs @@ -16,8 +16,10 @@ use crate::error::{CoreError, Result}; use crate::event::Event; use crate::machine::MachineConfig; use arcbox_constants::cmdline::{ - DEBUG_CONSOLE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, HV_EARLYCON_DIRECTIVE, + DEBUG_CONSOLE_KEY, DOCKER_METADATA_DEVICE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, + HV_EARLYCON_DIRECTIVE, }; +use arcbox_constants::devices::DOCKER_METADATA_BLOCK_DEVICE; use arcbox_error::CommonError; use super::actor::{Completion, InternalEvent, LifecycleShared}; @@ -459,6 +461,20 @@ impl LifecycleShared { } } + // Declare the ext4 metadata device this machine attaches as vdc. + // Unlike the data device (auto-detected for its HVC fast path), the + // declaration is authoritative: key present → the agent waits for + // the node and hard-fails if it never appears; key absent (older + // daemon) → the agent skips the metadata volume without probing. + if !cmdline + .split_whitespace() + .any(|token| token.starts_with(DOCKER_METADATA_DEVICE_KEY)) + { + cmdline.push(' '); + cmdline.push_str(DOCKER_METADATA_DEVICE_KEY); + cmdline.push_str(DOCKER_METADATA_BLOCK_DEVICE); + } + // Always attach an interactive debug console on the custom-HV backend. // An operator can `socat - UNIX-CONNECT:` to get a serial root // shell into the guest even when early boot hangs before networking diff --git a/guest/arcbox-agent/src/agent/linux/cmdline.rs b/guest/arcbox-agent/src/agent/linux/cmdline.rs index 7652ab5ad..91708c127 100644 --- a/guest/arcbox-agent/src/agent/linux/cmdline.rs +++ b/guest/arcbox-agent/src/agent/linux/cmdline.rs @@ -6,10 +6,7 @@ use arcbox_constants::cmdline::{ DOCKER_DATA_DEVICE_KEY as DOCKER_DATA_DEVICE_CMDLINE_KEY, DOCKER_METADATA_DEVICE_KEY as DOCKER_METADATA_DEVICE_CMDLINE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, }; -use arcbox_constants::devices::{ - DOCKER_DATA_BLOCK_DEVICE as DOCKER_DATA_DEVICE_DEFAULT, - DOCKER_METADATA_BLOCK_DEVICE as DOCKER_METADATA_DEVICE_DEFAULT, -}; +use arcbox_constants::devices::DOCKER_DATA_BLOCK_DEVICE as DOCKER_DATA_DEVICE_DEFAULT; use arcbox_constants::env::GUEST_DOCKER_VSOCK_PORT as GUEST_DOCKER_VSOCK_PORT_ENV; use arcbox_constants::ports::{DOCKER_API_VSOCK_PORT, KUBERNETES_API_VSOCK_PORT}; @@ -63,15 +60,11 @@ pub(super) fn docker_data_device() -> String { DOCKER_DATA_DEVICE_DEFAULT.to_string() } -pub(super) fn docker_metadata_device() -> String { - // Prefer explicit kernel cmdline override; no HVC fast-path analogue — - // metadata I/O is tiny, plain virtio-blk is plenty. - if let Some(v) = cmdline_value(DOCKER_METADATA_DEVICE_CMDLINE_KEY) { - if !v.trim().is_empty() { - return v; - } - } - DOCKER_METADATA_DEVICE_DEFAULT.to_string() +/// The metadata device path the host declared on the cmdline when it +/// attached the disk (there is no HVC fast path to auto-detect, so the host +/// declares instead). `None` means an older daemon that never attached one. +pub(super) fn declared_docker_metadata_device() -> Option { + cmdline_value(DOCKER_METADATA_DEVICE_CMDLINE_KEY).filter(|v| !v.trim().is_empty()) } pub(super) fn kubernetes_api_vsock_port() -> u32 { diff --git a/guest/arcbox-agent/src/agent/linux/metadata_volume.rs b/guest/arcbox-agent/src/agent/linux/metadata_volume.rs index 1ab81a9dd..8ac631044 100644 --- a/guest/arcbox-agent/src/agent/linux/metadata_volume.rs +++ b/guest/arcbox-agent/src/agent/linux/metadata_volume.rs @@ -13,7 +13,9 @@ use std::path::Path; use arcbox_constants::paths::{CONTAINERD_DATA_MOUNT_POINT, DOCKER_DATA_MOUNT_POINT}; -use super::cmdline::docker_metadata_device; +use arcbox_constants::devices::DOCKER_METADATA_BLOCK_DEVICE; + +use super::cmdline::declared_docker_metadata_device; use crate::metadata_migrate::{EntryKind, Prepared, prepare_entry}; /// Mount point of the raw ext4 volume (`/run` is tmpfs, writable). @@ -79,21 +81,36 @@ fn mappings() -> Vec { /// files must be closed while entries migrate). /// /// Failure policy (version-skew safe, see the plan doc): -/// - device absent (daemon without the third disk) → `Ok`, btrfs-only boot; +/// - no cmdline declaration and no default node (older daemon without the +/// third disk) → `Ok`, btrfs-only boot, zero probe delay; /// - mkfs binary absent AND device blank (older rootfs) → `Ok`, skip; -/// - device present with ext4 but mount/prepare/bind fails → `Err` — booting -/// dockerd against the stale shadowed btrfs state would fork it. +/// - device declared but never appears, or present but unusable after an +/// `e2fsck -y` retry → `Err` — booting dockerd against the stale shadowed +/// btrfs state would fork it. pub(super) fn ensure_metadata_mount() -> Result { let maps = mappings(); if maps.iter().all(|m| crate::mount::is_mounted(&m.target)) { return Ok("metadata binds already mounted".to_string()); } - let device = docker_metadata_device(); - if !wait_for_device(&device) { - tracing::warn!(device, "metadata device absent; running btrfs-only layout"); - return Ok(format!("metadata device {device} absent; skipped")); - } + let device = match declared_docker_metadata_device() { + Some(device) => { + if !wait_for_device(&device) { + return Err(format!("declared metadata device {device} never appeared")); + } + device + } + // No declaration (older daemon). An already-present default node is + // still honored so bespoke configs (e2e probes) can attach the disk + // without a cmdline; otherwise run the btrfs-only layout. + None if Path::new(DOCKER_METADATA_BLOCK_DEVICE).exists() => { + DOCKER_METADATA_BLOCK_DEVICE.to_string() + } + None => { + tracing::warn!("no metadata device declared or present; btrfs-only layout"); + return Ok("metadata volume skipped (no device)".to_string()); + } + }; let mut notes = Vec::new(); diff --git a/internal-docs/plans/ext4-metadata-volume.md b/internal-docs/plans/ext4-metadata-volume.md index 9e51c8563..1bdd6b73a 100644 --- a/internal-docs/plans/ext4-metadata-volume.md +++ b/internal-docs/plans/ext4-metadata-volume.md @@ -36,7 +36,7 @@ current btrfs locations. Everything else stays on btrfs unchanged. | Storage for ext4 | Second virtio-blk disk image | Loop file on btrfs rejected: loop FLUSH re-enters the btrfs commit path, keeping the 9.5 ms cost. GPT-partitioning docker.img rejected: destructive migration of existing whole-disk btrfs images. | | Image name | `-meta.img` derived from the data image (`docker.img` → `docker-meta.img`, `docker-rosetta.img` → `docker-rosetta-meta.img`) | Follows the existing per-VM data-image parameterization (`vm_lifecycle/mod.rs::for_machine`). | | Virtual size | 2 GiB, sparse (`ensure_sparse_block_image`), no resize path | Holds only bolt DBs + small configs; 2 GiB is ~100× headroom. No resize2fs in guest; fixed size avoids that dependency. | -| Guest device | `/dev/vdc` (third `BlockDeviceConfig` in the Vec; both backends attach in Vec order) + cmdline override key `arcbox.docker_metadata_device=` mirroring the existing data-device pattern | HV additionally exposes serial `arcbox-blk-docker-meta.img` via GET_ID, but ordering is already the load-bearing contract for vdb. No HVC fast-path analogue — metadata I/O is tiny. | +| Guest device | `/dev/vdc` (third `BlockDeviceConfig` in the Vec; both backends attach in Vec order), **declared** by the host via `arcbox.docker_metadata_device=` on the cmdline when it attaches the disk | Declaration is authoritative: key present → the agent waits for the node and hard-fails if it never appears; key absent (older daemon) → zero-delay skip (an already-present default node is still honored for bespoke e2e configs). Unlike the data device there is no HVC fast path to auto-detect, so no probe-timeout heuristics. HV additionally exposes serial `arcbox-blk-docker-meta.img` via GET_ID. | | Formatter | `mkfs.ext4` (static e2fsprogs) added to the guest rootfs via boot-assets, invoked guest-side like `mkfs.btrfs` | `arcbox-ext4` crate rejected: its formatter sets no `HAS_JOURNAL` (registry source, `formatter.rs:1107-1111`), and journal-less ext4 has no crash replay — VM force-stop is a routine event and these DBs are the system of record. Host-side formatting impossible (no mke2fs on macOS). | | mkfs options | `mkfs.ext4 -F -O fast_commit -E lazy_itable_init=0,lazy_journal_init=0 -L arcbox-meta` | fast_commit reduces exactly our fsync pattern (small metadata commits); kernel is 6.x with `CONFIG_EXT4_FS=y` on both arches. Lazy init off: one-time cost at first format, no background trickle. | | Mount options | `noatime` (defaults otherwise: `data=ordered`, barriers on) | `discard` skipped — bolt files are stable in size, few deletes. | @@ -91,8 +91,10 @@ New module `agent/linux/metadata_volume.rs`, called from `ensure_data_mount()` — i.e. after btrfs targets exist, before the NFS export and before containerd/dockerd start (bolt files guaranteed closed): -1. Resolve device: cmdline override → `/dev/vdc`; wait up to 5 s for the node - (same loop as `btrfs.rs`). +1. Resolve device from the host's cmdline declaration; wait up to 5 s for + the declared node (same loop as `btrfs.rs`) and hard-fail if it never + appears. No declaration → skip immediately (older daemon), unless the + default node already exists (bespoke e2e configs). 2. Probe ext4 superblock magic (0xEF53 at offset 0x438). If absent → `mkfs.ext4` with the locked options. 3. Mount at `/run/arcbox/metadata` (`noatime`). On failure: `e2fsck -y`, @@ -119,7 +121,8 @@ Failure policy (version-skew safe): | Condition | Behavior | |---|---| -| Device node absent (older daemon without the third disk) | warn + continue on btrfs — perf-only degradation | +| No cmdline declaration and no default node (older daemon without the third disk) | warn + continue on btrfs, zero probe delay — perf-only degradation | +| Device declared but the node never appears | **hard fail** — the host promised the disk; a silent skip could boot dockerd against stale shadowed state | | mkfs binary absent AND device blank (older rootfs) | warn + continue — nothing was ever migrated, no split-brain possible | | Device present with ext4 but mount/fsck fails | **hard fail** runtime start — booting dockerd against the stale shadowed btrfs DBs would fork state | From 1cdb1a012819c0a24e70d071117c54187e552d92 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 00:26:26 +0800 Subject: [PATCH 07/10] chore(assets): pin boot assets v0.6.11 (rootfs ships mkfs.ext4 + e2fsck) --- assets.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assets.lock b/assets.lock index 6673fa1c1..0e519d68b 100644 --- a/assets.lock +++ b/assets.lock @@ -14,9 +14,9 @@ # and the [boot] bump. [boot] -version = "0.6.10" +version = "0.6.11" cdn = "https://boot.arcboxcdn.com" -manifest_sha256 = "f417bfb4ac2f9c0ba6926266ea8f7902e5b53eb0762e423df780fd5bd333122f" +manifest_sha256 = "fc5130e3e64e75bd2df7bae2de7ff4fe790cb18d04db8f01c441156a662d59b5" [[tools]] name = "docker" From d9d5beb0d28d3e86c0273198f61cb187e5cc8bee Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 01:11:06 +0800 Subject: [PATCH 08/10] perf(vz): use fsync-level disk synchronization instead of F_FULLFSYNC (ABX-496) A/B on the same guest layout and host: a guest fsync costs ~10 ms under the default .full mode vs ~2 ms under .fsync and ~2 ms on the custom-HV backend, whose block worker has always used plain fsync for guest FLUSH. .fsync aligns both backends on the same power-loss window and matches the durability level Colima/OrbStack ship; measured guest fsync drops 10.3 -> 1.7 ms on the ext4 metadata volume. --- .../shim/Sources/ArcBoxVZShim/Devices.swift | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/virt/arcbox-vz/shim/Sources/ArcBoxVZShim/Devices.swift b/virt/arcbox-vz/shim/Sources/ArcBoxVZShim/Devices.swift index 14d3e6996..ff28395f2 100644 --- a/virt/arcbox-vz/shim/Sources/ArcBoxVZShim/Devices.swift +++ b/virt/arcbox-vz/shim/Sources/ArcBoxVZShim/Devices.swift @@ -12,7 +12,18 @@ func storageDiskImageNew( ) -> UnsafeMutableRawPointer? { let url = URL(fileURLWithPath: String(cString: path)) do { - let attachment = try VZDiskImageStorageDeviceAttachment(url: url, readOnly: readOnly) + // fsync-level durability, NOT the default .full (F_FULLFSYNC): a + // guest FLUSH under .full costs ~10 ms of host F_FULLFSYNC per + // barrier — the dominant cost of every guest fsync (ABX-496). The + // custom-HV backend's block worker has always used plain fsync for + // guest FLUSH, so .fsync makes both backends give the same + // power-loss window; Colima/OrbStack ship the same durability level. + let attachment = try VZDiskImageStorageDeviceAttachment( + url: url, + readOnly: readOnly, + cachingMode: .automatic, + synchronizationMode: .fsync + ) return abxRetainedHandle(VZVirtioBlockDeviceConfiguration(attachment: attachment)) } catch { errorOut?.pointee = abxErrorString(error) From 101ac9bc1cc8dc0895d41149b9b2dfdf9e17d97b Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 01:15:53 +0800 Subject: [PATCH 09/10] docs(plans): record ext4 metadata volume implementation results and VZ sync-mode finding --- internal-docs/plans/ext4-metadata-volume.md | 50 ++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/internal-docs/plans/ext4-metadata-volume.md b/internal-docs/plans/ext4-metadata-volume.md index 1bdd6b73a..ca60eb1e3 100644 --- a/internal-docs/plans/ext4-metadata-volume.md +++ b/internal-docs/plans/ext4-metadata-volume.md @@ -1,6 +1,6 @@ # ext4 Metadata Volume — snapshot-prepare fsync fix (ABX-496) -Status: **Planned** (design locked, awaiting implementation) +Status: **Implemented** (arcbox#497 + boot-assets#45 / v0.6.11); results in §9 Owner: ABX-496 (docker build performance vs Colima) Companion fix already shipped separately: `rcu_expedited` for runc-create (arcbox#496). @@ -11,10 +11,17 @@ on Colima. Profiling (static strace with `-y` fd resolution, dockerd debug lifecycle timeline, `dd conv=fsync` microbench) pinned the cost: - A single `fsync` on the guest data volume costs **9.5 ms on ArcBox (btrfs)** - vs **1.0 ms on Colima (ext4)** on the same virtio-blk stack. The cost is the + vs **1.0 ms on Colima (ext4)**. The cost is the btrfs commit path (`write_all_supers`: superblock write + FLUSH barriers), not COW extent allocation — `chattr +C` (NOCOW) was measured at 9.5 ms vs 10.5 ms, i.e. no help. + **Attribution correction found during implementation (§9):** most of those + 9.5 ms were not filesystem weight but the *host-side cost of each guest + FLUSH* — the VZ disk attachment defaulted to `synchronizationMode = .full` + (F_FULLFSYNC ≈ 10 ms per barrier), while Colima ships fsync-level + durability. btrfs still amplifies (more FLUSHes per fsync than ext4 + fast_commit) and the ~90 %-boltdb fsync profile below stands, but the + single biggest lever turned out to be the VZ synchronization mode. - ~90 % of the fsyncs during a container start hit small boltdb metadata files: containerd `io.containerd.metadata.v1.bolt/meta.db` (dominant), overlayfs snapshotter `metadata.db`, dockerd `network/files/local-kv.db` @@ -214,3 +221,42 @@ use case. No new exports, no fsid allocation. as separate atomic commits. 3. Bench + record results on ABX-496; close the snapshot-prepare half of the issue. + +## 9. Implementation results (2026-07-22) + +Shipped as boot-assets#45 (released v0.6.11) + arcbox#497. Two deviations +from the plan, both improvements: + +- **Device discovery**: instead of guest-side probing with a 5 s timeout, + the host *declares* the device via `arcbox.docker_metadata_device=/dev/vdc` + on the cmdline when it attaches the disk (§2 table updated). Declared but + missing → hard fail; undeclared → zero-delay skip. +- **VZ synchronization mode** (`arcbox-vz` shim): the disk attachment now + uses `synchronizationMode: .fsync` instead of the default `.full` + (F_FULLFSYNC per guest FLUSH). This aligns VZ with the custom-HV backend, + whose block worker has always used plain fsync, and with the durability + level Colima/OrbStack ship. Discovered because the metadata volume alone + moved the microbench barely at all — see the A/B below. + +Measured on the same host, guest `dd bs=1M count=1 conv=fsync`, avg of 20: + +| Configuration | ext4 metadata volume | btrfs data volume | +|---|---|---| +| VZ, `.full` (before) | 10.3 ms | 11.5 ms | +| VZ, `.fsync` (after) | **1.7 ms** | 2.1 ms | +| HV (plain fsync, unchanged) | 2.2 ms | 2.6 ms | +| Colima (reference) | — | 1.0 ms (ext4) | + +End-to-end (10-run averages, same day, same host, VZ): + +| Metric | ArcBox before campaign | ArcBox after | Colima | +|---|---|---|---| +| `docker create` (contains snapshot-prepare, was ~182 ms alone) | — | **53 ms** | 48 ms | +| `docker start` | — | 416 ms | 82 ms | +| `docker run --rm alpine true` | ~700 ms | 496 ms | 173 ms | + +Acceptance met: snapshot-prepare cost is gone from the create path +(53 ms total vs the < 60 ms target for the prepare step alone) and guest +fsync is at Colima's order of magnitude. The remaining `docker start` gap +(~330 ms vs Colima) sits in the network-endpoint/iptables portion of start +— outside both ABX-496 root causes; file separately. From 96603f25c20d649215778002f48184841b5e430a Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 01:46:55 +0800 Subject: [PATCH 10/10] fix(agent): make migration publish durable before retiring the source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on #497: the publish rename (ext4 volume) and the retire rename (btrfs source) live on different filesystems, so the volume-root sync must be mandatory or a crash could persist the retire while losing the publish — next boot would start empty. Also fsync each copied directory (per-file fsync does not contractually persist dentries) and document the metadata images in the daemon-lifecycle cleanup contract. --- docs/daemon-lifecycle.md | 18 +++++++++--------- guest/arcbox-agent/src/metadata_migrate.rs | 16 ++++++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/daemon-lifecycle.md b/docs/daemon-lifecycle.md index e8371a584..4a81838a9 100644 --- a/docs/daemon-lifecycle.md +++ b/docs/daemon-lifecycle.md @@ -16,7 +16,7 @@ acquire_daemon_lease flock(daemon.lock), terminate stale ~instant or ≤ start_control_plane Bind arcbox.sock, SystemService up ~instant │ Desktop can connect from this point on. │ -release_stale_resources Wait for docker.img holders to release 0–10 s +release_stale_resources Wait for disk-image holders to release 0–10 s │ Reported as CLEANING_UP via gRPC. │ prepare_assets Seed/download boot assets variable @@ -120,7 +120,7 @@ signal received | `daemon.lock` | exists, old PID, **lock released** | `try_flock` succeeds instantly | | `docker.sock` | deleted | — | | `arcbox.sock` | deleted | — | -| `docker.img` | exists, no holders | — | +| disk images (`docker.img` + `docker-meta.img`, Rosetta counterparts) | exist, no holders | — | | VM | gracefully stopped | — | No manual intervention needed. @@ -141,7 +141,7 @@ signal received during startup │ graceful Stop behind itself, so an unbounded │ wait could last the whole boot timeout └─ on timeout / second signal → runtime.shutdown_force() → VM killed, - no orphaned XPC helpers holding docker.img + no orphaned XPC helpers holding disk images ``` `early_runtime` is filled right after `Runtime` construction, before the @@ -161,7 +161,7 @@ When the daemon is killed without graceful shutdown: - Socket files are **not** cleaned up. - VM is **not** gracefully stopped. - Container subnet route is **not** removed. -- `docker.img` may still be held by Virtualization.framework XPC helpers. +- The disk images (`docker.img`, `docker-meta.img`, Rosetta counterparts) may still be held by Virtualization.framework XPC helpers. ### Residual state after crash @@ -170,7 +170,7 @@ When the daemon is killed without graceful shutdown: | `daemon.lock` | exists, old PID, **lock released** | `try_flock` succeeds instantly | | `docker.sock` | **stale** | `DockerApiServer::run` removes before bind | | `arcbox.sock` | **stale** | `start_grpc` removes before bind | -| `docker.img` | **possibly held by XPC helpers** | `wait_for_resources` waits up to 10 s | +| disk images | **possibly held by XPC helpers** | `wait_for_resources` waits up to 10 s | | VM | non-graceful termination | Virtualization.framework cleans up | | Route | **stale** | `recovery::run()` rebuilds | @@ -189,10 +189,10 @@ When a new daemon starts while an old one is still running: 6. Falls back to SIGKILL if unresponsive. 7. Acquires the lock once released. 8. `start_grpc` removes any stale sockets before binding. -9. `wait_for_resources` waits for `docker.img` holders to release. +9. `wait_for_resources` waits for disk-image holders to release. The old daemon's graceful shutdown runs its full sequence (drain, VM stop, -socket cleanup). The new daemon only needs to handle the `docker.img` +socket cleanup). The new daemon only needs to handle the disk-image holdover case. ## Socket Lifecycle @@ -210,10 +210,10 @@ socket that another component has already bound. ## Edge Cases -### docker.img held by orphaned XPC helpers +### Disk images held by orphaned XPC helpers Virtualization.framework spawns XPC helper processes that may outlive the -daemon. These processes hold `docker.img` open. The daemon waits up to +daemon. These processes hold the disk images (`docker.img`, `docker-meta.img`, Rosetta counterparts) open. The daemon waits up to 10 s for them to exit (`wait_for_resources`), then proceeds. If they persist, `init_runtime` may fail because the disk image is locked. diff --git a/guest/arcbox-agent/src/metadata_migrate.rs b/guest/arcbox-agent/src/metadata_migrate.rs index f4c683538..c84005ed5 100644 --- a/guest/arcbox-agent/src/metadata_migrate.rs +++ b/guest/arcbox-agent/src/metadata_migrate.rs @@ -73,11 +73,12 @@ pub fn prepare_entry( if has_content(target, kind) { copy_entry(target, &partial, kind)?; fs::rename(&partial, &final_path)?; - // Persist the rename itself; best-effort (a lost rename is - // redone on the next boot). - if let Ok(dir) = fs::File::open(volume_root) { - let _ = dir.sync_all(); - } + // The publish rename MUST be durable before the retire below: + // the two renames live on different filesystems (volume vs + // btrfs source), so without this barrier a crash could persist + // the retire while losing the publish — next boot would then + // see neither and start empty. + fs::File::open(volume_root)?.sync_all()?; Prepared::Migrated } else { create_entry(&final_path, kind)?; @@ -159,7 +160,10 @@ fn copy_dir_synced(src: &Path, dst: &Path) -> io::Result<()> { copy_file_synced(&entry.path(), &to)?; } } - Ok(()) + // Per-file fsync does not contractually persist this directory's + // entries; sync the dir itself so the published tree can never be + // durable-but-hollow (replay trusts the final path's existence). + fs::File::open(dst)?.sync_all() } /// `fs::copy` + fsync: the copy must be durable before the rename that