diff --git a/app/arcbox-cli/src/commands/disk.rs b/app/arcbox-cli/src/commands/disk.rs index 5d7b1579d..ee1e2ec44 100644 --- a/app/arcbox-cli/src/commands/disk.rs +++ b/app/arcbox-cli/src/commands/disk.rs @@ -103,6 +103,22 @@ async fn execute_usage() -> Result<()> { ); println!(" Reclaimable: {:.1} GiB", usage.reclaimable_gib()); + // The metadata image is small but part of the data set; report it so + // "what is on disk" is complete. + let meta_path = config.docker_meta_img_path(); + if meta_path.exists() { + let meta = read_disk_usage(&meta_path)?; + println!(); + println!("Docker metadata disk:"); + println!(" Path: {}", meta_path.display()); + println!(" Logical: {:.1} GiB", meta.logical_gib()); + println!( + " Physical: {:.1} GiB ({:.1}%)", + meta.physical_gib(), + meta.usage_pct() + ); + } + Ok(()) } diff --git a/app/arcbox-core/src/config.rs b/app/arcbox-core/src/config.rs index 2ad951d2f..850a077c6 100644 --- a/app/arcbox-core/src/config.rs +++ b/app/arcbox-core/src/config.rs @@ -179,6 +179,15 @@ impl Config { pub fn docker_img_path(&self) -> PathBuf { self.data_subdir().join("docker.img") } + + /// Returns the path to the Docker metadata image (`data/docker-meta.img`). + /// + /// Paired with [`Self::docker_img_path`]: the ext4 volume carrying the + /// fsync-hot boltdb metadata (see internal-docs/plans/ext4-metadata-volume.md). + #[must_use] + pub fn docker_meta_img_path(&self) -> PathBuf { + self.data_subdir().join("docker-meta.img") + } } /// Default VM configuration. diff --git a/app/arcbox-core/src/vm_lifecycle/boot.rs b/app/arcbox-core/src/vm_lifecycle/boot.rs index d55d4ee12..50410db77 100644 --- a/app/arcbox-core/src/vm_lifecycle/boot.rs +++ b/app/arcbox-core/src/vm_lifecycle/boot.rs @@ -16,13 +16,15 @@ use crate::error::{CoreError, Result}; use crate::event::Event; use crate::machine::MachineConfig; use arcbox_constants::cmdline::{ - DEBUG_CONSOLE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, HV_EARLYCON_DIRECTIVE, + DEBUG_CONSOLE_KEY, DOCKER_METADATA_DEVICE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, + HV_EARLYCON_DIRECTIVE, }; +use arcbox_constants::devices::DOCKER_METADATA_BLOCK_DEVICE; use arcbox_error::CommonError; use super::actor::{Completion, InternalEvent, LifecycleShared}; -use super::types::{DesiredBoot, machine_drift_reason}; -use super::{DOCKER_DATA_IMAGE_SIZE_BYTES, RecoveryAction}; +use super::types::{DesiredBoot, machine_drift_reason, metadata_image_filename}; +use super::{DOCKER_DATA_IMAGE_SIZE_BYTES, DOCKER_METADATA_IMAGE_SIZE_BYTES, RecoveryAction}; impl LifecycleShared { /// Boots the VM end-to-end (create if needed, start with retries, wait for @@ -342,7 +344,8 @@ impl LifecycleShared { /// /// Block devices: /// - vda: rootfs.erofs (read-only) - /// - vdb: docker-data.img (read-write) + /// - vdb: docker-data.img (read-write, btrfs bulk data) + /// - vdc: docker-meta.img (read-write, ext4 metadata volume) async fn create_default_machine(&self) -> Result<()> { let boot = self.resolve_desired_boot().await?; let rootfs_path = boot.rootfs_image.to_string_lossy().to_string(); @@ -368,6 +371,20 @@ impl LifecycleShared { read_only: false, }); + // Attach the ext4 metadata volume (vdc): the fsync-hot boltdb + // metadata lives there while bulk data stays on the btrfs data disk. + // The two images are a paired set — see + // internal-docs/plans/ext4-metadata-volume.md. + let metadata_image = self + .data_dir + .join(arcbox_constants::paths::host::DATA) + .join(metadata_image_filename(&self.data_image_filename)); + ensure_sparse_block_image(&metadata_image, DOCKER_METADATA_IMAGE_SIZE_BYTES)?; + block_devices.push(crate::vm::BlockDeviceConfig { + path: metadata_image.to_string_lossy().to_string(), + read_only: false, + }); + let config = MachineConfig { name: self.machine_name.clone(), cpus: self.config.default_vm.cpus, @@ -444,6 +461,20 @@ impl LifecycleShared { } } + // Declare the ext4 metadata device this machine attaches as vdc. + // Unlike the data device (auto-detected for its HVC fast path), the + // declaration is authoritative: key present → the agent waits for + // the node and hard-fails if it never appears; key absent (older + // daemon) → the agent skips the metadata volume without probing. + if !cmdline + .split_whitespace() + .any(|token| token.starts_with(DOCKER_METADATA_DEVICE_KEY)) + { + cmdline.push(' '); + cmdline.push_str(DOCKER_METADATA_DEVICE_KEY); + cmdline.push_str(DOCKER_METADATA_BLOCK_DEVICE); + } + // Always attach an interactive debug console on the custom-HV backend. // An operator can `socat - UNIX-CONNECT:` to get a serial root // shell into the guest even when early boot hangs before networking diff --git a/app/arcbox-core/src/vm_lifecycle/mod.rs b/app/arcbox-core/src/vm_lifecycle/mod.rs index 1ff5a1615..3f7133d0c 100644 --- a/app/arcbox-core/src/vm_lifecycle/mod.rs +++ b/app/arcbox-core/src/vm_lifecycle/mod.rs @@ -84,6 +84,12 @@ const DOCKER_DATA_IMAGE_NAME: &str = "docker.img"; /// only consumes actual disk space for written blocks. 8 TiB matches OrbStack /// and prevents users from hitting artificial limits. const DOCKER_DATA_IMAGE_SIZE_BYTES: u64 = 8 * 1024 * 1024 * 1024 * 1024; +/// Persistent guest metadata image size (2 GiB sparse file). +/// +/// The ext4 volume holds only the fsync-hot boltdb metadata directories — +/// bulk data stays on the btrfs data image — so 2 GiB is ~100x headroom. +/// See internal-docs/plans/ext4-metadata-volume.md. +const DOCKER_METADATA_IMAGE_SIZE_BYTES: u64 = 2 * 1024 * 1024 * 1024; pub(crate) use boot::ensure_sparse_block_image; pub use health::HealthMonitor; diff --git a/app/arcbox-core/src/vm_lifecycle/tests.rs b/app/arcbox-core/src/vm_lifecycle/tests.rs index d1a355d2c..9e7f1835b 100644 --- a/app/arcbox-core/src/vm_lifecycle/tests.rs +++ b/app/arcbox-core/src/vm_lifecycle/tests.rs @@ -1,5 +1,5 @@ use super::boot::{agent_timeout_error, ensure_earlycon, ensure_sparse_block_image}; -use super::types::{DesiredBoot, machine_drift_reason}; +use super::types::{DesiredBoot, machine_drift_reason, metadata_image_filename}; use super::*; use crate::machine::MachineState; use arcbox_constants::cmdline::HV_EARLYCON_DIRECTIVE; @@ -152,7 +152,20 @@ fn sample_machine(cpus: u32, memory_mb: u64, kernel: &str, cmdline: &str) -> Mac disk_gb: 50, kernel: Some(kernel.to_string()), cmdline: Some(cmdline.to_string()), - block_devices: Vec::new(), + block_devices: vec![ + crate::vm::BlockDeviceConfig { + path: "/rootfs.erofs".to_string(), + read_only: true, + }, + crate::vm::BlockDeviceConfig { + path: "/data/docker.img".to_string(), + read_only: false, + }, + crate::vm::BlockDeviceConfig { + path: "/data/docker-meta.img".to_string(), + read_only: false, + }, + ], distro: None, distro_version: None, disk_path: None, @@ -197,7 +210,25 @@ fn machine_drift_detects_each_overridable_field() { // The cmdline gap that previously slipped through (e.g. arm64.nosve // added/removed without bumping the boot-asset version). - let mut m = current; + let mut m = current.clone(); m.cmdline = Some("console=hvc0 earlycon arm64.nosve".to_string()); assert_eq!(machine_drift_reason(&m, &want, &boot), Some("cmdline")); + + // A machine persisted before the ext4 metadata volume (two disks) must + // be recreated so the guest receives vdc. + let mut m = current; + m.block_devices.pop(); + assert_eq!( + machine_drift_reason(&m, &want, &boot), + Some("block_devices") + ); +} + +#[test] +fn metadata_image_filename_pairs_with_data_image() { + assert_eq!(metadata_image_filename("docker.img"), "docker-meta.img"); + assert_eq!( + metadata_image_filename("docker-rosetta.img"), + "docker-rosetta-meta.img" + ); } diff --git a/app/arcbox-core/src/vm_lifecycle/types.rs b/app/arcbox-core/src/vm_lifecycle/types.rs index 55b7aba68..05558c5ed 100644 --- a/app/arcbox-core/src/vm_lifecycle/types.rs +++ b/app/arcbox-core/src/vm_lifecycle/types.rs @@ -188,7 +188,26 @@ pub(super) fn machine_drift_reason( Some("kernel") } else if persisted.cmdline.as_deref() != Some(boot.cmdline.as_str()) { Some("cmdline") + } else if persisted.block_devices.len() != DEFAULT_MACHINE_DISK_COUNT { + // A machine persisted before the ext4 metadata volume carries only + // two disks; recreating rewrites the machine record (image files are + // untouched) so the guest receives vdc and can migrate. + Some("block_devices") } else { None } } + +/// Number of block devices `create_default_machine` attaches: EROFS rootfs +/// (vda), btrfs data image (vdb), ext4 metadata image (vdc). +pub(super) const DEFAULT_MACHINE_DISK_COUNT: usize = 3; + +/// Derives the metadata-volume image filename paired with a data image: +/// `docker.img` → `docker-meta.img`, `docker-rosetta.img` → +/// `docker-rosetta-meta.img`. +pub(super) fn metadata_image_filename(data_image_filename: &str) -> String { + let stem = data_image_filename + .strip_suffix(".img") + .unwrap_or(data_image_filename); + format!("{stem}-meta.img") +} diff --git a/app/arcbox-daemon/src/startup/resource_cleanup.rs b/app/arcbox-daemon/src/startup/resource_cleanup.rs index 96e75de78..3dde37f32 100644 --- a/app/arcbox-daemon/src/startup/resource_cleanup.rs +++ b/app/arcbox-daemon/src/startup/resource_cleanup.rs @@ -4,7 +4,12 @@ use std::path::{Path, PathBuf}; use arcbox_core::persistence::MachinePersistence; -const DISK_IMAGE_NAMES: [&str; 2] = ["docker.img", "docker-rosetta.img"]; +const DISK_IMAGE_NAMES: [&str; 4] = [ + "docker.img", + "docker-meta.img", + "docker-rosetta.img", + "docker-rosetta-meta.img", +]; /// Why startup must scan for stale disk-image holders. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/assets.lock b/assets.lock index 6673fa1c1..0e519d68b 100644 --- a/assets.lock +++ b/assets.lock @@ -14,9 +14,9 @@ # and the [boot] bump. [boot] -version = "0.6.10" +version = "0.6.11" cdn = "https://boot.arcboxcdn.com" -manifest_sha256 = "f417bfb4ac2f9c0ba6926266ea8f7902e5b53eb0762e423df780fd5bd333122f" +manifest_sha256 = "fc5130e3e64e75bd2df7bae2de7ff4fe790cb18d04db8f01c441156a662d59b5" [[tools]] name = "docker" diff --git a/common/arcbox-constants/src/cmdline.rs b/common/arcbox-constants/src/cmdline.rs index 2646337ec..922f527d5 100644 --- a/common/arcbox-constants/src/cmdline.rs +++ b/common/arcbox-constants/src/cmdline.rs @@ -4,6 +4,9 @@ pub const GUEST_DOCKER_VSOCK_PORT_KEY: &str = "arcbox.guest_docker_vsock_port="; /// Kernel cmdline key for guest Docker data block-device path. pub const DOCKER_DATA_DEVICE_KEY: &str = "arcbox.docker_data_device="; +/// Kernel cmdline key for guest Docker metadata block-device path. +pub const DOCKER_METADATA_DEVICE_KEY: &str = "arcbox.docker_metadata_device="; + /// Kernel cmdline key carrying the host path of the interactive debug-console /// Unix socket (custom-HV backend). /// diff --git a/common/arcbox-constants/src/devices.rs b/common/arcbox-constants/src/devices.rs index 06fa9fde1..7c19ee0a6 100644 --- a/common/arcbox-constants/src/devices.rs +++ b/common/arcbox-constants/src/devices.rs @@ -3,3 +3,9 @@ pub const ROOT_BLOCK_DEVICE: &str = "/dev/vda"; /// Default Docker data block device path in guest. pub const DOCKER_DATA_BLOCK_DEVICE: &str = "/dev/vdb"; + +/// Default Docker metadata block device path in guest. +/// +/// The ext4 volume carrying the fsync-hot boltdb metadata directories; paired +/// with the btrfs data device (see internal-docs/plans/ext4-metadata-volume.md). +pub const DOCKER_METADATA_BLOCK_DEVICE: &str = "/dev/vdc"; diff --git a/docs/daemon-lifecycle.md b/docs/daemon-lifecycle.md index e8371a584..4a81838a9 100644 --- a/docs/daemon-lifecycle.md +++ b/docs/daemon-lifecycle.md @@ -16,7 +16,7 @@ acquire_daemon_lease flock(daemon.lock), terminate stale ~instant or ≤ start_control_plane Bind arcbox.sock, SystemService up ~instant │ Desktop can connect from this point on. │ -release_stale_resources Wait for docker.img holders to release 0–10 s +release_stale_resources Wait for disk-image holders to release 0–10 s │ Reported as CLEANING_UP via gRPC. │ prepare_assets Seed/download boot assets variable @@ -120,7 +120,7 @@ signal received | `daemon.lock` | exists, old PID, **lock released** | `try_flock` succeeds instantly | | `docker.sock` | deleted | — | | `arcbox.sock` | deleted | — | -| `docker.img` | exists, no holders | — | +| disk images (`docker.img` + `docker-meta.img`, Rosetta counterparts) | exist, no holders | — | | VM | gracefully stopped | — | No manual intervention needed. @@ -141,7 +141,7 @@ signal received during startup │ graceful Stop behind itself, so an unbounded │ wait could last the whole boot timeout └─ on timeout / second signal → runtime.shutdown_force() → VM killed, - no orphaned XPC helpers holding docker.img + no orphaned XPC helpers holding disk images ``` `early_runtime` is filled right after `Runtime` construction, before the @@ -161,7 +161,7 @@ When the daemon is killed without graceful shutdown: - Socket files are **not** cleaned up. - VM is **not** gracefully stopped. - Container subnet route is **not** removed. -- `docker.img` may still be held by Virtualization.framework XPC helpers. +- The disk images (`docker.img`, `docker-meta.img`, Rosetta counterparts) may still be held by Virtualization.framework XPC helpers. ### Residual state after crash @@ -170,7 +170,7 @@ When the daemon is killed without graceful shutdown: | `daemon.lock` | exists, old PID, **lock released** | `try_flock` succeeds instantly | | `docker.sock` | **stale** | `DockerApiServer::run` removes before bind | | `arcbox.sock` | **stale** | `start_grpc` removes before bind | -| `docker.img` | **possibly held by XPC helpers** | `wait_for_resources` waits up to 10 s | +| disk images | **possibly held by XPC helpers** | `wait_for_resources` waits up to 10 s | | VM | non-graceful termination | Virtualization.framework cleans up | | Route | **stale** | `recovery::run()` rebuilds | @@ -189,10 +189,10 @@ When a new daemon starts while an old one is still running: 6. Falls back to SIGKILL if unresponsive. 7. Acquires the lock once released. 8. `start_grpc` removes any stale sockets before binding. -9. `wait_for_resources` waits for `docker.img` holders to release. +9. `wait_for_resources` waits for disk-image holders to release. The old daemon's graceful shutdown runs its full sequence (drain, VM stop, -socket cleanup). The new daemon only needs to handle the `docker.img` +socket cleanup). The new daemon only needs to handle the disk-image holdover case. ## Socket Lifecycle @@ -210,10 +210,10 @@ socket that another component has already bound. ## Edge Cases -### docker.img held by orphaned XPC helpers +### Disk images held by orphaned XPC helpers Virtualization.framework spawns XPC helper processes that may outlive the -daemon. These processes hold `docker.img` open. The daemon waits up to +daemon. These processes hold the disk images (`docker.img`, `docker-meta.img`, Rosetta counterparts) open. The daemon waits up to 10 s for them to exit (`wait_for_resources`), then proceeds. If they persist, `init_runtime` may fail because the disk image is locked. diff --git a/docs/data-directories.md b/docs/data-directories.md index 18eda7997..dfa4fe141 100644 --- a/docs/data-directories.md +++ b/docs/data-directories.md @@ -57,6 +57,7 @@ Defined in `app/arcbox-core/src/config.rs`. | `data/machines/` | Virtual machine data | daemon | | `data/volumes/` | Named volumes | daemon | | `data/docker.img` | Docker persistent disk image (Btrfs) | daemon | +| `data/docker-meta.img` | Docker metadata disk image (ext4, fsync-hot boltdb state); paired with `docker.img` — back up or move the two together | daemon | ### 1.4 `boot/` — Boot Asset Cache diff --git a/guest/arcbox-agent/src/agent/linux/btrfs.rs b/guest/arcbox-agent/src/agent/linux/btrfs.rs index f66d81e3a..150102fb8 100644 --- a/guest/arcbox-agent/src/agent/linux/btrfs.rs +++ b/guest/arcbox-agent/src/agent/linux/btrfs.rs @@ -2,8 +2,9 @@ //! //! On first boot the data device is formatted as Btrfs with five subvolumes //! (`@docker`, `@containerd`, `@k3s`, `@kubelet`, `@cni`), each bind-mounted -//! to its canonical path. Metadata-heavy directories get `NOCOW` to avoid -//! Btrfs + APFS double write amplification. +//! to its canonical path. The fsync-hot boltdb metadata is NOT kept here — +//! it lives on the ext4 metadata volume (`metadata_volume.rs`); btrfs holds +//! the bulk, compression-friendly data. use std::io::{Read as _, Seek as _, SeekFrom}; use std::path::Path; @@ -247,11 +248,6 @@ pub(super) fn ensure_data_mount() -> Result { .status() { Ok(s) if s.success() => { - // Disable Btrfs COW on metadata-heavy subdirectories. - // BoltDB (containerd/dockerd) does frequent fdatasync on - // small pages. Without NOCOW, each write triggers Btrfs - // copy-on-write + APFS COW on the host = double amplification. - disable_cow_on_metadata_dirs(target); notes.push(format!("mounted {} -> {}", subvol, target)); } Ok(s) => { @@ -273,65 +269,6 @@ pub(super) fn ensure_data_mount() -> Result { } } -/// Disables Btrfs COW (sets NOCOW attribute) on metadata-heavy subdirectories. -/// -/// BoltDB and other metadata stores do frequent fdatasync on small pages. -/// Btrfs COW amplifies each write (copy 16KB metadata page + update B-tree), -/// and the host's APFS does another COW on top — double write amplification. -/// NOCOW converts these to in-place overwrites at the Btrfs layer. -fn disable_cow_on_metadata_dirs(mount_point: &str) { - // FS_IOC_SETFLAGS = _IOW('f', 2, long) - // FS_NOCOW_FL = 0x00800000 - const FS_NOCOW_FL: libc::c_long = 0x0080_0000; - - // Subdirectories that contain BoltDB or other fsync-heavy metadata. - // The NOCOW attribute is inherited by new files created in these dirs. - let metadata_subdirs = [ - "io.containerd.metadata.v1.bolt", - "io.containerd.snapshotter.v1.overlayfs", - "containerd", - "network", - "builder", - "buildkit", - "image", - "trust", - ]; - - for subdir in &metadata_subdirs { - let path = format!("{}/{}", mount_point, subdir); - let _ = std::fs::create_dir_all(&path); - - let Ok(cpath) = std::ffi::CString::new(path.as_str()) else { - continue; - }; - // SAFETY: valid path, O_RDONLY | O_DIRECTORY. - let fd = unsafe { libc::open(cpath.as_ptr(), libc::O_RDONLY | libc::O_DIRECTORY) }; - if fd < 0 { - continue; - } - - let mut flags: libc::c_long = 0; - // Get current flags, then set NOCOW. - // SAFETY: FS_IOC_GETFLAGS/SETFLAGS on a valid directory fd. - // NOTE: `libc::Ioctl` differs per target — `c_ulong` on - // Linux GNU, `c_int` on Linux musl. Using the typedef keeps - // the cast right for whichever target we cross-compile to. - unsafe { - #[allow(clippy::cast_possible_wrap)] - let get_flags = 0x8008_6601u32 as libc::Ioctl; // FS_IOC_GETFLAGS - #[allow(clippy::cast_possible_wrap)] - let set_flags = 0x4008_6602u32 as libc::Ioctl; // FS_IOC_SETFLAGS - if libc::ioctl(fd, get_flags, &mut flags) == 0 { - flags |= FS_NOCOW_FL; - if libc::ioctl(fd, set_flags, &flags) == 0 { - tracing::debug!("set NOCOW on {}", path); - } - } - libc::close(fd); - } - } -} - // BTRFS_IOC_SUBVOL_CREATE = _IOW(0x94, 14, struct btrfs_ioctl_vol_args) // struct btrfs_ioctl_vol_args { __s64 fd; char name[4088]; } total = 4096 bytes // diff --git a/guest/arcbox-agent/src/agent/linux/cmdline.rs b/guest/arcbox-agent/src/agent/linux/cmdline.rs index 29d843465..91708c127 100644 --- a/guest/arcbox-agent/src/agent/linux/cmdline.rs +++ b/guest/arcbox-agent/src/agent/linux/cmdline.rs @@ -3,7 +3,8 @@ use std::path::Path; use arcbox_constants::cmdline::{ - DOCKER_DATA_DEVICE_KEY as DOCKER_DATA_DEVICE_CMDLINE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, + DOCKER_DATA_DEVICE_KEY as DOCKER_DATA_DEVICE_CMDLINE_KEY, + DOCKER_METADATA_DEVICE_KEY as DOCKER_METADATA_DEVICE_CMDLINE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, }; use arcbox_constants::devices::DOCKER_DATA_BLOCK_DEVICE as DOCKER_DATA_DEVICE_DEFAULT; use arcbox_constants::env::GUEST_DOCKER_VSOCK_PORT as GUEST_DOCKER_VSOCK_PORT_ENV; @@ -59,6 +60,13 @@ pub(super) fn docker_data_device() -> String { DOCKER_DATA_DEVICE_DEFAULT.to_string() } +/// The metadata device path the host declared on the cmdline when it +/// attached the disk (there is no HVC fast path to auto-detect, so the host +/// declares instead). `None` means an older daemon that never attached one. +pub(super) fn declared_docker_metadata_device() -> Option { + cmdline_value(DOCKER_METADATA_DEVICE_CMDLINE_KEY).filter(|v| !v.trim().is_empty()) +} + pub(super) fn kubernetes_api_vsock_port() -> u32 { KUBERNETES_API_VSOCK_PORT } diff --git a/guest/arcbox-agent/src/agent/linux/metadata_volume.rs b/guest/arcbox-agent/src/agent/linux/metadata_volume.rs new file mode 100644 index 000000000..8ac631044 --- /dev/null +++ b/guest/arcbox-agent/src/agent/linux/metadata_volume.rs @@ -0,0 +1,270 @@ +//! ext4 metadata volume: format, mount, migrate, and bind the fsync-hot +//! boltdb metadata onto a small journaled ext4 disk. +//! +//! Container-start profiling (ABX-496) put ~90 % of fsyncs on these boltdb +//! files, and one fsync costs ~9.5 ms on btrfs vs ~1 ms on ext4 over the +//! same virtio-blk stack — so the hot metadata moves to ext4 while bulk data +//! (layers, blobs, volumes) stays on the compressed btrfs data volume. The +//! crash-safe migration state machine lives in `crate::metadata_migrate`; +//! design and failure policy: internal-docs/plans/ext4-metadata-volume.md. + +use std::io::{Read as _, Seek as _, SeekFrom}; +use std::path::Path; + +use arcbox_constants::paths::{CONTAINERD_DATA_MOUNT_POINT, DOCKER_DATA_MOUNT_POINT}; + +use arcbox_constants::devices::DOCKER_METADATA_BLOCK_DEVICE; + +use super::cmdline::declared_docker_metadata_device; +use crate::metadata_migrate::{EntryKind, Prepared, prepare_entry}; + +/// Mount point of the raw ext4 volume (`/run` is tmpfs, writable). +const METADATA_MOUNT: &str = "/run/arcbox/metadata"; +/// Both binaries are baked into the EROFS rootfs (static e2fsprogs). +const MKFS_EXT4: &str = "/sbin/mkfs.ext4"; +const E2FSCK: &str = "/sbin/e2fsck"; + +/// ext4 superblock magic `0xEF53`, little-endian at byte 56 of the +/// superblock (which starts at byte 1024). +const EXT4_MAGIC_OFFSET: u64 = 1024 + 56; +const EXT4_MAGIC: [u8; 2] = [0x53, 0xEF]; + +/// One fsync-hot metadata location: an entry on the volume bound over its +/// canonical btrfs-side path. The set is exactly the profiled hot set — +/// everything else (containers/, volumes/, builder/, trust/) stays on btrfs. +struct Mapping { + /// Entry name inside the metadata volume. + name: &'static str, + /// Canonical path the runtime opens (bind target). + target: String, + kind: EntryKind, +} + +fn mappings() -> Vec { + vec![ + Mapping { + name: "containerd-bolt", + target: format!("{CONTAINERD_DATA_MOUNT_POINT}/io.containerd.metadata.v1.bolt"), + kind: EntryKind::Dir, + }, + // The snapshotter dir also holds snapshots/ (bulk layer data), so + // only its boltdb is bound — a file bind is safe for bolt, which + // writes in place and never renames its database file. + Mapping { + name: "snapshotter-metadata.db", + target: format!( + "{CONTAINERD_DATA_MOUNT_POINT}/io.containerd.snapshotter.v1.overlayfs/metadata.db" + ), + kind: EntryKind::File, + }, + Mapping { + name: "docker-network", + target: format!("{DOCKER_DATA_MOUNT_POINT}/network"), + kind: EntryKind::Dir, + }, + Mapping { + name: "docker-image", + target: format!("{DOCKER_DATA_MOUNT_POINT}/image"), + kind: EntryKind::Dir, + }, + Mapping { + name: "docker-buildkit", + target: format!("{DOCKER_DATA_MOUNT_POINT}/buildkit"), + kind: EntryKind::Dir, + }, + ] +} + +/// Mounts the ext4 metadata volume and binds the hot metadata dirs over +/// their btrfs-side paths. Must run after `ensure_data_mount` (targets live +/// on the data subvolumes) and before containerd/dockerd start (their boltdb +/// files must be closed while entries migrate). +/// +/// Failure policy (version-skew safe, see the plan doc): +/// - no cmdline declaration and no default node (older daemon without the +/// third disk) → `Ok`, btrfs-only boot, zero probe delay; +/// - mkfs binary absent AND device blank (older rootfs) → `Ok`, skip; +/// - device declared but never appears, or present but unusable after an +/// `e2fsck -y` retry → `Err` — booting dockerd against the stale shadowed +/// btrfs state would fork it. +pub(super) fn ensure_metadata_mount() -> Result { + let maps = mappings(); + if maps.iter().all(|m| crate::mount::is_mounted(&m.target)) { + return Ok("metadata binds already mounted".to_string()); + } + + let device = match declared_docker_metadata_device() { + Some(device) => { + if !wait_for_device(&device) { + return Err(format!("declared metadata device {device} never appeared")); + } + device + } + // No declaration (older daemon). An already-present default node is + // still honored so bespoke configs (e2e probes) can attach the disk + // without a cmdline; otherwise run the btrfs-only layout. + None if Path::new(DOCKER_METADATA_BLOCK_DEVICE).exists() => { + DOCKER_METADATA_BLOCK_DEVICE.to_string() + } + None => { + tracing::warn!("no metadata device declared or present; btrfs-only layout"); + return Ok("metadata volume skipped (no device)".to_string()); + } + }; + + let mut notes = Vec::new(); + + if !has_ext4_superblock(&device) { + if !Path::new(MKFS_EXT4).exists() { + // Older rootfs without e2fsprogs and a never-used disk: nothing + // was ever migrated, so a btrfs-only boot is consistent. + tracing::warn!("mkfs.ext4 missing and metadata device blank; skipping metadata volume"); + return Ok("metadata volume skipped (no mkfs.ext4)".to_string()); + } + notes.push(format_ext4(&device)?); + } + + mount_metadata(&device, &mut notes)?; + + for mapping in &maps { + if crate::mount::is_mounted(&mapping.target) { + continue; + } + let volume_entry = Path::new(METADATA_MOUNT).join(mapping.name); + match prepare_entry( + Path::new(METADATA_MOUNT), + Path::new(&mapping.target), + mapping.name, + mapping.kind, + ) { + Ok(Prepared::Migrated) => notes.push(format!("migrated {}", mapping.target)), + Ok(_) => {} + Err(e) => return Err(format!("prepare {} failed: {e}", mapping.target)), + } + bind(&volume_entry, &mapping.target)?; + } + + if notes.is_empty() { + Ok("metadata volume mounted".to_string()) + } else { + Ok(notes.join("; ")) + } +} + +/// Waits up to 5 s for the VirtIO block device node (same budget and +/// rationale as the data-device wait in `btrfs.rs`). +fn wait_for_device(device: &str) -> bool { + for attempt in 0..50 { + if Path::new(device).exists() { + if attempt > 0 { + tracing::info!(device, attempt, "waited for metadata device"); + } + return true; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + false +} + +fn has_ext4_superblock(device: &str) -> bool { + let Ok(mut file) = std::fs::File::open(device) else { + return false; + }; + if file.seek(SeekFrom::Start(EXT4_MAGIC_OFFSET)).is_err() { + return false; + } + let mut magic = [0_u8; 2]; + file.read_exact(&mut magic).is_ok() && magic == EXT4_MAGIC +} + +fn format_ext4(device: &str) -> Result { + // Explicit feature list so the result is deterministic regardless of + // any mke2fs.conf; fast_commit targets exactly the small-metadata-commit + // fsync pattern boltdb produces, lazy init off pays the one-time cost at + // format instead of trickling background writes into first boot. + match std::process::Command::new(MKFS_EXT4) + .args([ + "-F", + "-t", + "ext4", + "-O", + "has_journal,extent,huge_file,flex_bg,metadata_csum,64bit,dir_nlink,extra_isize,fast_commit", + "-E", + "lazy_itable_init=0,lazy_journal_init=0", + "-L", + "arcbox-meta", + device, + ]) + .status() + { + Ok(status) if status.success() => Ok(format!("formatted {device} as ext4")), + Ok(status) => Err(format!( + "mkfs.ext4 failed on {device} (exit={})", + status.code().unwrap_or(-1) + )), + Err(e) => Err(format!("failed to execute mkfs.ext4: {e}")), + } +} + +/// Mounts the volume; on failure runs `e2fsck -y` once (journal replay is +/// in-kernel — fsck covers the residual corruption class) and retries once. +fn mount_metadata(device: &str, notes: &mut Vec) -> Result<(), String> { + if crate::mount::is_mounted(METADATA_MOUNT) { + return Ok(()); + } + std::fs::create_dir_all(METADATA_MOUNT) + .map_err(|e| format!("failed to create {METADATA_MOUNT}: {e}"))?; + + if try_mount(device) { + return Ok(()); + } + + if !Path::new(E2FSCK).exists() { + return Err(format!( + "mount {device} on {METADATA_MOUNT} failed and {E2FSCK} is unavailable" + )); + } + // e2fsck exit codes 0/1/2 mean clean or corrected; >=4 is a real failure. + match std::process::Command::new(E2FSCK) + .args(["-y", device]) + .status() + { + Ok(status) if status.code().is_some_and(|c| c <= 2) => { + notes.push(format!("e2fsck repaired {device}")); + } + Ok(status) => { + return Err(format!( + "e2fsck failed on {device} (exit={})", + status.code().unwrap_or(-1) + )); + } + Err(e) => return Err(format!("failed to execute e2fsck: {e}")), + } + if try_mount(device) { + Ok(()) + } else { + Err(format!( + "mount {device} on {METADATA_MOUNT} failed even after e2fsck" + )) + } +} + +fn try_mount(device: &str) -> bool { + matches!( + std::process::Command::new("/bin/busybox") + .args(["mount", "-t", "ext4", "-o", "noatime", device, METADATA_MOUNT]) + .status(), + Ok(status) if status.success() + ) +} + +fn bind(source: &Path, target: &str) -> Result<(), String> { + nix::mount::mount( + Some(source), + target, + None::<&str>, + nix::mount::MsFlags::MS_BIND, + None::<&str>, + ) + .map_err(|e| format!("bind {} -> {target} failed: {e}", source.display())) +} diff --git a/guest/arcbox-agent/src/agent/linux/mod.rs b/guest/arcbox-agent/src/agent/linux/mod.rs index ac43e71d0..5913796a0 100644 --- a/guest/arcbox-agent/src/agent/linux/mod.rs +++ b/guest/arcbox-agent/src/agent/linux/mod.rs @@ -10,6 +10,7 @@ mod disk; mod kubernetes; mod machine_exec; mod memory_pressure; +mod metadata_volume; mod port_forward; mod probe; mod proxy; diff --git a/guest/arcbox-agent/src/agent/linux/runtime.rs b/guest/arcbox-agent/src/agent/linux/runtime.rs index eb211cc4a..8e765698e 100644 --- a/guest/arcbox-agent/src/agent/linux/runtime.rs +++ b/guest/arcbox-agent/src/agent/linux/runtime.rs @@ -774,6 +774,15 @@ async fn try_start_bundled_runtime() -> String { Err(e) => return format!("data volume setup failed: {}", e), } + // Bind the fsync-hot metadata dirs onto the ext4 volume before the + // daemons open their boltdb files. A hard error means the volume exists + // but is unusable — starting dockerd against the stale shadowed btrfs + // state would fork it, so abort instead. + match super::metadata_volume::ensure_metadata_mount() { + Ok(note) => notes.push(note), + Err(e) => return format!("metadata volume setup failed: {}", e), + } + // The docker data mount now exists; export it read-only over NFS. setup_nfs_export(&mut notes); diff --git a/guest/arcbox-agent/src/lib.rs b/guest/arcbox-agent/src/lib.rs index 2f450b6f3..f57d6264b 100644 --- a/guest/arcbox-agent/src/lib.rs +++ b/guest/arcbox-agent/src/lib.rs @@ -13,6 +13,7 @@ pub mod dns_server; #[cfg(target_os = "linux")] pub mod error; pub mod memory_pressure; +pub mod metadata_migrate; pub mod rootfs_builder; #[cfg(target_os = "linux")] pub mod sandbox; diff --git a/guest/arcbox-agent/src/main.rs b/guest/arcbox-agent/src/main.rs index 8898fe151..456ce1741 100644 --- a/guest/arcbox-agent/src/main.rs +++ b/guest/arcbox-agent/src/main.rs @@ -23,6 +23,11 @@ mod memory_pressure; #[cfg(target_os = "linux")] mod stats; +// Same arrangement for the ext4 metadata-volume migration state machine +// (pure std::fs; the mount syscalls live in agent/linux/metadata_volume.rs). +#[cfg(target_os = "linux")] +mod metadata_migrate; + #[cfg(target_os = "linux")] mod error; diff --git a/guest/arcbox-agent/src/metadata_migrate.rs b/guest/arcbox-agent/src/metadata_migrate.rs new file mode 100644 index 000000000..c84005ed5 --- /dev/null +++ b/guest/arcbox-agent/src/metadata_migrate.rs @@ -0,0 +1,387 @@ +//! Crash-safe migrate-then-retire state machine for the ext4 metadata volume. +//! +//! The Linux agent (`agent/linux/metadata_volume.rs`) moves the fsync-hot +//! boltdb metadata directories from the btrfs data volume onto a small ext4 +//! volume and bind-mounts them back over their canonical paths. This module +//! owns the on-disk state machine — copy, retire, mountpoint stub — and is +//! kept free of mount syscalls so every crash window stays unit-testable on +//! any host. Design: internal-docs/plans/ext4-metadata-volume.md. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +/// What a migration entry is on disk. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntryKind { + /// A directory bind-mounted as a whole. + Dir, + /// A single file (boltdb) bind-mounted over its canonical path. + File, +} + +/// How [`prepare_entry`] left the volume side. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Prepared { + /// Existing btrfs-side content was copied onto the volume. + Migrated, + /// No source content existed; the volume entry was created empty. + Fresh, + /// The volume entry already existed (nothing copied). + Existing, +} + +/// Suffix of the retired btrfs-side source after a successful migration. +/// +/// Renaming the source — rather than leaving it shadowed under the bind — is +/// what prevents a later blank metadata volume (deleted image, recreated by +/// the host) from silently re-migrating stale pre-upgrade state against a +/// data store that has moved on. The renamed copy doubles as a manual +/// recovery artifact for the (unsupported) downgrade path. +pub const RETIRED_SUFFIX: &str = ".pre-ext4"; + +/// Suffix of an in-progress copy on the volume; never bound, discarded and +/// redone on re-entry. +const PARTIAL_SUFFIX: &str = ".partial"; + +/// Prepares one metadata entry on the volume and retires its btrfs-side +/// source. Idempotent and crash-safe: +/// +/// 1. If `/` is absent: discard any stale `.partial`, +/// then either copy the source into `.partial` and atomically rename it +/// into place, or create the entry empty when no source content exists. +/// 2. If the final entry exists while the source still has content at its +/// canonical path, rename the source to `*.pre-ext4` (also covers a crash +/// between the copy and this retire). +/// 3. Ensure an empty mountpoint stub exists at the canonical path. +/// +/// Every interruption re-converges: a torn copy is redone from scratch, a +/// completed copy is never redone, and the retire is retried until it lands. +pub fn prepare_entry( + volume_root: &Path, + target: &Path, + name: &str, + kind: EntryKind, +) -> io::Result { + let final_path = volume_root.join(name); + let partial = volume_root.join(format!("{name}{PARTIAL_SUFFIX}")); + + let prepared = if final_path.symlink_metadata().is_ok() { + Prepared::Existing + } else { + remove_existing(&partial)?; + if has_content(target, kind) { + copy_entry(target, &partial, kind)?; + fs::rename(&partial, &final_path)?; + // The publish rename MUST be durable before the retire below: + // the two renames live on different filesystems (volume vs + // btrfs source), so without this barrier a crash could persist + // the retire while losing the publish — next boot would then + // see neither and start empty. + fs::File::open(volume_root)?.sync_all()?; + Prepared::Migrated + } else { + create_entry(&final_path, kind)?; + Prepared::Fresh + } + }; + + if has_content(target, kind) { + fs::rename(target, free_retired_path(target)?)?; + } + ensure_stub(target, kind)?; + Ok(prepared) +} + +/// Whether the canonical source still carries data worth migrating/retiring: +/// a non-empty directory or a non-empty file. +fn has_content(path: &Path, kind: EntryKind) -> bool { + match kind { + EntryKind::Dir => fs::read_dir(path).is_ok_and(|mut dir| dir.next().is_some()), + EntryKind::File => fs::metadata(path).is_ok_and(|meta| meta.len() > 0), + } +} + +fn remove_existing(path: &Path) -> io::Result<()> { + match path.symlink_metadata() { + Ok(meta) if meta.is_dir() => fs::remove_dir_all(path), + Ok(_) => fs::remove_file(path), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +fn create_entry(path: &Path, kind: EntryKind) -> io::Result<()> { + match kind { + EntryKind::Dir => fs::create_dir_all(path), + EntryKind::File => { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map(|_| ()) + } + } +} + +/// Creates the (empty) mountpoint at the canonical path without touching +/// existing content — an existing empty file/dir is reused as the stub. +fn ensure_stub(target: &Path, kind: EntryKind) -> io::Result<()> { + create_entry(target, kind) +} + +fn copy_entry(src: &Path, dst: &Path, kind: EntryKind) -> io::Result<()> { + match kind { + EntryKind::Dir => copy_dir_synced(src, dst), + EntryKind::File => { + if let Some(parent) = dst.parent() { + fs::create_dir_all(parent)?; + } + copy_file_synced(src, dst) + } + } +} + +fn copy_dir_synced(src: &Path, dst: &Path) -> io::Result<()> { + fs::create_dir_all(dst)?; + fs::set_permissions(dst, fs::metadata(src)?.permissions())?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let to = dst.join(entry.file_name()); + let file_type = entry.file_type()?; + if file_type.is_dir() { + copy_dir_synced(&entry.path(), &to)?; + } else if file_type.is_symlink() { + std::os::unix::fs::symlink(fs::read_link(entry.path())?, &to)?; + } else { + copy_file_synced(&entry.path(), &to)?; + } + } + // Per-file fsync does not contractually persist this directory's + // entries; sync the dir itself so the published tree can never be + // durable-but-hollow (replay trusts the final path's existence). + fs::File::open(dst)?.sync_all() +} + +/// `fs::copy` + fsync: the copy must be durable before the rename that +/// publishes it, or a crash could publish a hollow entry. +fn copy_file_synced(src: &Path, dst: &Path) -> io::Result<()> { + fs::copy(src, dst)?; + fs::File::open(dst)?.sync_all() +} + +/// First free `*.pre-ext4[.N]` sibling for retiring the source. A numbered +/// fallback covers state re-accumulated at the canonical path by boots that +/// ran without the metadata volume. +fn free_retired_path(target: &Path) -> io::Result { + let base = path_with_suffix(target, RETIRED_SUFFIX); + if base.symlink_metadata().is_err() { + return Ok(base); + } + for n in 1..100u32 { + let candidate = path_with_suffix(target, &format!("{RETIRED_SUFFIX}.{n}")); + if candidate.symlink_metadata().is_err() { + return Ok(candidate); + } + } + Err(io::Error::other(format!( + "no free {RETIRED_SUFFIX} backup name next to {}", + target.display() + ))) +} + +fn path_with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut os = path.as_os_str().to_owned(); + os.push(suffix); + PathBuf::from(os) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn setup() -> (tempfile::TempDir, PathBuf, PathBuf) { + let tmp = tempfile::tempdir().unwrap(); + let volume = tmp.path().join("volume"); + let data = tmp.path().join("data"); + fs::create_dir_all(&volume).unwrap(); + fs::create_dir_all(&data).unwrap(); + (tmp, volume, data) + } + + fn seed_dir(dir: &Path) { + fs::create_dir_all(dir.join("sub")).unwrap(); + fs::write(dir.join("meta.db"), b"bolt").unwrap(); + fs::write(dir.join("sub/inner.json"), b"{}").unwrap(); + } + + #[test] + fn fresh_install_creates_empty_entries() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Fresh); + assert!(volume.join("docker-network").is_dir()); + assert!(target.is_dir(), "mountpoint stub must exist"); + + let file_target = data.join("overlayfs/metadata.db"); + let out = prepare_entry( + &volume, + &file_target, + "snapshotter-metadata.db", + EntryKind::File, + ) + .unwrap(); + assert_eq!(out, Prepared::Fresh); + assert_eq!( + fs::metadata(volume.join("snapshotter-metadata.db")) + .unwrap() + .len(), + 0 + ); + assert_eq!(fs::metadata(&file_target).unwrap().len(), 0); + } + + #[test] + fn populated_source_is_migrated_and_retired() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Migrated); + let migrated = volume.join("docker-network"); + assert_eq!(fs::read(migrated.join("meta.db")).unwrap(), b"bolt"); + assert_eq!(fs::read(migrated.join("sub/inner.json")).unwrap(), b"{}"); + // Source retired, empty stub in its place. + assert!(data.join("network.pre-ext4").join("meta.db").exists()); + assert!(fs::read_dir(&target).unwrap().next().is_none()); + } + + #[test] + fn torn_copy_is_discarded_and_redone() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + // A crashed previous copy left a partial with garbage. + fs::create_dir_all(volume.join("docker-network.partial")).unwrap(); + fs::write(volume.join("docker-network.partial/meta.db"), b"torn").unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Migrated); + assert!(!volume.join("docker-network.partial").exists()); + assert_eq!( + fs::read(volume.join("docker-network/meta.db")).unwrap(), + b"bolt" + ); + } + + #[test] + fn crash_between_copy_and_retire_converges() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + // The copy landed (final exists) but the retire never ran. + fs::create_dir_all(volume.join("docker-network")).unwrap(); + fs::write(volume.join("docker-network/meta.db"), b"bolt").unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + // Nothing re-copied, but the retire and stub landed. + assert_eq!(out, Prepared::Existing); + assert!(data.join("network.pre-ext4/meta.db").exists()); + assert!(fs::read_dir(&target).unwrap().next().is_none()); + } + + #[test] + fn replay_after_full_migration_is_a_noop() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Existing); + assert!( + !data.join("network.pre-ext4.1").exists(), + "no second backup" + ); + } + + #[test] + fn blank_volume_after_retire_starts_fresh_not_stale() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + // The metadata image was deleted and recreated blank: volume side + // empty, retired backup still on btrfs. + fs::remove_dir_all(volume.join("docker-network")).unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + // The retired backup must NOT be resurrected — that state predates + // whatever the data store has since moved on to. + assert_eq!(out, Prepared::Fresh); + assert!( + fs::read_dir(volume.join("docker-network")) + .unwrap() + .next() + .is_none() + ); + } + + #[test] + fn interim_state_retires_under_numbered_backup() { + let (_tmp, volume, data) = setup(); + let target = data.join("network"); + seed_dir(&target); + prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + // Boots without the volume re-accumulated state at the canonical path. + fs::write(target.join("meta.db"), b"interim").unwrap(); + + let out = prepare_entry(&volume, &target, "docker-network", EntryKind::Dir).unwrap(); + + assert_eq!(out, Prepared::Existing); + assert_eq!( + fs::read(data.join("network.pre-ext4.1/meta.db")).unwrap(), + b"interim" + ); + assert!(fs::read_dir(&target).unwrap().next().is_none()); + } + + #[test] + fn file_entry_migrates_and_stubs() { + let (_tmp, volume, data) = setup(); + let target = data.join("overlayfs/metadata.db"); + fs::create_dir_all(target.parent().unwrap()).unwrap(); + fs::write(&target, b"boltdb-pages").unwrap(); + // A torn file copy is discarded too. + fs::write(volume.join("snapshotter-metadata.db.partial"), b"torn").unwrap(); + + let out = + prepare_entry(&volume, &target, "snapshotter-metadata.db", EntryKind::File).unwrap(); + + assert_eq!(out, Prepared::Migrated); + assert!(!volume.join("snapshotter-metadata.db.partial").exists()); + assert_eq!( + fs::read(volume.join("snapshotter-metadata.db")).unwrap(), + b"boltdb-pages" + ); + assert_eq!( + fs::read(data.join("overlayfs/metadata.db.pre-ext4")).unwrap(), + b"boltdb-pages" + ); + assert_eq!(fs::metadata(&target).unwrap().len(), 0, "0-byte stub"); + } +} diff --git a/internal-docs/plans/ext4-metadata-volume.md b/internal-docs/plans/ext4-metadata-volume.md new file mode 100644 index 000000000..ca60eb1e3 --- /dev/null +++ b/internal-docs/plans/ext4-metadata-volume.md @@ -0,0 +1,262 @@ +# ext4 Metadata Volume — snapshot-prepare fsync fix (ABX-496) + +Status: **Implemented** (arcbox#497 + boot-assets#45 / v0.6.11); results in §9 +Owner: ABX-496 (docker build performance vs Colima) +Companion fix already shipped separately: `rcu_expedited` for runc-create (arcbox#496). + +## 1. Problem and evidence + +Container start on ArcBox spends ~182 ms in containerd snapshot-prepare vs ~20 ms +on Colima. Profiling (static strace with `-y` fd resolution, dockerd debug +lifecycle timeline, `dd conv=fsync` microbench) pinned the cost: + +- A single `fsync` on the guest data volume costs **9.5 ms on ArcBox (btrfs)** + vs **1.0 ms on Colima (ext4)**. The cost is the + btrfs commit path (`write_all_supers`: superblock write + FLUSH barriers), + not COW extent allocation — `chattr +C` (NOCOW) was measured at 9.5 ms vs + 10.5 ms, i.e. no help. + **Attribution correction found during implementation (§9):** most of those + 9.5 ms were not filesystem weight but the *host-side cost of each guest + FLUSH* — the VZ disk attachment defaulted to `synchronizationMode = .full` + (F_FULLFSYNC ≈ 10 ms per barrier), while Colima ships fsync-level + durability. btrfs still amplifies (more FLUSHes per fsync than ext4 + fast_commit) and the ~90 %-boltdb fsync profile below stands, but the + single biggest lever turned out to be the VZ synchronization mode. +- ~90 % of the fsyncs during a container start hit small boltdb metadata + files: containerd `io.containerd.metadata.v1.bolt/meta.db` (dominant), + overlayfs snapshotter `metadata.db`, dockerd `network/files/local-kv.db` + and `image/*.db`. Content-store blobs and snapshot dirs are a small + minority of fsyncs. + +The fsync-hot files are small, incompressible, and rewritten in place; the +bulk data (extracted layers, blobs, volumes) is large and benefits from btrfs +zstd compression. They have opposite filesystem needs — so we split them. + +## 2. Locked design + +Add a second, small disk image formatted **ext4 (journaled, fast_commit)** +that carries only the fsync-hot metadata directories, bind-mounted over their +current btrfs locations. Everything else stays on btrfs unchanged. + +| Decision | Choice | Rationale / rejected alternatives | +|---|---|---| +| Storage for ext4 | Second virtio-blk disk image | Loop file on btrfs rejected: loop FLUSH re-enters the btrfs commit path, keeping the 9.5 ms cost. GPT-partitioning docker.img rejected: destructive migration of existing whole-disk btrfs images. | +| Image name | `-meta.img` derived from the data image (`docker.img` → `docker-meta.img`, `docker-rosetta.img` → `docker-rosetta-meta.img`) | Follows the existing per-VM data-image parameterization (`vm_lifecycle/mod.rs::for_machine`). | +| Virtual size | 2 GiB, sparse (`ensure_sparse_block_image`), no resize path | Holds only bolt DBs + small configs; 2 GiB is ~100× headroom. No resize2fs in guest; fixed size avoids that dependency. | +| Guest device | `/dev/vdc` (third `BlockDeviceConfig` in the Vec; both backends attach in Vec order), **declared** by the host via `arcbox.docker_metadata_device=` on the cmdline when it attaches the disk | Declaration is authoritative: key present → the agent waits for the node and hard-fails if it never appears; key absent (older daemon) → zero-delay skip (an already-present default node is still honored for bespoke e2e configs). Unlike the data device there is no HVC fast path to auto-detect, so no probe-timeout heuristics. HV additionally exposes serial `arcbox-blk-docker-meta.img` via GET_ID. | +| Formatter | `mkfs.ext4` (static e2fsprogs) added to the guest rootfs via boot-assets, invoked guest-side like `mkfs.btrfs` | `arcbox-ext4` crate rejected: its formatter sets no `HAS_JOURNAL` (registry source, `formatter.rs:1107-1111`), and journal-less ext4 has no crash replay — VM force-stop is a routine event and these DBs are the system of record. Host-side formatting impossible (no mke2fs on macOS). | +| mkfs options | `mkfs.ext4 -F -O fast_commit -E lazy_itable_init=0,lazy_journal_init=0 -L arcbox-meta` | fast_commit reduces exactly our fsync pattern (small metadata commits); kernel is 6.x with `CONFIG_EXT4_FS=y` on both arches. Lazy init off: one-time cost at first format, no background trickle. | +| Mount options | `noatime` (defaults otherwise: `data=ordered`, barriers on) | `discard` skipped — bolt files are stable in size, few deletes. | +| Recovery | Ship static `e2fsck` alongside; on mount failure run `e2fsck -y` once, retry mount once, then fail hard | Journal replay is in-kernel; e2fsck covers the residual corruption class. | + +### Directories moved to ext4 (the profiled hot set, nothing more) + +| ext4 volume path | Bind target | Kind | +|---|---|---| +| `containerd-bolt/` | `/var/lib/containerd/io.containerd.metadata.v1.bolt` | dir bind | +| `snapshotter-metadata.db` | `/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/metadata.db` | **file** bind (the dir also holds `snapshots/`, which stays on btrfs) | +| `docker-network/` | `/var/lib/docker/network` | dir bind | +| `docker-image/` | `/var/lib/docker/image` | dir bind | +| `docker-buildkit/` | `/var/lib/docker/buildkit` | dir bind | + +The file bind is safe for boltdb: bolt opens, mmaps, writes in place and +fsyncs — it never renames or recreates its DB file. Both sides of the bind +are pre-created (empty files) before containerd starts; bolt initializes a +0-byte file as a fresh DB. + +EXDEV audit: none of the five targets receives cross-boundary renames. Bolt +never renames; libnetwork and docker's image store use same-directory +tmp+rename atomic writes; buildkit DBs are bolt. Content-store ingest→blobs +and snapshotter tmp→snapshots renames stay entirely within btrfs subtrees. +Any regression here fails loudly (EXDEV errors the operation) and is covered +by the validation suite. + +### Explicitly excluded (with reasons) + +- `/var/lib/docker/containers` — config fsyncs are a minor cost; the dir also + holds json-file logs, which are unbounded and would risk filling the 2 GiB + volume. +- `/var/lib/docker/volumes/metadata.db`, `builder/`, `trust/` — cold paths, + not in the profile. +- Distro machines (`data.img`, machine-init.sh) — out of scope; System VM + and Rosetta VM only (both go through `create_default_machine`). + +### NOCOW removal rides along + +`btrfs.rs::disable_cow_on_metadata_dirs` is removed in the same change. Its +premise (NOCOW reduces bolt fsync amplification) is disproven by measurement +(9.5 vs 10.5 ms), the dirs it targeted either move to ext4 or are cold, and +its inherited NOCOW flag on `io.containerd.snapshotter.v1.overlayfs` silently +disables zstd compression for every extracted layer file — the one thing +btrfs is being kept for. Existing installs keep the on-disk flag on old dirs +(only re-flagging stops); new snapshot dirs regain compression. + +## 3. Guest changes (`guest/arcbox-agent`) + +New module `agent/linux/metadata_volume.rs`, called from +`try_start_bundled_runtime` (`runtime.rs`) immediately after +`ensure_data_mount()` — i.e. after btrfs targets exist, before the NFS export +and before containerd/dockerd start (bolt files guaranteed closed): + +1. Resolve device from the host's cmdline declaration; wait up to 5 s for + the declared node (same loop as `btrfs.rs`) and hard-fail if it never + appears. No declaration → skip immediately (older daemon), unless the + default node already exists (bespoke e2e configs). +2. Probe ext4 superblock magic (0xEF53 at offset 0x438). If absent → + `mkfs.ext4` with the locked options. +3. Mount at `/run/arcbox/metadata` (`noatime`). On failure: `e2fsck -y`, + retry once, else hard-fail. +4. Per-mapping migrate-then-bind, idempotent and crash-safe per entry: + 1. If the ext4-side final entry is absent: if the btrfs-side source + exists at its canonical path, copy it (recursive, pure Rust) to + `.partial` on the volume, fsync, then rename to final — + an interrupted copy leaves only a `.partial` that is discarded and + redone next boot, never a half-populated final that would get bound. + If no source exists, create the entry empty (fresh install). + 2. If the ext4-side final exists AND the btrfs-side source is still at + its canonical path, rename the source to `.pre-ext4` (covers a + crash between step 1 and this rename). The rename — rather than + leaving the source in place — is load-bearing: it is what prevents a + later blank metadata volume (user deleted the image; host recreated + it) from silently re-migrating stale pre-upgrade metadata against a + data store that has moved on. The renamed copy doubles as a manual + recovery artifact. + 3. Ensure an empty mountpoint stub exists at the canonical path + (dir, or 0-byte file for the metadata.db mapping), then bind. + +Failure policy (version-skew safe): + +| Condition | Behavior | +|---|---| +| No cmdline declaration and no default node (older daemon without the third disk) | warn + continue on btrfs, zero probe delay — perf-only degradation | +| Device declared but the node never appears | **hard fail** — the host promised the disk; a silent skip could boot dockerd against stale shadowed state | +| mkfs binary absent AND device blank (older rootfs) | warn + continue — nothing was ever migrated, no split-brain possible | +| Device present with ext4 but mount/fsck fails | **hard fail** runtime start — booting dockerd against the stale shadowed btrfs DBs would fork state | + +Constants: `DOCKER_METADATA_BLOCK_DEVICE = "/dev/vdc"` +(`arcbox-constants/src/devices.rs`) and the cmdline key +(`arcbox-constants/src/cmdline.rs`), mirroring the data-device pair. + +The Kubernetes path (`kubernetes.rs`) is untouched — k3s state lives under +`/var/lib/rancher` and none of the five targets. + +### User migration matrix + +| Scenario | Outcome | +|---|---| +| Fresh install | Format + empty entries; no migration. | +| Upgrade with populated docker.img | Atomic per-mapping migration (above); docker state fully preserved; sources renamed to `*.pre-ext4` on btrfs. | +| metadata.img deleted or corrupted by the user after migration | Host recreates a blank image; guest formats; sources are renamed away, so the result is a clean **empty** docker state (images re-pull; orphaned blobs/snapshots leak on btrfs but state is consistent) — never a resurrection of stale pre-upgrade metadata. | +| Downgrade to a pre-feature release | **Unsupported (one-way door).** Old binaries mount btrfs only and see the empty mountpoint stubs → empty docker state, consistent. Manual recovery: rename the `*.pre-ext4` dirs back, restoring the state as of the moment of upgrade. Document this in the release notes. | +| Backups / moving data dirs by hand | `docker.img` and `docker-meta.img` are a paired set; copying one without the other loses metadata or orphans data. Documented in `docs/data-directories.md`; `resource_cleanup` already treats them as a unit. | + +## 4. Host changes (`app/`) + +- `vm_lifecycle/mod.rs`: `DOCKER_METADATA_IMAGE_SIZE_BYTES` (2 GiB); + metadata filename derived next to `data_image_filename`. +- `vm_lifecycle/boot.rs::create_default_machine`: + `ensure_sparse_block_image` + third `block_devices.push` (rw); update the + vda/vdb doc comment to cover vdc. +- `arcbox-daemon/src/startup/resource_cleanup.rs::DISK_IMAGE_NAMES`: add + `docker-meta.img`, `docker-rosetta-meta.img` (the pair is a unit — deleting + one without the other orphans state). +- `arcbox-core/src/config.rs`: `docker_meta_img_path()` accessor; + `arcbox-cli/commands/disk.rs`: include it in usage reporting. +- `docs/data-directories.md`: add the `data/docker-meta.img` row. + +No dockerd/containerd config changes — the split is invisible to both. + +## 5. boot-assets changes (sibling repo) + +- `src/build/scripts/build-rootfs-binaries.sh`: static e2fsprogs build from + the upstream source tarball with `LDFLAGS=-static`, mirroring the + btrfs-progs recipe (e2fsprogs vendors its own libuuid/libblkid); stage + `mke2fs` (installed as `/sbin/mkfs.ext4`) and `/sbin/e2fsck`. +- `src/build/rootfs.rs`: extend `CORE_STATIC_BINARIES` + `copy_executable` + lines; the static-check loop picks them up. +- Tag `v0.6.11` → release publishes → arcbox bumps `assets.lock` + (`[boot] version` + `manifest_sha256`) in the same PR that ships the + daemon/agent changes (the lock is compile-time embedded; daemon rebuild + required). + +## 6. NFS / `~/ArcBox` interplay + +None required. The docker export is a **non-recursive** read-only bind +(`nfs.rs::bind_readonly`, plain `MS_BIND`), so the new submounts do not +surface over NFS; `~/ArcBox/network|image|buildkit` show the (empty, +shadowed) underlying @docker dirs. These are internal dirs with no browsing +use case. No new exports, no fsid allocation. + +## 7. Validation and acceptance + +1. Unit: migrate-then-bind decision logic and superblock probe + (pure functions, `cargo test -p arcbox-agent` host-side where possible; + musl cross-compile for the rest). Must cover every crash window of the + migration algorithm: interruption during copy (`.partial` present), + between ext4 rename and btrfs source rename, and re-entry after each — + each replay must converge on the same final state. +2. e2e: `docker_build` suite D1–D10 green (VZ), `boot_assets` green on VZ + **and** HV (`ARCBOX_VM_BACKEND=hv`) — covers full Docker lifecycle over + the new mounts; any EXDEV or bind mistake fails these loudly. +3. Migration: boot a data dir populated by current master (images + + containers present) with the new build; `docker images` / `docker ps -a` + intact; bolt DBs physically on the ext4 volume (`findmnt`, file sizes). +4. Version-skew: new agent + two-disk-less daemon boots with the warn path; + fresh install formats and mounts cleanly. +5. Perf acceptance (the point of it all), measured via dockerd debug + lifecycle timeline as in the ABX-496 campaign: + - guest `dd conv=fsync` on the metadata mount ≈ 1 ms (vs 9.5 ms); + - snapshot-prepare 182 ms → **< 60 ms**; + - combined with the shipped rcu_expedited fix, `docker run` container + start approaches Colima parity; re-run the bench table in + `internal-docs/plans/docker-build-e2e-matrix.md` and record the delta + on ABX-496. + +## 8. Rollout sequence + +1. **PR 1 (boot-assets)**: e2fsprogs in rootfs; tag `v0.6.11`. Inert for + existing arcbox releases. +2. **PR 2 (arcbox)**: constants + host attach/cleanup/CLI + guest + `metadata_volume.rs` + NOCOW removal + `assets.lock` bump + doc updates, + as separate atomic commits. +3. Bench + record results on ABX-496; close the snapshot-prepare half of the + issue. + +## 9. Implementation results (2026-07-22) + +Shipped as boot-assets#45 (released v0.6.11) + arcbox#497. Two deviations +from the plan, both improvements: + +- **Device discovery**: instead of guest-side probing with a 5 s timeout, + the host *declares* the device via `arcbox.docker_metadata_device=/dev/vdc` + on the cmdline when it attaches the disk (§2 table updated). Declared but + missing → hard fail; undeclared → zero-delay skip. +- **VZ synchronization mode** (`arcbox-vz` shim): the disk attachment now + uses `synchronizationMode: .fsync` instead of the default `.full` + (F_FULLFSYNC per guest FLUSH). This aligns VZ with the custom-HV backend, + whose block worker has always used plain fsync, and with the durability + level Colima/OrbStack ship. Discovered because the metadata volume alone + moved the microbench barely at all — see the A/B below. + +Measured on the same host, guest `dd bs=1M count=1 conv=fsync`, avg of 20: + +| Configuration | ext4 metadata volume | btrfs data volume | +|---|---|---| +| VZ, `.full` (before) | 10.3 ms | 11.5 ms | +| VZ, `.fsync` (after) | **1.7 ms** | 2.1 ms | +| HV (plain fsync, unchanged) | 2.2 ms | 2.6 ms | +| Colima (reference) | — | 1.0 ms (ext4) | + +End-to-end (10-run averages, same day, same host, VZ): + +| Metric | ArcBox before campaign | ArcBox after | Colima | +|---|---|---|---| +| `docker create` (contains snapshot-prepare, was ~182 ms alone) | — | **53 ms** | 48 ms | +| `docker start` | — | 416 ms | 82 ms | +| `docker run --rm alpine true` | ~700 ms | 496 ms | 173 ms | + +Acceptance met: snapshot-prepare cost is gone from the create path +(53 ms total vs the < 60 ms target for the prepare step alone) and guest +fsync is at Colima's order of magnitude. The remaining `docker start` gap +(~330 ms vs Colima) sits in the network-endpoint/iptables portion of start +— outside both ABX-496 root causes; file separately. diff --git a/virt/arcbox-vz/shim/Sources/ArcBoxVZShim/Devices.swift b/virt/arcbox-vz/shim/Sources/ArcBoxVZShim/Devices.swift index 14d3e6996..ff28395f2 100644 --- a/virt/arcbox-vz/shim/Sources/ArcBoxVZShim/Devices.swift +++ b/virt/arcbox-vz/shim/Sources/ArcBoxVZShim/Devices.swift @@ -12,7 +12,18 @@ func storageDiskImageNew( ) -> UnsafeMutableRawPointer? { let url = URL(fileURLWithPath: String(cString: path)) do { - let attachment = try VZDiskImageStorageDeviceAttachment(url: url, readOnly: readOnly) + // fsync-level durability, NOT the default .full (F_FULLFSYNC): a + // guest FLUSH under .full costs ~10 ms of host F_FULLFSYNC per + // barrier — the dominant cost of every guest fsync (ABX-496). The + // custom-HV backend's block worker has always used plain fsync for + // guest FLUSH, so .fsync makes both backends give the same + // power-loss window; Colima/OrbStack ship the same durability level. + let attachment = try VZDiskImageStorageDeviceAttachment( + url: url, + readOnly: readOnly, + cachingMode: .automatic, + synchronizationMode: .fsync + ) return abxRetainedHandle(VZVirtioBlockDeviceConfiguration(attachment: attachment)) } catch { errorOut?.pointee = abxErrorString(error)