perf(storage): ext4 metadata volume for fsync-hot boltdb state (ABX-496)

app/arcbox-cli/src/commands/disk.rs

@@ -103,6 +103,22 @@ async fn execute_usage() -> Result<()> {
     );
     println!("  Reclaimable: {:.1} GiB", usage.reclaimable_gib());
 
+    // The metadata image is small but part of the data set; report it so
+    // "what is on disk" is complete.
+    let meta_path = config.docker_meta_img_path();
+    if meta_path.exists() {
+        let meta = read_disk_usage(&meta_path)?;
+        println!();
+        println!("Docker metadata disk:");
+        println!("  Path:        {}", meta_path.display());
+        println!("  Logical:     {:.1} GiB", meta.logical_gib());
+        println!(
+            "  Physical:    {:.1} GiB   ({:.1}%)",
+            meta.physical_gib(),
+            meta.usage_pct()
+        );
+    }
+
     Ok(())
 }
 

app/arcbox-core/src/config.rs

@@ -179,6 +179,15 @@ impl Config {
     pub fn docker_img_path(&self) -> PathBuf {
         self.data_subdir().join("docker.img")
     }
+
+    /// Returns the path to the Docker metadata image (`data/docker-meta.img`).
+    ///
+    /// Paired with [`Self::docker_img_path`]: the ext4 volume carrying the
+    /// fsync-hot boltdb metadata (see internal-docs/plans/ext4-metadata-volume.md).
+    #[must_use]
+    pub fn docker_meta_img_path(&self) -> PathBuf {
+        self.data_subdir().join("docker-meta.img")
+    }
 }
 
 /// Default VM configuration.

app/arcbox-core/src/vm_lifecycle/boot.rs

@@ -16,13 +16,15 @@ use crate::error::{CoreError, Result};
 use crate::event::Event;
 use crate::machine::MachineConfig;
 use arcbox_constants::cmdline::{
-    DEBUG_CONSOLE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY, HV_EARLYCON_DIRECTIVE,
+    DEBUG_CONSOLE_KEY, DOCKER_METADATA_DEVICE_KEY, GUEST_DOCKER_VSOCK_PORT_KEY,
+    HV_EARLYCON_DIRECTIVE,
 };
+use arcbox_constants::devices::DOCKER_METADATA_BLOCK_DEVICE;
 use arcbox_error::CommonError;
 
 use super::actor::{Completion, InternalEvent, LifecycleShared};
-use super::types::{DesiredBoot, machine_drift_reason};
-use super::{DOCKER_DATA_IMAGE_SIZE_BYTES, RecoveryAction};
+use super::types::{DesiredBoot, machine_drift_reason, metadata_image_filename};
+use super::{DOCKER_DATA_IMAGE_SIZE_BYTES, DOCKER_METADATA_IMAGE_SIZE_BYTES, RecoveryAction};
 
 impl LifecycleShared {
     /// Boots the VM end-to-end (create if needed, start with retries, wait for
@@ -342,7 +344,8 @@ impl LifecycleShared {
     ///
     /// Block devices:
     /// - vda: rootfs.erofs (read-only)
-    /// - vdb: docker-data.img (read-write)
+    /// - vdb: docker-data.img (read-write, btrfs bulk data)
+    /// - vdc: docker-meta.img (read-write, ext4 metadata volume)
     async fn create_default_machine(&self) -> Result<()> {
         let boot = self.resolve_desired_boot().await?;
         let rootfs_path = boot.rootfs_image.to_string_lossy().to_string();
@@ -368,6 +371,20 @@ impl LifecycleShared {
             read_only: false,
         });
 
+        // Attach the ext4 metadata volume (vdc): the fsync-hot boltdb
+        // metadata lives there while bulk data stays on the btrfs data disk.
+        // The two images are a paired set — see
+        // internal-docs/plans/ext4-metadata-volume.md.
+        let metadata_image = self
+            .data_dir
+            .join(arcbox_constants::paths::host::DATA)
+            .join(metadata_image_filename(&self.data_image_filename));
+        ensure_sparse_block_image(&metadata_image, DOCKER_METADATA_IMAGE_SIZE_BYTES)?;
+        block_devices.push(crate::vm::BlockDeviceConfig {
+            path: metadata_image.to_string_lossy().to_string(),
+            read_only: false,
+        });
+
         let config = MachineConfig {
             name: self.machine_name.clone(),
             cpus: self.config.default_vm.cpus,
@@ -444,6 +461,20 @@ impl LifecycleShared {
             }
         }
 
+        // Declare the ext4 metadata device this machine attaches as vdc.
+        // Unlike the data device (auto-detected for its HVC fast path), the
+        // declaration is authoritative: key present → the agent waits for
+        // the node and hard-fails if it never appears; key absent (older
+        // daemon) → the agent skips the metadata volume without probing.
+        if !cmdline
+            .split_whitespace()
+            .any(|token| token.starts_with(DOCKER_METADATA_DEVICE_KEY))
+        {
+            cmdline.push(' ');
+            cmdline.push_str(DOCKER_METADATA_DEVICE_KEY);
+            cmdline.push_str(DOCKER_METADATA_BLOCK_DEVICE);
+        }
+
         // Always attach an interactive debug console on the custom-HV backend.
         // An operator can `socat - UNIX-CONNECT:<sock>` to get a serial root
         // shell into the guest even when early boot hangs before networking

app/arcbox-core/src/vm_lifecycle/mod.rs

@@ -84,6 +84,12 @@ const DOCKER_DATA_IMAGE_NAME: &str = "docker.img";
 /// only consumes actual disk space for written blocks. 8 TiB matches OrbStack
 /// and prevents users from hitting artificial limits.
 const DOCKER_DATA_IMAGE_SIZE_BYTES: u64 = 8 * 1024 * 1024 * 1024 * 1024;
+/// Persistent guest metadata image size (2 GiB sparse file).
+///
+/// The ext4 volume holds only the fsync-hot boltdb metadata directories —
+/// bulk data stays on the btrfs data image — so 2 GiB is ~100x headroom.
+/// See internal-docs/plans/ext4-metadata-volume.md.
+const DOCKER_METADATA_IMAGE_SIZE_BYTES: u64 = 2 * 1024 * 1024 * 1024;
 
 pub(crate) use boot::ensure_sparse_block_image;
 pub use health::HealthMonitor;

app/arcbox-core/src/vm_lifecycle/tests.rs

@@ -1,5 +1,5 @@
 use super::boot::{agent_timeout_error, ensure_earlycon, ensure_sparse_block_image};
-use super::types::{DesiredBoot, machine_drift_reason};
+use super::types::{DesiredBoot, machine_drift_reason, metadata_image_filename};
 use super::*;
 use crate::machine::MachineState;
 use arcbox_constants::cmdline::HV_EARLYCON_DIRECTIVE;
@@ -152,7 +152,20 @@ fn sample_machine(cpus: u32, memory_mb: u64, kernel: &str, cmdline: &str) -> Mac
         disk_gb: 50,
         kernel: Some(kernel.to_string()),
         cmdline: Some(cmdline.to_string()),
-        block_devices: Vec::new(),
+        block_devices: vec![
+            crate::vm::BlockDeviceConfig {
+                path: "/rootfs.erofs".to_string(),
+                read_only: true,
+            },
+            crate::vm::BlockDeviceConfig {
+                path: "/data/docker.img".to_string(),
+                read_only: false,
+            },
+            crate::vm::BlockDeviceConfig {
+                path: "/data/docker-meta.img".to_string(),
+                read_only: false,
+            },
+        ],
         distro: None,
         distro_version: None,
         disk_path: None,
@@ -197,7 +210,25 @@ fn machine_drift_detects_each_overridable_field() {
 
     // The cmdline gap that previously slipped through (e.g. arm64.nosve
     // added/removed without bumping the boot-asset version).
-    let mut m = current;
+    let mut m = current.clone();
     m.cmdline = Some("console=hvc0 earlycon arm64.nosve".to_string());
     assert_eq!(machine_drift_reason(&m, &want, &boot), Some("cmdline"));
+
+    // A machine persisted before the ext4 metadata volume (two disks) must
+    // be recreated so the guest receives vdc.
+    let mut m = current;
+    m.block_devices.pop();
+    assert_eq!(
+        machine_drift_reason(&m, &want, &boot),
+        Some("block_devices")
+    );
+}
+
+#[test]
+fn metadata_image_filename_pairs_with_data_image() {
+    assert_eq!(metadata_image_filename("docker.img"), "docker-meta.img");
+    assert_eq!(
+        metadata_image_filename("docker-rosetta.img"),
+        "docker-rosetta-meta.img"
+    );
 }

app/arcbox-core/src/vm_lifecycle/types.rs

@@ -188,7 +188,26 @@ pub(super) fn machine_drift_reason(
         Some("kernel")
     } else if persisted.cmdline.as_deref() != Some(boot.cmdline.as_str()) {
         Some("cmdline")
+    } else if persisted.block_devices.len() != DEFAULT_MACHINE_DISK_COUNT {
+        // A machine persisted before the ext4 metadata volume carries only
+        // two disks; recreating rewrites the machine record (image files are
+        // untouched) so the guest receives vdc and can migrate.
+        Some("block_devices")
     } else {
         None
     }
 }
+
+/// Number of block devices `create_default_machine` attaches: EROFS rootfs
+/// (vda), btrfs data image (vdb), ext4 metadata image (vdc).
+pub(super) const DEFAULT_MACHINE_DISK_COUNT: usize = 3;
+
+/// Derives the metadata-volume image filename paired with a data image:
+/// `docker.img` → `docker-meta.img`, `docker-rosetta.img` →
+/// `docker-rosetta-meta.img`.
+pub(super) fn metadata_image_filename(data_image_filename: &str) -> String {
+    let stem = data_image_filename
+        .strip_suffix(".img")
+        .unwrap_or(data_image_filename);
+    format!("{stem}-meta.img")
+}

app/arcbox-daemon/src/startup/resource_cleanup.rs

@@ -4,7 +4,12 @@ use std::path::{Path, PathBuf};
 
 use arcbox_core::persistence::MachinePersistence;
 
-const DISK_IMAGE_NAMES: [&str; 2] = ["docker.img", "docker-rosetta.img"];
+const DISK_IMAGE_NAMES: [&str; 4] = [
+    "docker.img",
+    "docker-meta.img",
+    "docker-rosetta.img",
+    "docker-rosetta-meta.img",
+];
 
 /// Why startup must scan for stale disk-image holders.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]

assets.lock

@@ -14,9 +14,9 @@
 # and the [boot] bump.
 
 [boot]
-version = "0.6.10"
+version = "0.6.11"
 cdn = "https://boot.arcboxcdn.com"
-manifest_sha256 = "f417bfb4ac2f9c0ba6926266ea8f7902e5b53eb0762e423df780fd5bd333122f"
+manifest_sha256 = "fc5130e3e64e75bd2df7bae2de7ff4fe790cb18d04db8f01c441156a662d59b5"
 
 [[tools]]
 name = "docker"

common/arcbox-constants/src/cmdline.rs

@@ -4,6 +4,9 @@ pub const GUEST_DOCKER_VSOCK_PORT_KEY: &str = "arcbox.guest_docker_vsock_port=";
 /// Kernel cmdline key for guest Docker data block-device path.
 pub const DOCKER_DATA_DEVICE_KEY: &str = "arcbox.docker_data_device=";
 
+/// Kernel cmdline key for guest Docker metadata block-device path.
+pub const DOCKER_METADATA_DEVICE_KEY: &str = "arcbox.docker_metadata_device=";
+
 /// Kernel cmdline key carrying the host path of the interactive debug-console
 /// Unix socket (custom-HV backend).
 ///

common/arcbox-constants/src/devices.rs

@@ -3,3 +3,9 @@ pub const ROOT_BLOCK_DEVICE: &str = "/dev/vda";
 
 /// Default Docker data block device path in guest.
 pub const DOCKER_DATA_BLOCK_DEVICE: &str = "/dev/vdb";
+
+/// Default Docker metadata block device path in guest.
+///
+/// The ext4 volume carrying the fsync-hot boltdb metadata directories; paired
+/// with the btrfs data device (see internal-docs/plans/ext4-metadata-volume.md).
+pub const DOCKER_METADATA_BLOCK_DEVICE: &str = "/dev/vdc";

docs/daemon-lifecycle.md

@@ -16,7 +16,7 @@ acquire_daemon_lease     flock(daemon.lock), terminate stale     ~instant or ≤
 start_control_plane      Bind arcbox.sock, SystemService up      ~instant
     │                    Desktop can connect from this point on.
     │
-release_stale_resources  Wait for docker.img holders to release  0–10 s
+release_stale_resources  Wait for disk-image holders to release  0–10 s
     │                    Reported as CLEANING_UP via gRPC.
     │
 prepare_assets           Seed/download boot assets               variable
@@ -120,7 +120,7 @@ signal received
 | `daemon.lock` | exists, old PID, **lock released** | `try_flock` succeeds instantly |
 | `docker.sock` | deleted | — |
 | `arcbox.sock` | deleted | — |
-| `docker.img` | exists, no holders | — |
+| disk images (`docker.img` + `docker-meta.img`, Rosetta counterparts) | exist, no holders | — |
 | VM | gracefully stopped | — |
 
 No manual intervention needed.
@@ -141,7 +141,7 @@ signal received during startup
   │                          graceful Stop behind itself, so an unbounded
   │                          wait could last the whole boot timeout
   └─ on timeout / second signal → runtime.shutdown_force()  → VM killed,
-                                  no orphaned XPC helpers holding docker.img
+                                  no orphaned XPC helpers holding disk images
 ```
 
 `early_runtime` is filled right after `Runtime` construction, before the
@@ -161,7 +161,7 @@ When the daemon is killed without graceful shutdown:
 - Socket files are **not** cleaned up.
 - VM is **not** gracefully stopped.
 - Container subnet route is **not** removed.
-- `docker.img` may still be held by Virtualization.framework XPC helpers.
+- The disk images (`docker.img`, `docker-meta.img`, Rosetta counterparts) may still be held by Virtualization.framework XPC helpers.
 
 ### Residual state after crash
 
@@ -170,7 +170,7 @@ When the daemon is killed without graceful shutdown:
 | `daemon.lock` | exists, old PID, **lock released** | `try_flock` succeeds instantly |
 | `docker.sock` | **stale** | `DockerApiServer::run` removes before bind |
 | `arcbox.sock` | **stale** | `start_grpc` removes before bind |
-| `docker.img` | **possibly held by XPC helpers** | `wait_for_resources` waits up to 10 s |
+| disk images | **possibly held by XPC helpers** | `wait_for_resources` waits up to 10 s |
 | VM | non-graceful termination | Virtualization.framework cleans up |
 | Route | **stale** | `recovery::run()` rebuilds |
 
@@ -189,10 +189,10 @@ When a new daemon starts while an old one is still running:
 6. Falls back to SIGKILL if unresponsive.
 7. Acquires the lock once released.
 8. `start_grpc` removes any stale sockets before binding.
-9. `wait_for_resources` waits for `docker.img` holders to release.
+9. `wait_for_resources` waits for disk-image holders to release.
 
 The old daemon's graceful shutdown runs its full sequence (drain, VM stop,
-socket cleanup). The new daemon only needs to handle the `docker.img`
+socket cleanup). The new daemon only needs to handle the disk-image
 holdover case.
 
 ## Socket Lifecycle
@@ -210,10 +210,10 @@ socket that another component has already bound.
 
 ## Edge Cases
 
-### docker.img held by orphaned XPC helpers
+### Disk images held by orphaned XPC helpers
 
 Virtualization.framework spawns XPC helper processes that may outlive the
-daemon. These processes hold `docker.img` open. The daemon waits up to
+daemon. These processes hold the disk images (`docker.img`, `docker-meta.img`, Rosetta counterparts) open. The daemon waits up to
 10 s for them to exit (`wait_for_resources`), then proceeds. If they
 persist, `init_runtime` may fail because the disk image is locked.
 

docs/data-directories.md

@@ -57,6 +57,7 @@ Defined in `app/arcbox-core/src/config.rs`.
 | `data/machines/` | Virtual machine data | daemon |
 | `data/volumes/` | Named volumes | daemon |
 | `data/docker.img` | Docker persistent disk image (Btrfs) | daemon |
+| `data/docker-meta.img` | Docker metadata disk image (ext4, fsync-hot boltdb state); paired with `docker.img` — back up or move the two together | daemon |
 
 ### 1.4 `boot/` — Boot Asset Cache