diff --git a/example/bcache-subvolumes.nix b/example/bcache-subvolumes.nix new file mode 100644 index 000000000..b552b2875 --- /dev/null +++ b/example/bcache-subvolumes.nix @@ -0,0 +1,74 @@ +{ + disko.devices = { + disk = { + fast = { + type = "disk"; + device = "/dev/cache-disk"; + content = { + type = "gpt"; + partitions = { + boot = { + size = "500M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + }; + }; + root = { + size = "2G"; + content = { + type = "btrfs"; + mountpoint = "/"; + }; + }; + cache = { + size = "100%"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + }; + }; + }; + + large = { + type = "disk"; + device = "/dev/backing-disk"; + content = { + type = "gpt"; + partitions = { + storage = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + }; + }; + }; + + bcache = { + main = { + type = "bcache"; + device = "/dev/bcache0"; + cacheMode = "writeback"; + content = { + type = "btrfs"; + subvolumes = { + "@data" = { + mountpoint = "/data"; + }; + "@nix-store" = { + mountpoint = "/nix/store"; + }; + }; + }; + }; + }; + }; +} diff --git a/example/bcache.nix b/example/bcache.nix new file mode 100644 index 000000000..485a97cc9 --- /dev/null +++ b/example/bcache.nix @@ -0,0 +1,62 @@ +{ + disko.devices = { + disk = { + cache-disk = { + type = "disk"; + device = "/dev/my-cache-disk"; + content = { + type = "gpt"; + partitions = { + boot = { + size = "1M"; + type = "EF02"; # for grub MBR + }; + boot-fs = { + size = "500M"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/boot"; + }; + }; + cache = { + size = "100%"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + }; + }; + }; + backing-disk = { + type = "disk"; + device = "/dev/my-backing-disk"; + content = { + type = "gpt"; + partitions = { + backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + }; + }; + }; + bcache = { + main = { + type = "bcache"; + device = "/dev/bcache0"; + cacheMode = "writeback"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }; + }; +} diff --git a/lib/default.nix b/lib/default.nix index 39ccf849d..c727b88a8 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -79,6 +79,8 @@ let _partitionTypes = { inherit (diskoLib.types) bcachefs + bcache_cache + bcache_backing btrfs filesystem zfs @@ -105,6 +107,8 @@ let _deviceTypes = { inherit (diskoLib.types) bcachefs + bcache_cache + bcache_backing table gpt btrfs @@ -684,6 +688,7 @@ let let devices = { inherit (cfg.config) + bcache bcachefs_filesystems disk mdadm @@ -692,9 +697,133 @@ let nodev ; }; + stopBcacheDevices = lib.optionalString (devices.bcache != { }) '' + modprobe bcache || true + # shellcheck disable=SC2043 + for dev in ${ + lib.escapeShellArgs ( + map (bcache: builtins.baseNameOf bcache.device) (lib.attrValues devices.bcache) + ) + }; do + if [ -e "/sys/block/$dev/bcache/detach" ]; then + echo 1 > "/sys/block/$dev/bcache/detach" 2>/dev/null || true + fi + if [ -e "/sys/block/$dev/bcache/stop" ]; then + echo 1 > "/sys/block/$dev/bcache/stop" 2>/dev/null || true + fi + done + # Destructive paths wipe configured disks next, so unregister all + # active bcache cache sets to release any member devices still held. + find /sys/fs/bcache -maxdepth 1 -mindepth 1 -type d -exec sh -c ' + for cset; do + if [ -e "$cset/unregister" ]; then + echo 1 > "$cset/unregister" 2>/dev/null || true + fi + done + ' _ {} + 2>/dev/null || true + udevadm settle --timeout=10 || true + # shellcheck disable=SC2043 + for dev in ${ + lib.escapeShellArgs ( + map (bcache: builtins.baseNameOf bcache.device) (lib.attrValues devices.bcache) + ) + }; do + for i in $(seq 1 30); do + [ ! -e "/sys/block/$dev" ] && break + udevadm settle --timeout=1 || true + sleep 1 + done + if [ -e "/sys/block/$dev" ]; then + printf "\033[31mERROR:\033[0m bcache device /dev/%s did not stop\n" "$dev" >&2 + exit 1 + fi + done + ''; + bcacheMeta = cfg.config._meta.bcache or { }; + bcacheCacheMembers = bcacheMeta.cache or [ ]; + bcacheBackingMembers = bcacheMeta.backing or [ ]; + bcacheMembers = + (map (member: member // { role = "bcache_cache"; }) bcacheCacheMembers) + ++ (map (member: member // { role = "bcache_backing"; }) bcacheBackingMembers); + bcacheSetNames = lib.attrNames devices.bcache; + referencedBcacheSetNames = lib.unique (map (member: member.set) bcacheMembers); + bcacheDevicesByPath = lib.groupBy (bcache: bcache.device) (lib.attrValues devices.bcache); + duplicateBcacheDevices = lib.filterAttrs (_device: sets: lib.length sets != 1) bcacheDevicesByPath; + bcacheMembersByPath = lib.groupBy (member: member.device) bcacheMembers; + duplicateBcacheMemberDevices = lib.filterAttrs ( + _device: members: lib.length members != 1 + ) bcacheMembersByPath; + invalidBcacheDevices = lib.concatLists ( + lib.mapAttrsToList ( + set: bcache: + lib.optional ( + builtins.match "/dev/bcache[0-9]+" bcache.device == null + ) "bcache set \"${set}\" device must be /dev/bcacheN, got \"${bcache.device}\"" + ) devices.bcache + ); + invalidBcacheMemberDevices = lib.concatMap ( + member: + (lib.optional (builtins.match "/dev/.+" member.device == null) + "${member.role} for bcache set \"${member.set}\" device must be an absolute /dev/... path, got \"${member.device}\"" + ) + ++ (lib.optional (builtins.match "/dev/bcache[0-9]+" member.device != null) + "${member.role} for bcache set \"${member.set}\" must use a member device, not bcache output device \"${member.device}\"" + ) + ) bcacheMembers; + bcacheMemberErrors = + (lib.concatMap ( + set: + lib.optional ( + !(lib.elem set bcacheSetNames) + ) "bcache member references undefined bcache set \"${set}\"" + ) referencedBcacheSetNames) + ++ (lib.concatMap ( + set: + let + caches = lib.filter (member: member.set == set) bcacheCacheMembers; + backings = lib.filter (member: member.set == set) bcacheBackingMembers; + cacheDevices = map (member: member.device) caches; + backingDevices = map (member: member.device) backings; + in + (lib.optional (lib.length caches != 1) + "bcache set \"${set}\" needs exactly one bcache_cache partition, found ${toString (lib.length caches)}" + ) + ++ (lib.optional (lib.length backings != 1) + "bcache set \"${set}\" needs exactly one bcache_backing partition, found ${toString (lib.length backings)}" + ) + ++ (lib.optional + ( + lib.length caches == 1 + && lib.length backings == 1 + && lib.head cacheDevices == lib.head backingDevices + ) + "bcache set \"${set}\" uses the same member device for cache and backing: \"${lib.head cacheDevices}\"" + ) + ) bcacheSetNames) + ++ (lib.mapAttrsToList ( + device: sets: "bcache device \"${device}\" is used by ${toString (lib.length sets)} bcache sets" + ) duplicateBcacheDevices) + ++ (lib.mapAttrsToList ( + device: members: + "bcache member device \"${device}\" is used by ${toString (lib.length members)} bcache roles" + ) duplicateBcacheMemberDevices) + ++ invalidBcacheDevices + ++ invalidBcacheMemberDevices; + validateBcacheMembers = + if bcacheMemberErrors == [ ] then + true + else + throw "Invalid bcache disko configuration:\n${ + lib.concatMapStringsSep "\n" (msg: " - ${msg}") bcacheMemberErrors + }"; in { options = { + bcache = lib.mkOption { + type = lib.types.attrsOf diskoLib.types.bcache; + default = { }; + description = "bcache device (cache + backing)"; + }; bcachefs_filesystems = lib.mkOption { type = lib.types.attrsOf diskoLib.types.bcachefs_filesystem; default = { }; @@ -769,6 +898,8 @@ let # @todo Do we need to add bcachefs-tools or not? destroyDependencies = with pkgs; [ util-linux + bcache-tools + kmod e2fsprogs mdadm zfs @@ -780,160 +911,162 @@ let coreutils-full ]; in - lib.mapAttrs throwIfNoDisksDetected { - destroy = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-destroy" '' - export PATH=${lib.makeBinPath destroyDependencies}:$PATH - ${cfg.config._destroy} - ''; - format = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-format" '' - export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH - ${cfg.config._create} - ''; - mount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-mount" '' - export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH - ${cfg.config._mount} - ''; - unmount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-unmount" '' - export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH - ${cfg.config._unmount} - ''; - formatMount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-format-mount" '' - export PATH=${lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ])}:$PATH - ${cfg.config._formatMount} - ''; - destroyFormatMount = - (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-destroy-format-mount" - '' - export PATH=${ - lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ] ++ destroyDependencies) - }:$PATH - ${cfg.config._destroyFormatMount} - ''; - - # These are useful to skip copying executables uploading a script to an in-memory installer - destroyNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "/bin/disko-destroy" - '' - ${cfg.config._destroy} - ''; - formatNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "/bin/disko-format" - '' - ${cfg.config._create} - ''; - mountNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "/bin/disko-mount" - '' - ${cfg.config._mount} - ''; - unmountNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "/bin/disko-unmount" - '' - ${cfg.config._unmount} - ''; - formatMountNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "/bin/disko-format-mount" - '' - ${cfg.config._formatMount} - ''; - destroyFormatMountNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "/bin/disko-destroy-format-mount" - '' - ${cfg.config._destroyFormatMount} - ''; - - # Legacy scripts, to be removed in version 2.0.0 - # They are generally less useful, because the scripts are directly written to their $out path instead of - # into the $out/bin directory, which makes them incompatible with `nix run` - # (see https://github.com/nix-community/disko/pull/78), `lib.buildEnv` and thus `environment.systemPackages`, - # `user.users..packages` and `home.packages`, see https://github.com/nix-community/disko/issues/454 - destroyScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko-destroy" '' - export PATH=${lib.makeBinPath destroyDependencies}:$PATH - ${cfg.config._legacyDestroy} - ''; - - formatScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko-format" '' - export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH - ${cfg.config._create} - ''; - - mountScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko-mount" '' - export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH - ${cfg.config._mount} - ''; - - diskoScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko" '' - export PATH=${ - lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ] ++ destroyDependencies) - }:$PATH - ${cfg.config._disko} - ''; - - # These are useful to skip copying executables uploading a script to an in-memory installer - destroyScriptNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "disko-destroy" - '' - ${cfg.config._legacyDestroy} - ''; - - formatScriptNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "disko-format" - '' - ${cfg.config._create} - ''; - - mountScriptNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "disko-mount" - '' - ${cfg.config._mount} - ''; - - diskoScriptNoDeps = - (diskoLib.writeCheckedBash { - inherit pkgs checked; - noDeps = true; - }) - "disko" - '' - ${cfg.config._disko} - ''; - }; + lib.mapAttrs throwIfNoDisksDetected ( + builtins.seq validateBcacheMembers { + destroy = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-destroy" '' + export PATH=${lib.makeBinPath destroyDependencies}:$PATH + ${cfg.config._destroy} + ''; + format = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-format" '' + export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH + ${cfg.config._create} + ''; + mount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-mount" '' + export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH + ${cfg.config._mount} + ''; + unmount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-unmount" '' + export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH + ${cfg.config._unmount} + ''; + formatMount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-format-mount" '' + export PATH=${lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ])}:$PATH + ${cfg.config._formatMount} + ''; + destroyFormatMount = + (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-destroy-format-mount" + '' + export PATH=${ + lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ] ++ destroyDependencies) + }:$PATH + ${cfg.config._destroyFormatMount} + ''; + + # These are useful to skip copying executables uploading a script to an in-memory installer + destroyNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "/bin/disko-destroy" + '' + ${cfg.config._destroy} + ''; + formatNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "/bin/disko-format" + '' + ${cfg.config._create} + ''; + mountNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "/bin/disko-mount" + '' + ${cfg.config._mount} + ''; + unmountNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "/bin/disko-unmount" + '' + ${cfg.config._unmount} + ''; + formatMountNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "/bin/disko-format-mount" + '' + ${cfg.config._formatMount} + ''; + destroyFormatMountNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "/bin/disko-destroy-format-mount" + '' + ${cfg.config._destroyFormatMount} + ''; + + # Legacy scripts, to be removed in version 2.0.0 + # They are generally less useful, because the scripts are directly written to their $out path instead of + # into the $out/bin directory, which makes them incompatible with `nix run` + # (see https://github.com/nix-community/disko/pull/78), `lib.buildEnv` and thus `environment.systemPackages`, + # `user.users..packages` and `home.packages`, see https://github.com/nix-community/disko/issues/454 + destroyScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko-destroy" '' + export PATH=${lib.makeBinPath destroyDependencies}:$PATH + ${cfg.config._legacyDestroy} + ''; + + formatScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko-format" '' + export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH + ${cfg.config._create} + ''; + + mountScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko-mount" '' + export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH + ${cfg.config._mount} + ''; + + diskoScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko" '' + export PATH=${ + lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ] ++ destroyDependencies) + }:$PATH + ${cfg.config._disko} + ''; + + # These are useful to skip copying executables uploading a script to an in-memory installer + destroyScriptNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "disko-destroy" + '' + ${cfg.config._legacyDestroy} + ''; + + formatScriptNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "disko-format" + '' + ${cfg.config._create} + ''; + + mountScriptNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "disko-mount" + '' + ${cfg.config._mount} + ''; + + diskoScriptNoDeps = + (diskoLib.writeCheckedBash { + inherit pkgs checked; + noDeps = true; + }) + "disko" + '' + ${cfg.config._disko} + ''; + } + ); }; _legacyDestroy = lib.mkOption { internal = true; @@ -945,6 +1078,8 @@ let default = '' umount -Rv "${rootMountPoint}" || : + ${stopBcacheDevices} + # shellcheck disable=SC2043,2041 for dev in ${ toString ( @@ -990,6 +1125,8 @@ let umount -Rv "${rootMountPoint}" || : + ${stopBcacheDevices} + # shellcheck disable=SC2043,2041 for dev in ${toString (lib.catAttrs "device" disksToWipe)}; do $BASH ${../disk-deactivate}/disk-deactivate "$dev" @@ -1008,11 +1145,13 @@ let sortedDeviceList = diskoLib.sortDevicesByDependencies (cfg.config._meta.deviceDependencies or { } ) devices; in - '' + builtins.seq validateBcacheMembers '' set -efux - disko_devices_dir=$(mktemp -d) - trap 'rm -rf "$disko_devices_dir"' EXIT + if [ -z "''${disko_devices_dir:-}" ]; then + disko_devices_dir=$(mktemp -d) + trap 'rm -rf "$disko_devices_dir"' EXIT + fi mkdir -p "$disko_devices_dir" ${concatMapStrings (dev: (attrByPath (dev ++ [ "_create" ]) { } devices)) sortedDeviceList} @@ -1049,8 +1188,14 @@ let sortedDeviceList = diskoLib.sortDevicesByDependencies (cfg.config._meta.deviceDependencies or { } ) devices; in - '' + builtins.seq validateBcacheMembers '' set -efux + if [ -z "''${disko_devices_dir:-}" ]; then + disko_devices_dir=$(mktemp -d) + trap 'rm -rf "$disko_devices_dir"' EXIT + fi + mkdir -p "$disko_devices_dir" + # first create the necessary devices ${concatMapStrings (dev: (attrByPath (dev ++ [ "_mount" ]) { } devices).dev or "") sortedDeviceList} @@ -1133,7 +1278,9 @@ let ); collectedConfigs = flatten (map (dev: dev._config) (flatten (map attrValues (attrValues devices)))); in - genAttrs configKeys (key: mkMerge (catAttrs key collectedConfigs)); + builtins.seq validateBcacheMembers ( + genAttrs configKeys (key: mkMerge (catAttrs key collectedConfigs)) + ); }; }; } diff --git a/lib/tests.nix b/lib/tests.nix index 75d437415..7b9f0227a 100644 --- a/lib/tests.nix +++ b/lib/tests.nix @@ -299,6 +299,11 @@ let '' import shlex + disko_format = ${builtins.toJSON (lib.getExe tsp-format)} + disko_mount = ${builtins.toJSON (lib.getExe tsp-mount)} + disko_unmount = ${builtins.toJSON (lib.getExe tsp-unmount)} + disko_destroy_format_mount = ${builtins.toJSON (lib.getExe tsp-disko)} + def disks(oldmachine, num_disks): disk_flags = [] for i in range(num_disks): diff --git a/lib/types/bcache.nix b/lib/types/bcache.nix new file mode 100644 index 000000000..8b8327431 --- /dev/null +++ b/lib/types/bcache.nix @@ -0,0 +1,393 @@ +{ + config, + options, + lib, + diskoLib, + rootMountPoint, + ... +}: +{ + options = { + name = lib.mkOption { + type = lib.types.str; + default = config._module.args.name; + description = "Name of the bcache set."; + example = "main"; + }; + type = lib.mkOption { + type = lib.types.enum [ "bcache" ]; + internal = true; + description = "Type"; + }; + device = lib.mkOption { + type = lib.types.str; + default = "/dev/bcache0"; + description = '' + The bcache block device path. This must be a concrete `/dev/bcacheN` + path because generated scripts use `/sys/block/`. + + The current implementation supports exactly one `bcache_cache` + partition and one `bcache_backing` partition per bcache set. + Member devices must be unique across all bcache sets and cannot be + reused between cache and backing roles. + Mount-only assembly requires existing bcache superblocks on both + members and never creates bcache metadata. + ''; + }; + cacheMode = lib.mkOption { + type = lib.types.enum [ + "writethrough" + "writeback" + "writearound" + "none" + ]; + default = "writethrough"; + description = "Cache mode for the bcache device."; + }; + extraCacheArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra arguments passed to `make-bcache -C` for the cache device."; + }; + extraBackingArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra arguments passed to `make-bcache -B` for the backing device."; + }; + content = diskoLib.deviceType { + parent = config; + device = config.device; + }; + _meta = lib.mkOption { + internal = true; + readOnly = true; + type = diskoLib.jsonType; + default = lib.optionalAttrs (config.content != null) ( + config.content._meta [ + "bcache" + config.name + ] + ); + description = "Metadata"; + }; + _create = diskoLib.mkCreateOption { + inherit config options; + default = + let + devBasename = builtins.baseNameOf config.device; + in + '' + # Read cache and backing device paths from temp directory + if ! test -s "$disko_devices_dir/bcache_cache_${lib.escapeShellArg config.name}"; then + printf "\033[31mERROR:\033[0m No cache device found for bcache set \"${config.name}\"!\n" >&2 + exit 1 + fi + if ! test -s "$disko_devices_dir/bcache_backing_${lib.escapeShellArg config.name}"; then + printf "\033[31mERROR:\033[0m No backing device found for bcache set \"${config.name}\"!\n" >&2 + exit 1 + fi + + cache_dev=$(head -n1 "$disko_devices_dir/bcache_cache_${lib.escapeShellArg config.name}") + backing_dev=$(head -n1 "$disko_devices_dir/bcache_backing_${lib.escapeShellArg config.name}") + + modprobe bcache + + # Resolve symlink to kernel device name for sysfs matching + cache_resolved="$(readlink -f "$cache_dev")" + backing_resolved="$(readlink -f "$backing_dev")" + if [ "$cache_resolved" = "$backing_resolved" ]; then + printf "\033[31mERROR:\033[0m bcache set \"${config.name}\" uses the same block device for cache and backing: %s\n" "$cache_resolved" >&2 + exit 1 + fi + backing_basename="$(basename "$backing_resolved")" + + verify_bcache_superblock() { + local role="$1" + local dev="$2" + local resolved + + resolved="$(readlink -f "$dev")" + echo "Verifying bcache $role superblock on $dev -> $resolved" >&2 + if ! bcache-super-show "$resolved" >&2; then + printf "\033[31mERROR:\033[0m make-bcache did not create a valid bcache %s superblock on %s (%s)\n" "$role" "$dev" "$resolved" >&2 + exit 1 + fi + } + + # Idempotency: if the bcache device already exists with our backing device, skip creation + if [ -b "${config.device}" ] && \ + [ -e "/sys/block/${devBasename}/bcache/backing_dev_name" ] && \ + [ "$(cat "/sys/block/${devBasename}/bcache/backing_dev_name")" = "$backing_basename" ]; then + verify_bcache_superblock cache "$cache_dev" + verify_bcache_superblock backing "$backing_dev" + else + # Teardown stale bcache device if it exists (destroy-format-mount scenario) + if [ -e "/sys/block/${devBasename}/bcache/stop" ]; then + echo 1 > "/sys/block/${devBasename}/bcache/stop" 2>/dev/null || true + fi + # Unregister any active cache sets + find /sys/fs/bcache -maxdepth 1 -mindepth 1 -type d -exec sh -c ' + for cset; do + if [ -e "$cset/unregister" ]; then + echo 1 > "$cset/unregister" 2>/dev/null || true + fi + done + ' _ {} + 2>/dev/null || true + udevadm settle --timeout=10 + # Wait for the bcache device to fully disappear + for i in $(seq 1 30); do + [ -b "${config.device}" ] || break + sleep 1 + done + + # Create bcache set — --force handles pre-existing superblocks + echo "Creating bcache set \"${config.name}\" with cache $cache_dev ($cache_resolved) and backing $backing_dev ($backing_resolved)" >&2 + make-bcache \ + -C "$cache_dev" \ + -B "$backing_dev" \ + --force \ + ${lib.optionalString (config.cacheMode == "writeback") "--writeback"} \ + ${lib.concatStringsSep " " (map lib.escapeShellArg config.extraCacheArgs)} \ + ${lib.concatStringsSep " " (map lib.escapeShellArg config.extraBackingArgs)} + + verify_bcache_superblock cache "$cache_dev" + verify_bcache_superblock backing "$backing_dev" + + # Wait for the bcache block device to appear + for i in $(seq 1 60); do + [ -b "${config.device}" ] && break + sleep 1 + done + + # A freshly assembled bcache device shifts backing data by + # `data_offset` sectors but does not zero the resulting mapping. + # Stale filesystem signatures from prior content on the backing + # member can therefore surface on the new bcache device, causing + # downstream content `_create` checks (e.g. btrfs' "skip mkfs if + # blkid reports any TYPE=") to skip formatting and then fail at + # mount time. Wipe the new device so it is unambiguously empty. + if [ -b "${config.device}" ]; then + wipefs --all --force "${config.device}" >&2 || true + fi + fi + + if [ ! -b "${config.device}" ]; then + printf "\033[31mERROR:\033[0m bcache device ${config.device} did not appear\n" >&2 + exit 1 + fi + + if [ ! -e "/sys/block/${devBasename}/bcache/backing_dev_name" ]; then + printf "\033[31mERROR:\033[0m bcache device ${config.device} has no backing_dev_name in sysfs\n" >&2 + exit 1 + fi + + actual_backing_basename="$(cat "/sys/block/${devBasename}/bcache/backing_dev_name")" + if [ "$actual_backing_basename" != "$backing_basename" ]; then + printf "\033[31mERROR:\033[0m bcache device ${config.device} is attached to backing %s, expected %s\n" "$actual_backing_basename" "$backing_basename" >&2 + exit 1 + fi + + # Set cache mode + echo "${config.cacheMode}" > "/sys/block/${devBasename}/bcache/cache_mode" + + ${lib.optionalString (config.content != null) config.content._create} + ''; + }; + _mount = diskoLib.mkMountOption { + inherit config options; + default = + let + devBasename = builtins.baseNameOf config.device; + content = lib.optionalAttrs (config.content != null) config.content._mount; + in + { + dev = '' + if ! test -s "$disko_devices_dir/bcache_cache_${lib.escapeShellArg config.name}"; then + printf "\033[31mERROR:\033[0m No cache device found for bcache set \"${config.name}\"!\n" >&2 + exit 1 + fi + if ! test -s "$disko_devices_dir/bcache_backing_${lib.escapeShellArg config.name}"; then + printf "\033[31mERROR:\033[0m No backing device found for bcache set \"${config.name}\"!\n" >&2 + exit 1 + fi + + cache_dev=$(head -n1 "$disko_devices_dir/bcache_cache_${lib.escapeShellArg config.name}") + backing_dev=$(head -n1 "$disko_devices_dir/bcache_backing_${lib.escapeShellArg config.name}") + + wait_for_path() { + local path="$1" + local i + + for i in $(seq 1 30); do + [ -e "$path" ] && return 0 + udevadm settle --timeout=1 || true + sleep 1 + done + + printf "\033[31mERROR:\033[0m timed out waiting for %s\n" "$path" >&2 + return 1 + } + + wait_for_block() { + local path="$1" + local i + + for i in $(seq 1 60); do + [ -b "$path" ] && return 0 + udevadm settle --timeout=1 || true + sleep 1 + done + + printf "\033[31mERROR:\033[0m timed out waiting for bcache device %s\n" "$path" >&2 + return 1 + } + + wait_for_member_registration() { + local block_name="$1" + local i + + for i in $(seq 1 30); do + if [ -e "/sys/class/block/$block_name/bcache" ] || [ -e "/sys/block/$block_name/bcache" ]; then + return 0 + fi + udevadm settle --timeout=1 || true + sleep 1 + done + + printf "\033[31mERROR:\033[0m bcache member %s did not register\n" "$block_name" >&2 + return 1 + } + + register_bcache_member() { + local dev="$1" + local resolved + local block_name + + wait_for_block "$dev" + resolved="$(readlink -f "$dev")" + if [ ! -b "$resolved" ]; then + printf "\033[31mERROR:\033[0m resolved bcache member %s is not a block device\n" "$resolved" >&2 + return 1 + fi + block_name="$(basename "$resolved")" + + if ! bcache-super-show "$resolved" >/dev/null; then + printf "\033[31mERROR:\033[0m %s does not contain a bcache superblock\n" "$resolved" >&2 + return 1 + fi + + if [ -e "/sys/class/block/$block_name/bcache" ] || [ -e "/sys/block/$block_name/bcache" ]; then + return 0 + fi + + if ! printf '%s\n' "$resolved" > /sys/fs/bcache/register; then + if [ -e "/sys/class/block/$block_name/bcache" ] || [ -e "/sys/block/$block_name/bcache" ]; then + return 0 + fi + if [ -b "${config.device}" ] && \ + [ -e "/sys/block/${devBasename}/bcache/backing_dev_name" ] && \ + [ "$(cat "/sys/block/${devBasename}/bcache/backing_dev_name")" = "$block_name" ]; then + return 0 + fi + printf "\033[31mERROR:\033[0m failed to register bcache member %s\n" "$resolved" >&2 + return 1 + fi + + wait_for_member_registration "$block_name" + } + + modprobe bcache + wait_for_path /sys/fs/bcache/register + wait_for_block "$cache_dev" + wait_for_block "$backing_dev" + expected_backing_basename="$(basename "$(readlink -f "$backing_dev")")" + register_bcache_member "$cache_dev" + register_bcache_member "$backing_dev" + udevadm settle --timeout=10 || true + + if [ ! -b "${config.device}" ]; then + wait_for_block "${config.device}" + fi + + if [ ! -b "${config.device}" ]; then + printf "\033[31mERROR:\033[0m bcache device ${config.device} did not appear\n" >&2 + exit 1 + fi + + if [ ! -e "/sys/block/${devBasename}/bcache/backing_dev_name" ]; then + printf "\033[31mERROR:\033[0m bcache device ${config.device} has no backing_dev_name in sysfs\n" >&2 + exit 1 + fi + + actual_backing_basename="$(cat "/sys/block/${devBasename}/bcache/backing_dev_name")" + if [ "$actual_backing_basename" != "$expected_backing_basename" ]; then + printf "\033[31mERROR:\033[0m bcache device ${config.device} is attached to backing %s, expected %s\n" "$actual_backing_basename" "$expected_backing_basename" >&2 + exit 1 + fi + + if [ -e "/sys/block/${devBasename}/bcache/cache_mode" ]; then + echo "${config.cacheMode}" > "/sys/block/${devBasename}/bcache/cache_mode" + fi + ${content.dev or ""} + ''; + fs = content.fs or { }; + }; + }; + _unmount = diskoLib.mkUnmountOption { + inherit config options; + default = + let + devBasename = builtins.baseNameOf config.device; + content = lib.optionalAttrs (config.content != null) config.content._unmount; + in + { + fs = content.fs or { }; + dev = '' + ${content.dev or ""} + # Stop the bcache device to release cache and backing devices + if [ -e "/sys/block/${devBasename}/bcache/detach" ]; then + echo 1 > "/sys/block/${devBasename}/bcache/detach" 2>/dev/null || true + fi + if [ -e "/sys/block/${devBasename}/bcache/stop" ]; then + echo 1 > "/sys/block/${devBasename}/bcache/stop" 2>/dev/null || true + fi + # Unregister cache sets + find /sys/fs/bcache -maxdepth 1 -mindepth 1 -type d -exec sh -c ' + for cset; do + if [ -e "$cset/unregister" ]; then + echo 1 > "$cset/unregister" 2>/dev/null || true + fi + done + ' _ {} + 2>/dev/null || true + udevadm settle --timeout=10 + ''; + }; + }; + _config = lib.mkOption { + internal = true; + readOnly = true; + default = [ + { + boot.bcache.enable = true; + boot.initrd.kernelModules = [ "bcache" ]; + } + ] + ++ lib.optional (config.content != null) config.content._config; + description = "NixOS configuration"; + }; + _pkgs = lib.mkOption { + internal = true; + readOnly = true; + type = lib.types.functionTo (lib.types.listOf lib.types.package); + default = + pkgs: + [ + pkgs.bcache-tools + pkgs.kmod + pkgs.util-linux + ] + ++ lib.optionals (config.content != null) (config.content._pkgs pkgs); + description = "Packages"; + }; + }; +} diff --git a/lib/types/bcache_backing.nix b/lib/types/bcache_backing.nix new file mode 100644 index 000000000..cc900c1af --- /dev/null +++ b/lib/types/bcache_backing.nix @@ -0,0 +1,80 @@ +{ + config, + options, + lib, + diskoLib, + parent, + device, + ... +}: +{ + options = { + type = lib.mkOption { + type = lib.types.enum [ "bcache_backing" ]; + internal = true; + description = "Type"; + }; + device = lib.mkOption { + type = lib.types.str; + default = device; + description = '' + Device to use as bcache backing device. This must be an absolute + `/dev/...` member device path, not a `/dev/bcacheN` output device. + ''; + }; + set = lib.mkOption { + type = lib.types.str; + description = "Name of the bcache set this backing device belongs to."; + example = "main"; + }; + _parent = lib.mkOption { + internal = true; + default = parent; + }; + _meta = lib.mkOption { + internal = true; + readOnly = true; + type = lib.types.functionTo diskoLib.jsonType; + default = dev: { + deviceDependencies.bcache.${config.set} = [ dev ]; + bcache.backing = [ + { + inherit (config) set device; + } + ]; + }; + description = "Metadata"; + }; + _create = diskoLib.mkCreateOption { + inherit config options; + default = '' + echo "${config.device}" >> "$disko_devices_dir/bcache_backing_${lib.escapeShellArg config.set}" + ''; + }; + _mount = diskoLib.mkMountOption { + inherit config options; + default = { + dev = '' + echo "${config.device}" >> "$disko_devices_dir/bcache_backing_${lib.escapeShellArg config.set}" + ''; + }; + }; + _unmount = diskoLib.mkUnmountOption { + inherit config options; + default = { }; + }; + _config = lib.mkOption { + internal = true; + readOnly = true; + default = [ ]; + description = "NixOS configuration"; + }; + _pkgs = lib.mkOption { + internal = true; + readOnly = true; + type = lib.types.functionTo (lib.types.listOf lib.types.package); + default = pkgs: [ pkgs.bcache-tools ]; + description = "Packages"; + }; + }; +} diff --git a/lib/types/bcache_cache.nix b/lib/types/bcache_cache.nix new file mode 100644 index 000000000..06a24f46b --- /dev/null +++ b/lib/types/bcache_cache.nix @@ -0,0 +1,91 @@ +{ + config, + options, + lib, + diskoLib, + parent, + device, + ... +}: +{ + options = { + type = lib.mkOption { + type = lib.types.enum [ "bcache_cache" ]; + internal = true; + description = "Type"; + }; + device = lib.mkOption { + type = lib.types.str; + default = device; + description = '' + Device to use as bcache cache. This must be an absolute `/dev/...` + member device path, not a `/dev/bcacheN` output device. + ''; + }; + set = lib.mkOption { + type = lib.types.str; + description = "Name of the bcache set this cache device belongs to."; + example = "main"; + }; + bucketSize = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Bucket size for the cache device."; + example = "512k"; + }; + extraArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra arguments passed to `make-bcache -C`."; + }; + _parent = lib.mkOption { + internal = true; + default = parent; + }; + _meta = lib.mkOption { + internal = true; + readOnly = true; + type = lib.types.functionTo diskoLib.jsonType; + default = dev: { + deviceDependencies.bcache.${config.set} = [ dev ]; + bcache.cache = [ + { + inherit (config) set device; + } + ]; + }; + description = "Metadata"; + }; + _create = diskoLib.mkCreateOption { + inherit config options; + default = '' + echo "${config.device}" >> "$disko_devices_dir/bcache_cache_${lib.escapeShellArg config.set}" + ''; + }; + _mount = diskoLib.mkMountOption { + inherit config options; + default = { + dev = '' + echo "${config.device}" >> "$disko_devices_dir/bcache_cache_${lib.escapeShellArg config.set}" + ''; + }; + }; + _unmount = diskoLib.mkUnmountOption { + inherit config options; + default = { }; + }; + _config = lib.mkOption { + internal = true; + readOnly = true; + default = [ ]; + description = "NixOS configuration"; + }; + _pkgs = lib.mkOption { + internal = true; + readOnly = true; + type = lib.types.functionTo (lib.types.listOf lib.types.package); + default = pkgs: [ pkgs.bcache-tools ]; + description = "Packages"; + }; + }; +} diff --git a/lib/types/gpt.nix b/lib/types/gpt.nix index 5f7b26d34..b4e813c6c 100644 --- a/lib/types/gpt.nix +++ b/lib/types/gpt.nix @@ -269,7 +269,7 @@ in type = lib.types.functionTo diskoLib.jsonType; default = dev: - lib.foldr lib.recursiveUpdate { } ( + lib.foldr diskoLib.recursiveUpdate { } ( map (partition: lib.optionalAttrs (partition.content != null) (partition.content._meta dev)) ( lib.attrValues config.partitions ) @@ -342,30 +342,32 @@ in inherit config options; default = let - partMounts = lib.foldr lib.recursiveUpdate { } ( + partitionMounts = ( map (partition: lib.optionalAttrs (partition.content != null) partition.content._mount) ( lib.attrValues config.partitions ) ); + partMountFs = lib.foldr lib.recursiveUpdate { } (map (mount: mount.fs or { }) partitionMounts); in { - dev = partMounts.dev or ""; - fs = partMounts.fs or { }; + dev = lib.concatMapStrings (mount: mount.dev or "") partitionMounts; + fs = partMountFs; }; }; _unmount = diskoLib.mkUnmountOption { inherit config options; default = let - partMounts = lib.foldr lib.recursiveUpdate { } ( + partitionMounts = ( map (partition: lib.optionalAttrs (partition.content != null) partition.content._unmount) ( lib.attrValues config.partitions ) ); + partMountFs = lib.foldr lib.recursiveUpdate { } (map (mount: mount.fs or { }) partitionMounts); in { - dev = partMounts.dev or ""; - fs = partMounts.fs or { }; + dev = lib.concatMapStrings (mount: mount.dev or "") partitionMounts; + fs = partMountFs; }; }; _config = lib.mkOption { diff --git a/lib/types/table.nix b/lib/types/table.nix index 99371ea4c..51bcf5a57 100644 --- a/lib/types/table.nix +++ b/lib/types/table.nix @@ -126,7 +126,7 @@ type = lib.types.functionTo diskoLib.jsonType; default = dev: - lib.foldr lib.recursiveUpdate { } ( + lib.foldr diskoLib.recursiveUpdate { } ( lib.imap ( _index: partition: lib.optionalAttrs (partition.content != null) (partition.content._meta dev) ) config.partitions @@ -174,30 +174,32 @@ inherit config options; default = let - partMounts = lib.foldr lib.recursiveUpdate { } ( + partitionMounts = ( map ( partition: lib.optionalAttrs (partition.content != null) partition.content._mount ) config.partitions ); + partMountFs = lib.foldr lib.recursiveUpdate { } (map (mount: mount.fs or { }) partitionMounts); in { - dev = partMounts.dev or ""; - fs = partMounts.fs or { }; + dev = lib.concatMapStrings (mount: mount.dev or "") partitionMounts; + fs = partMountFs; }; }; _unmount = diskoLib.mkUnmountOption { inherit config options; default = let - partMounts = lib.foldr lib.recursiveUpdate { } ( + partitionMounts = ( map ( partition: lib.optionalAttrs (partition.content != null) partition.content._unmount ) config.partitions ); + partMountFs = lib.foldr lib.recursiveUpdate { } (map (mount: mount.fs or { }) partitionMounts); in { - dev = partMounts.dev or ""; - fs = partMounts.fs or { }; + dev = lib.concatMapStrings (mount: mount.dev or "") partitionMounts; + fs = partMountFs; }; }; _config = lib.mkOption { diff --git a/tests/bcache-validation.nix b/tests/bcache-validation.nix new file mode 100644 index 000000000..cfa1e3b5c --- /dev/null +++ b/tests/bcache-validation.nix @@ -0,0 +1,577 @@ +{ + pkgs ? import { }, + diskoLib ? pkgs.callPackage ../lib { }, + lib ? pkgs.lib, +}: +let + generator = pkgs.callPackage ../. { checked = true; }; + + mkDisk = content: { + disko.devices.disk.main = { + type = "disk"; + device = "/dev/vdb"; + content = { + type = "gpt"; + partitions = content; + }; + }; + }; + + mkBcache = + { + partitions, + bcache ? { + main = { + type = "bcache"; + device = "/dev/bcache0"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }, + }: + let + diskConfig = mkDisk partitions; + in + { + disko.devices = diskConfig.disko.devices // { + inherit bcache; + }; + }; + + scriptEvaluates = config: (builtins.tryEval (generator._cliMount config pkgs).drvPath).success; + configEvaluates = + config: (builtins.tryEval (builtins.deepSeq (generator.config config) true)).success; + evaluates = config: (scriptEvaluates config) && (configEvaluates config); + + invalidCases = { + missing-cache = mkBcache { + partitions.backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + + missing-backing = mkBcache { + partitions.cache = { + size = "100%"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + }; + + cache-undefined-set = mkBcache { + bcache = { }; + partitions.cache = { + size = "100%"; + content = { + type = "bcache_cache"; + set = "missing"; + }; + }; + }; + + backing-undefined-set = mkBcache { + bcache = { }; + partitions.backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "missing"; + }; + }; + }; + + duplicate-cache = mkBcache { + partitions = { + cache-a = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + cache-b = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + }; + + duplicate-backing = mkBcache { + partitions = { + cache = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + backing-a = { + size = "1G"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + backing-b = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + }; + + duplicate-device = mkBcache { + bcache = { + main = { + type = "bcache"; + device = "/dev/bcache0"; + }; + other = { + type = "bcache"; + device = "/dev/bcache0"; + }; + }; + partitions = { + cache-main = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + backing-main = { + size = "1G"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + cache-other = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "other"; + }; + }; + backing-other = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "other"; + }; + }; + }; + }; + + invalid-device-path = mkBcache { + bcache.main = { + type = "bcache"; + device = "/dev/disk/by-id/not-supported"; + }; + partitions = { + cache = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + }; + + duplicate-cache-same-device = mkBcache { + partitions = { + cache-a = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + device = "/dev/disk/by-partlabel/shared-cache"; + }; + }; + cache-b = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + device = "/dev/disk/by-partlabel/shared-cache"; + }; + }; + backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + }; + + duplicate-backing-same-device = mkBcache { + partitions = { + cache = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + backing-a = { + size = "1G"; + content = { + type = "bcache_backing"; + set = "main"; + device = "/dev/disk/by-partlabel/shared-backing"; + }; + }; + backing-b = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + device = "/dev/disk/by-partlabel/shared-backing"; + }; + }; + }; + }; + + same-cache-and-backing-device = mkBcache { + partitions = { + cache = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + device = "/dev/disk/by-partlabel/shared-member"; + }; + }; + backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + device = "/dev/disk/by-partlabel/shared-member"; + }; + }; + }; + }; + + member-device-not-absolute = mkBcache { + partitions = { + cache = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + device = "cache0"; + }; + }; + backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + }; + + member-device-is-bcache-output = mkBcache { + partitions = { + cache = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + device = "/dev/bcache0"; + }; + }; + backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + }; + + reused-cache-across-sets = mkBcache { + bcache = { + main = { + type = "bcache"; + device = "/dev/bcache0"; + }; + other = { + type = "bcache"; + device = "/dev/bcache1"; + }; + }; + partitions = { + cache-main = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + device = "/dev/disk/by-partlabel/shared-cache"; + }; + }; + backing-main = { + size = "1G"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + cache-other = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "other"; + device = "/dev/disk/by-partlabel/shared-cache"; + }; + }; + backing-other = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "other"; + }; + }; + }; + }; + + reused-backing-across-sets = mkBcache { + bcache = { + main = { + type = "bcache"; + device = "/dev/bcache0"; + }; + other = { + type = "bcache"; + device = "/dev/bcache1"; + }; + }; + partitions = { + cache-main = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + backing-main = { + size = "1G"; + content = { + type = "bcache_backing"; + set = "main"; + device = "/dev/disk/by-partlabel/shared-backing"; + }; + }; + cache-other = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "other"; + }; + }; + backing-other = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "other"; + device = "/dev/disk/by-partlabel/shared-backing"; + }; + }; + }; + }; + + reused-cache-as-backing = mkBcache { + bcache = { + main = { + type = "bcache"; + device = "/dev/bcache0"; + }; + other = { + type = "bcache"; + device = "/dev/bcache1"; + }; + }; + partitions = { + cache-main = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + device = "/dev/disk/by-partlabel/shared-member"; + }; + }; + backing-main = { + size = "1G"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + cache-other = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "other"; + }; + }; + backing-other = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "other"; + device = "/dev/disk/by-partlabel/shared-member"; + }; + }; + }; + }; + }; + + validCase = mkBcache { + partitions = { + cache = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + backing = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + }; + }; + + validTwoSets = mkBcache { + bcache = { + main = { + type = "bcache"; + device = "/dev/bcache0"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/main"; + }; + }; + other = { + type = "bcache"; + device = "/dev/bcache1"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/other"; + }; + }; + }; + partitions = { + cache-main = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "main"; + }; + }; + backing-main = { + size = "1G"; + content = { + type = "bcache_backing"; + set = "main"; + }; + }; + cache-other = { + size = "1G"; + content = { + type = "bcache_cache"; + set = "other"; + }; + }; + backing-other = { + size = "100%"; + content = { + type = "bcache_backing"; + set = "other"; + }; + }; + }; + }; + + failedInvalidCases = lib.filterAttrs (_: evaluates) invalidCases; + failedInvalidConfigCases = lib.filterAttrs (_: configEvaluates) invalidCases; + validMountScript = generator._cliMount validCase pkgs; + validFormatScript = generator._cliFormat validCase pkgs; +in +pkgs.runCommand "bcache-validation" { } '' + grep=${pkgs.gnugrep}/bin/grep + mount_script=${lib.getExe validMountScript} + format_script=${lib.getExe validFormatScript} + + ${lib.optionalString (failedInvalidCases != { }) '' + echo "Expected invalid bcache configs to fail evaluation: ${lib.concatStringsSep ", " (lib.attrNames failedInvalidCases)}" >&2 + exit 1 + ''} + ${lib.optionalString (failedInvalidConfigCases != { }) '' + echo "Expected invalid bcache configs to fail config evaluation: ${lib.concatStringsSep ", " (lib.attrNames failedInvalidConfigCases)}" >&2 + exit 1 + ''} + ${lib.optionalString (!(evaluates validCase)) '' + echo "Expected valid bcache config to evaluate" >&2 + exit 1 + ''} + ${lib.optionalString (!(evaluates validTwoSets)) '' + echo "Expected valid two-set bcache config to evaluate" >&2 + exit 1 + ''} + "$grep" -q 'bcache_cache_main' "$mount_script" + "$grep" -q 'bcache_backing_main' "$mount_script" + "$grep" -q 'bcache-super-show' "$mount_script" + "$grep" -q '/sys/fs/bcache/register' "$mount_script" + "$grep" -q 'backing_dev_name' "$mount_script" + "$grep" -q 'make-bcache' "$format_script" + "$grep" -q 'verify_bcache_superblock cache' "$format_script" + "$grep" -q 'verify_bcache_superblock backing' "$format_script" + "$grep" -q 'did not create a valid bcache' "$format_script" + "$grep" -q 'backing_dev_name' "$format_script" + "$grep" -q 'uses the same block device for cache and backing' "$format_script" + if "$grep" -q 'make-bcache' "$mount_script"; then + echo "Mount-only script must not run make-bcache" >&2 + exit 1 + fi + touch "$out" +'' diff --git a/tests/bcache.nix b/tests/bcache.nix new file mode 100644 index 000000000..d040cbe38 --- /dev/null +++ b/tests/bcache.nix @@ -0,0 +1,140 @@ +{ + pkgs ? import { }, + diskoLib ? pkgs.callPackage ../lib { }, +}: +let + lib = pkgs.lib; + generator = pkgs.callPackage ../. { checked = true; }; + wrongBackingConfig = { + disko.devices = { + disk = { + cache-disk = { + type = "disk"; + device = "/dev/my-cache-disk"; + content = { + type = "gpt"; + partitions = { + boot = { + size = "1M"; + type = "EF02"; + }; + boot-fs = { + size = "500M"; + content = { + type = "bcache_backing"; + set = "wrong"; + }; + }; + cache = { + size = "100%"; + content = { + type = "bcache_cache"; + set = "wrong"; + }; + }; + }; + }; + }; + }; + bcache.wrong = { + type = "bcache"; + device = "/dev/bcache0"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/wrong"; + }; + }; + }; + }; + wrongBackingMount = generator._cliMount (diskoLib.testLib.prepareDiskoConfig wrongBackingConfig (lib.tail diskoLib.testLib.devices)) pkgs; +in +diskoLib.testLib.makeDiskoTest { + inherit pkgs; + name = "bcache"; + disko-config = ../example/bcache.nix; + extraInstallerConfig = { + boot.kernelModules = [ "bcache" ]; + }; + extraSystemConfig = { + boot.bcache.enable = true; + }; + # bcache root requires initrd support that NixOS doesn't fully provide yet; + # test format/mount/destroy/idempotency only + testBoot = false; + testMode = "direct"; + extraTestScript = '' + # Verify bcache device exists and is a block device + machine.succeed("test -b /dev/bcache0") + + # Verify make-bcache wrote metadata to both member devices. + machine.succeed("bcache-super-show /dev/disk/by-partlabel/disk-cache-disk-cache >&2") + machine.succeed("bcache-super-show /dev/disk/by-partlabel/disk-backing-disk-backing >&2") + + # Verify root is mounted on bcache + machine.succeed("mountpoint /mnt") + + # Verify cache mode is writeback + machine.succeed("cat /sys/block/bcache0/bcache/cache_mode | grep -q writeback") + + # Verify the cache is attached (state should be "clean" or "dirty", not "no cache") + machine.succeed("cat /sys/block/bcache0/bcache/state | grep -qE 'clean|dirty'") + + # Verify a backing device is attached (non-empty name) + machine.succeed("test -n \"$(cat /sys/block/bcache0/bcache/backing_dev_name)\"") + + # Verify data read/write through bcache works + machine.succeed("echo 'bcache test data' > /mnt/testfile") + machine.succeed("grep -q 'bcache test data' /mnt/testfile") + + # Verify the filesystem on bcache is ext4 + machine.succeed("findmnt -n -o FSTYPE /mnt | grep -q ext4") + + # Cold-style mount-only reassembly: stop the active set, unregister cache + # sets, and ensure the generated mount script brings /dev/bcache0 back. + expected_backing = machine.succeed("basename $(readlink -f /dev/disk/by-partlabel/disk-backing-disk-backing)").strip() + machine.succeed(disko_unmount) + machine.succeed("test ! -b /dev/bcache0") + machine.succeed("find /sys/fs/bcache -maxdepth 1 -mindepth 1 -type d -exec sh -c 'for cset; do [ ! -e \"$cset/unregister\" ] || echo 1 > \"$cset/unregister\"; done' _ {} + 2>/dev/null || true") + machine.succeed("udevadm settle --timeout=10 || true") + machine.succeed(disko_mount) + machine.succeed("test -b /dev/bcache0") + machine.succeed(f'test "$(cat /sys/block/bcache0/bcache/backing_dev_name)" = "{expected_backing}"') + machine.succeed("mountpoint /mnt") + + # Regression: a stale filesystem signature on the backing partition must + # not surface through /dev/bcache0 and trick content `_create` into + # skipping format. Tear the set down, wipe bcache metadata, pre-poison the + # backing partition with a btrfs filesystem, then re-run format. The + # bcache device must come back as the configured ext4, not the stale + # btrfs. + machine.succeed("umount /mnt/boot") + machine.succeed(disko_unmount) + machine.succeed("test ! -b /dev/bcache0") + machine.succeed("find /sys/fs/bcache -maxdepth 1 -mindepth 1 -type d -exec sh -c 'for cset; do [ ! -e \"$cset/unregister\" ] || echo 1 > \"$cset/unregister\"; done' _ {} + 2>/dev/null || true") + machine.succeed("udevadm settle --timeout=10 || true") + machine.succeed("wipefs --all --force /dev/disk/by-partlabel/disk-backing-disk-backing") + machine.succeed("wipefs --all --force /dev/disk/by-partlabel/disk-cache-disk-cache") + machine.succeed("mkfs.btrfs -f /dev/disk/by-partlabel/disk-backing-disk-backing") + machine.succeed("blkid -o export /dev/disk/by-partlabel/disk-backing-disk-backing | grep -q '^TYPE=btrfs$'") + machine.succeed(disko_format) + machine.succeed(disko_mount) + machine.succeed("test -b /dev/bcache0") + machine.succeed("blkid -o export /dev/bcache0 | grep -q '^TYPE=ext4$'") + machine.succeed("findmnt -n -o FSTYPE /mnt | grep -q ext4") + + # A mount-only config expecting a different backing device must not reuse an + # unrelated active /dev/bcache0. + machine.succeed("umount /mnt/boot") + machine.succeed("wipefs -a /dev/disk/by-partlabel/disk-cache-disk-boot-fs") + machine.succeed("${pkgs.bcache-tools}/bin/make-bcache -B /dev/disk/by-partlabel/disk-cache-disk-boot-fs --force") + machine.succeed( + "set +e; " + "${lib.getExe wrongBackingMount} >/tmp/wrong-bcache-mount.log 2>&1; " + "status=$?; " + "test $status -ne 0; " + "grep -q 'is attached to backing' /tmp/wrong-bcache-mount.log" + ) + ''; + efi = false; +}