ci: fix arm64 QEMU block hotplug coverage

.github/workflows/capos-full-qemu.yml

@@ -27,7 +27,7 @@ permissions:
 
 env:
   CARGO_TERM_COLOR: always
-  ARGOSFS_CAPOS_REF: 9e636b9b45941e9d9219af97054b4c661d416294
+  ARGOSFS_CAPOS_REF: 168f01fc44f627ab07f23dd9590453f0ba5fe7fc
 
 jobs:
   capos-full-build:

scripts/qemu/block_lifecycle_stress.sh

@@ -9,6 +9,7 @@ argosfs_qemu_select_arch
 argosfs_qemu_require_binary
 
 monitor="$artifacts/qemu-monitor.sock"
+monitor_log="$artifacts/qemu-monitor.log"
 log="$artifacts/qemu-block-lifecycle-stress-$arch.log"
 commands="$artifacts/qemu-block-lifecycle-stress.commands"
 reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:|segfault|I/O error}"
@@ -32,8 +33,10 @@ awk '$2=="/"{print "ARGOSFS_ROOT_MOUNT " $1 " " $3 " " $4; exit}' /proc/mounts
 test -e /run/argosfs-root-active && echo ARGOSFS_ROOT_MARKER_OK
 echo ARGOSFS_WAIT_LIFECYCLE_HOTPLUG
 for dev in /dev/vdb /dev/vdc /dev/vdd /dev/vde /dev/vdf; do
-  for i in $(seq 1 60); do [ -b "$dev" ] && break; sleep 1; done
+  name="${dev##*/}"
+  for i in $(seq 1 60); do [ -b "$dev" ] && [ -e "/sys/class/block/$name" ] && break; sleep 1; done
   test -b "$dev"
+  test -e "/sys/class/block/$name"
 done
 echo ARGOSFS_LIFECYCLE_BLOCK_DEVICES_OK
 src=/tmp/argosfs-lifecycle-src
@@ -63,28 +66,37 @@ all="$all,/dev/vdf"
 argosfs fsck --backend raw --devices "$all" --repair --remove-orphans >/tmp/argosfs-lifecycle-fsck-final.json
 argosfs scrub --backend raw --devices "$all" >/tmp/argosfs-lifecycle-scrub-final.json
 argosfs export-tree --backend raw --devices "$all" "$out"
-cmp "$src/data/meta-37.txt" "$out/data/meta-37.txt"
-cmp "$src/data/blob-111.bin" "$out/data/blob-111.bin"
-echo ARGOSFS_BLOCK_LIFECYCLE_CONTENT_OK
+if cmp "$src/data/meta-37.txt" "$out/data/meta-37.txt" &&
+   cmp "$src/data/blob-111.bin" "$out/data/blob-111.bin"; then
+  echo ARGOSFS_BLOCK_LIFECYCLE_CONTENT_OK
+else
+  echo ARGOSFS_BLOCK_LIFECYCLE_CONTENT_FAILED
+fi
 sync
 echo ARGOSFS_QEMU_BLOCK_LIFECYCLE_STRESS_DONE
 poweroff -f || reboot -f || halt -f
 CMDS
 
 argosfs_qemu_build_args
+argosfs_qemu_add_hotplug_ports 5 life
 qemu_args+=(-monitor "unix:$monitor,server,nowait")
+: >"$monitor_log"
 
 qemu_device_add() {
   local idx="$1" path="$2"
-  printf 'drive_add 0 if=none,file=%s,format=raw,id=life%s\n' "$path" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-  if [ "$arch" = "arm64" ]; then
-    printf 'device_add virtio-blk-pci,drive=life%s,id=lifedisk%s,romfile=\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-  else
-    printf 'device_add virtio-blk-pci,drive=life%s,id=lifedisk%s\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-  fi
+  local bus_arg rom_arg
+  argosfs_qemu_monitor_command "$monitor" \
+    "drive_add 0 if=none,file=$path,format=raw,id=life$idx" "$monitor_log"
+  bus_arg="$(argosfs_qemu_hotplug_bus_arg life "$idx")"
+  rom_arg=""
+  [ "$arch" != "arm64" ] || rom_arg=",romfile="
+  argosfs_qemu_monitor_command "$monitor" \
+    "device_add virtio-blk-pci,drive=life$idx,id=lifedisk$idx${bus_arg}${rom_arg}" "$monitor_log"
 }
 
 set +e
+# QEMU output is intentionally polled while this pipeline appends to the log.
+# shellcheck disable=SC2094
 (
   sleep "$login_delay_s"
   printf '\r'
@@ -93,6 +105,7 @@ set +e
     printf '%s\r' "$line"
     sleep "$command_delay_s"
     if [ "$line" = "echo ARGOSFS_WAIT_LIFECYCLE_HOTPLUG" ]; then
+      argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_LIFECYCLE_HOTPLUG 180
       for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
       idx=0
       for disk in "${disks[@]}"; do qemu_device_add "$idx" "$disk"; idx=$((idx + 1)); done

scripts/qemu/build_capos_image.sh

@@ -38,7 +38,7 @@ ARGOSFS_CAPOS_MAKE_JOBS="$make_jobs" \
 ARGOSFS_CAPOS_MAKE_TARGET="$make_target" \
 ARGOSFS_CAPOS_MAKE_V="${ARGOSFS_CAPOS_MAKE_V:-}" \
 CAPOS_REPO="${CAPOS_REPO:-https://github.com/fwerkor/capos.git}" \
-CAPOS_REF="${CAPOS_REF:-9e636b9b45941e9d9219af97054b4c661d416294}" \
+CAPOS_REF="${CAPOS_REF:-168f01fc44f627ab07f23dd9590453f0ba5fe7fc}" \
 CAPOS_LOCAL_SOURCE="${CAPOS_LOCAL_SOURCE:-}" \
 "$repo/scripts/qemu/capos_build.sh"
 

scripts/qemu/capos_build.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 artifacts="${ARGOSFS_TEST_ARTIFACTS:-$repo/target/argosfs-test-artifacts/capos-build}"
 capos_repo="${CAPOS_REPO:-https://github.com/fwerkor/capos.git}"
-capos_ref="${CAPOS_REF:-9e636b9b45941e9d9219af97054b4c661d416294}"
+capos_ref="${CAPOS_REF:-168f01fc44f627ab07f23dd9590453f0ba5fe7fc}"
 capos_local_source="${CAPOS_LOCAL_SOURCE:-}"
 capos_full_compile="${ARGOSFS_CAPOS_FULL_COMPILE:-1}"
 capos_make_jobs="${ARGOSFS_CAPOS_MAKE_JOBS:-$(nproc 2>/dev/null || echo 2)}"

scripts/qemu/crash_recovery.sh

@@ -9,6 +9,7 @@ argosfs_qemu_select_arch
 argosfs_qemu_require_binary
 
 monitor="$artifacts/qemu-monitor.sock"
+monitor_log="$artifacts/qemu-monitor.log"
 log1="$artifacts/qemu-crash-recovery-phase1-$arch.log"
 log2="$artifacts/qemu-crash-recovery-phase2-$arch.log"
 commands1="$artifacts/qemu-crash-recovery-phase1.commands"
@@ -34,8 +35,10 @@ awk '$2=="/"{print "ARGOSFS_ROOT_MOUNT_PHASE1 " $1 " " $3 " " $4; exit}' /proc/m
 test -e /run/argosfs-root-active && echo ARGOSFS_CRASH_ROOT_MARKER_PHASE1_OK
 echo ARGOSFS_WAIT_CRASH_HOTPLUG
 for dev in /dev/vdb /dev/vdc /dev/vdd; do
-  for i in $(seq 1 60); do [ -b "$dev" ] && break; sleep 1; done
+  name="${dev##*/}"
+  for i in $(seq 1 60); do [ -b "$dev" ] && [ -e "/sys/class/block/$name" ] && break; sleep 1; done
   test -b "$dev"
+  test -e "/sys/class/block/$name"
 done
 base=/tmp/argosfs-crash-base
 next=/tmp/argosfs-crash-next
@@ -87,20 +90,26 @@ CMDS
 
 qemu_device_add() {
   local idx="$1" path="$2"
-  printf 'drive_add 0 if=none,file=%s,format=raw,id=crash%s\n' "$path" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-  if [ "$arch" = "arm64" ]; then
-    printf 'device_add virtio-blk-pci,drive=crash%s,id=crashdisk%s,romfile=\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-  else
-    printf 'device_add virtio-blk-pci,drive=crash%s,id=crashdisk%s\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-  fi
+  local bus_arg rom_arg
+  argosfs_qemu_monitor_command "$monitor" \
+    "drive_add 0 if=none,file=$path,format=raw,id=crash$idx" "$monitor_log"
+  bus_arg="$(argosfs_qemu_hotplug_bus_arg crash "$idx")"
+  rom_arg=""
+  [ "$arch" != "arm64" ] || rom_arg=",romfile="
+  argosfs_qemu_monitor_command "$monitor" \
+    "device_add virtio-blk-pci,drive=crash$idx,id=crashdisk$idx${bus_arg}${rom_arg}" "$monitor_log"
 }
 
 run_phase1_until_kill_marker() {
   rm -f "$monitor"
   argosfs_qemu_build_args
+  argosfs_qemu_add_hotplug_ports 3 crash
   qemu_args+=(-monitor "unix:$monitor,server,nowait")
+  : >"$monitor_log"
   : >"$log1"
   set +e
+  # QEMU output is intentionally polled while this pipeline appends to the log.
+  # shellcheck disable=SC2094
   (
     sleep "$login_delay_s"
     printf '\r'
@@ -109,6 +118,7 @@ run_phase1_until_kill_marker() {
       printf '%s\r' "$line"
       sleep "$command_delay_s"
       if [ "$line" = "echo ARGOSFS_WAIT_CRASH_HOTPLUG" ]; then
+        argosfs_qemu_wait_log_marker "$log1" ARGOSFS_WAIT_CRASH_HOTPLUG 180
         for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
         idx=0
         for disk in "${disks[@]}"; do qemu_device_add "$idx" "$disk"; idx=$((idx + 1)); done
@@ -136,6 +146,8 @@ run_phase1_until_kill_marker() {
 run_phase2() {
   argosfs_qemu_build_args
   set +e
+  # QEMU output is intentionally polled while this pipeline appends to the log.
+  # shellcheck disable=SC2094
   (
     sleep "$login_delay_s"
     printf '\r'

scripts/qemu/degraded_rootfs.sh

@@ -9,6 +9,7 @@ argosfs_qemu_select_arch
 argosfs_qemu_require_binary
 
 monitor="$artifacts/qemu-monitor.sock"
+monitor_log="$artifacts/qemu-monitor.log"
 log="$artifacts/qemu-degraded-rootfs-$arch.log"
 commands="$artifacts/qemu-degraded-rootfs.commands"
 reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:|segfault|I/O error}"
@@ -32,8 +33,10 @@ awk '$2=="/"{print "ARGOSFS_ROOT_MOUNT " $1 " " $3 " " $4; exit}' /proc/mounts
 test -e /run/argosfs-root-active && echo ARGOSFS_ROOT_MARKER_OK
 echo ARGOSFS_WAIT_DEGRADED_HOTPLUG
 for dev in /dev/vdb /dev/vdc /dev/vdd; do
-  for i in $(seq 1 60); do [ -b "$dev" ] && break; sleep 1; done
+  name="${dev##*/}"
+  for i in $(seq 1 60); do [ -b "$dev" ] && [ -e "/sys/class/block/$name" ] && break; sleep 1; done
   test -b "$dev"
+  test -e "/sys/class/block/$name"
 done
 src=/tmp/argosfs-degraded-src
 mnt=/mnt/argosfs-degraded-root
@@ -70,19 +73,25 @@ poweroff -f || reboot -f || halt -f
 CMDS
 
 argosfs_qemu_build_args
+argosfs_qemu_add_hotplug_ports 3 deg
 qemu_args+=(-monitor "unix:$monitor,server,nowait")
+: >"$monitor_log"
 
 qemu_device_add() {
   local idx="$1" path="$2"
-  printf 'drive_add 0 if=none,file=%s,format=raw,id=deg%s\n' "$path" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-  if [ "$arch" = "arm64" ]; then
-    printf 'device_add virtio-blk-pci,drive=deg%s,id=degdisk%s,romfile=\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-  else
-    printf 'device_add virtio-blk-pci,drive=deg%s,id=degdisk%s\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-  fi
+  local bus_arg rom_arg
+  argosfs_qemu_monitor_command "$monitor" \
+    "drive_add 0 if=none,file=$path,format=raw,id=deg$idx" "$monitor_log"
+  bus_arg="$(argosfs_qemu_hotplug_bus_arg deg "$idx")"
+  rom_arg=""
+  [ "$arch" != "arm64" ] || rom_arg=",romfile="
+  argosfs_qemu_monitor_command "$monitor" \
+    "device_add virtio-blk-pci,drive=deg$idx,id=degdisk$idx${bus_arg}${rom_arg}" "$monitor_log"
 }
 
 set +e
+# QEMU output is intentionally polled while this pipeline appends to the log.
+# shellcheck disable=SC2094
 (
   sleep "$login_delay_s"
   printf '\r'
@@ -91,6 +100,7 @@ set +e
     printf '%s\r' "$line"
     sleep "$command_delay_s"
     if [ "$line" = "echo ARGOSFS_WAIT_DEGRADED_HOTPLUG" ]; then
+      argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_DEGRADED_HOTPLUG 180
       for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
       idx=0
       for disk in "${disks[@]}"; do qemu_device_add "$idx" "$disk"; idx=$((idx + 1)); done

scripts/qemu/hotplug.sh

@@ -13,9 +13,10 @@ argosfs_qemu_require_binary
 hotplug_disk="$artifacts/hotplug.img"
 qemu-img create -f raw "$hotplug_disk" "${ARGOSFS_QEMU_HOTPLUG_SIZE:-64M}" >/dev/null
 monitor="$artifacts/qemu-monitor.sock"
+monitor_log="$artifacts/qemu-monitor.log"
 log="$artifacts/qemu-hotplug.log"
 commands="$artifacts/qemu-hotplug.commands"
-reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:}"
+reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:|I/O error|missing device}"
 timeout_s="${ARGOSFS_QEMU_TIMEOUT:-260}"
 login_delay_s="${ARGOSFS_QEMU_HOTPLUG_LOGIN_DELAY:-90}"
 login_delay_s="$(argosfs_qemu_adjust_login_delay "$login_delay_s")"
@@ -27,25 +28,28 @@ echo ARGOSFS_QEMU_HOTPLUG_BEGIN
 awk '$2=="/"{print "ARGOSFS_ROOT_MOUNT " $1 " " $3 " " $4; exit}' /proc/mounts
 ls -l /dev/vd* 2>/dev/null || true
 echo ARGOSFS_WAIT_HOTPLUG
-for i in $(seq 1 30); do [ -b /dev/vdb ] && break; sleep 1; done
+for i in $(seq 1 30); do [ -b /dev/vdb ] && [ -e /sys/class/block/vdb ] && break; sleep 1; done
 ls -l /dev/vd* 2>/dev/null || true
-test -b /dev/vdb && echo ARGOSFS_HOTPLUG_BLOCK_DEVICE_OK
+test -b /dev/vdb && test -e /sys/class/block/vdb && echo ARGOSFS_HOTPLUG_BLOCK_DEVICE_OK
 mkdir -p /tmp/argosfs-hotplug/src /tmp/argosfs-hotplug/out
 printf 'hotplug payload\n' >/tmp/argosfs-hotplug/payload.txt
 printf 'raw hotplug payload\n' >/tmp/argosfs-hotplug/src/raw.txt
 if command -v argosfs >/dev/null; then
-  argosfs mkfs --backend raw --devices /dev/vdb --k 1 --m 0 --chunk-size 32768 --compression zstd --force >/tmp/argosfs-hotplug/mkfs-raw.json
-  argosfs import-tree --backend raw --devices /dev/vdb /tmp/argosfs-hotplug/src /
-  argosfs export-tree --backend raw --devices /dev/vdb /tmp/argosfs-hotplug/out
-  cmp /tmp/argosfs-hotplug/src/raw.txt /tmp/argosfs-hotplug/out/raw.txt
-  argosfs fsck --backend raw --devices /dev/vdb --repair --remove-orphans >/tmp/argosfs-hotplug/fsck-raw.json
-  argosfs scrub --backend raw --devices /dev/vdb >/tmp/argosfs-hotplug/scrub-raw.json
-  echo ARGOSFS_HOTPLUG_RAW_ARGOSFS_OK
+  if argosfs mkfs --backend raw --devices /dev/vdb --k 1 --m 0 --chunk-size 32768 --compression zstd --force >/tmp/argosfs-hotplug/mkfs-raw.json &&
+     argosfs import-tree --backend raw --devices /dev/vdb /tmp/argosfs-hotplug/src / &&
+     argosfs export-tree --backend raw --devices /dev/vdb /tmp/argosfs-hotplug/out &&
+     cmp /tmp/argosfs-hotplug/src/raw.txt /tmp/argosfs-hotplug/out/raw.txt &&
+     argosfs fsck --backend raw --devices /dev/vdb --repair --remove-orphans >/tmp/argosfs-hotplug/fsck-raw.json &&
+     argosfs scrub --backend raw --devices /dev/vdb >/tmp/argosfs-hotplug/scrub-raw.json; then
+    echo ARGOSFS_HOTPLUG_RAW_ARGOSFS_OK
+  else
+    echo ARGOSFS_HOTPLUG_RAW_ARGOSFS_FAILED
+  fi
 fi
 sync
 echo ARGOSFS_WAIT_UNPLUG
-for i in $(seq 1 30); do [ ! -b /dev/vdb ] && break; sleep 1; done
-if [ ! -b /dev/vdb ]; then echo ARGOSFS_UNPLUG_BLOCK_DEVICE_OK; fi
+for i in $(seq 1 30); do [ ! -e /sys/class/block/vdb ] && break; sleep 1; done
+if [ ! -e /sys/class/block/vdb ]; then echo ARGOSFS_UNPLUG_BLOCK_DEVICE_OK; fi
 test "$(cat /tmp/argosfs-hotplug/payload.txt)" = "hotplug payload" && echo ARGOSFS_ROOT_SURVIVED_HOTPLUG_OK
 rm -rf /tmp/argosfs-hotplug
 sync
@@ -54,9 +58,13 @@ poweroff -f || reboot -f || halt -f
 CMDS
 
 argosfs_qemu_build_args
+argosfs_qemu_add_hotplug_ports 1 hot
 qemu_args+=(-monitor "unix:$monitor,server,nowait")
+: >"$monitor_log"
 
 set +e
+# QEMU output is intentionally polled while this pipeline appends to the log.
+# shellcheck disable=SC2094
 (
 	sleep "$login_delay_s"
 	printf '\r'
@@ -65,18 +73,21 @@ set +e
 		printf '%s\r' "$line"
 		sleep "$command_delay_s"
 		if [ "$line" = "echo ARGOSFS_WAIT_HOTPLUG" ]; then
+			argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_HOTPLUG 180
 			for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
-			printf 'drive_add 0 if=none,file=%s,format=raw,id=hot0\n' "$hotplug_disk" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-			if [ "$arch" = "arm64" ]; then
-				printf 'device_add virtio-blk-pci,drive=hot0,id=hotdisk0,romfile=\n' | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-			else
-				printf 'device_add virtio-blk-pci,drive=hot0,id=hotdisk0\n' | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
-			fi
+			argosfs_qemu_monitor_command "$monitor" \
+				"drive_add 0 if=none,file=$hotplug_disk,format=raw,id=hot0" "$monitor_log"
+			bus_arg="$(argosfs_qemu_hotplug_bus_arg hot 0)"
+			rom_arg=""
+			[ "$arch" != "arm64" ] || rom_arg=",romfile="
+			argosfs_qemu_monitor_command "$monitor" \
+				"device_add virtio-blk-pci,drive=hot0,id=hotdisk0${bus_arg}${rom_arg}" "$monitor_log"
 		fi
 		if [ "$line" = "echo ARGOSFS_WAIT_UNPLUG" ]; then
-			printf 'device_del hotdisk0\n' | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
+			argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_UNPLUG 600
+			argosfs_qemu_monitor_command "$monitor" "device_del hotdisk0" "$monitor_log"
 			sleep 2
-			printf 'drive_del hot0\n' | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true
+			argosfs_qemu_monitor_command "$monitor" "drive_del hot0" "$monitor_log"
 		fi
 	done <"$commands"
 ) | timeout "$timeout_s" "$qemu_bin" "${qemu_args[@]}" >"$log" 2>&1