From 49b1c63be9388cc21174284fb1e1289df06fa673 Mon Sep 17 00:00:00 2001 From: Cao Yuhang Date: Sun, 12 Jul 2026 16:55:49 +0000 Subject: [PATCH 1/2] ci: make QEMU block hotplug observable --- scripts/qemu/block_lifecycle_stress.sh | 33 +++++++---- scripts/qemu/crash_recovery.sh | 26 ++++++--- scripts/qemu/degraded_rootfs.sh | 24 +++++--- scripts/qemu/hotplug.sh | 51 +++++++++------- scripts/qemu/lib/common.sh | 80 ++++++++++++++++++++++++++ scripts/qemu/mixed_chaos.sh | 41 +++++++++---- 6 files changed, 199 insertions(+), 56 deletions(-) diff --git a/scripts/qemu/block_lifecycle_stress.sh b/scripts/qemu/block_lifecycle_stress.sh index 30b327c..00ccc35 100755 --- a/scripts/qemu/block_lifecycle_stress.sh +++ b/scripts/qemu/block_lifecycle_stress.sh @@ -9,6 +9,7 @@ argosfs_qemu_select_arch argosfs_qemu_require_binary monitor="$artifacts/qemu-monitor.sock" +monitor_log="$artifacts/qemu-monitor.log" log="$artifacts/qemu-block-lifecycle-stress-$arch.log" commands="$artifacts/qemu-block-lifecycle-stress.commands" reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:|segfault|I/O error}" @@ -32,8 +33,10 @@ awk '$2=="/"{print "ARGOSFS_ROOT_MOUNT " $1 " " $3 " " $4; exit}' /proc/mounts test -e /run/argosfs-root-active && echo ARGOSFS_ROOT_MARKER_OK echo ARGOSFS_WAIT_LIFECYCLE_HOTPLUG for dev in /dev/vdb /dev/vdc /dev/vdd /dev/vde /dev/vdf; do - for i in $(seq 1 60); do [ -b "$dev" ] && break; sleep 1; done + name="${dev##*/}" + for i in $(seq 1 60); do [ -b "$dev" ] && [ -e "/sys/class/block/$name" ] && break; sleep 1; done test -b "$dev" + test -e "/sys/class/block/$name" done echo ARGOSFS_LIFECYCLE_BLOCK_DEVICES_OK src=/tmp/argosfs-lifecycle-src @@ -63,28 +66,37 @@ all="$all,/dev/vdf" argosfs fsck --backend raw --devices "$all" --repair --remove-orphans >/tmp/argosfs-lifecycle-fsck-final.json argosfs scrub --backend raw --devices "$all" >/tmp/argosfs-lifecycle-scrub-final.json argosfs export-tree --backend raw --devices "$all" "$out" -cmp "$src/data/meta-37.txt" "$out/data/meta-37.txt" -cmp "$src/data/blob-111.bin" "$out/data/blob-111.bin" -echo ARGOSFS_BLOCK_LIFECYCLE_CONTENT_OK +if cmp "$src/data/meta-37.txt" "$out/data/meta-37.txt" && + cmp "$src/data/blob-111.bin" "$out/data/blob-111.bin"; then + echo ARGOSFS_BLOCK_LIFECYCLE_CONTENT_OK +else + echo ARGOSFS_BLOCK_LIFECYCLE_CONTENT_FAILED +fi sync echo ARGOSFS_QEMU_BLOCK_LIFECYCLE_STRESS_DONE poweroff -f || reboot -f || halt -f CMDS argosfs_qemu_build_args +argosfs_qemu_add_hotplug_ports 5 life qemu_args+=(-monitor "unix:$monitor,server,nowait") +: >"$monitor_log" qemu_device_add() { local idx="$1" path="$2" - printf 'drive_add 0 if=none,file=%s,format=raw,id=life%s\n' "$path" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - if [ "$arch" = "arm64" ]; then - printf 'device_add virtio-blk-pci,drive=life%s,id=lifedisk%s,romfile=\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - else - printf 'device_add virtio-blk-pci,drive=life%s,id=lifedisk%s\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - fi + local bus_arg rom_arg + argosfs_qemu_monitor_command "$monitor" \ + "drive_add 0 if=none,file=$path,format=raw,id=life$idx" "$monitor_log" + bus_arg="$(argosfs_qemu_hotplug_bus_arg life "$idx")" + rom_arg="" + [ "$arch" != "arm64" ] || rom_arg=",romfile=" + argosfs_qemu_monitor_command "$monitor" \ + "device_add virtio-blk-pci,drive=life$idx,id=lifedisk$idx${bus_arg}${rom_arg}" "$monitor_log" } set +e +# QEMU output is intentionally polled while this pipeline appends to the log. +# shellcheck disable=SC2094 ( sleep "$login_delay_s" printf '\r' @@ -93,6 +105,7 @@ set +e printf '%s\r' "$line" sleep "$command_delay_s" if [ "$line" = "echo ARGOSFS_WAIT_LIFECYCLE_HOTPLUG" ]; then + argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_LIFECYCLE_HOTPLUG 180 for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done idx=0 for disk in "${disks[@]}"; do qemu_device_add "$idx" "$disk"; idx=$((idx + 1)); done diff --git a/scripts/qemu/crash_recovery.sh b/scripts/qemu/crash_recovery.sh index 78de752..4ce0de5 100755 --- a/scripts/qemu/crash_recovery.sh +++ b/scripts/qemu/crash_recovery.sh @@ -9,6 +9,7 @@ argosfs_qemu_select_arch argosfs_qemu_require_binary monitor="$artifacts/qemu-monitor.sock" +monitor_log="$artifacts/qemu-monitor.log" log1="$artifacts/qemu-crash-recovery-phase1-$arch.log" log2="$artifacts/qemu-crash-recovery-phase2-$arch.log" commands1="$artifacts/qemu-crash-recovery-phase1.commands" @@ -34,8 +35,10 @@ awk '$2=="/"{print "ARGOSFS_ROOT_MOUNT_PHASE1 " $1 " " $3 " " $4; exit}' /proc/m test -e /run/argosfs-root-active && echo ARGOSFS_CRASH_ROOT_MARKER_PHASE1_OK echo ARGOSFS_WAIT_CRASH_HOTPLUG for dev in /dev/vdb /dev/vdc /dev/vdd; do - for i in $(seq 1 60); do [ -b "$dev" ] && break; sleep 1; done + name="${dev##*/}" + for i in $(seq 1 60); do [ -b "$dev" ] && [ -e "/sys/class/block/$name" ] && break; sleep 1; done test -b "$dev" + test -e "/sys/class/block/$name" done base=/tmp/argosfs-crash-base next=/tmp/argosfs-crash-next @@ -87,20 +90,26 @@ CMDS qemu_device_add() { local idx="$1" path="$2" - printf 'drive_add 0 if=none,file=%s,format=raw,id=crash%s\n' "$path" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - if [ "$arch" = "arm64" ]; then - printf 'device_add virtio-blk-pci,drive=crash%s,id=crashdisk%s,romfile=\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - else - printf 'device_add virtio-blk-pci,drive=crash%s,id=crashdisk%s\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - fi + local bus_arg rom_arg + argosfs_qemu_monitor_command "$monitor" \ + "drive_add 0 if=none,file=$path,format=raw,id=crash$idx" "$monitor_log" + bus_arg="$(argosfs_qemu_hotplug_bus_arg crash "$idx")" + rom_arg="" + [ "$arch" != "arm64" ] || rom_arg=",romfile=" + argosfs_qemu_monitor_command "$monitor" \ + "device_add virtio-blk-pci,drive=crash$idx,id=crashdisk$idx${bus_arg}${rom_arg}" "$monitor_log" } run_phase1_until_kill_marker() { rm -f "$monitor" argosfs_qemu_build_args + argosfs_qemu_add_hotplug_ports 3 crash qemu_args+=(-monitor "unix:$monitor,server,nowait") + : >"$monitor_log" : >"$log1" set +e + # QEMU output is intentionally polled while this pipeline appends to the log. + # shellcheck disable=SC2094 ( sleep "$login_delay_s" printf '\r' @@ -109,6 +118,7 @@ run_phase1_until_kill_marker() { printf '%s\r' "$line" sleep "$command_delay_s" if [ "$line" = "echo ARGOSFS_WAIT_CRASH_HOTPLUG" ]; then + argosfs_qemu_wait_log_marker "$log1" ARGOSFS_WAIT_CRASH_HOTPLUG 180 for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done idx=0 for disk in "${disks[@]}"; do qemu_device_add "$idx" "$disk"; idx=$((idx + 1)); done @@ -136,6 +146,8 @@ run_phase1_until_kill_marker() { run_phase2() { argosfs_qemu_build_args set +e + # QEMU output is intentionally polled while this pipeline appends to the log. + # shellcheck disable=SC2094 ( sleep "$login_delay_s" printf '\r' diff --git a/scripts/qemu/degraded_rootfs.sh b/scripts/qemu/degraded_rootfs.sh index 6a53c1c..fe58c1f 100755 --- a/scripts/qemu/degraded_rootfs.sh +++ b/scripts/qemu/degraded_rootfs.sh @@ -9,6 +9,7 @@ argosfs_qemu_select_arch argosfs_qemu_require_binary monitor="$artifacts/qemu-monitor.sock" +monitor_log="$artifacts/qemu-monitor.log" log="$artifacts/qemu-degraded-rootfs-$arch.log" commands="$artifacts/qemu-degraded-rootfs.commands" reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:|segfault|I/O error}" @@ -32,8 +33,10 @@ awk '$2=="/"{print "ARGOSFS_ROOT_MOUNT " $1 " " $3 " " $4; exit}' /proc/mounts test -e /run/argosfs-root-active && echo ARGOSFS_ROOT_MARKER_OK echo ARGOSFS_WAIT_DEGRADED_HOTPLUG for dev in /dev/vdb /dev/vdc /dev/vdd; do - for i in $(seq 1 60); do [ -b "$dev" ] && break; sleep 1; done + name="${dev##*/}" + for i in $(seq 1 60); do [ -b "$dev" ] && [ -e "/sys/class/block/$name" ] && break; sleep 1; done test -b "$dev" + test -e "/sys/class/block/$name" done src=/tmp/argosfs-degraded-src mnt=/mnt/argosfs-degraded-root @@ -70,19 +73,25 @@ poweroff -f || reboot -f || halt -f CMDS argosfs_qemu_build_args +argosfs_qemu_add_hotplug_ports 3 deg qemu_args+=(-monitor "unix:$monitor,server,nowait") +: >"$monitor_log" qemu_device_add() { local idx="$1" path="$2" - printf 'drive_add 0 if=none,file=%s,format=raw,id=deg%s\n' "$path" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - if [ "$arch" = "arm64" ]; then - printf 'device_add virtio-blk-pci,drive=deg%s,id=degdisk%s,romfile=\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - else - printf 'device_add virtio-blk-pci,drive=deg%s,id=degdisk%s\n' "$idx" "$idx" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - fi + local bus_arg rom_arg + argosfs_qemu_monitor_command "$monitor" \ + "drive_add 0 if=none,file=$path,format=raw,id=deg$idx" "$monitor_log" + bus_arg="$(argosfs_qemu_hotplug_bus_arg deg "$idx")" + rom_arg="" + [ "$arch" != "arm64" ] || rom_arg=",romfile=" + argosfs_qemu_monitor_command "$monitor" \ + "device_add virtio-blk-pci,drive=deg$idx,id=degdisk$idx${bus_arg}${rom_arg}" "$monitor_log" } set +e +# QEMU output is intentionally polled while this pipeline appends to the log. +# shellcheck disable=SC2094 ( sleep "$login_delay_s" printf '\r' @@ -91,6 +100,7 @@ set +e printf '%s\r' "$line" sleep "$command_delay_s" if [ "$line" = "echo ARGOSFS_WAIT_DEGRADED_HOTPLUG" ]; then + argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_DEGRADED_HOTPLUG 180 for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done idx=0 for disk in "${disks[@]}"; do qemu_device_add "$idx" "$disk"; idx=$((idx + 1)); done diff --git a/scripts/qemu/hotplug.sh b/scripts/qemu/hotplug.sh index b3acb2f..6031893 100755 --- a/scripts/qemu/hotplug.sh +++ b/scripts/qemu/hotplug.sh @@ -13,9 +13,10 @@ argosfs_qemu_require_binary hotplug_disk="$artifacts/hotplug.img" qemu-img create -f raw "$hotplug_disk" "${ARGOSFS_QEMU_HOTPLUG_SIZE:-64M}" >/dev/null monitor="$artifacts/qemu-monitor.sock" +monitor_log="$artifacts/qemu-monitor.log" log="$artifacts/qemu-hotplug.log" commands="$artifacts/qemu-hotplug.commands" -reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:}" +reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:|I/O error|missing device}" timeout_s="${ARGOSFS_QEMU_TIMEOUT:-260}" login_delay_s="${ARGOSFS_QEMU_HOTPLUG_LOGIN_DELAY:-90}" login_delay_s="$(argosfs_qemu_adjust_login_delay "$login_delay_s")" @@ -27,25 +28,28 @@ echo ARGOSFS_QEMU_HOTPLUG_BEGIN awk '$2=="/"{print "ARGOSFS_ROOT_MOUNT " $1 " " $3 " " $4; exit}' /proc/mounts ls -l /dev/vd* 2>/dev/null || true echo ARGOSFS_WAIT_HOTPLUG -for i in $(seq 1 30); do [ -b /dev/vdb ] && break; sleep 1; done +for i in $(seq 1 30); do [ -b /dev/vdb ] && [ -e /sys/class/block/vdb ] && break; sleep 1; done ls -l /dev/vd* 2>/dev/null || true -test -b /dev/vdb && echo ARGOSFS_HOTPLUG_BLOCK_DEVICE_OK +test -b /dev/vdb && test -e /sys/class/block/vdb && echo ARGOSFS_HOTPLUG_BLOCK_DEVICE_OK mkdir -p /tmp/argosfs-hotplug/src /tmp/argosfs-hotplug/out printf 'hotplug payload\n' >/tmp/argosfs-hotplug/payload.txt printf 'raw hotplug payload\n' >/tmp/argosfs-hotplug/src/raw.txt if command -v argosfs >/dev/null; then - argosfs mkfs --backend raw --devices /dev/vdb --k 1 --m 0 --chunk-size 32768 --compression zstd --force >/tmp/argosfs-hotplug/mkfs-raw.json - argosfs import-tree --backend raw --devices /dev/vdb /tmp/argosfs-hotplug/src / - argosfs export-tree --backend raw --devices /dev/vdb /tmp/argosfs-hotplug/out - cmp /tmp/argosfs-hotplug/src/raw.txt /tmp/argosfs-hotplug/out/raw.txt - argosfs fsck --backend raw --devices /dev/vdb --repair --remove-orphans >/tmp/argosfs-hotplug/fsck-raw.json - argosfs scrub --backend raw --devices /dev/vdb >/tmp/argosfs-hotplug/scrub-raw.json - echo ARGOSFS_HOTPLUG_RAW_ARGOSFS_OK + if argosfs mkfs --backend raw --devices /dev/vdb --k 1 --m 0 --chunk-size 32768 --compression zstd --force >/tmp/argosfs-hotplug/mkfs-raw.json && + argosfs import-tree --backend raw --devices /dev/vdb /tmp/argosfs-hotplug/src / && + argosfs export-tree --backend raw --devices /dev/vdb /tmp/argosfs-hotplug/out && + cmp /tmp/argosfs-hotplug/src/raw.txt /tmp/argosfs-hotplug/out/raw.txt && + argosfs fsck --backend raw --devices /dev/vdb --repair --remove-orphans >/tmp/argosfs-hotplug/fsck-raw.json && + argosfs scrub --backend raw --devices /dev/vdb >/tmp/argosfs-hotplug/scrub-raw.json; then + echo ARGOSFS_HOTPLUG_RAW_ARGOSFS_OK + else + echo ARGOSFS_HOTPLUG_RAW_ARGOSFS_FAILED + fi fi sync echo ARGOSFS_WAIT_UNPLUG -for i in $(seq 1 30); do [ ! -b /dev/vdb ] && break; sleep 1; done -if [ ! -b /dev/vdb ]; then echo ARGOSFS_UNPLUG_BLOCK_DEVICE_OK; fi +for i in $(seq 1 30); do [ ! -e /sys/class/block/vdb ] && break; sleep 1; done +if [ ! -e /sys/class/block/vdb ]; then echo ARGOSFS_UNPLUG_BLOCK_DEVICE_OK; fi test "$(cat /tmp/argosfs-hotplug/payload.txt)" = "hotplug payload" && echo ARGOSFS_ROOT_SURVIVED_HOTPLUG_OK rm -rf /tmp/argosfs-hotplug sync @@ -54,9 +58,13 @@ poweroff -f || reboot -f || halt -f CMDS argosfs_qemu_build_args +argosfs_qemu_add_hotplug_ports 1 hot qemu_args+=(-monitor "unix:$monitor,server,nowait") +: >"$monitor_log" set +e +# QEMU output is intentionally polled while this pipeline appends to the log. +# shellcheck disable=SC2094 ( sleep "$login_delay_s" printf '\r' @@ -65,18 +73,21 @@ set +e printf '%s\r' "$line" sleep "$command_delay_s" if [ "$line" = "echo ARGOSFS_WAIT_HOTPLUG" ]; then + argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_HOTPLUG 180 for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done - printf 'drive_add 0 if=none,file=%s,format=raw,id=hot0\n' "$hotplug_disk" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - if [ "$arch" = "arm64" ]; then - printf 'device_add virtio-blk-pci,drive=hot0,id=hotdisk0,romfile=\n' | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - else - printf 'device_add virtio-blk-pci,drive=hot0,id=hotdisk0\n' | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - fi + argosfs_qemu_monitor_command "$monitor" \ + "drive_add 0 if=none,file=$hotplug_disk,format=raw,id=hot0" "$monitor_log" + bus_arg="$(argosfs_qemu_hotplug_bus_arg hot 0)" + rom_arg="" + [ "$arch" != "arm64" ] || rom_arg=",romfile=" + argosfs_qemu_monitor_command "$monitor" \ + "device_add virtio-blk-pci,drive=hot0,id=hotdisk0${bus_arg}${rom_arg}" "$monitor_log" fi if [ "$line" = "echo ARGOSFS_WAIT_UNPLUG" ]; then - printf 'device_del hotdisk0\n' | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true + argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_UNPLUG 600 + argosfs_qemu_monitor_command "$monitor" "device_del hotdisk0" "$monitor_log" sleep 2 - printf 'drive_del hot0\n' | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true + argosfs_qemu_monitor_command "$monitor" "drive_del hot0" "$monitor_log" fi done <"$commands" ) | timeout "$timeout_s" "$qemu_bin" "${qemu_args[@]}" >"$log" 2>&1 diff --git a/scripts/qemu/lib/common.sh b/scripts/qemu/lib/common.sh index d3a7850..995000b 100755 --- a/scripts/qemu/lib/common.sh +++ b/scripts/qemu/lib/common.sh @@ -111,6 +111,86 @@ argosfs_qemu_wait_process_gone() { return 1 } +argosfs_qemu_add_hotplug_ports() { + local count="$1" + local prefix="$2" + local index=0 + local slot + + [ "${arch:-}" = "arm64" ] || return 0 + while [ "$index" -lt "$count" ]; do + slot=$((index + 2)) + qemu_args+=( + -device "pcie-root-port,id=${prefix}port${index},chassis=${slot},slot=${slot}" + ) + index=$((index + 1)) + done +} + +argosfs_qemu_hotplug_bus_arg() { + local prefix="$1" + local index="$2" + if [ "${arch:-}" = "arm64" ]; then + printf ',bus=%sport%s' "$prefix" "$index" + fi +} + +argosfs_qemu_wait_log_marker() { + local log="$1" + local marker="$2" + local timeout_s="${3:-120}" + local deadline=$((SECONDS + timeout_s)) + + while [ "$SECONDS" -lt "$deadline" ]; do + if [ -f "$log" ] && awk -v marker="$marker" \ + '{ sub(/\r$/, ""); if ($0 == marker) found = 1 } END { exit !found }' "$log"; then + return 0 + fi + sleep 1 + done + echo "timed out waiting for QEMU guest marker: $marker" >&2 + return 1 +} + +argosfs_qemu_monitor_command() { + local monitor="$1" + local command="$2" + local monitor_log="$3" + local attempt=1 + local response + local status + + while [ "$attempt" -le 3 ]; do + if [ ! -S "$monitor" ]; then + sleep 1 + attempt=$((attempt + 1)) + continue + fi + + if response="$(printf '%s\n' "$command" | timeout 10 socat - "UNIX-CONNECT:$monitor" 2>&1)"; then + status=0 + else + status=$? + fi + { + printf '>>> %s\n' "$command" + printf '%s\n' "$response" | tail -c 8192 + printf '\n' + } >>"$monitor_log" + + if [ "$status" -eq 0 ] && ! printf '%s\n' "$response" | grep -Eiq \ + 'Error:|unknown command|invalid parameter|not found|not supported|does not support hotplugging|already in use'; then + return 0 + fi + sleep 1 + attempt=$((attempt + 1)) + done + + echo "QEMU monitor command failed: $command" >&2 + tail -n 80 "$monitor_log" >&2 || true + return 1 +} + argosfs_qemu_build_args() { kernel="${ARGOSFS_QEMU_KERNEL:-}" rootfs="${ARGOSFS_QEMU_ROOTFS:-}" diff --git a/scripts/qemu/mixed_chaos.sh b/scripts/qemu/mixed_chaos.sh index 12e8b94..d70182c 100755 --- a/scripts/qemu/mixed_chaos.sh +++ b/scripts/qemu/mixed_chaos.sh @@ -9,6 +9,7 @@ argosfs_qemu_select_arch argosfs_qemu_require_binary monitor="$artifacts/qemu-monitor.sock" +monitor_log="$artifacts/qemu-monitor.log" log1="$artifacts/qemu-mixed-chaos-phase1-$arch.log" log2="$artifacts/qemu-mixed-chaos-phase2-$arch.log" commands1="$artifacts/qemu-mixed-chaos-phase1.commands" @@ -42,8 +43,10 @@ awk '\$2=="/"{print "ARGOSFS_ROOT_MOUNT_PHASE1 " \$1 " " \$3 " " \$4; exit}' /pr test -e /run/argosfs-root-active && echo ARGOSFS_CHAOS_ROOT_MARKER_PHASE1_OK echo ARGOSFS_WAIT_CHAOS_HOTPLUG for dev in /dev/vdb /dev/vdc /dev/vdd /dev/vde /dev/vdf /dev/vdg; do - for i in \$(seq 1 90); do [ -b "\$dev" ] && break; sleep 1; done + name="\${dev##*/}" + for i in \$(seq 1 90); do [ -b "\$dev" ] && [ -e "/sys/class/block/\$name" ] && break; sleep 1; done test -b "\$dev" + test -e "/sys/class/block/\$name" done echo ARGOSFS_CHAOS_BLOCK_DEVICES_OK src=/root/argosfs-chaos-source @@ -94,8 +97,8 @@ while [ "\$id" -le "$worker_count" ]; do chaos_worker "\$id" & id=\$((id + 1)); sleep 3 echo ARGOSFS_CHAOS_WORKLOAD_STARTED echo ARGOSFS_WAIT_CHAOS_UNPLUG -for i in \$(seq 1 90); do [ ! -b /dev/vdc ] && break; sleep 1; done -test ! -b /dev/vdc +for i in \$(seq 1 90); do [ ! -e /sys/class/block/vdc ] && break; sleep 1; done +test ! -e /sys/class/block/vdc echo ARGOSFS_CHAOS_DEVICE_LOSS_OBSERVED survivors=/dev/vdb,/dev/vdd,/dev/vde,/dev/vdf minimum=/dev/vdb,/dev/vdd,/dev/vdf @@ -145,8 +148,10 @@ grep -q 'mixed chaos persistent sentinel' /root/argosfs-chaos-source/sentinel.tx echo ARGOSFS_CHAOS_ROOT_REPLAY_OK echo ARGOSFS_WAIT_CHAOS_REATTACH for dev in /dev/vdb /dev/vdc /dev/vdd /dev/vde /dev/vdf; do - for i in $(seq 1 90); do [ -b "$dev" ] && break; sleep 1; done + name="${dev##*/}" + for i in $(seq 1 90); do [ -b "$dev" ] && [ -e "/sys/class/block/$name" ] && break; sleep 1; done test -b "$dev" + test -e "/sys/class/block/$name" done devs=/dev/vdb,/dev/vdc,/dev/vdd,/dev/vde,/dev/vdf argosfs list-devices --backend raw --devices "$devs" >/tmp/argosfs-chaos-devices-after-reboot.json @@ -170,27 +175,33 @@ CMDS qemu_device_add() { local prefix="$1" index="$2" path="$3" - printf 'drive_add 0 if=none,file=%s,format=raw,id=%s%s\n' "$path" "$prefix" "$index" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - if [ "$arch" = "arm64" ]; then - printf 'device_add virtio-blk-pci,drive=%s%s,id=%sdisk%s,romfile=\n' "$prefix" "$index" "$prefix" "$index" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - else - printf 'device_add virtio-blk-pci,drive=%s%s,id=%sdisk%s\n' "$prefix" "$index" "$prefix" "$index" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true - fi + local bus_arg rom_arg + argosfs_qemu_monitor_command "$monitor" \ + "drive_add 0 if=none,file=$path,format=raw,id=$prefix$index" "$monitor_log" + bus_arg="$(argosfs_qemu_hotplug_bus_arg "$prefix" "$index")" + rom_arg="" + [ "$arch" != "arm64" ] || rom_arg=",romfile=" + argosfs_qemu_monitor_command "$monitor" \ + "device_add virtio-blk-pci,drive=$prefix$index,id=${prefix}disk$index${bus_arg}${rom_arg}" "$monitor_log" } qemu_device_del() { local prefix="$1" index="$2" - printf 'device_del %sdisk%s\n' "$prefix" "$index" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true + argosfs_qemu_monitor_command "$monitor" "device_del ${prefix}disk$index" "$monitor_log" sleep 3 - printf 'drive_del %s%s\n' "$prefix" "$index" | socat - "UNIX-CONNECT:$monitor" >/dev/null 2>&1 || true + argosfs_qemu_monitor_command "$monitor" "drive_del $prefix$index" "$monitor_log" } run_phase1_until_kill_marker() { rm -f "$monitor" argosfs_qemu_build_args + argosfs_qemu_add_hotplug_ports 6 chaos qemu_args+=(-monitor "unix:$monitor,server,nowait") + : >"$monitor_log" : >"$log1" set +e + # QEMU output is intentionally polled while this pipeline appends to the log. + # shellcheck disable=SC2094 ( sleep "$login_delay_s" printf '\r' @@ -199,6 +210,7 @@ run_phase1_until_kill_marker() { printf '%s\r' "$line" sleep "$command_delay_s" if [ "$line" = "echo ARGOSFS_WAIT_CHAOS_HOTPLUG" ]; then + argosfs_qemu_wait_log_marker "$log1" ARGOSFS_WAIT_CHAOS_HOTPLUG 180 for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done idx=0 for disk in "${disks[@]}"; do qemu_device_add chaos "$idx" "$disk"; idx=$((idx + 1)); done @@ -229,8 +241,12 @@ run_phase1_until_kill_marker() { run_phase2() { rm -f "$monitor" argosfs_qemu_build_args + argosfs_qemu_add_hotplug_ports 5 recover qemu_args+=(-monitor "unix:$monitor,server,nowait") + : >"$monitor_log" set +e + # QEMU output is intentionally polled while this pipeline appends to the log. + # shellcheck disable=SC2094 ( sleep "$login_delay_s" printf '\r' @@ -239,6 +255,7 @@ run_phase2() { printf '%s\r' "$line" sleep "$command_delay_s" if [ "$line" = "echo ARGOSFS_WAIT_CHAOS_REATTACH" ]; then + argosfs_qemu_wait_log_marker "$log2" ARGOSFS_WAIT_CHAOS_REATTACH 180 for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done qemu_device_add recover 0 "${disks[0]}" qemu_device_add recover 1 "${disks[2]}" From cfbe0cc0d8e6764281dcd41fc07292941fc5321f Mon Sep 17 00:00:00 2001 From: Cao Yuhang Date: Sun, 12 Jul 2026 16:55:49 +0000 Subject: [PATCH 2/2] ci: pin CapOS arm64 PCIe hotplug support --- .github/workflows/capos-full-qemu.yml | 2 +- scripts/qemu/build_capos_image.sh | 2 +- scripts/qemu/capos_build.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/capos-full-qemu.yml b/.github/workflows/capos-full-qemu.yml index 01b8f31..cf6d4aa 100644 --- a/.github/workflows/capos-full-qemu.yml +++ b/.github/workflows/capos-full-qemu.yml @@ -27,7 +27,7 @@ permissions: env: CARGO_TERM_COLOR: always - ARGOSFS_CAPOS_REF: 9e636b9b45941e9d9219af97054b4c661d416294 + ARGOSFS_CAPOS_REF: 168f01fc44f627ab07f23dd9590453f0ba5fe7fc jobs: capos-full-build: diff --git a/scripts/qemu/build_capos_image.sh b/scripts/qemu/build_capos_image.sh index 54744d8..1c04224 100755 --- a/scripts/qemu/build_capos_image.sh +++ b/scripts/qemu/build_capos_image.sh @@ -38,7 +38,7 @@ ARGOSFS_CAPOS_MAKE_JOBS="$make_jobs" \ ARGOSFS_CAPOS_MAKE_TARGET="$make_target" \ ARGOSFS_CAPOS_MAKE_V="${ARGOSFS_CAPOS_MAKE_V:-}" \ CAPOS_REPO="${CAPOS_REPO:-https://github.com/fwerkor/capos.git}" \ -CAPOS_REF="${CAPOS_REF:-9e636b9b45941e9d9219af97054b4c661d416294}" \ +CAPOS_REF="${CAPOS_REF:-168f01fc44f627ab07f23dd9590453f0ba5fe7fc}" \ CAPOS_LOCAL_SOURCE="${CAPOS_LOCAL_SOURCE:-}" \ "$repo/scripts/qemu/capos_build.sh" diff --git a/scripts/qemu/capos_build.sh b/scripts/qemu/capos_build.sh index a0059a9..77c8ba3 100755 --- a/scripts/qemu/capos_build.sh +++ b/scripts/qemu/capos_build.sh @@ -4,7 +4,7 @@ set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" artifacts="${ARGOSFS_TEST_ARTIFACTS:-$repo/target/argosfs-test-artifacts/capos-build}" capos_repo="${CAPOS_REPO:-https://github.com/fwerkor/capos.git}" -capos_ref="${CAPOS_REF:-9e636b9b45941e9d9219af97054b4c661d416294}" +capos_ref="${CAPOS_REF:-168f01fc44f627ab07f23dd9590453f0ba5fe7fc}" capos_local_source="${CAPOS_LOCAL_SOURCE:-}" capos_full_compile="${ARGOSFS_CAPOS_FULL_COMPILE:-1}" capos_make_jobs="${ARGOSFS_CAPOS_MAKE_JOBS:-$(nproc 2>/dev/null || echo 2)}"