ci: pin CapOS baseline and stabilize QEMU guest scripts

.github/workflows/capos-full-qemu.yml

@@ -24,7 +24,7 @@ permissions:
 
 env:
   CARGO_TERM_COLOR: always
-  ARGOSFS_CAPOS_REF: 168f01fc44f627ab07f23dd9590453f0ba5fe7fc
+  ARGOSFS_CAPOS_REF: 3903b2968a692fdc77e05f63eb3a1cfa9a739999
 
 jobs:
   capos-full-build:

.github/workflows/weekly-randomized-model.yml

@@ -49,7 +49,7 @@ permissions:
 
 env:
   CARGO_TERM_COLOR: always
-  ARGOSFS_CAPOS_REF: 168f01fc44f627ab07f23dd9590453f0ba5fe7fc
+  ARGOSFS_CAPOS_REF: 3903b2968a692fdc77e05f63eb3a1cfa9a739999
 
 jobs:
   prepare-matrix:

scripts/qemu/build_capos_image.sh

@@ -38,7 +38,7 @@ ARGOSFS_CAPOS_MAKE_JOBS="$make_jobs" \
 ARGOSFS_CAPOS_MAKE_TARGET="$make_target" \
 ARGOSFS_CAPOS_MAKE_V="${ARGOSFS_CAPOS_MAKE_V:-}" \
 CAPOS_REPO="${CAPOS_REPO:-https://github.com/fwerkor/capos.git}" \
-CAPOS_REF="${CAPOS_REF:-168f01fc44f627ab07f23dd9590453f0ba5fe7fc}" \
+CAPOS_REF="${CAPOS_REF:-3903b2968a692fdc77e05f63eb3a1cfa9a739999}" \
 CAPOS_LOCAL_SOURCE="${CAPOS_LOCAL_SOURCE:-}" \
 "$repo/scripts/qemu/capos_build.sh"
 

scripts/qemu/capos_build.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 artifacts="${ARGOSFS_TEST_ARTIFACTS:-$repo/target/argosfs-test-artifacts/capos-build}"
 capos_repo="${CAPOS_REPO:-https://github.com/fwerkor/capos.git}"
-capos_ref="${CAPOS_REF:-168f01fc44f627ab07f23dd9590453f0ba5fe7fc}"
+capos_ref="${CAPOS_REF:-3903b2968a692fdc77e05f63eb3a1cfa9a739999}"
 capos_local_source="${CAPOS_LOCAL_SOURCE:-}"
 capos_full_compile="${ARGOSFS_CAPOS_FULL_COMPILE:-1}"
 capos_make_jobs="${ARGOSFS_CAPOS_MAKE_JOBS:-$(nproc 2>/dev/null || echo 2)}"

scripts/qemu/crash_recovery.sh

@@ -17,7 +17,6 @@ commands2="$artifacts/qemu-crash-recovery-phase2.commands"
 reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:|segfault}"
 timeout_s="${ARGOSFS_QEMU_TIMEOUT:-1800}"
 console_timeout_s="${ARGOSFS_QEMU_CRASH_CONSOLE_TIMEOUT:-600}"
-command_delay_s="${ARGOSFS_QEMU_CRASH_COMMAND_DELAY:-1}"
 done_marker="ARGOSFS_QEMU_CRASH_RECOVERY_DONE"
 
 disks=()
@@ -111,18 +110,11 @@ run_phase1_until_kill_marker() {
   # shellcheck disable=SC2094
   (
     argosfs_qemu_wait_console_prompt "$log1" 1 "$console_timeout_s" "$reject" "crash-recovery phase1 console prompt" || exit $?
-    printf '\r'
-    sleep "$command_delay_s"
-    while IFS= read -r line; do
-      printf '%s\r' "$line" || exit 0
-      sleep "$command_delay_s"
-      if [ "$line" = "echo ARGOSFS_WAIT_CRASH_HOTPLUG" ]; then
-        argosfs_qemu_wait_log_marker "$log1" ARGOSFS_WAIT_CRASH_HOTPLUG 180
-        for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
-        idx=0
-        for disk in "${disks[@]}"; do qemu_device_add "$idx" "$disk"; idx=$((idx + 1)); done
-      fi
-    done <"$commands1"
+    argosfs_qemu_stream_script "$commands1" 1 /tmp/argosfs-qemu-crash-phase1.sh "$log1"
+    argosfs_qemu_wait_log_marker "$log1" ARGOSFS_WAIT_CRASH_HOTPLUG 180
+    for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
+    idx=0
+    for disk in "${disks[@]}"; do qemu_device_add "$idx" "$disk"; idx=$((idx + 1)); done
   ) | timeout "$timeout_s" "$qemu_bin" "${qemu_args[@]}" >"$log1" 2>&1 &
   qemu_pid=$!
   deadline=$((SECONDS + timeout_s))
@@ -149,12 +141,7 @@ run_phase2() {
   # shellcheck disable=SC2094
   (
     argosfs_qemu_wait_console_prompt "$log2" 1 "$console_timeout_s" "$reject" "crash-recovery phase2 console prompt" || exit $?
-    printf '\r'
-    sleep "$command_delay_s"
-    while IFS= read -r line; do
-      printf '%s\r' "$line" || exit 0
-      sleep "$command_delay_s"
-    done <"$commands2"
+    argosfs_qemu_stream_script "$commands2" 1 /tmp/argosfs-qemu-crash-phase2.sh "$log2"
   ) | timeout "$timeout_s" "$qemu_bin" "${qemu_args[@]}" >"$log2" 2>&1
   status=${PIPESTATUS[1]}
   set -e

scripts/qemu/hotplug.sh

@@ -19,7 +19,6 @@ commands="$artifacts/qemu-hotplug.commands"
 reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:|I/O error|missing device}"
 timeout_s="${ARGOSFS_QEMU_TIMEOUT:-260}"
 console_timeout_s="${ARGOSFS_QEMU_HOTPLUG_CONSOLE_TIMEOUT:-420}"
-command_delay_s="${ARGOSFS_QEMU_HOTPLUG_COMMAND_DELAY:-1}"
 done_marker="ARGOSFS_QEMU_HOTPLUG_DONE"
 
 cat >"$commands" <<'CMDS'
@@ -66,29 +65,20 @@ set +e
 # shellcheck disable=SC2094
 (
 	argosfs_qemu_wait_console_prompt "$log" 1 "$console_timeout_s" "$reject" "hotplug console prompt" || exit $?
-	printf '\r'
-	sleep "$command_delay_s"
-	while IFS= read -r line; do
-		printf '%s\r' "$line"
-		sleep "$command_delay_s"
-		if [ "$line" = "echo ARGOSFS_WAIT_HOTPLUG" ]; then
-			argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_HOTPLUG 180
-			for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
-			argosfs_qemu_monitor_command "$monitor" \
-				"drive_add 0 if=none,file=$hotplug_disk,format=raw,id=hot0" "$monitor_log"
-			bus_arg="$(argosfs_qemu_hotplug_bus_arg hot 0)"
-			rom_arg=""
-			[ "$arch" != "arm64" ] || rom_arg=",romfile="
-			argosfs_qemu_monitor_command "$monitor" \
-				"device_add virtio-blk-pci,drive=hot0,id=hotdisk0${bus_arg}${rom_arg}" "$monitor_log"
-		fi
-		if [ "$line" = "echo ARGOSFS_WAIT_UNPLUG" ]; then
-			argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_UNPLUG 600
-			argosfs_qemu_monitor_command "$monitor" "device_del hotdisk0" "$monitor_log"
-			sleep 2
-			argosfs_qemu_monitor_command "$monitor" "drive_del hot0" "$monitor_log" "Device '[^']+' not found"
-		fi
-	done <"$commands"
+	argosfs_qemu_stream_script "$commands" 1 /tmp/argosfs-qemu-hotplug.sh "$log"
+	argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_HOTPLUG 180
+	for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
+	argosfs_qemu_monitor_command "$monitor" \
+		"drive_add 0 if=none,file=$hotplug_disk,format=raw,id=hot0" "$monitor_log"
+	bus_arg="$(argosfs_qemu_hotplug_bus_arg hot 0)"
+	rom_arg=""
+	[ "$arch" != "arm64" ] || rom_arg=",romfile="
+	argosfs_qemu_monitor_command "$monitor" \
+		"device_add virtio-blk-pci,drive=hot0,id=hotdisk0${bus_arg}${rom_arg}" "$monitor_log"
+	argosfs_qemu_wait_log_marker "$log" ARGOSFS_WAIT_UNPLUG 600
+	argosfs_qemu_monitor_command "$monitor" "device_del hotdisk0" "$monitor_log"
+	sleep 2
+	argosfs_qemu_monitor_command "$monitor" "drive_del hot0" "$monitor_log" "Device '[^']+' not found"
 ) | timeout "$timeout_s" "$qemu_bin" "${qemu_args[@]}" >"$log" 2>&1
 status=${PIPESTATUS[1]}
 set -e

scripts/qemu/lib/common.sh

@@ -152,6 +152,61 @@ argosfs_qemu_wait_log_marker() {
 	return 1
 }
 
+argosfs_qemu_stream_script() {
+	local script="$1"
+	local fd="${2:-1}"
+	local remote="${3:-/tmp/argosfs-qemu-script.sh}"
+	local log="${4:-}"
+	local line_delay="${ARGOSFS_QEMU_SCRIPT_LINE_DELAY:-0.01}"
+	local delimiter="ARGOSFS_QEMU_SCRIPT_EOF"
+	local marker_id
+	local shell_marker
+	local upload_marker
+	local deadline
+	local line
+
+	if grep -Fxq "$delimiter" "$script"; then
+		echo "QEMU guest script contains reserved delimiter: $delimiter" >&2
+		return 2
+	fi
+	marker_id="$(printf '%s' "$remote" | cksum | awk '{print $1}')"
+	shell_marker="ARGOSFS_QEMU_SHELL_READY_$marker_id"
+	upload_marker="ARGOSFS_QEMU_UPLOAD_READY_$marker_id"
+
+	printf '\r' >&"$fd"
+	if [ -n "$log" ]; then
+		deadline=$((SECONDS + ${ARGOSFS_QEMU_SHELL_READY_TIMEOUT:-60}))
+		while [ "$SECONDS" -lt "$deadline" ]; do
+			if awk -v marker="$shell_marker" \
+				'{ sub(/\r$/, ""); if ($0 == marker) found = 1 } END { exit !found }' "$log" 2>/dev/null; then
+				break
+			fi
+			printf 'echo %s\r' "$shell_marker" >&"$fd"
+			sleep 1
+		done
+		if ! awk -v marker="$shell_marker" \
+			'{ sub(/\r$/, ""); if ($0 == marker) found = 1 } END { exit !found }' "$log" 2>/dev/null; then
+			echo "timed out waiting for QEMU guest shell: $shell_marker" >&2
+			return 1
+		fi
+	else
+		sleep "${ARGOSFS_QEMU_SCRIPT_START_DELAY:-1}"
+	fi
+
+	printf 'stty -echo; echo %s\r' "$upload_marker" >&"$fd"
+	if [ -n "$log" ]; then
+		argosfs_qemu_wait_log_marker "$log" "$upload_marker" "${ARGOSFS_QEMU_SCRIPT_READY_TIMEOUT:-30}" || return $?
+	else
+		sleep "${ARGOSFS_QEMU_SCRIPT_STTY_DELAY:-0.2}"
+	fi
+	printf "cat >'%s' <<'%s'\r" "$remote" "$delimiter" >&"$fd"
+	while IFS= read -r line || [ -n "$line" ]; do
+		printf '%s\r' "$line" >&"$fd"
+		sleep "$line_delay"
+	done <"$script"
+	printf '%s\r' "$delimiter" >&"$fd"
+	printf "stty echo; sh '%s'\r" "$remote" >&"$fd"
+}
 argosfs_qemu_wait_console_prompt() {
 	local log="$1"
 	local min_count="${2:-1}"

scripts/qemu/mixed_chaos.sh

@@ -17,7 +17,6 @@ commands2="$artifacts/qemu-mixed-chaos-phase2.commands"
 reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd: emergency|Oops:|BUG:|segfault}"
 timeout_s="${ARGOSFS_QEMU_TIMEOUT:-3000}"
 console_timeout_s="${ARGOSFS_QEMU_CHAOS_CONSOLE_TIMEOUT:-600}"
-command_delay_s="${ARGOSFS_QEMU_CHAOS_COMMAND_DELAY:-1}"
 worker_count="${ARGOSFS_QEMU_CHAOS_WORKERS:-6}"
 file_count="${ARGOSFS_QEMU_CHAOS_FILES:-160}"
 if [ "$file_count" -lt 150 ]; then
@@ -203,18 +202,11 @@ run_phase1_until_kill_marker() {
   # shellcheck disable=SC2094
   (
     argosfs_qemu_wait_console_prompt "$log1" 1 "$console_timeout_s" "$reject" "mixed-chaos phase1 console prompt" || exit $?
-    printf '\r'
-    sleep "$command_delay_s"
-    while IFS= read -r line; do
-      printf '%s\r' "$line"
-      sleep "$command_delay_s"
-      if [ "$line" = "echo ARGOSFS_WAIT_CHAOS_HOTPLUG" ]; then
-        argosfs_qemu_wait_log_marker "$log1" ARGOSFS_WAIT_CHAOS_HOTPLUG 180
-        for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
-        idx=0
-        for disk in "${disks[@]}"; do qemu_device_add chaos "$idx" "$disk"; idx=$((idx + 1)); done
-      fi
-    done <"$commands1"
+    argosfs_qemu_stream_script "$commands1" 1 /tmp/argosfs-qemu-mixed-phase1.sh "$log1"
+    argosfs_qemu_wait_log_marker "$log1" ARGOSFS_WAIT_CHAOS_HOTPLUG 180
+    for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
+    idx=0
+    for disk in "${disks[@]}"; do qemu_device_add chaos "$idx" "$disk"; idx=$((idx + 1)); done
   ) | timeout "$timeout_s" "$qemu_bin" "${qemu_args[@]}" >"$log1" 2>&1 &
   qemu_pid=$!
   deadline=$((SECONDS + timeout_s))
@@ -248,21 +240,14 @@ run_phase2() {
   # shellcheck disable=SC2094
   (
     argosfs_qemu_wait_console_prompt "$log2" 1 "$console_timeout_s" "$reject" "mixed-chaos phase2 console prompt" || exit $?
-    printf '\r'
-    sleep "$command_delay_s"
-    while IFS= read -r line; do
-      printf '%s\r' "$line"
-      sleep "$command_delay_s"
-      if [ "$line" = "echo ARGOSFS_WAIT_CHAOS_REATTACH" ]; then
-        argosfs_qemu_wait_log_marker "$log2" ARGOSFS_WAIT_CHAOS_REATTACH 180
-        for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
-        qemu_device_add recover 0 "${disks[0]}"
-        qemu_device_add recover 1 "${disks[2]}"
-        qemu_device_add recover 2 "${disks[3]}"
-        qemu_device_add recover 3 "${disks[4]}"
-        qemu_device_add recover 4 "${disks[5]}"
-      fi
-    done <"$commands2"
+    argosfs_qemu_stream_script "$commands2" 1 /tmp/argosfs-qemu-mixed-phase2.sh "$log2"
+    argosfs_qemu_wait_log_marker "$log2" ARGOSFS_WAIT_CHAOS_REATTACH 180
+    for _ in $(seq 1 30); do [ -S "$monitor" ] && break; sleep 1; done
+    qemu_device_add recover 0 "${disks[0]}"
+    qemu_device_add recover 1 "${disks[2]}"
+    qemu_device_add recover 2 "${disks[3]}"
+    qemu_device_add recover 3 "${disks[4]}"
+    qemu_device_add recover 4 "${disks[5]}"
   ) | timeout "$timeout_s" "$qemu_bin" "${qemu_args[@]}" >"$log2" 2>&1
   status=${PIPESTATUS[1]}
   set -e

scripts/qemu/reboot_persistence.sh

@@ -17,7 +17,6 @@ reject="${ARGOSFS_QEMU_REJECT:-Kernel panic|Bad file descriptor|argosfs-initrd:
 timeout_s="${ARGOSFS_QEMU_TIMEOUT:-600}"
 login_delay_s="${ARGOSFS_QEMU_REBOOT_LOGIN_DELAY:-420}"
 reboot_delay_s="${ARGOSFS_QEMU_REBOOT_DELAY:-600}"
-command_delay_s="${ARGOSFS_QEMU_REBOOT_COMMAND_DELAY:-1}"
 done_marker="ARGOSFS_QEMU_REBOOT_DONE"
 
 cat >"$commands1" <<'CMDS'
@@ -51,12 +50,8 @@ argosfs_qemu_build_args
 
 send_command_file() {
 	local file="$1"
-	printf '\r' >&3
-	sleep "$command_delay_s"
-	while IFS= read -r line; do
-		printf '%s\r' "$line" >&3
-		sleep "$command_delay_s"
-	done <"$file"
+	local remote="$2"
+	argosfs_qemu_stream_script "$file" 3 "$remote" "$log"
 }
 
 wait_for_log_count() {
@@ -94,14 +89,14 @@ exec 3>"$stdin_fifo"
 wait_status=0
 wait_for_log_count 'Please press Enter to activate this console\.' 1 "$login_delay_s" 'first login prompt' || wait_status=$?
 if [ "$wait_status" -eq 0 ]; then
-	send_command_file "$commands1"
+	send_command_file "$commands1" /tmp/argosfs-qemu-reboot-phase1.sh
 	wait_for_log_count 'ARGOSFS_REBOOT_REQUESTED' 1 60 'phase1 reboot request' || wait_status=$?
 fi
 if [ "$wait_status" -eq 0 ]; then
 	wait_for_log_count 'Please press Enter to activate this console\.' 2 "$reboot_delay_s" 'second login prompt' || wait_status=$?
 fi
 if [ "$wait_status" -eq 0 ]; then
-	send_command_file "$commands2"
+	send_command_file "$commands2" /tmp/argosfs-qemu-reboot-phase2.sh
 	wait_for_log_count "$done_marker" 1 120 'reboot persistence completion' || wait_status=$?
 fi
 exec 3>&-