ci: run tmt e2e tests on GitHub-hosted runners with nested KVM

.github/workflows/tmt-e2e-tests.yml

@@ -0,0 +1,50 @@
+name: KAB tmt e2e tests
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  # ── Full e2e tests  ────
+  #
+  # Uses GitHub-hosted runners with nested KVM. Each plan provisions
+  # two Fedora VMs via testcloud and runs the bisection end-to-end.
+  #
+  tmt-run:
+    runs-on: ubuntu-latest
+    container:
+      image: quay.io/fedora/fedora:latest
+      options: "--privileged -v /dev:/dev --volume /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host"
+    strategy:
+      fail-fast: false
+      matrix:
+        plan: [ssh, ssh_src, ssh_auto, criu]
+    timeout-minutes: 480
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install tmt and start libvirtd
+        run: |
+          dnf install tmt+provision-virtual -yq
+          virtlogd -d
+          libvirtd -d
+          # verify KVM is available
+          test -e /dev/kvm && echo "KVM OK" || echo "WARNING: /dev/kvm missing"
+          virsh list --all
+
+
+      - name: Run e2e test (${{ matrix.plan }})
+        run: |
+          # "-B finish" to skip shutting down KVM guests thus to bypass "libvirt:  error : Failed to terminate process with SIGKILL: Device or resource busy" e.g.
+          # https://github.com/coiby/kernel-auto-bisect/actions/runs/27753332468/job/82108959294
+          tmt run -B finish provision -h virtual -c system -i fedora:43 execute -vvv plans --name '/plans/${{ matrix.plan }}$'
+
+      - name: Upload tmt artifacts
+        uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: tmt-run-${{ matrix.plan }}
+          path: /var/tmp/tmt/run*
+          if-no-files-found: ignore

.shellspec

@@ -0,0 +1 @@
+--require spec_helper

spec/lib_spec.sh

@@ -49,9 +49,21 @@ Describe 'kdump-lib'
 				#|
 				#|
 			}
+
+			is_sigpipe_ignored() {
+				trap -p SIGPIPE | grep -q "''"
+			}
+
 			When call run_cmd yes '' "|" head -3
 			The output should equal "$(three_newlines)"
 			The status should be success
+			# When running inside a git hook (where Git somehow ignores SIGPIPE),
+			# 'yes' writes a 'Broken pipe' message to stderr instead
+			# of terminating silently. We assert on the error stream to
+			# suppress ShellSpec warnings.
+			if is_sigpipe_ignored; then
+				The error should be defined
+			fi
 		End
 
 		It "should handle sed command correctly"

spec/spec_helper.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Git exports repository-local environment variables to hooks. They can make
+# nested test repositories operate on the outer repository instead.
+if command -v git >/dev/null 2>&1; then
+	unset $(git rev-parse --local-env-vars 2>/dev/null || :)
+fi

tests/kab_criu/test.sh

@@ -1,8 +1,6 @@
 #!/bin/bash
 # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
-set -x
-
-. ./tmt.sh
+. ../test_lib.sh
 
 [[ -z $ARCH ]] && ARCH=$(uname -m)
 
@@ -20,16 +18,22 @@ TARGET_HOST="${SERVERS}"
 TMT_TEST_PLAN_ROOT=${TMT_PLAN_DATA%data}
 SERVER_SSH_KEY=${TMT_TEST_PLAN_ROOT}/provision/server/id_ecdsa
 
+ssh_opts=(-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ChannelTimeout=session=2s)
+
+if [[ -f "$SERVER_SSH_KEY" ]]; then
+	ssh_opts+=(-o IdentitiesOnly=yes -i "$SERVER_SSH_KEY")
+fi
+
 # ssh_cmd wrapper to handle local and remote execution
 ssh_cmd() {
-	local _opts="-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10"
-	if [[ -f "$SERVER_SSH_KEY" ]]; then
-		_opts="$_opts -o IdentitiesOnly=yes -i $SERVER_SSH_KEY"
-	fi
-	ssh $_opts "$TARGET_HOST" "$@"
+	ssh "${ssh_opts[@]}" "$TARGET_HOST" "$@"
 	return $?
 }
 
+copy_xtrace_log() {
+	scp "${ssh_opts[@]}" "${TARGET_HOST}:${XTRACE_LOG}" "$XTRACE_LOG"
+}
+
 if ! ssh_cmd "cd $TMT_TREE && make install"; then
 	echo "Failed to install KAB on ${SERVERS}"
 	exit 1
@@ -83,29 +87,42 @@ END
 
 # For idempotence
 ssh_cmd "rm -rf $GIT_REPO"
+
 # 2. Start kab.sh on Target if not already running and no checkpoint exists
 if ! ssh_cmd "pgrep -f $KAB_SCRIPT" >/dev/null 2>&1 && ! ssh_cmd "ls /var/local/kernel-auto-bisect/dump/core-*.img" >/dev/null 2>&1; then
 	echo "Starting kab.sh..."
-	ssh_cmd "setsid bash -x $KAB_SCRIPT </dev/null &>/root/test.log &"
+	ssh_cmd "setsid bash -x $KAB_SCRIPT </dev/null &>$XTRACE_LOG &"
 fi
 
 # 3. Wait for result
 MAX_WAIT_TIME=600 # 10 minutes
 wait_time=0
+MAIN_LOG_REMOTE=/var/local/kernel-auto-bisect/main.log
+MAIN_LOG_LOCAL=/tmp/kab-main.log
+touch "$MAIN_LOG_LOCAL"
+printed_lines=0
+
 while [[ $wait_time -lt $MAX_WAIT_TIME ]]; do
 	# Try to check if finished
-	output=$(ssh_cmd "git -C $GIT_REPO bisect log | grep 'first bad commit' | grep -q '$BAD_COMMIT'")
+	output=$(ssh_cmd "git -C $GIT_REPO bisect log 2>/dev/null | grep 'first bad commit' | grep -q '$BAD_COMMIT'")
 	ret=$?
 
 	if [[ $ret -eq 0 ]]; then
+		copy_xtrace_log
 		exit 0
-	else
-		echo "Target ($TARGET_HOST) is down or unreachable (exit code: $ret), waiting..."
 	fi
 
+	# Rsync the remote log and print only new lines
+	rsync --timeout=20 -e "ssh ${ssh_opts[*]}" "${TARGET_HOST}:${MAIN_LOG_REMOTE}" "$MAIN_LOG_LOCAL" 2>/dev/null
+	current_total=$(wc -l <"$MAIN_LOG_LOCAL" 2>/dev/null || echo 0)
+	if [[ $current_total -gt $printed_lines ]]; then
+		tail -n +$((printed_lines + 1)) "$MAIN_LOG_LOCAL"
+		printed_lines=$current_total
+	fi
 	sleep 10
 	wait_time=$((wait_time + 10))
 done
 
+copy_xtrace_log
 echo "Failed to get 1st bad commit within $MAX_WAIT_TIME seconds"
 exit 1

tests/kab_ssh/test.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
 # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
-set -x
 
-. ./tmt.sh
+. ../test_lib.sh
 
 [[ -z $ARCH ]] && ARCH=$(uname -m)
 
@@ -62,7 +61,7 @@ on_test() {
 }
 END
 
-	bash -x $KAB_SCRIPT </dev/null &>/root/test.log
+	bash -x $KAB_SCRIPT </dev/null 2>"$XTRACE_LOG"
 	GIT_REPO=/var/local/kernel-auto-bisect/git_repo
 	MAX_WAIT_TIME=600
 	wait_time=0

tests/kab_ssh_auto/test.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
 # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
-set -x
 
-. ./tmt.sh
+. ../test_lib.sh
 
 [[ -z $ARCH ]] && ARCH=$(uname -m)
 
@@ -16,7 +15,7 @@ if echo "${CLIENTS}" | grep -qi "${HOSTNAME}"; then
 	KERNEL_RPM_LIST=/usr/local/bin/kernel-auto-bisect/kernel_list
 	WORK_DIR=/var/local/kernel-auto-bisect
 	GIT_REPO=$WORK_DIR/git_repo
-	GOOD_COMMIT=6.19.5-300.fc44.${ARCH}
+	GOOD_COMMIT=6.19.2-300.fc44.${ARCH}
 	BAD_COMMIT=6.19.8-200.fc43.${ARCH}
 
 	TMT_TEST_PLAN_ROOT=${TMT_PLAN_DATA%data}
@@ -50,7 +49,7 @@ END
 	fi
 
 	cat <<END >$KERNEL_RPM_LIST
-https://kojipkgs.fedoraproject.org/packages/kernel/6.19.5/300.fc44/${ARCH}/kernel-core-6.19.5-300.fc44.${ARCH}.rpm
+https://kojipkgs.fedoraproject.org/packages/kernel/6.19.2/300.fc44/${ARCH}/kernel-core-6.19.2-300.fc44.${ARCH}.rpm
 https://kojipkgs.fedoraproject.org/packages/kernel/6.19.6/200.fc43/${ARCH}/kernel-core-6.19.6-200.fc43.${ARCH}.rpm
 https://kojipkgs.fedoraproject.org/packages/kernel/6.19.7/200.fc43/${ARCH}/kernel-core-6.19.7-200.fc43.${ARCH}.rpm
 https://kojipkgs.fedoraproject.org/packages/kernel/6.19.8/200.fc43/${ARCH}/kernel-core-6.19.8-200.fc43.${ARCH}.rpm
@@ -72,7 +71,7 @@ on_test() {
 }
 END
 
-	bash -x $KAB_SCRIPT </dev/null &>/root/test.log
+	bash -x $KAB_SCRIPT </dev/null 2>"$XTRACE_LOG"
 
 	if [[ -f "$SERVER_SSH_KEY" ]]; then
 		ssh_cmd="ssh -o IdentitiesOnly=yes -i $SERVER_SSH_KEY"

tests/kab_ssh_src/test.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
 # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
-set -x
 
-. ./tmt.sh
+. ../test_lib.sh
 
 [[ -z $ARCH ]] && ARCH=$(uname -m)
 
@@ -66,14 +65,14 @@ on_test() {
 }
 END
 
-	bash -x "$KAB_SCRIPT" </dev/null &>test.log
+	bash -x "$KAB_SCRIPT" </dev/null 2>"$XTRACE_LOG"
 	KAB_EXIT=$?
 
 	LOCAL_STATE_DIR=~/.local/state/kernel-auto-bisect
 
 	if [[ $KAB_EXIT -ne 0 ]]; then
 		echo "FAIL: kab.sh failed as non-root user (exit=$KAB_EXIT)"
-		cat "test.log"
+		cat "$XTRACE_LOG"
 		cat "$LOCAL_STATE_DIR/main.log" 2>/dev/null
 		exit 1
 	fi

tests/kab_criu/tmt.sh → tests/test_lib.sh

@@ -1,4 +1,10 @@
 #!/bin/bash
+if [[ -z $TMT_PLAN_DATA ]]; then
+    echo "TMT_PLAN_DATA not defined"
+    exit 1
+fi
+XTRACE_LOG="${TMT_PLAN_DATA}/test.log"
+
 assign_server_roles() {
 	if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then
 		# assign roles based on tmt topology data