From 3e880ca87ee40dca72b3eec85fde67b8d408c161 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Sat, 20 Jun 2026 17:45:47 +0800 Subject: [PATCH 01/10] Clean Git hook env for ShellSpec When pre-commit runs during git commit --amend in a worktree, Git exports repository-local variables such as GIT_DIR and GIT_WORK_TREE into the hook environment. ShellSpec tests that create nested temporary repositories then inherit those variables, so git init/config/commit can operate on the outer repository instead of the test repo. Load a shared ShellSpec helper and unset Git's local environment variables before specs run, keeping nested test repositories isolated from the invoking repository. --- .shellspec | 1 + spec/spec_helper.sh | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 spec/spec_helper.sh diff --git a/.shellspec b/.shellspec index e69de29..c99d2e7 100644 --- a/.shellspec +++ b/.shellspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/spec/spec_helper.sh b/spec/spec_helper.sh new file mode 100644 index 0000000..cf06b17 --- /dev/null +++ b/spec/spec_helper.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Git exports repository-local environment variables to hooks. They can make +# nested test repositories operate on the outer repository instead. +if command -v git >/dev/null 2>&1; then + unset $(git rev-parse --local-env-vars 2>/dev/null || :) +fi From c61f62feee9350c5c693a5f499090f3bf32758cb Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Tue, 9 Jun 2026 11:01:12 +0800 Subject: [PATCH 02/10] Fix ShellSpec stderr warning on yes command when run in git hook --- spec/lib_spec.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/spec/lib_spec.sh b/spec/lib_spec.sh index cbf0314..b4f4c1b 100644 --- a/spec/lib_spec.sh +++ b/spec/lib_spec.sh @@ -49,9 +49,21 @@ Describe 'kdump-lib' #| #| } + + is_sigpipe_ignored() { + trap -p SIGPIPE | grep -q "''" + } + When call run_cmd yes '' "|" head -3 The output should equal "$(three_newlines)" The status should be success + # When running inside a git hook (where Git somehow ignores SIGPIPE), + # 'yes' writes a 'Broken pipe' message to stderr instead + # of terminating silently. We assert on the error stream to + # suppress ShellSpec warnings. + if is_sigpipe_ignored; then + The error should be defined + fi End It "should handle sed command correctly" From 823c8477c6d26c722e1ddf54e3e3f2bbb2bb28e1 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Sun, 14 Jun 2026 11:30:07 +0800 Subject: [PATCH 03/10] ci: run tmt e2e tests on GitHub-hosted runners with nested KVM - Provision two Fedora VMs per plan via testcloud/libvirt and runs the bisection end-to-end. Uses matrix strategy for parallel execution across all 4 plans (ssh, ssh_src, ssh_auto, criu). 8-hour timeout per plan, fail-fast: false. - Uploads /var/tmp/tmt/run* for debugging. --- .github/workflows/tmt-e2e-tests.yml | 51 +++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/tmt-e2e-tests.yml diff --git a/.github/workflows/tmt-e2e-tests.yml b/.github/workflows/tmt-e2e-tests.yml new file mode 100644 index 0000000..1013515 --- /dev/null +++ b/.github/workflows/tmt-e2e-tests.yml @@ -0,0 +1,51 @@ +name: KAB tmt e2e tests + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +jobs: + # ── Full e2e tests ──── + # + # Uses GitHub-hosted runners with nested KVM. Each plan provisions + # two Fedora VMs via testcloud and runs the bisection end-to-end. + # + tmt-run: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + plan: [ssh, ssh_src, ssh_auto, criu] + timeout-minutes: 480 + steps: + - uses: actions/checkout@v6 + + - name: Install libvirt + QEMU + run: | + sudo apt-get update -qq + sudo apt-get install -y qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev pkg-config genisoimage python3-libvirt + sudo usermod -aG libvirt $USER + sudo systemctl start libvirtd + # verify KVM is available + test -e /dev/kvm && echo "KVM OK" || echo "WARNING: /dev/kvm missing" + sudo virsh list --all + + - name: Install tmt + testcloud + run: | + pip install --user tmt testcloud + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + tmt --version + + - name: Run e2e test (${{ matrix.plan }}) + run: | + sg libvirt -c "tmt run -a -vvv plans --name '/plans/${{ matrix.plan }}$'" + + - name: Upload tmt artifacts + uses: actions/upload-artifact@v7 + if: always() + with: + name: tmt-run-${{ matrix.plan }} + path: /var/tmp/tmt/run* + if-no-files-found: ignore From 8329efff84c5f5387bc8a0c3d62dbb837398424b Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Sun, 14 Jun 2026 11:32:39 +0800 Subject: [PATCH 04/10] ci: Use Fedora:43 to run the tests Fedora 44 Cloud /boot uses Btrfs subvolume for /boot [1]. Running the e2e tests can encounter issues like [2][3]. So use Fedora 43 to bypass these issues. [1] https://fedoraproject.org/wiki/Changes/BtrfsBootForCloud [2] https://bugzilla.redhat.com/show_bug.cgi?id=2483604 [3] https://github.com/rhkdump/kdump-utils/issues/155 --- .github/workflows/tmt-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tmt-e2e-tests.yml b/.github/workflows/tmt-e2e-tests.yml index 1013515..61b7e38 100644 --- a/.github/workflows/tmt-e2e-tests.yml +++ b/.github/workflows/tmt-e2e-tests.yml @@ -40,7 +40,7 @@ jobs: - name: Run e2e test (${{ matrix.plan }}) run: | - sg libvirt -c "tmt run -a -vvv plans --name '/plans/${{ matrix.plan }}$'" + sg libvirt -c "tmt run -a provision -h virtual -c system -i fedora:43 -vvv plans --name '/plans/${{ matrix.plan }}$'" - name: Upload tmt artifacts uses: actions/upload-artifact@v7 From d3cf1fc22f462fa91cc9aaeb20ef6de9b5e7e33d Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Sun, 14 Jun 2026 11:39:21 +0800 Subject: [PATCH 05/10] ci: 6.19.5-300.fc44 was deleted 6.19.{3..5}-300.fc44 were deleted. This can lead the failure of kab_ssh_auto test. Use 6.19.2-300.fc44 instead. --- tests/kab_ssh_auto/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kab_ssh_auto/test.sh b/tests/kab_ssh_auto/test.sh index 432607b..491c895 100755 --- a/tests/kab_ssh_auto/test.sh +++ b/tests/kab_ssh_auto/test.sh @@ -16,7 +16,7 @@ if echo "${CLIENTS}" | grep -qi "${HOSTNAME}"; then KERNEL_RPM_LIST=/usr/local/bin/kernel-auto-bisect/kernel_list WORK_DIR=/var/local/kernel-auto-bisect GIT_REPO=$WORK_DIR/git_repo - GOOD_COMMIT=6.19.5-300.fc44.${ARCH} + GOOD_COMMIT=6.19.2-300.fc44.${ARCH} BAD_COMMIT=6.19.8-200.fc43.${ARCH} TMT_TEST_PLAN_ROOT=${TMT_PLAN_DATA%data} @@ -50,7 +50,7 @@ END fi cat <$KERNEL_RPM_LIST -https://kojipkgs.fedoraproject.org/packages/kernel/6.19.5/300.fc44/${ARCH}/kernel-core-6.19.5-300.fc44.${ARCH}.rpm +https://kojipkgs.fedoraproject.org/packages/kernel/6.19.2/300.fc44/${ARCH}/kernel-core-6.19.2-300.fc44.${ARCH}.rpm https://kojipkgs.fedoraproject.org/packages/kernel/6.19.6/200.fc43/${ARCH}/kernel-core-6.19.6-200.fc43.${ARCH}.rpm https://kojipkgs.fedoraproject.org/packages/kernel/6.19.7/200.fc43/${ARCH}/kernel-core-6.19.7-200.fc43.${ARCH}.rpm https://kojipkgs.fedoraproject.org/packages/kernel/6.19.8/200.fc43/${ARCH}/kernel-core-6.19.8-200.fc43.${ARCH}.rpm From 0bb200297c8c064d4cfc06eaada19f01b3c18abb Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Sun, 14 Jun 2026 11:52:41 +0800 Subject: [PATCH 06/10] ci: Copy xtrace logs to TMT_PLAN_DATA for debug --- tests/kab_criu/test.sh | 22 ++++++++++++++++------ tests/kab_ssh/test.sh | 3 ++- tests/kab_ssh_auto/test.sh | 3 ++- tests/kab_ssh_src/test.sh | 5 +++-- tests/test_lib.sh | 7 +++++++ 5 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 tests/test_lib.sh diff --git a/tests/kab_criu/test.sh b/tests/kab_criu/test.sh index a669260..c880c26 100755 --- a/tests/kab_criu/test.sh +++ b/tests/kab_criu/test.sh @@ -2,6 +2,7 @@ # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k set -x +. ../test_lib.sh . ./tmt.sh [[ -z $ARCH ]] && ARCH=$(uname -m) @@ -20,16 +21,22 @@ TARGET_HOST="${SERVERS}" TMT_TEST_PLAN_ROOT=${TMT_PLAN_DATA%data} SERVER_SSH_KEY=${TMT_TEST_PLAN_ROOT}/provision/server/id_ecdsa +ssh_opts=(-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10) + +if [[ -f "$SERVER_SSH_KEY" ]]; then + ssh_opts+=(-o IdentitiesOnly=yes -i $SERVER_SSH_KEY) +fi + # ssh_cmd wrapper to handle local and remote execution ssh_cmd() { - local _opts="-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10" - if [[ -f "$SERVER_SSH_KEY" ]]; then - _opts="$_opts -o IdentitiesOnly=yes -i $SERVER_SSH_KEY" - fi - ssh $_opts "$TARGET_HOST" "$@" + ssh "${ssh_opts[@]}" "$TARGET_HOST" "$@" return $? } +copy_xtrace_log() { + scp "${ssh_opts[@]}" "${TARGET_HOST}:${XTRACE_LOG}" "$XTRACE_LOG" +} + if ! ssh_cmd "cd $TMT_TREE && make install"; then echo "Failed to install KAB on ${SERVERS}" exit 1 @@ -83,10 +90,11 @@ END # For idempotence ssh_cmd "rm -rf $GIT_REPO" + # 2. Start kab.sh on Target if not already running and no checkpoint exists if ! ssh_cmd "pgrep -f $KAB_SCRIPT" >/dev/null 2>&1 && ! ssh_cmd "ls /var/local/kernel-auto-bisect/dump/core-*.img" >/dev/null 2>&1; then echo "Starting kab.sh..." - ssh_cmd "setsid bash -x $KAB_SCRIPT /root/test.log &" + ssh_cmd "setsid bash -x $KAB_SCRIPT $XTRACE_LOG &" fi # 3. Wait for result @@ -98,6 +106,7 @@ while [[ $wait_time -lt $MAX_WAIT_TIME ]]; do ret=$? if [[ $ret -eq 0 ]]; then + copy_xtrace_log exit 0 else echo "Target ($TARGET_HOST) is down or unreachable (exit code: $ret), waiting..." @@ -107,5 +116,6 @@ while [[ $wait_time -lt $MAX_WAIT_TIME ]]; do wait_time=$((wait_time + 10)) done +copy_xtrace_log echo "Failed to get 1st bad commit within $MAX_WAIT_TIME seconds" exit 1 diff --git a/tests/kab_ssh/test.sh b/tests/kab_ssh/test.sh index a98ff92..bfebf00 100755 --- a/tests/kab_ssh/test.sh +++ b/tests/kab_ssh/test.sh @@ -2,6 +2,7 @@ # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k set -x +. ../test_lib.sh . ./tmt.sh [[ -z $ARCH ]] && ARCH=$(uname -m) @@ -62,7 +63,7 @@ on_test() { } END - bash -x $KAB_SCRIPT /root/test.log + bash -x $KAB_SCRIPT "$XTRACE_LOG" GIT_REPO=/var/local/kernel-auto-bisect/git_repo MAX_WAIT_TIME=600 wait_time=0 diff --git a/tests/kab_ssh_auto/test.sh b/tests/kab_ssh_auto/test.sh index 491c895..3c67d9e 100755 --- a/tests/kab_ssh_auto/test.sh +++ b/tests/kab_ssh_auto/test.sh @@ -2,6 +2,7 @@ # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k set -x +. ../test_lib.sh . ./tmt.sh [[ -z $ARCH ]] && ARCH=$(uname -m) @@ -72,7 +73,7 @@ on_test() { } END - bash -x $KAB_SCRIPT /root/test.log + bash -x $KAB_SCRIPT "$XTRACE_LOG" if [[ -f "$SERVER_SSH_KEY" ]]; then ssh_cmd="ssh -o IdentitiesOnly=yes -i $SERVER_SSH_KEY" diff --git a/tests/kab_ssh_src/test.sh b/tests/kab_ssh_src/test.sh index d07c56b..00a447f 100755 --- a/tests/kab_ssh_src/test.sh +++ b/tests/kab_ssh_src/test.sh @@ -2,6 +2,7 @@ # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k set -x +. ../test_lib.sh . ./tmt.sh [[ -z $ARCH ]] && ARCH=$(uname -m) @@ -66,14 +67,14 @@ on_test() { } END - bash -x "$KAB_SCRIPT" test.log + bash -x "$KAB_SCRIPT" "$XTRACE_LOG" KAB_EXIT=$? LOCAL_STATE_DIR=~/.local/state/kernel-auto-bisect if [[ $KAB_EXIT -ne 0 ]]; then echo "FAIL: kab.sh failed as non-root user (exit=$KAB_EXIT)" - cat "test.log" + cat "$XTRACE_LOG" cat "$LOCAL_STATE_DIR/main.log" 2>/dev/null exit 1 fi diff --git a/tests/test_lib.sh b/tests/test_lib.sh new file mode 100644 index 0000000..6d5450d --- /dev/null +++ b/tests/test_lib.sh @@ -0,0 +1,7 @@ +#!/bin/bash +if [[ -z $TMT_PLAN_DATA ]]; then + echo "TMT_PLAN_DATA not defined" + exit 1 +fi +XTRACE_LOG="${TMT_PLAN_DATA}/test.log" + From 9ec2318b8237b153ea9a8669f88788f9c0b44642 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Tue, 16 Jun 2026 15:54:55 +0800 Subject: [PATCH 07/10] tests: include tmt.sh in test_lib.sh --- tests/kab_criu/test.sh | 1 - tests/kab_criu/tmt.sh | 17 ----------------- tests/kab_ssh/test.sh | 1 - tests/kab_ssh/tmt.sh | 17 ----------------- tests/kab_ssh_auto/test.sh | 1 - tests/kab_ssh_auto/tmt.sh | 17 ----------------- tests/kab_ssh_src/test.sh | 1 - tests/kab_ssh_src/tmt.sh | 17 ----------------- tests/test_lib.sh | 16 ++++++++++++++++ 9 files changed, 16 insertions(+), 72 deletions(-) delete mode 100644 tests/kab_criu/tmt.sh delete mode 100644 tests/kab_ssh/tmt.sh delete mode 100755 tests/kab_ssh_auto/tmt.sh delete mode 100644 tests/kab_ssh_src/tmt.sh diff --git a/tests/kab_criu/test.sh b/tests/kab_criu/test.sh index c880c26..7a6c14b 100755 --- a/tests/kab_criu/test.sh +++ b/tests/kab_criu/test.sh @@ -3,7 +3,6 @@ set -x . ../test_lib.sh -. ./tmt.sh [[ -z $ARCH ]] && ARCH=$(uname -m) diff --git a/tests/kab_criu/tmt.sh b/tests/kab_criu/tmt.sh deleted file mode 100644 index 4d1266e..0000000 --- a/tests/kab_criu/tmt.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -assign_server_roles() { - if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then - # assign roles based on tmt topology data - # shellcheck source=/dev/null - . "${TMT_TOPOLOGY_BASH}" - - if [[ $(grep -Ec "TMT_GUESTS\[.*role\]" "${TMT_TOPOLOGY_BASH}") -gt 1 ]]; then - export SERVERS=${TMT_GUESTS[server.hostname]} - export CLIENTS=${TMT_GUESTS[client.hostname]} - fi - - export HOSTNAME=${TMT_GUEST[hostname]} - fi -} - -assign_server_roles diff --git a/tests/kab_ssh/test.sh b/tests/kab_ssh/test.sh index bfebf00..512c58a 100755 --- a/tests/kab_ssh/test.sh +++ b/tests/kab_ssh/test.sh @@ -3,7 +3,6 @@ set -x . ../test_lib.sh -. ./tmt.sh [[ -z $ARCH ]] && ARCH=$(uname -m) diff --git a/tests/kab_ssh/tmt.sh b/tests/kab_ssh/tmt.sh deleted file mode 100644 index 4d1266e..0000000 --- a/tests/kab_ssh/tmt.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -assign_server_roles() { - if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then - # assign roles based on tmt topology data - # shellcheck source=/dev/null - . "${TMT_TOPOLOGY_BASH}" - - if [[ $(grep -Ec "TMT_GUESTS\[.*role\]" "${TMT_TOPOLOGY_BASH}") -gt 1 ]]; then - export SERVERS=${TMT_GUESTS[server.hostname]} - export CLIENTS=${TMT_GUESTS[client.hostname]} - fi - - export HOSTNAME=${TMT_GUEST[hostname]} - fi -} - -assign_server_roles diff --git a/tests/kab_ssh_auto/test.sh b/tests/kab_ssh_auto/test.sh index 3c67d9e..e8d5849 100755 --- a/tests/kab_ssh_auto/test.sh +++ b/tests/kab_ssh_auto/test.sh @@ -3,7 +3,6 @@ set -x . ../test_lib.sh -. ./tmt.sh [[ -z $ARCH ]] && ARCH=$(uname -m) diff --git a/tests/kab_ssh_auto/tmt.sh b/tests/kab_ssh_auto/tmt.sh deleted file mode 100755 index 4d1266e..0000000 --- a/tests/kab_ssh_auto/tmt.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -assign_server_roles() { - if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then - # assign roles based on tmt topology data - # shellcheck source=/dev/null - . "${TMT_TOPOLOGY_BASH}" - - if [[ $(grep -Ec "TMT_GUESTS\[.*role\]" "${TMT_TOPOLOGY_BASH}") -gt 1 ]]; then - export SERVERS=${TMT_GUESTS[server.hostname]} - export CLIENTS=${TMT_GUESTS[client.hostname]} - fi - - export HOSTNAME=${TMT_GUEST[hostname]} - fi -} - -assign_server_roles diff --git a/tests/kab_ssh_src/test.sh b/tests/kab_ssh_src/test.sh index 00a447f..9067983 100755 --- a/tests/kab_ssh_src/test.sh +++ b/tests/kab_ssh_src/test.sh @@ -3,7 +3,6 @@ set -x . ../test_lib.sh -. ./tmt.sh [[ -z $ARCH ]] && ARCH=$(uname -m) diff --git a/tests/kab_ssh_src/tmt.sh b/tests/kab_ssh_src/tmt.sh deleted file mode 100644 index 4d1266e..0000000 --- a/tests/kab_ssh_src/tmt.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -assign_server_roles() { - if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then - # assign roles based on tmt topology data - # shellcheck source=/dev/null - . "${TMT_TOPOLOGY_BASH}" - - if [[ $(grep -Ec "TMT_GUESTS\[.*role\]" "${TMT_TOPOLOGY_BASH}") -gt 1 ]]; then - export SERVERS=${TMT_GUESTS[server.hostname]} - export CLIENTS=${TMT_GUESTS[client.hostname]} - fi - - export HOSTNAME=${TMT_GUEST[hostname]} - fi -} - -assign_server_roles diff --git a/tests/test_lib.sh b/tests/test_lib.sh index 6d5450d..ee9082f 100644 --- a/tests/test_lib.sh +++ b/tests/test_lib.sh @@ -5,3 +5,19 @@ if [[ -z $TMT_PLAN_DATA ]]; then fi XTRACE_LOG="${TMT_PLAN_DATA}/test.log" +assign_server_roles() { + if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then + # assign roles based on tmt topology data + # shellcheck source=/dev/null + . "${TMT_TOPOLOGY_BASH}" + + if [[ $(grep -Ec "TMT_GUESTS\[.*role\]" "${TMT_TOPOLOGY_BASH}") -gt 1 ]]; then + export SERVERS=${TMT_GUESTS[server.hostname]} + export CLIENTS=${TMT_GUESTS[client.hostname]} + fi + + export HOSTNAME=${TMT_GUEST[hostname]} + fi +} + +assign_server_roles From fe21199bedb1e1bbc58897a50ab6b663de26481c Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Wed, 17 Jun 2026 11:57:19 +0800 Subject: [PATCH 08/10] ci: Only output kernel-auto-bisect/main.log for criu e2e test kernel-auto-bisect/main.log is the what we care most to know the progress. For criu e2e test, kab.sh will be checkpointed by CRIU and then the system will get rebooted. To continuously display main.log, we append remote main.log to local file and use "tail -f" to show the content. --- tests/kab_criu/test.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/kab_criu/test.sh b/tests/kab_criu/test.sh index 7a6c14b..f1a8deb 100755 --- a/tests/kab_criu/test.sh +++ b/tests/kab_criu/test.sh @@ -1,7 +1,5 @@ #!/bin/bash # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k -set -x - . ../test_lib.sh [[ -z $ARCH ]] && ARCH=$(uname -m) @@ -20,10 +18,10 @@ TARGET_HOST="${SERVERS}" TMT_TEST_PLAN_ROOT=${TMT_PLAN_DATA%data} SERVER_SSH_KEY=${TMT_TEST_PLAN_ROOT}/provision/server/id_ecdsa -ssh_opts=(-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10) +ssh_opts=(-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ChannelTimeout=session=2s) if [[ -f "$SERVER_SSH_KEY" ]]; then - ssh_opts+=(-o IdentitiesOnly=yes -i $SERVER_SSH_KEY) + ssh_opts+=(-o IdentitiesOnly=yes -i "$SERVER_SSH_KEY") fi # ssh_cmd wrapper to handle local and remote execution @@ -99,18 +97,28 @@ fi # 3. Wait for result MAX_WAIT_TIME=600 # 10 minutes wait_time=0 +MAIN_LOG_REMOTE=/var/local/kernel-auto-bisect/main.log +MAIN_LOG_LOCAL=/tmp/kab-main.log +touch "$MAIN_LOG_LOCAL" +printed_lines=0 + while [[ $wait_time -lt $MAX_WAIT_TIME ]]; do # Try to check if finished - output=$(ssh_cmd "git -C $GIT_REPO bisect log | grep 'first bad commit' | grep -q '$BAD_COMMIT'") + output=$(ssh_cmd "git -C $GIT_REPO bisect log 2>/dev/null | grep 'first bad commit' | grep -q '$BAD_COMMIT'") ret=$? if [[ $ret -eq 0 ]]; then copy_xtrace_log exit 0 - else - echo "Target ($TARGET_HOST) is down or unreachable (exit code: $ret), waiting..." fi + # Rsync the remote log and print only new lines + rsync --timeout=20 -e "ssh ${ssh_opts[*]}" "${TARGET_HOST}:${MAIN_LOG_REMOTE}" "$MAIN_LOG_LOCAL" 2>/dev/null + current_total=$(wc -l <"$MAIN_LOG_LOCAL" 2>/dev/null || echo 0) + if [[ $current_total -gt $printed_lines ]]; then + tail -n +$((printed_lines + 1)) "$MAIN_LOG_LOCAL" + printed_lines=$current_total + fi sleep 10 wait_time=$((wait_time + 10)) done From 994d58629f9687274943aa949a7475ecccebf860 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Wed, 17 Jun 2026 07:50:07 +0800 Subject: [PATCH 09/10] ci: Only output kernel-auto-bisect/main.log for other e2e tests kernel-auto-bisect/main.log is the what we care most to know the progress. So use "tmt run execute -vvv" so the main logs can be displayed. --- .github/workflows/tmt-e2e-tests.yml | 2 +- tests/kab_ssh/test.sh | 3 +-- tests/kab_ssh_auto/test.sh | 3 +-- tests/kab_ssh_src/test.sh | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tmt-e2e-tests.yml b/.github/workflows/tmt-e2e-tests.yml index 61b7e38..e0361c7 100644 --- a/.github/workflows/tmt-e2e-tests.yml +++ b/.github/workflows/tmt-e2e-tests.yml @@ -40,7 +40,7 @@ jobs: - name: Run e2e test (${{ matrix.plan }}) run: | - sg libvirt -c "tmt run -a provision -h virtual -c system -i fedora:43 -vvv plans --name '/plans/${{ matrix.plan }}$'" + sg libvirt -c "tmt run -a provision -h virtual -c system -i fedora:43 execute -vvv plans --name '/plans/${{ matrix.plan }}$'" - name: Upload tmt artifacts uses: actions/upload-artifact@v7 diff --git a/tests/kab_ssh/test.sh b/tests/kab_ssh/test.sh index 512c58a..efd508b 100755 --- a/tests/kab_ssh/test.sh +++ b/tests/kab_ssh/test.sh @@ -1,6 +1,5 @@ #!/bin/bash # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k -set -x . ../test_lib.sh @@ -62,7 +61,7 @@ on_test() { } END - bash -x $KAB_SCRIPT "$XTRACE_LOG" + bash -x $KAB_SCRIPT "$XTRACE_LOG" GIT_REPO=/var/local/kernel-auto-bisect/git_repo MAX_WAIT_TIME=600 wait_time=0 diff --git a/tests/kab_ssh_auto/test.sh b/tests/kab_ssh_auto/test.sh index e8d5849..8f32bdd 100755 --- a/tests/kab_ssh_auto/test.sh +++ b/tests/kab_ssh_auto/test.sh @@ -1,6 +1,5 @@ #!/bin/bash # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k -set -x . ../test_lib.sh @@ -72,7 +71,7 @@ on_test() { } END - bash -x $KAB_SCRIPT "$XTRACE_LOG" + bash -x $KAB_SCRIPT "$XTRACE_LOG" if [[ -f "$SERVER_SSH_KEY" ]]; then ssh_cmd="ssh -o IdentitiesOnly=yes -i $SERVER_SSH_KEY" diff --git a/tests/kab_ssh_src/test.sh b/tests/kab_ssh_src/test.sh index 9067983..6144cff 100755 --- a/tests/kab_ssh_src/test.sh +++ b/tests/kab_ssh_src/test.sh @@ -1,6 +1,5 @@ #!/bin/bash # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k -set -x . ../test_lib.sh @@ -66,7 +65,7 @@ on_test() { } END - bash -x "$KAB_SCRIPT" "$XTRACE_LOG" + bash -x "$KAB_SCRIPT" "$XTRACE_LOG" KAB_EXIT=$? LOCAL_STATE_DIR=~/.local/state/kernel-auto-bisect From 13ec03034e08335a3bc6289054abfb940b54d773 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Sat, 20 Jun 2026 17:56:30 +0800 Subject: [PATCH 10/10] ci: Run e2e tests on Fedora container tmt is natively supported on Fedora. By using Fedora container, some benefits includes bypassing issue [1]. [1] https://github.com/teemtee/testcloud/pull/30 --- .github/workflows/tmt-e2e-tests.yml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tmt-e2e-tests.yml b/.github/workflows/tmt-e2e-tests.yml index e0361c7..e98694d 100644 --- a/.github/workflows/tmt-e2e-tests.yml +++ b/.github/workflows/tmt-e2e-tests.yml @@ -14,6 +14,9 @@ jobs: # tmt-run: runs-on: ubuntu-latest + container: + image: quay.io/fedora/fedora:latest + options: "--privileged -v /dev:/dev --volume /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host" strategy: fail-fast: false matrix: @@ -22,25 +25,21 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Install libvirt + QEMU + - name: Install tmt and start libvirtd run: | - sudo apt-get update -qq - sudo apt-get install -y qemu-kvm libvirt-daemon-system libvirt-clients libvirt-dev pkg-config genisoimage python3-libvirt - sudo usermod -aG libvirt $USER - sudo systemctl start libvirtd + dnf install tmt+provision-virtual -yq + virtlogd -d + libvirtd -d # verify KVM is available test -e /dev/kvm && echo "KVM OK" || echo "WARNING: /dev/kvm missing" - sudo virsh list --all + virsh list --all - - name: Install tmt + testcloud - run: | - pip install --user tmt testcloud - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - tmt --version - name: Run e2e test (${{ matrix.plan }}) run: | - sg libvirt -c "tmt run -a provision -h virtual -c system -i fedora:43 execute -vvv plans --name '/plans/${{ matrix.plan }}$'" + # "-B finish" to skip shutting down KVM guests thus to bypass "libvirt: error : Failed to terminate process with SIGKILL: Device or resource busy" e.g. + # https://github.com/coiby/kernel-auto-bisect/actions/runs/27753332468/job/82108959294 + tmt run -B finish provision -h virtual -c system -i fedora:43 execute -vvv plans --name '/plans/${{ matrix.plan }}$' - name: Upload tmt artifacts uses: actions/upload-artifact@v7