diff --git a/.github/workflows/tmt-e2e-tests.yml b/.github/workflows/tmt-e2e-tests.yml new file mode 100644 index 0000000..e98694d --- /dev/null +++ b/.github/workflows/tmt-e2e-tests.yml @@ -0,0 +1,50 @@ +name: KAB tmt e2e tests + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +jobs: + # ── Full e2e tests ──── + # + # Uses GitHub-hosted runners with nested KVM. Each plan provisions + # two Fedora VMs via testcloud and runs the bisection end-to-end. + # + tmt-run: + runs-on: ubuntu-latest + container: + image: quay.io/fedora/fedora:latest + options: "--privileged -v /dev:/dev --volume /sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host" + strategy: + fail-fast: false + matrix: + plan: [ssh, ssh_src, ssh_auto, criu] + timeout-minutes: 480 + steps: + - uses: actions/checkout@v6 + + - name: Install tmt and start libvirtd + run: | + dnf install tmt+provision-virtual -yq + virtlogd -d + libvirtd -d + # verify KVM is available + test -e /dev/kvm && echo "KVM OK" || echo "WARNING: /dev/kvm missing" + virsh list --all + + + - name: Run e2e test (${{ matrix.plan }}) + run: | + # "-B finish" to skip shutting down KVM guests thus to bypass "libvirt: error : Failed to terminate process with SIGKILL: Device or resource busy" e.g. + # https://github.com/coiby/kernel-auto-bisect/actions/runs/27753332468/job/82108959294 + tmt run -B finish provision -h virtual -c system -i fedora:43 execute -vvv plans --name '/plans/${{ matrix.plan }}$' + + - name: Upload tmt artifacts + uses: actions/upload-artifact@v7 + if: always() + with: + name: tmt-run-${{ matrix.plan }} + path: /var/tmp/tmt/run* + if-no-files-found: ignore diff --git a/.shellspec b/.shellspec index e69de29..c99d2e7 100644 --- a/.shellspec +++ b/.shellspec @@ -0,0 +1 @@ +--require spec_helper diff --git a/spec/lib_spec.sh b/spec/lib_spec.sh index cbf0314..b4f4c1b 100644 --- a/spec/lib_spec.sh +++ b/spec/lib_spec.sh @@ -49,9 +49,21 @@ Describe 'kdump-lib' #| #| } + + is_sigpipe_ignored() { + trap -p SIGPIPE | grep -q "''" + } + When call run_cmd yes '' "|" head -3 The output should equal "$(three_newlines)" The status should be success + # When running inside a git hook (where Git somehow ignores SIGPIPE), + # 'yes' writes a 'Broken pipe' message to stderr instead + # of terminating silently. We assert on the error stream to + # suppress ShellSpec warnings. + if is_sigpipe_ignored; then + The error should be defined + fi End It "should handle sed command correctly" diff --git a/spec/spec_helper.sh b/spec/spec_helper.sh new file mode 100644 index 0000000..cf06b17 --- /dev/null +++ b/spec/spec_helper.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Git exports repository-local environment variables to hooks. They can make +# nested test repositories operate on the outer repository instead. +if command -v git >/dev/null 2>&1; then + unset $(git rev-parse --local-env-vars 2>/dev/null || :) +fi diff --git a/tests/kab_criu/test.sh b/tests/kab_criu/test.sh index a669260..f1a8deb 100755 --- a/tests/kab_criu/test.sh +++ b/tests/kab_criu/test.sh @@ -1,8 +1,6 @@ #!/bin/bash # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k -set -x - -. ./tmt.sh +. ../test_lib.sh [[ -z $ARCH ]] && ARCH=$(uname -m) @@ -20,16 +18,22 @@ TARGET_HOST="${SERVERS}" TMT_TEST_PLAN_ROOT=${TMT_PLAN_DATA%data} SERVER_SSH_KEY=${TMT_TEST_PLAN_ROOT}/provision/server/id_ecdsa +ssh_opts=(-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ChannelTimeout=session=2s) + +if [[ -f "$SERVER_SSH_KEY" ]]; then + ssh_opts+=(-o IdentitiesOnly=yes -i "$SERVER_SSH_KEY") +fi + # ssh_cmd wrapper to handle local and remote execution ssh_cmd() { - local _opts="-o StrictHostKeyChecking=accept-new -o ConnectTimeout=10" - if [[ -f "$SERVER_SSH_KEY" ]]; then - _opts="$_opts -o IdentitiesOnly=yes -i $SERVER_SSH_KEY" - fi - ssh $_opts "$TARGET_HOST" "$@" + ssh "${ssh_opts[@]}" "$TARGET_HOST" "$@" return $? } +copy_xtrace_log() { + scp "${ssh_opts[@]}" "${TARGET_HOST}:${XTRACE_LOG}" "$XTRACE_LOG" +} + if ! ssh_cmd "cd $TMT_TREE && make install"; then echo "Failed to install KAB on ${SERVERS}" exit 1 @@ -83,29 +87,42 @@ END # For idempotence ssh_cmd "rm -rf $GIT_REPO" + # 2. Start kab.sh on Target if not already running and no checkpoint exists if ! ssh_cmd "pgrep -f $KAB_SCRIPT" >/dev/null 2>&1 && ! ssh_cmd "ls /var/local/kernel-auto-bisect/dump/core-*.img" >/dev/null 2>&1; then echo "Starting kab.sh..." - ssh_cmd "setsid bash -x $KAB_SCRIPT /root/test.log &" + ssh_cmd "setsid bash -x $KAB_SCRIPT $XTRACE_LOG &" fi # 3. Wait for result MAX_WAIT_TIME=600 # 10 minutes wait_time=0 +MAIN_LOG_REMOTE=/var/local/kernel-auto-bisect/main.log +MAIN_LOG_LOCAL=/tmp/kab-main.log +touch "$MAIN_LOG_LOCAL" +printed_lines=0 + while [[ $wait_time -lt $MAX_WAIT_TIME ]]; do # Try to check if finished - output=$(ssh_cmd "git -C $GIT_REPO bisect log | grep 'first bad commit' | grep -q '$BAD_COMMIT'") + output=$(ssh_cmd "git -C $GIT_REPO bisect log 2>/dev/null | grep 'first bad commit' | grep -q '$BAD_COMMIT'") ret=$? if [[ $ret -eq 0 ]]; then + copy_xtrace_log exit 0 - else - echo "Target ($TARGET_HOST) is down or unreachable (exit code: $ret), waiting..." fi + # Rsync the remote log and print only new lines + rsync --timeout=20 -e "ssh ${ssh_opts[*]}" "${TARGET_HOST}:${MAIN_LOG_REMOTE}" "$MAIN_LOG_LOCAL" 2>/dev/null + current_total=$(wc -l <"$MAIN_LOG_LOCAL" 2>/dev/null || echo 0) + if [[ $current_total -gt $printed_lines ]]; then + tail -n +$((printed_lines + 1)) "$MAIN_LOG_LOCAL" + printed_lines=$current_total + fi sleep 10 wait_time=$((wait_time + 10)) done +copy_xtrace_log echo "Failed to get 1st bad commit within $MAX_WAIT_TIME seconds" exit 1 diff --git a/tests/kab_ssh/test.sh b/tests/kab_ssh/test.sh index a98ff92..efd508b 100755 --- a/tests/kab_ssh/test.sh +++ b/tests/kab_ssh/test.sh @@ -1,8 +1,7 @@ #!/bin/bash # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k -set -x -. ./tmt.sh +. ../test_lib.sh [[ -z $ARCH ]] && ARCH=$(uname -m) @@ -62,7 +61,7 @@ on_test() { } END - bash -x $KAB_SCRIPT /root/test.log + bash -x $KAB_SCRIPT "$XTRACE_LOG" GIT_REPO=/var/local/kernel-auto-bisect/git_repo MAX_WAIT_TIME=600 wait_time=0 diff --git a/tests/kab_ssh/tmt.sh b/tests/kab_ssh/tmt.sh deleted file mode 100644 index 4d1266e..0000000 --- a/tests/kab_ssh/tmt.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -assign_server_roles() { - if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then - # assign roles based on tmt topology data - # shellcheck source=/dev/null - . "${TMT_TOPOLOGY_BASH}" - - if [[ $(grep -Ec "TMT_GUESTS\[.*role\]" "${TMT_TOPOLOGY_BASH}") -gt 1 ]]; then - export SERVERS=${TMT_GUESTS[server.hostname]} - export CLIENTS=${TMT_GUESTS[client.hostname]} - fi - - export HOSTNAME=${TMT_GUEST[hostname]} - fi -} - -assign_server_roles diff --git a/tests/kab_ssh_auto/test.sh b/tests/kab_ssh_auto/test.sh index 432607b..8f32bdd 100755 --- a/tests/kab_ssh_auto/test.sh +++ b/tests/kab_ssh_auto/test.sh @@ -1,8 +1,7 @@ #!/bin/bash # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k -set -x -. ./tmt.sh +. ../test_lib.sh [[ -z $ARCH ]] && ARCH=$(uname -m) @@ -16,7 +15,7 @@ if echo "${CLIENTS}" | grep -qi "${HOSTNAME}"; then KERNEL_RPM_LIST=/usr/local/bin/kernel-auto-bisect/kernel_list WORK_DIR=/var/local/kernel-auto-bisect GIT_REPO=$WORK_DIR/git_repo - GOOD_COMMIT=6.19.5-300.fc44.${ARCH} + GOOD_COMMIT=6.19.2-300.fc44.${ARCH} BAD_COMMIT=6.19.8-200.fc43.${ARCH} TMT_TEST_PLAN_ROOT=${TMT_PLAN_DATA%data} @@ -50,7 +49,7 @@ END fi cat <$KERNEL_RPM_LIST -https://kojipkgs.fedoraproject.org/packages/kernel/6.19.5/300.fc44/${ARCH}/kernel-core-6.19.5-300.fc44.${ARCH}.rpm +https://kojipkgs.fedoraproject.org/packages/kernel/6.19.2/300.fc44/${ARCH}/kernel-core-6.19.2-300.fc44.${ARCH}.rpm https://kojipkgs.fedoraproject.org/packages/kernel/6.19.6/200.fc43/${ARCH}/kernel-core-6.19.6-200.fc43.${ARCH}.rpm https://kojipkgs.fedoraproject.org/packages/kernel/6.19.7/200.fc43/${ARCH}/kernel-core-6.19.7-200.fc43.${ARCH}.rpm https://kojipkgs.fedoraproject.org/packages/kernel/6.19.8/200.fc43/${ARCH}/kernel-core-6.19.8-200.fc43.${ARCH}.rpm @@ -72,7 +71,7 @@ on_test() { } END - bash -x $KAB_SCRIPT /root/test.log + bash -x $KAB_SCRIPT "$XTRACE_LOG" if [[ -f "$SERVER_SSH_KEY" ]]; then ssh_cmd="ssh -o IdentitiesOnly=yes -i $SERVER_SSH_KEY" diff --git a/tests/kab_ssh_auto/tmt.sh b/tests/kab_ssh_auto/tmt.sh deleted file mode 100755 index 4d1266e..0000000 --- a/tests/kab_ssh_auto/tmt.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -assign_server_roles() { - if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then - # assign roles based on tmt topology data - # shellcheck source=/dev/null - . "${TMT_TOPOLOGY_BASH}" - - if [[ $(grep -Ec "TMT_GUESTS\[.*role\]" "${TMT_TOPOLOGY_BASH}") -gt 1 ]]; then - export SERVERS=${TMT_GUESTS[server.hostname]} - export CLIENTS=${TMT_GUESTS[client.hostname]} - fi - - export HOSTNAME=${TMT_GUEST[hostname]} - fi -} - -assign_server_roles diff --git a/tests/kab_ssh_src/test.sh b/tests/kab_ssh_src/test.sh index d07c56b..6144cff 100755 --- a/tests/kab_ssh_src/test.sh +++ b/tests/kab_ssh_src/test.sh @@ -1,8 +1,7 @@ #!/bin/bash # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k -set -x -. ./tmt.sh +. ../test_lib.sh [[ -z $ARCH ]] && ARCH=$(uname -m) @@ -66,14 +65,14 @@ on_test() { } END - bash -x "$KAB_SCRIPT" test.log + bash -x "$KAB_SCRIPT" "$XTRACE_LOG" KAB_EXIT=$? LOCAL_STATE_DIR=~/.local/state/kernel-auto-bisect if [[ $KAB_EXIT -ne 0 ]]; then echo "FAIL: kab.sh failed as non-root user (exit=$KAB_EXIT)" - cat "test.log" + cat "$XTRACE_LOG" cat "$LOCAL_STATE_DIR/main.log" 2>/dev/null exit 1 fi diff --git a/tests/kab_ssh_src/tmt.sh b/tests/kab_ssh_src/tmt.sh deleted file mode 100644 index 4d1266e..0000000 --- a/tests/kab_ssh_src/tmt.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -assign_server_roles() { - if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then - # assign roles based on tmt topology data - # shellcheck source=/dev/null - . "${TMT_TOPOLOGY_BASH}" - - if [[ $(grep -Ec "TMT_GUESTS\[.*role\]" "${TMT_TOPOLOGY_BASH}") -gt 1 ]]; then - export SERVERS=${TMT_GUESTS[server.hostname]} - export CLIENTS=${TMT_GUESTS[client.hostname]} - fi - - export HOSTNAME=${TMT_GUEST[hostname]} - fi -} - -assign_server_roles diff --git a/tests/kab_criu/tmt.sh b/tests/test_lib.sh similarity index 78% rename from tests/kab_criu/tmt.sh rename to tests/test_lib.sh index 4d1266e..ee9082f 100644 --- a/tests/kab_criu/tmt.sh +++ b/tests/test_lib.sh @@ -1,4 +1,10 @@ #!/bin/bash +if [[ -z $TMT_PLAN_DATA ]]; then + echo "TMT_PLAN_DATA not defined" + exit 1 +fi +XTRACE_LOG="${TMT_PLAN_DATA}/test.log" + assign_server_roles() { if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then # assign roles based on tmt topology data