Merge pull request #115 from gardenlinux/feat/ci-emergency-logging

feat: Store Logs from CI Runs for Debugging and Offline Analysis

Author: anokfireball

.github/actions/test/integration/collect-logs/action.yml

@@ -0,0 +1,42 @@
+name: collect-logs
+description: Store debugging information for offline analysis
+runs:
+  using: "composite"
+  steps:
+    - name: Obtain logs from test runner
+      shell: bash
+      run: |
+        sudo cp /var/log/libvirt/qemu/HV1.log "libvirt-HV1.log" || true
+        sudo cp /var/log/libvirt/qemu/HV2.log "libvirt-HV2.log" || true
+        sudo journalctl --no-pager > journal-RUNNER.log || true
+
+    - name: Obtain logs from hypervisors
+      shell: bash
+      run: |
+        KEY=/opt/ssh_host_ed25519_key
+        SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+        USER="root"
+        HV1_IP="192.168.122.2"
+        HV2_IP="192.168.122.3"
+
+        ssh $SSH_OPTS $USER@$HV1_IP "journalctl --no-pager" > "journal-HV1.log" || true
+        scp $SSH_OPTS $USER@$HV1_IP:/var/log/libvirt/qemu/VM-QEMU.log "libvirt-VM-QEMU-HV1.log" || true
+        scp $SSH_OPTS $USER@$HV1_IP:/var/log/libvirt/ch/VM-CHV.log "libvirt-VM-CHV-HV1.log" || true
+        scp $SSH_OPTS $USER@$HV1_IP:/var/log/QEMU.log "serial-VM-QEMU-HV1.log" || true
+
+        ssh $SSH_OPTS $USER@$HV2_IP "journalctl --no-pager" > "journal-HV2.log" || true
+        scp $SSH_OPTS $USER@$HV2_IP:/var/log/libvirt/qemu/VM-QEMU.log "libvirt-VM-QEMU-HV2.log" || true
+        scp $SSH_OPTS $USER@$HV2_IP:/var/log/libvirt/ch/VM-CHV.log "libvirt-VM-CHV-HV2.log" || true
+        scp $SSH_OPTS $USER@$HV2_IP:/var/log/QEMU.log "serial-VM-QEMU-HV2.log" || true
+
+    - name: Set permissions
+      shell: bash
+      run: |
+        sudo chmod 644 *.log
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: debug-logs
+        path: "*.log"
+

.github/actions/test/integration/test/cloudhypervisor/action.yml

@@ -47,7 +47,7 @@ runs:
           SSH_OPTS="-i \$KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
           USER="root"
 
-          MAX_ITER=40
+          MAX_ITER=60
           for ((i=1; i<=MAX_ITER; i++)); do
             IP=\$(virsh -c ch:///session domifaddr VM-CHV | awk '/ipv4/ {print \$4}' | cut -d'/' -f1)
             if [ -z "\$IP" ]; then
@@ -60,7 +60,6 @@ runs:
             fi
             if (( i == MAX_ITER )); then
               echo "Timeout waiting for VM to respond to SSH."
-              cat /var/log/CHV.log
               exit 1
             fi
             echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
@@ -110,7 +109,7 @@ runs:
           # the IP is static, so we can just use it directly
           IP="192.168.222.3"
 
-          MAX_ITER=40
+          MAX_ITER=15
           for ((i=1; i<=MAX_ITER; i++)); do
             if ssh \$SSH_OPTS "\$USER@\$IP" 'exit' ; then
               echo "VM is up at IP: \$IP"
@@ -125,7 +124,6 @@ runs:
             fi
             if (( i == MAX_ITER )); then
               echo "Timeout waiting for VM to respond to SSH."
-              cat /var/log/CHV.log
               exit 1
             fi
             echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"

.github/actions/test/integration/test/qemu/action.yml

@@ -45,7 +45,7 @@ runs:
           SSH_OPTS="-i \$KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
           USER="root"
 
-          MAX_ITER=40
+          MAX_ITER=60
           for ((i=1; i<=MAX_ITER; i++)); do
             IP=\$(virsh domifaddr VM-QEMU | awk '/ipv4/ {print \$4}' | cut -d'/' -f1)
             if [ -z "\$IP" ]; then
@@ -58,7 +58,6 @@ runs:
             fi
             if (( i == MAX_ITER )); then
               echo "Timeout waiting for VM to respond to SSH."
-              cat /var/log/QEMU.log
               exit 1
             fi
             echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
@@ -108,7 +107,7 @@ runs:
           # the IP is static, so we can just use it directly
           IP="192.168.222.2"
 
-          MAX_ITER=40
+          MAX_ITER=15
           for ((i=1; i<=MAX_ITER; i++)); do
             if ssh \$SSH_OPTS "\$USER@\$IP" 'exit' 2>/dev/null; then
               echo "VM is up at IP: \$IP"
@@ -123,32 +122,9 @@ runs:
             fi
             if (( i == MAX_ITER )); then
               echo "Timeout waiting for VM to respond to SSH."
-              cat /var/log/QEMU.log
               exit 1
             fi
             echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
             sleep 10
           done
         EOF
-
-    - name: Clean up HyperVisors
-      shell: bash
-      run: |
-        KEY=/opt/ssh_host_ed25519_key
-        SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
-        USER="root"
-        IP_HV1="192.168.122.2"
-        IP_HV2="192.168.122.3"
-
-        cleanup() {
-          local IP="$1"
-          ssh $SSH_OPTS "$USER@$IP" << EOF
-            virsh destroy VM-QEMU || true
-            virsh undefine VM-QEMU --nvram --tpm --remove-all-storage
-            rm -f /var/lib/libvirt/images/ubuntu.qcow2
-            rm -f /opt/vm-qemu.xml
-        EOF
-        }
-
-        cleanup "$IP_HV1"
-        cleanup "$IP_HV2"

.github/workflows/test.yml

@@ -40,3 +40,6 @@ jobs:
         uses: ./.github/actions/test/integration/test/qemu
       - name: Test CloudHypervisor
         uses: ./.github/actions/test/integration/test/cloudhypervisor
+      - name: Collect Debug Logs
+        if: always()
+        uses: ./.github/actions/test/integration/collect-logs