autoqa-template.yml

name: Auto QA Test Runner Template

on:
  workflow_call:
    inputs:
      jan_app_windows_source:
        description: 'Windows app source - can be URL or local path'
        required: true
        type: string
      jan_app_ubuntu_source:
        description: 'Ubuntu app source - can be URL or local path'
        required: true
        type: string
      jan_app_macos_source:
        description: 'macOS app source - can be URL or local path'
        required: true
        type: string
      is_nightly:
        description: 'Is this a nightly build?'
        required: true
        type: boolean
        default: true
      source_type:
        description: 'Source type: url or local'
        required: true
        type: string
        default: 'url'
      artifact_name_windows:
        description: 'Windows artifact name (only needed for local)'
        required: false
        type: string
        default: ''
      artifact_name_ubuntu:
        description: 'Ubuntu artifact name (only needed for local)'
        required: false
        type: string
        default: ''
      artifact_name_macos:
        description: 'macOS artifact name (only needed for local)'
        required: false
        type: string
        default: ''
    secrets:
      RP_TOKEN:
        description: 'ReportPortal API token'
        required: true

jobs:
  windows:
    runs-on: windows-11-nvidia-gpu
    timeout-minutes: 60

    env:
      DEFAULT_JAN_APP_URL: 'https://catalog.jan.ai/windows/Jan-nightly_0.6.5-758_x64-setup.exe'
      DEFAULT_IS_NIGHTLY: 'true'

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Setup Python 3.13
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: '3.13'

      - name: Download artifact (if source_type is local)
        if: inputs.source_type == 'local'
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: ${{ inputs.artifact_name_windows }}
          path: ${{ runner.temp }}/windows-artifact

      - name: Clean existing Jan installations
        shell: powershell
        run: |
          .\autoqa\scripts\windows_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}"

      - name: Download/Prepare Jan app
        shell: powershell
        run: |
          if ("${{ inputs.source_type }}" -eq "local") {
            # Find the exe file in the artifact
            $exeFile = Get-ChildItem -Path "${{ runner.temp }}/windows-artifact" -Recurse -Filter "*.exe" | Select-Object -First 1
            if ($exeFile) {
              Write-Host "[SUCCESS] Found local installer: $($exeFile.FullName)"
              Copy-Item -Path $exeFile.FullName -Destination "$env:TEMP\jan-installer.exe" -Force
              Write-Host "[SUCCESS] Installer copied to: $env:TEMP\jan-installer.exe"
              # Don't set JAN_APP_PATH here - let the install script set it to the correct installed app path
              echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $env:GITHUB_ENV
            } else {
              Write-Error "[FAILED] No .exe file found in artifact"
              exit 1
            }
          } else {
            # Use the existing download script for URLs
            .\autoqa\scripts\windows_download.ps1 `
              -WorkflowInputUrl "${{ inputs.jan_app_windows_source }}" `
              -WorkflowInputIsNightly "${{ inputs.is_nightly }}" `
              -RepoVariableUrl "${{ vars.JAN_APP_URL }}" `
              -RepoVariableIsNightly "${{ vars.IS_NIGHTLY }}" `
              -DefaultUrl "$env:DEFAULT_JAN_APP_URL" `
              -DefaultIsNightly "$env:DEFAULT_IS_NIGHTLY"
          }

      - name: Install Jan app
        shell: powershell
        run: |
          .\autoqa\scripts\windows_install.ps1 -IsNightly "$env:IS_NIGHTLY"

      - name: Install Python dependencies
        working-directory: autoqa
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Run Auto QA Tests
        working-directory: autoqa
        shell: powershell
        env:
          RP_TOKEN: ${{ secrets.RP_TOKEN }}
          ENABLE_REPORTPORTAL: 'true'
          RP_ENDPOINT: 'https://reportportal.menlo.ai'
          RP_PROJECT: 'default_personal'
          MAX_TURNS: '50'
          DELAY_BETWEEN_TESTS: '3'
          LAUNCH_NAME: 'CI AutoQA Run Windows - ${{ github.run_number }} - ${{ github.ref_name }}'
        run: |
          .\scripts\run_tests.ps1 -JanAppPath "$env:JAN_APP_PATH" -ProcessName "$env:JAN_PROCESS_NAME" -RpToken "$env:RP_TOKEN"

      - name: Collect Jan logs for artifact upload
        if: always()
        shell: powershell
        run: |
          $logDirs = @(
            "$env:APPDATA\Jan-nightly\data\logs",
            "$env:APPDATA\Jan\data\logs"
          )
          $dest = "autoqa\jan-logs"
          mkdir $dest -Force | Out-Null
          foreach ($dir in $logDirs) {
            if (Test-Path $dir) {
              Copy-Item "$dir\*.log" $dest -Force -ErrorAction SilentlyContinue
            }
          }

      - name: Upload screen recordings
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        continue-on-error: true
        with:
          name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-recordings-${{ github.run_number }}-${{ runner.os }}
          path: autoqa/recordings/

      - name: Upload Jan logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-logs-${{ github.run_number }}-${{ runner.os }}
          path: autoqa/jan-logs/

      - name: Cleanup after tests
        if: always()
        shell: powershell
        run: |
          .\autoqa\scripts\windows_post_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}"

  ubuntu:
    runs-on: ubuntu-22-04-nvidia-gpu
    timeout-minutes: 60

    env:
      DEFAULT_JAN_APP_URL: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_amd64.deb'
      DEFAULT_IS_NIGHTLY: 'true'

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Setup Python 3.13
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: '3.13'

      - name: Download artifact (if source_type is local)
        if: inputs.source_type == 'local'
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: ${{ inputs.artifact_name_ubuntu }}
          path: ${{ runner.temp }}/ubuntu-artifact

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y \
            x11-utils \
            python3-tk \
            python3-dev \
            wmctrl \
            xdotool \
            libnss3-dev \
            libgconf-2-4 \
            libxss1 \
            libasound2 \
            libxtst6 \
            libgtk-3-0 \
            libgbm-dev \
            libxshmfence1 \
            libxrandr2 \
            libpangocairo-1.0-0 \
            libatk1.0-0 \
            libcairo-gobject2 \
            libgdk-pixbuf2.0-0 \
            gnome-screenshot

      - name: Setup script permissions
        run: |
          chmod +x autoqa/scripts/setup_permissions.sh
          ./autoqa/scripts/setup_permissions.sh

      - name: Clean existing Jan installations
        run: |
          ./autoqa/scripts/ubuntu_cleanup.sh

      - name: Download/Prepare Jan app
        run: |
          if [ "${{ inputs.source_type }}" = "local" ]; then
            # Find the deb file in the artifact
            DEB_FILE=$(find "${{ runner.temp }}/ubuntu-artifact" -name "*.deb" -type f | head -1)
            if [ -n "$DEB_FILE" ]; then
              echo "[SUCCESS] Found local installer: $DEB_FILE"
              cp "$DEB_FILE" "/tmp/jan-installer.deb"
              echo "[SUCCESS] Installer copied to: /tmp/jan-installer.deb"
              echo "JAN_APP_PATH=/tmp/jan-installer.deb" >> $GITHUB_ENV
              echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $GITHUB_ENV
              if [ "${{ inputs.is_nightly }}" = "true" ]; then
                echo "JAN_PROCESS_NAME=Jan-Desktop-nightly" >> $GITHUB_ENV
              else
                echo "JAN_PROCESS_NAME=Jan-Desktop" >> $GITHUB_ENV
              fi
            else
              echo "[FAILED] No .deb file found in artifact"
              exit 1
            fi
          else
            # Use the existing download script for URLs
            ./autoqa/scripts/ubuntu_download.sh \
              "${{ inputs.jan_app_ubuntu_source }}" \
              "${{ inputs.is_nightly }}" \
              "${{ vars.JAN_APP_URL_LINUX }}" \
              "${{ vars.IS_NIGHTLY }}" \
              "$DEFAULT_JAN_APP_URL" \
              "$DEFAULT_IS_NIGHTLY"
            
            # Set the correct environment variables for the test runner
            echo "JAN_APP_PATH=/tmp/jan-installer.deb" >> $GITHUB_ENV
            if [ "${{ inputs.is_nightly }}" = "true" ]; then
              echo "JAN_PROCESS_NAME=Jan-Desktop-nightly" >> $GITHUB_ENV
            else
              echo "JAN_PROCESS_NAME=Jan-Desktop" >> $GITHUB_ENV
            fi
          fi

      - name: Install Jan app
        run: |
          ./autoqa/scripts/ubuntu_install.sh "$IS_NIGHTLY"

      - name: Install Python dependencies
        working-directory: autoqa
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Run Auto QA Tests
        working-directory: autoqa
        env:
          RP_TOKEN: ${{ secrets.RP_TOKEN }}
          ENABLE_REPORTPORTAL: 'true'
          RP_ENDPOINT: 'https://reportportal.menlo.ai'
          RP_PROJECT: 'default_personal'
          MAX_TURNS: '50'
          DELAY_BETWEEN_TESTS: '3'
          LAUNCH_NAME: 'CI AutoQA Run Ubuntu - ${{ github.run_number }} - ${{ github.ref_name }}'
        run: |
          ./scripts/run_tests.sh "$JAN_APP_PATH" "$JAN_PROCESS_NAME" "$RP_TOKEN" "ubuntu"

      - name: Collect Jan logs for artifact upload
        if: always()
        run: |
          mkdir -p autoqa/jan-logs
          cp ~/.local/share/Jan-nightly/data/logs/*.log autoqa/jan-logs/ 2>/dev/null || true
          cp ~/.local/share/Jan/data/logs/*.log autoqa/jan-logs/ 2>/dev/null || true

      - name: Upload screen recordings
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        continue-on-error: true
        with:
          name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-recordings-${{ github.run_number }}-${{ runner.os }}
          path: autoqa/recordings/

      - name: Upload Jan logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-logs-${{ github.run_number }}-${{ runner.os }}
          path: autoqa/jan-logs/

      - name: Cleanup after tests
        if: always()
        run: |
          ./autoqa/scripts/ubuntu_post_cleanup.sh "$IS_NIGHTLY"

  macos:
    runs-on: macos-selfhosted-15-arm64-cua
    timeout-minutes: 60

    env:
      DEFAULT_JAN_APP_URL: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_universal.dmg'
      DEFAULT_IS_NIGHTLY: 'true'

    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

      - name: Setup Python 3.13
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: '3.13'

      - name: Download artifact (if source_type is local)
        if: inputs.source_type == 'local'
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: ${{ inputs.artifact_name_macos }}
          path: ${{ runner.temp }}/macos-artifact

      - name: Setup script permissions
        run: |
          chmod +x autoqa/scripts/setup_permissions.sh
          ./autoqa/scripts/setup_permissions.sh

      - name: Clean existing Jan installations
        run: |
          ./autoqa/scripts/macos_cleanup.sh

      - name: Download/Prepare Jan app
        run: |
          if [ "${{ inputs.source_type }}" = "local" ]; then
            # Find the dmg file in the artifact
            DMG_FILE=$(find "${{ runner.temp }}/macos-artifact" -name "*.dmg" -type f | head -1)
            if [ -n "$DMG_FILE" ]; then
              echo "[SUCCESS] Found local installer: $DMG_FILE"
              cp "$DMG_FILE" "/tmp/jan-installer.dmg"
              echo "[SUCCESS] Installer copied to: /tmp/jan-installer.dmg"
              echo "JAN_APP_PATH=/tmp/jan-installer.dmg" >> $GITHUB_ENV
              echo "IS_NIGHTLY=${{ inputs.is_nightly }}" >> $GITHUB_ENV
              if [ "${{ inputs.is_nightly }}" = "true" ]; then
                echo "PROCESS_NAME=Jan-Desktop-nightly" >> $GITHUB_ENV
              else
                echo "PROCESS_NAME=Jan-Desktop" >> $GITHUB_ENV
              fi
            else
              echo "[FAILED] No .dmg file found in artifact"
              exit 1
            fi
          else
            # Use the existing download script for URLs
            ./autoqa/scripts/macos_download.sh \
              "${{ inputs.jan_app_macos_source }}" \
              "${{ inputs.is_nightly }}" \
              "${{ vars.JAN_APP_URL }}" \
              "${{ vars.IS_NIGHTLY }}" \
              "$DEFAULT_JAN_APP_URL" \
              "$DEFAULT_IS_NIGHTLY"
            
            # Set the correct environment variables for the test runner
            echo "JAN_APP_PATH=/tmp/jan-installer.dmg" >> $GITHUB_ENV
            if [ "${{ inputs.is_nightly }}" = "true" ]; then
              echo "PROCESS_NAME=Jan-Desktop-nightly" >> $GITHUB_ENV
            else
              echo "PROCESS_NAME=Jan-Desktop" >> $GITHUB_ENV
            fi
          fi

      - name: Install Jan app
        run: |
          ./autoqa/scripts/macos_install.sh

      - name: Install system dependencies
        run: |
          echo "Installing system dependencies for macOS..."

          # Check if Homebrew is available
          if command -v brew >/dev/null 2>&1; then
            echo "Homebrew is available"
            
            # Install python-tk if not available
            python3 -c "import tkinter" 2>/dev/null || {
              echo "Installing python-tk via Homebrew..."
              brew install python-tk || true
            }
          else
            echo "Homebrew not available, checking if tkinter works..."
            python3 -c "import tkinter" || {
              echo "[WARNING] tkinter not available and Homebrew not found"
              echo "This may cause issues with mouse control"
            }
          fi

          echo "System dependencies check completed"

      - name: Install Python dependencies
        run: |
          cd autoqa
          echo "Installing Python dependencies..."
          pip install --upgrade pip
          pip install -r requirements.txt
          echo "[SUCCESS] Python dependencies installed"

      - name: Setup ReportPortal environment
        run: |
          echo "Setting up ReportPortal environment..."
          echo "RP_TOKEN=${{ secrets.RP_TOKEN }}" >> $GITHUB_ENV
          echo "ReportPortal environment configured"

      - name: Run E2E tests
        env:
          RP_TOKEN: ${{ secrets.RP_TOKEN }}
          ENABLE_REPORTPORTAL: 'true'
          RP_ENDPOINT: 'https://reportportal.menlo.ai'
          RP_PROJECT: 'default_personal'
          MAX_TURNS: '50'
          DELAY_BETWEEN_TESTS: '3'
          LAUNCH_NAME: 'CI AutoQA Run Macos - ${{ github.run_number }} - ${{ github.ref_name }}'
        run: |
          cd autoqa
          echo "Starting E2E test execution..."

          echo "Environment variables:"
          echo "JAN_APP_PATH: $JAN_APP_PATH"
          echo "PROCESS_NAME: $PROCESS_NAME"
          echo "IS_NIGHTLY: $IS_NIGHTLY"

          ./scripts/run_tests.sh "$JAN_APP_PATH" "$PROCESS_NAME" "$RP_TOKEN" "macos"

      - name: Collect Jan logs for artifact upload
        if: always()
        run: |
          mkdir -p autoqa/jan-logs
          cp ~/Library/Application\ Support/Jan-nightly/data/logs/*.log autoqa/jan-logs/ 2>/dev/null || true
          cp ~/Library/Application\ Support/Jan/data/logs/*.log autoqa/jan-logs/ 2>/dev/null || true

      - name: Upload screen recordings
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        continue-on-error: true
        with:
          name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-recordings-${{ github.run_number }}-${{ runner.os }}
          path: autoqa/recordings/

      - name: Upload Jan logs
        if: always()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: ${{ inputs.is_nightly && 'jan-nightly' || 'jan' }}-logs-${{ github.run_number }}-${{ runner.os }}
          path: autoqa/jan-logs/

      - name: Cleanup after tests
        if: always()
        run: |
          ./autoqa/scripts/macos_post_cleanup.sh