python-test-e2e.yml

name: Manual E2E Tests

on:
  workflow_dispatch:
    inputs:
      models:
        description: "Models to test"
        required: false
        default: "gpt-4.1-mini,gpt-4o-mini"
        type: string

permissions:
  contents: read
  actions: write

jobs:
  test:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd

      - name: Validate inputs
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        with:
          script: |
            const models = '${{ inputs.models }}';
            console.log('Validating models input:', models);

            if (!/^[a-zA-Z0-9,.\-_]+$/.test(models)) {
              core.setFailed('Models input contains invalid characters. Only alphanumeric, hyphens, commas, periods, and underscores are allowed.');
              return;
            }
            console.log('Input validation passed');

      - name: Set up Python 3.12
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
        with:
          python-version: "3.12"
      - name: Install the latest version of uv
        uses: astral-sh/setup-uv@39eb6c9dde236bbc368681611e63120a6eb4afac
        with:
          version: "0.7.13"
          activate-environment: true

      - name: Install dependencies
        run: |
          uv sync --extra dev

      - name: Test with pytest
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          MODELS_TO_TEST: ${{ inputs.models }}
          MCP_USE_ANONYMIZED_TELEMETRY: false
        run: |
          pytest --run-e2e

      - name: Generate HTML report
        if: always()
        run: |
          python scripts/generate_e2e_report.py

      - name: Upload HTML report artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        if: always()
        with:
          name: e2e-test-report
          path: ./static_test_report.html