tests.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:
    types: [opened, reopened, synchronize, labeled, unlabeled, edited]
  # Manually trigger a workflow for a branch
  workflow_dispatch:
  # Merge queue trigger
  merge_group:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  HF_DATASET_CACHE_DIR: /tmp/huggingface_datasets  # each job lazily caches its own subset here
  UV_LINK_MODE: symlink
  UV_LOCKED: 1

jobs:

  lint:
    runs-on: ubuntu-latest # default runner runs out of disk space due to hf cache
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v7

      - name: Setup uv
        uses: astral-sh/setup-uv@v10.0.1
        with:
          version: "~=0.8.16"

      - name: Run Pre-Commit
        run: uvx pre-commit run --all-files

      - name: Dependency check
        run: ./utils/dependency_check.sh

      - name: Run MyPy
        run: uv run --all-extras mypy

  test-extras:
    # Test uv installs (CPU)
    runs-on: ubuntu-latest
    needs: [lint]
    strategy:
      fail-fast: false
      matrix:
        extras: ['', 'determined', 'api', 'openai', 'transformers', 'accelerate', 'optional']
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v7

      - name: Setup uv
        uses: astral-sh/setup-uv@v10.0.1
        with:
          version: "~=0.8.16"

      - name: Verify install and functionality via uv --exact
        run: |
          if [ "${{ matrix.extras }}" != "" ]; then
            echo "Testing extra: ${{ matrix.extras }}"
            uv run --exact --extra ${{ matrix.extras }} pytest -v --noconftest tests/tests_eval_framework/installs/test_${{ matrix.extras }}.py
          else
            echo "Testing core install"
            uv run --exact pytest --noconftest -v tests/tests_eval_framework/installs/test_core.py
          fi

  test-cpu:
    runs-on: cpu-runner-8c-32gb-01
    container: derskythe/github-runner-base:ubuntu-noble
    needs: [test-extras]
    steps:
      - uses: actions/checkout@v7

      - name: Setup uv
        uses: astral-sh/setup-uv@v10.0.1
        with:
          version: "~=0.8.16"

      - name: Huggingface datasets cache
        uses: actions/cache@v6
        with:
          path: ${{ env.HF_DATASET_CACHE_DIR }}
          key: hf-datasets-${{ github.job }}-${{ github.run_id }}
          restore-keys: |
            hf-datasets-${{ github.job }}-

      - name: Run tests
        env:
          HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
        run: |
          uv run --all-extras python -c "import nltk; nltk.download('punkt_tab')"
          uv run --all-extras pytest -n auto --max-worker-restart=0 --durations=30 -v -m "not gpu and not cpu_slow and not external_api and not formatter_hash"

  test-cpu-slow:
    runs-on: cpu-runner-8c-32gb-01
    container: derskythe/github-runner-base:ubuntu-noble
    needs: [test-extras]
    steps:
      - uses: actions/checkout@v7

      - name: Setup uv
        uses: astral-sh/setup-uv@v10.0.1
        with:
          version: "~=0.8.16"

      - name: Huggingface datasets cache
        uses: actions/cache@v6
        with:
          path: ${{ env.HF_DATASET_CACHE_DIR }}
          key: hf-datasets-${{ github.job }}-${{ github.run_id }}
          restore-keys: |
            hf-datasets-${{ github.job }}-

      - name: Run tests
        env:
          HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
        run: |
          uv run --all-extras python -c "import nltk; nltk.download('punkt_tab')"
          uv run --all-extras pytest -n auto --max-worker-restart=0 --durations=30 -v -m "not gpu and cpu_slow and not slow_download and not external_api and not formatter_hash"

  test-formatter-hash:
    runs-on: cpu-runner-8c-32gb-01
    container: derskythe/github-runner-base:ubuntu-noble
    needs: [test-extras]
    permissions:
      pull-requests: read
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 2

      - name: Mark workspace as safe for git
        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - uses: dorny/paths-filter@v4
        id: filter
        with:
          filters: |
            task_related:
              - 'src/eval_framework/tasks/**'
              - 'tests/tests_eval_framework/tasks/benchmarks/**'
              - 'src/template_formatting/**'

      - name: Set formatter hash run flag
        id: should_run
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] || \
             [ "${{ github.event_name }}" = "merge_group" ] || \
             [ "${{ steps.filter.outputs.task_related }}" = "true" ]; then
            echo "value=true" >> "$GITHUB_OUTPUT"
          else
            echo "value=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Skip formatter hash tests (no relevant changes)
        if: steps.should_run.outputs.value != 'true'
        run: echo "Skipping formatter hash tests — no task-related paths changed."

      - name: Setup uv
        if: steps.should_run.outputs.value == 'true'
        uses: astral-sh/setup-uv@v10.0.1
        with:
          version: "~=0.8.16"

      - name: Huggingface datasets cache
        if: steps.should_run.outputs.value == 'true'
        uses: actions/cache@v6
        with:
          path: ${{ env.HF_DATASET_CACHE_DIR }}
          key: hf-datasets-${{ github.job }}-${{ github.run_id }}
          restore-keys: |
            hf-datasets-${{ github.job }}-

      - name: Run formatter hashing tests
        if: steps.should_run.outputs.value == 'true'
        env:
          HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
        run: |
          uv run --all-extras python -c "import nltk; nltk.download('punkt_tab')"
          uv run --all-extras pytest -n auto --max-worker-restart=0 --durations=30 -v -m "formatter_hash"

  test-gpu:
    # PRs: run GPU tests directly on the GPU runner (no Docker image).
    if: github.event_name == 'pull_request'
    runs-on: EvalFrameworkGPURunner
    needs: [test-cpu, test-cpu-slow, test-formatter-hash]
    permissions:
      pull-requests: read
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v7
        with:
          fetch-depth: 2

      - name: Set up Python
        uses: actions/setup-python@v7
        with:
          python-version: "3.11"

      - name: Setup uv
        uses: astral-sh/setup-uv@v10.0.1
        with:
          version: "~=0.8.16"

      - name: Huggingface datasets cache
        uses: actions/cache@v6
        with:
          path: ${{ env.HF_DATASET_CACHE_DIR }}
          key: hf-datasets-${{ github.job }}-${{ github.run_id }}
          restore-keys: |
            hf-datasets-${{ github.job }}-

      - name: Verify GPU installs via uv --exact
        run: |
          set -e
          echo "Testing all extras together"
          uv run --exact --all-extras pytest -v --noconftest tests/tests_eval_framework/installs/

      - name: Test GPU
        timeout-minutes: 20
        env:
          HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
        run: |
          uv run --all-extras python -c "import nltk; nltk.download('punkt_tab')"
          uv run --all-extras pytest -n auto --max-worker-restart=0 --durations=30 -v -m "gpu and not cpu_slow and not external_api"