chore(main): release 0.11.7 #4246
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled, unlabeled, edited] | |
| # Manually trigger a workflow for a branch | |
| workflow_dispatch: | |
| # Merge queue trigger | |
| merge_group: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| HF_DATASET_CACHE_DIR: /tmp/huggingface_datasets # each job lazily caches its own subset here | |
| UV_LINK_MODE: symlink | |
| UV_LOCKED: 1 | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest # default runner runs out of disk space due to hf cache | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v7 | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "~=0.8.16" | |
| - name: Run Pre-Commit | |
| run: uvx pre-commit run --all-files | |
| - name: Dependency check | |
| run: ./utils/dependency_check.sh | |
| - name: Run MyPy | |
| run: uv run --all-extras mypy | |
| test-extras: | |
| # Test uv installs (CPU) | |
| runs-on: ubuntu-latest | |
| needs: [lint] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| extras: ['', 'determined', 'api', 'openai', 'transformers', 'accelerate', 'optional'] | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v7 | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "~=0.8.16" | |
| - name: Verify install and functionality via uv --exact | |
| run: | | |
| if [ "${{ matrix.extras }}" != "" ]; then | |
| echo "Testing extra: ${{ matrix.extras }}" | |
| uv run --exact --extra ${{ matrix.extras }} pytest -v --noconftest tests/tests_eval_framework/installs/test_${{ matrix.extras }}.py | |
| else | |
| echo "Testing core install" | |
| uv run --exact pytest --noconftest -v tests/tests_eval_framework/installs/test_core.py | |
| fi | |
| test-cpu: | |
| runs-on: cpu-runner-8c-32gb-01 | |
| container: derskythe/github-runner-base:ubuntu-noble | |
| needs: [test-extras] | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "~=0.8.16" | |
| - name: Huggingface datasets cache | |
| uses: actions/cache@v6 | |
| with: | |
| path: ${{ env.HF_DATASET_CACHE_DIR }} | |
| key: hf-datasets-${{ github.job }}-${{ github.run_id }} | |
| restore-keys: | | |
| hf-datasets-${{ github.job }}- | |
| - name: Run tests | |
| env: | |
| HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }} | |
| run: | | |
| uv run --all-extras python -c "import nltk; nltk.download('punkt_tab')" | |
| uv run --all-extras pytest -n auto --max-worker-restart=0 --durations=30 -v -m "not gpu and not cpu_slow and not external_api and not formatter_hash" | |
| test-cpu-slow: | |
| runs-on: cpu-runner-8c-32gb-01 | |
| container: derskythe/github-runner-base:ubuntu-noble | |
| needs: [test-extras] | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "~=0.8.16" | |
| - name: Huggingface datasets cache | |
| uses: actions/cache@v6 | |
| with: | |
| path: ${{ env.HF_DATASET_CACHE_DIR }} | |
| key: hf-datasets-${{ github.job }}-${{ github.run_id }} | |
| restore-keys: | | |
| hf-datasets-${{ github.job }}- | |
| - name: Run tests | |
| env: | |
| HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }} | |
| run: | | |
| uv run --all-extras python -c "import nltk; nltk.download('punkt_tab')" | |
| uv run --all-extras pytest -n auto --max-worker-restart=0 --durations=30 -v -m "not gpu and cpu_slow and not slow_download and not external_api and not formatter_hash" | |
| test-formatter-hash: | |
| runs-on: cpu-runner-8c-32gb-01 | |
| container: derskythe/github-runner-base:ubuntu-noble | |
| needs: [test-extras] | |
| permissions: | |
| pull-requests: read | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 2 | |
| - name: Mark workspace as safe for git | |
| run: git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - uses: dorny/paths-filter@v4 | |
| id: filter | |
| with: | |
| filters: | | |
| task_related: | |
| - 'src/eval_framework/tasks/**' | |
| - 'tests/tests_eval_framework/tasks/benchmarks/**' | |
| - 'src/template_formatting/**' | |
| - name: Set formatter hash run flag | |
| id: should_run | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ] || \ | |
| [ "${{ github.event_name }}" = "merge_group" ] || \ | |
| [ "${{ steps.filter.outputs.task_related }}" = "true" ]; then | |
| echo "value=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "value=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Skip formatter hash tests (no relevant changes) | |
| if: steps.should_run.outputs.value != 'true' | |
| run: echo "Skipping formatter hash tests — no task-related paths changed." | |
| - name: Setup uv | |
| if: steps.should_run.outputs.value == 'true' | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "~=0.8.16" | |
| - name: Huggingface datasets cache | |
| if: steps.should_run.outputs.value == 'true' | |
| uses: actions/cache@v6 | |
| with: | |
| path: ${{ env.HF_DATASET_CACHE_DIR }} | |
| key: hf-datasets-${{ github.job }}-${{ github.run_id }} | |
| restore-keys: | | |
| hf-datasets-${{ github.job }}- | |
| - name: Run formatter hashing tests | |
| if: steps.should_run.outputs.value == 'true' | |
| env: | |
| HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }} | |
| run: | | |
| uv run --all-extras python -c "import nltk; nltk.download('punkt_tab')" | |
| uv run --all-extras pytest -n auto --max-worker-restart=0 --durations=30 -v -m "formatter_hash" | |
| test-gpu: | |
| # PRs: run GPU tests directly on the GPU runner (no Docker image). | |
| if: github.event_name == 'pull_request' | |
| runs-on: EvalFrameworkGPURunner | |
| needs: [test-cpu, test-cpu-slow, test-formatter-hash] | |
| permissions: | |
| pull-requests: read | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: "3.11" | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| version: "~=0.8.16" | |
| - name: Huggingface datasets cache | |
| uses: actions/cache@v6 | |
| with: | |
| path: ${{ env.HF_DATASET_CACHE_DIR }} | |
| key: hf-datasets-${{ github.job }}-${{ github.run_id }} | |
| restore-keys: | | |
| hf-datasets-${{ github.job }}- | |
| - name: Verify GPU installs via uv --exact | |
| run: | | |
| set -e | |
| echo "Testing all extras together" | |
| uv run --exact --all-extras pytest -v --noconftest tests/tests_eval_framework/installs/ | |
| - name: Test GPU | |
| timeout-minutes: 20 | |
| env: | |
| HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }} | |
| run: | | |
| uv run --all-extras python -c "import nltk; nltk.download('punkt_tab')" | |
| uv run --all-extras pytest -n auto --max-worker-restart=0 --durations=30 -v -m "gpu and not cpu_slow and not external_api" |