helm/.github/workflows/test-python.yml at main · AIQ-Kitware/helm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
name: Test Python
on:
  push:
    branches: [ main ]
    paths-ignore:
      - 'helm-frontend/**'
  pull_request:
    paths-ignore:
      - 'helm-frontend/**'

jobs:
  install:
    # Tests that the HELM CLI commands work when only installing required dependencies
    # without optional extra dependencies.
    name: Run HELM with minimal dependencies only
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.9.4"
      - name: Build
        run: uv build
      - name: helm-run
        run: uv run --isolated --no-project --with dist/*.whl helm-run --run-entries simple1:model=simple/model1 --max-eval-instances 10 --suite test
      - name: helm-summarize
        run: uv run --isolated --no-project --with dist/*.whl helm-summarize --suite test
      - name: helm-server
        run: uv run --isolated --no-project --with dist/*.whl helm-server --help

  test:
    name: Run all tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          version: "0.9.4"
      - name: Install dependencies
        run: uv sync --extra ci
      - name: Run pre-commit
        run: ./pre-commit.sh
      - name: Run tests
        run: uv run pytest --durations=20
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'main' }}
      - name: Run entire pipeline quickly without any data
        # Checking RunSpecs with openai/davinci should be comprehensive enough
        run: uv run helm-run --suite test -m 100 --skip-instances --models-to-run openai/davinci --exit-on-error