diff --git a/.github/workflows/nightly-evals.yml b/.github/workflows/nightly-evals.yml new file mode 100644 index 00000000..7e1168a6 --- /dev/null +++ b/.github/workflows/nightly-evals.yml @@ -0,0 +1,43 @@ +name: Nightly Evals + +on: + schedule: + # Midnight UTC every day + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + evals: + timeout-minutes: 120 + runs-on: + group: Large Runner Shared Public + labels: ubuntu-16-core-latest + steps: + - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Set up Python 3.10 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.10" + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + - name: Install dependencies + run: uv sync + + - name: Run evals + run: uv run evals/run.py --workers 16 + + - name: Upload combined report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: combined-report + path: evals/workdir/combined_report.html + if-no-files-found: error diff --git a/tests/core/test_github_action_project_scanner.py b/tests/core/test_github_action_project_scanner.py index 5b858912..99c50f17 100644 --- a/tests/core/test_github_action_project_scanner.py +++ b/tests/core/test_github_action_project_scanner.py @@ -59,6 +59,7 @@ def test_githubactions_find_requirements(): os.path.join(repo_root, ".github", "workflows", "checks.yml"), os.path.join(repo_root, ".github", "workflows", "docker-publish.yml"), os.path.join(repo_root, ".github", "workflows", "guarddog.yml"), + os.path.join(repo_root, ".github", "workflows", "nightly-evals.yml"), os.path.join(repo_root, ".github", "workflows", "pr.yml"), os.path.join(repo_root, ".github", "workflows", "pypi-publish.yml"), os.path.join(repo_root, ".github", "workflows", "scorecard.yml"),