From 29739061feb71fa21e09d26bebfd1775aa6bd302 Mon Sep 17 00:00:00 2001 From: Christophe Tafani-Dereeper Date: Mon, 15 Jun 2026 15:03:29 +0200 Subject: [PATCH 1/2] Add nightly evals workflow --- .github/workflows/nightly-evals.yml | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/nightly-evals.yml diff --git a/.github/workflows/nightly-evals.yml b/.github/workflows/nightly-evals.yml new file mode 100644 index 00000000..7e1168a6 --- /dev/null +++ b/.github/workflows/nightly-evals.yml @@ -0,0 +1,43 @@ +name: Nightly Evals + +on: + schedule: + # Midnight UTC every day + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + evals: + timeout-minutes: 120 + runs-on: + group: Large Runner Shared Public + labels: ubuntu-16-core-latest + steps: + - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Set up Python 3.10 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.10" + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + - name: Install dependencies + run: uv sync + + - name: Run evals + run: uv run evals/run.py --workers 16 + + - name: Upload combined report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: combined-report + path: evals/workdir/combined_report.html + if-no-files-found: error From 382ce9ca83376ed5ddac07ac6e06d36cfe3edd39 Mon Sep 17 00:00:00 2001 From: Christophe Tafani-Dereeper Date: Wed, 17 Jun 2026 10:24:40 +0200 Subject: [PATCH 2/2] Add nightly-evals.yml to workflow scanner test expectations --- tests/core/test_github_action_project_scanner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/test_github_action_project_scanner.py b/tests/core/test_github_action_project_scanner.py index 5b858912..99c50f17 100644 --- a/tests/core/test_github_action_project_scanner.py +++ b/tests/core/test_github_action_project_scanner.py @@ -59,6 +59,7 @@ def test_githubactions_find_requirements(): os.path.join(repo_root, ".github", "workflows", "checks.yml"), os.path.join(repo_root, ".github", "workflows", "docker-publish.yml"), os.path.join(repo_root, ".github", "workflows", "guarddog.yml"), + os.path.join(repo_root, ".github", "workflows", "nightly-evals.yml"), os.path.join(repo_root, ".github", "workflows", "pr.yml"), os.path.join(repo_root, ".github", "workflows", "pypi-publish.yml"), os.path.join(repo_root, ".github", "workflows", "scorecard.yml"),