Template-fit residuals + HF retry wrapper + pytest in CI

.claude/skills/create-eval/SKILL.md

@@ -309,7 +309,7 @@ Implement in dependency order: dataset -> prompts -> scorer -> solver -> @task.
 
 ### 4.1 Dataset Loading
 
-- Use `hf_dataset()` from `inspect_ai.dataset` for HuggingFace datasets
+- Use `hf_dataset()` from `utils.huggingface` for HuggingFace datasets — it wraps `inspect_ai.dataset.hf_dataset` with retry-on-rate-limit/transient-error and requires a `revision=` for reproducibility
 - Write a `record_to_sample` function that maps raw records to `Sample` objects
 - Include all useful fields in `sample.metadata` for debugging
 - Use stable, canonical IDs for `sample.id`

.claude/skills/investigate-dataset/SKILL.md

@@ -95,6 +95,6 @@ uv run python -c "from datasets import load_dataset; print(load_dataset('org/nam
 HuggingFace caches at `~/.cache/huggingface/datasets/`. To force re-download set `HF_DATASETS_OFFLINE=0` and pass `download_mode="force_redownload"` to `load_dataset`, or simply delete the cache directory.
 
 - **Gated dataset**: Run `huggingface-cli login` or set `HF_TOKEN`
-- **Rate limited**: The `hf_dataset` wrapper in `inspect_evals.utils.huggingface` has built-in retry with backoff
+- **Rate limited**: use the `hf_dataset` wrapper in `src/utils/huggingface.py` — it retries with exponential backoff on 429s and 502s. Authenticating with `HF_TOKEN` also raises HuggingFace's per-IP rate limit.
 - **Large dataset**: Use `streaming=True` or `split="train[:1000]"` for sampling
 - **Missing revision**: Check the dataset's "Files and versions" tab on HuggingFace

.github/workflows/checks.yml

@@ -21,6 +21,7 @@ jobs:
     outputs:
       enforce_ruff: ${{ steps.load.outputs.enforce_ruff }}
       enforce_mypy: ${{ steps.load.outputs.enforce_mypy }}
+      enforce_tests: ${{ steps.load.outputs.enforce_tests }}
       enforce_posix_check: ${{ steps.load.outputs.enforce_posix_check }}
       enforce_unlisted_evals: ${{ steps.load.outputs.enforce_unlisted_evals }}
       enforce_package: ${{ steps.load.outputs.enforce_package }}
@@ -80,6 +81,29 @@ jobs:
       - name: Run mypy
         run: uv run mypy --version && uv run mypy src tests
 
+  tests:
+    runs-on: ubuntu-latest
+    needs: config
+    continue-on-error: ${{ needs.config.outputs.enforce_tests == 'false' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    env:
+      UV_PYTHON: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v7
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: uv sync --frozen
+      - name: Run pytest
+        # Skip slow tests by default (use --runslow on a manual dispatch
+        # if you want them in CI). GIT_LFS_SKIP_SMUDGE matches make test.
+        run: GIT_LFS_SKIP_SMUDGE=1 uv run pytest -q
+
   posix-code-check:
     runs-on: ubuntu-latest
     needs: config

Makefile

@@ -1,8 +1,4 @@
 # MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md
-PYTHON_MINOR := $(shell uv run python --version 2>&1 | sed 's/Python 3\.\([0-9]*\).*/\1/')
-TEST_GROUP_PY311 := test_py311_or_lower
-TEST_GROUP_PY312 := test_py312_or_higher
-DEFAULT_TEST_GROUP := $(shell [ "$(PYTHON_MINOR)" -le "11" ] && echo "$(TEST_GROUP_PY311)" || echo "$(TEST_GROUP_PY312)")
 
 hooks:
 	uv run pre-commit install
@@ -11,15 +7,7 @@ check:
 	@bash tools/run_checks.sh
 
 TEST_ARGS ?=
-TEST_EXTRAS ?= test
-TEST_GROUPS ?= $(DEFAULT_TEST_GROUP)
 test:
-	@echo "PYTHON_MINOR=$(PYTHON_MINOR)"
-	@echo "TEST_GROUPS=$(TEST_GROUPS)"
-	@echo "TEST_EXTRAS=$(TEST_EXTRAS)"
-	GIT_LFS_SKIP_SMUDGE=1 uv run \
-		$(addprefix --extra ,$(TEST_EXTRAS)) \
-		$(addprefix --group ,$(TEST_GROUPS)) \
-		pytest $(TEST_ARGS)
+	GIT_LFS_SKIP_SMUDGE=1 uv run pytest $(TEST_ARGS)
 
 .PHONY: hooks check test

README.md

@@ -247,6 +247,7 @@ All toggles live in [`tools/enforcement.config`](tools/enforcement.config) — e
 
 - `ENFORCE_RUFF` — Ruff format + lint
 - `ENFORCE_MYPY` — Mypy static types
+- `ENFORCE_TESTS` — `pytest` against `tests/` (skips `@pytest.mark.slow` by default; use `--runslow` to include)
 - `ENFORCE_POSIX_CHECK` — POSIX-only Python idioms (`tools/check_posix_code.py`)
 - `ENFORCE_UV_LOCK` — `uv.lock` in sync with `pyproject.toml`
 - `ENFORCE_UNLISTED_EVALS` — eval directories must be registered in

pyproject.toml

@@ -113,6 +113,14 @@ ignore_missing_imports = true
 module = ["add_readme_section", "generate_readmes"]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+# transformers / sentence_transformers are lazy-imported inside the
+# utils.huggingface wrappers. They aren't a runtime requirement of the
+# template, only of evals that opt to use them, so let mypy treat them
+# as untyped rather than requiring the stub to be installed everywhere.
+module = ["transformers", "transformers.*", "sentence_transformers", "sentence_transformers.*"]
+ignore_missing_imports = true
+
 [project]
 name = "inspect-eval-template"
 description = "Template for Inspect AI evaluations"
@@ -134,6 +142,13 @@ classifiers = [
 dependencies = [
     "inspect_ai>=0.3.158",
     "pyyaml>=5.1.0",
+    # Powers src/utils/huggingface.py (retry-aware HF wrappers). If your
+    # eval doesn't touch HuggingFace and you want a leaner environment,
+    # remove these here and delete the wrapper section in
+    # src/utils/huggingface.py.
+    "backoff>=2.0",
+    "datasets>=2.14",
+    "huggingface_hub>=0.20",
     # USER: Add your eval's dependencies here
 ]
 
@@ -165,6 +180,7 @@ dev = [
     "ruff==0.13.2",
     "toml",
     "types-PyYAML",
+    "types-requests",
 ]
 
 [tool.check-wheel-contents]