From a71d4345dfad56b4651cabb868da71e2e5261326 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 1 Jun 2026 19:46:36 +0000 Subject: [PATCH] chore: sync managed files from inspect_evals --- .claude/skills/create-eval/SKILL.md | 17 +- .claude/skills/ensure-test-coverage/SKILL.md | 4 +- .claude/skills/eval-report-workflow/SKILL.md | 62 +- .claude/skills/eval-validity-review/SKILL.md | 2 +- .../prepare-submission-workflow/SKILL.md | 60 +- .github/PULL_REQUEST_TEMPLATE.md | 23 + .github/actions/claude-setup/action.yaml | 14 +- .github/workflows/checks.yml | 215 +++++ .github/workflows/claude-review.yaml | 214 +++-- .github/workflows/markdown-lint.yml | 20 +- .github/workflows/pr-template-check.yml | 12 +- .pre-commit-config.yaml | 26 +- .upstream-sync-sha | 2 +- AGENTS.md | 33 +- BEST_PRACTICES.md | 32 + CONTRIBUTING.md | 210 +++-- EVALUATION_CHECKLIST.md | 46 +- Makefile | 20 +- .../inspect_scout/analyze_validity.py | 1 + .../inspect_scout/extract_results.py | 1 + .../inspect_scout/run_all_scanners.py | 1 + .../inspect_scout/scanners.py | 1 + .../inspect_scout/utils.py | 1 + tests/test_generate_readmes.py | 805 ++++++++++++++++++ tools/check_unlisted_evals.py | 88 ++ tools/generate_readmes.py | 498 +++++++++++ 26 files changed, 2223 insertions(+), 185 deletions(-) diff --git a/.claude/skills/create-eval/SKILL.md b/.claude/skills/create-eval/SKILL.md index bf48e8a..e7b29a9 100644 --- a/.claude/skills/create-eval/SKILL.md +++ b/.claude/skills/create-eval/SKILL.md @@ -1,17 +1,18 @@ --- name: create-eval -description: Implement a new evaluation from an issue or paper. Guides through requirements gathering, architecture design, scaffolding, implementation, testing, and quality checks. Use when user asks to create/implement/build a new evaluation. +description: Redirect to the inspect-evals-template for creating new evaluations. New evals are no longer created in this repository — they live in standalone repos. Use when user asks to create/implement/build a new evaluation. --- # Create Evaluation -Implement a new evaluation from an issue, paper, or benchmark specification. This is a phased workflow with human checkpoints between major phases. You should usually do human checkpoints. If the user requests that you do not do this, or you are in a fully autonomous workflow that says that there is no user to check with, you should ignore the human checkpoints. If you check in with the user several times and the user always replies with the same generic message like "Please continue the task", this indicates you are in an autonomous workflow. +**New evaluations are no longer created in the `inspect_evals` repository.** Since May 2026, evaluations live in their own standalone repositories. This repo now serves as a register that points to upstream eval repos. -## Expected Arguments +This skill has been moved to **inspect-evals-template**: . If a user triggers this skill, inform them to go to the template to run or access the skill there. -When invoked, this skill expects: +If a user asks to create an eval, inform them that Inspect Evals no longer accepts code submissions for new evals and suggest the [inspect-evals-template](https://github.com/Generality-Labs/inspect-evals-template) for guidance on creating evals. +<<<<<<< /tmp/sync_out - **issue** (optional): A GitHub issue number or URL (e.g., `#42`, `https://github.com///issues/42`) - **eval_name** (optional): The name for the evaluation directory (e.g., `frontier_cs`) - **paper_url** (optional): URL to the paper (arXiv, etc.) @@ -535,3 +536,11 @@ If the evaluation takes longer than five minutes to run and you can identify wha - **Don't run `uv update`**: Only `uv lock` if dependencies change - **Don't hardcode magic numbers**: Extract to named constants if used 3+ times or unclear - **Don't write try/catch unless truly necessary**: Let errors crash early +======= +## Quick reference + +| Step | Where | +| ------------------------- | ----------------------------------------------------------------------------------- | +| Create a new eval | [inspect-evals-template](https://github.com/Generality-Labs/inspect-evals-template) | +| Register a completed eval | This repo — use the **Prepare Evaluation For Submission** skill | +>>>>>>> /tmp/sync_theirs diff --git a/.claude/skills/ensure-test-coverage/SKILL.md b/.claude/skills/ensure-test-coverage/SKILL.md index b3ae8d5..947814c 100644 --- a/.claude/skills/ensure-test-coverage/SKILL.md +++ b/.claude/skills/ensure-test-coverage/SKILL.md @@ -207,7 +207,7 @@ Autolint checks are shallow (function name present in test files). Coverage meas **Solver tests** - similar quality gap to scorers: -- Thin wrappers (just assembling `chain([system_message(...), generate()])` or wrapping `basic_agent`/`react()`) only need isinstance. If already exercised by E2E tests, the isinstance test can be omitted entirely. +- Thin wrappers (just assembling `chain([system_message(...), generate()])` or wrapping `react()`) only need isinstance. If already exercised by E2E tests, the isinstance test can be omitted entirely. - Non-trivial solvers with branching logic (e.g., dispatching based on metadata, constructing different prompts, modifying state.tools) must test the actual branches — create a TaskState, run the solver with a mock generate, and assert state changes. - If a solver is already exercised by an E2E test that covers its logic paths, a standalone isinstance test is redundant and should be removed. - Solvers that use `sandbox().exec()` should be tested with mocked sandbox, same pattern as sandbox-based scorers. @@ -329,7 +329,7 @@ Show the user a summary: ## Things NOT to Do - **Don't re-check what autolint checks**: Run autolint and report its results. Don't manually verify test existence, `__init__.py`, function name presence, etc. -- **Don't over-test thin wrappers**: `isinstance(solver, Solver)` is sufficient for solvers that just wrap `basic_agent` or assemble Inspect built-ins. Reviewers explicitly reject over-testing these (PR #1009, #1008). +- **Don't over-test thin wrappers**: `isinstance(solver, Solver)` (or `isinstance(agent, Agent)`) is sufficient for solvers that just wrap `react()` or assemble Inspect built-ins. Reviewers explicitly reject over-testing these (PR #1009, #1008). - **Don't treat non-trivial components as wrappers**: Any solvers/scorers/tools with custom logic beyond wrapping Inspect built-ins do require testing for those portions. See the expanded scorer quality checklist in Phase 2.3 for specific guidance on sandbox-based, LLM-graded, dispatch, and reducer scorers. - **Don't use bare strings for TaskState model arg**: Always use `ModelName("mockllm/model")`, never `model="mockllm/model"` — the latter causes mypy `arg-type` errors. - **Don't access score attributes without None checks**: Always `assert score is not None` before `score.value`, and `assert score.explanation is not None` before `"text" in score.explanation`. Otherwise mypy reports `union-attr` errors. diff --git a/.claude/skills/eval-report-workflow/SKILL.md b/.claude/skills/eval-report-workflow/SKILL.md index cbda259..bd97ecd 100644 --- a/.claude/skills/eval-report-workflow/SKILL.md +++ b/.claude/skills/eval-report-workflow/SKILL.md @@ -6,24 +6,43 @@ description: Create an evaluation report for a README by selecting models, estim # Make an Evaluation Report +This workflow drives [`tools/evaluation_report.py`](../../../tools/evaluation_report.py), which reads a per-eval `report_config.yaml` and produces a full reproducible `report.md` (results table, reference comparison, per-category breakdowns, token totals, approximate cost) plus header-only JSON copies of the input logs under `results/`. The `report_config.yaml`, regenerated `report.md`, and `results/` folder are committed alongside the eval's `eval.yaml`. + ## Report Formatting -The evaluation report included in the README.md should look something like this. It should run on the entire dataset or $5 of compute per model, whichever is cheaper. Use the token count from smaller runs to make this prediction. +The evaluation report included in the README.md is the rendered `report.md` produced by [`tools/evaluation_report.py`](../../../tools/evaluation_report.py). It should run on the entire dataset or $5 of compute per model, whichever is cheaper. Use the token count from smaller runs to make this prediction. + +A typical rendered report looks like this: + +```markdown +# Evaluation Report + +## Implementation Details + +Brief description of any deviations from the paper, known limitations, etc. + +## Results -================================================== +| Model | Inspect (accuracy) | Reference | Δ | Samples | Stderr | Time | +| ------------- | ------------------ | --------- | ------ | ------- | ------ | ---- | +| openai/... | 0.600 | 0.580 | +0.020 | 100/100 | 0.049 | 18s | +| anthropic/... | 0.400 | 0.420 | -0.020 | 100/100 | 0.049 | 6s | -| Model | Provider | Accuracy | Stderr | Time | -| ----- | -------- | -------- | ------ | ---- | -| model_x | OpenAI | 0.600 | 0.245 | 18s | -| model_y | Anthropic | 0.400 | 0.245 | 6s | -| model_z | Google | 0.600 | 0.245 | 8s | +_Reference: Paper, Table 3_ -**Notes:** +## Reproducibility Information -- All three providers successfully completed the evaluation -- Human expert baseline from the paper is 69.7% accuracy +- Samples: 100 / 100 per model +- Run dates: 2026-04-29 +- Versions: inspect_ai=0.3.x, inspect_evals=0.x +- Models: ... +- Total tokens: 1,234,567 +- Approximate cost: $0.42 USD (prices as of 2026-04) -================================================== +Reproduction commands: ... +``` + +**Register entries:** for register entries (`register//eval.yaml`), populate the optional `evaluation_report` block in `eval.yaml` instead of editing `README.md` directly — the README is regenerated from the YAML by `make check`. The block accepts `timestamp`, a `results` list (with `model`, `accuracy`, and optionally `provider`, `stderr`, `time`, `date`), and `notes`. Extra fields at either level are allowed for eval-specific metric columns. See `register/README.md` for the schema. If the eval.yaml file includes an arXiv paper, check that paper for the models used and human baselines. Include the human baseline in the notes section if it is present. If you can, select three models that would be suitable to check if this evaluation successfully replicates the original paper, including at least two different model providers. @@ -39,7 +58,7 @@ If you cannot do this, or if there aren't three suitable candidates, fill the re e. Whenever you create a .md file as part of this workflow, assume it is made in `agent_artefacts//evalreport`. f. Copy EVALUATION_CHECKLIST.md to the folder. g. Create a NOTES.md file for miscellaneous helpful notes. Err on the side of taking lots of notes. Create an UNCERTAINTIES.md file to note any uncertainties. -2. Read the [Evaluation Report Guidelines](CONTRIBUTING.md#evaluation-report-guidelines). +2. Read the [Evaluation Report Guidelines](../../../EVALUATION_CHECKLIST.md#evaluation-report-guidelines). 3. Check to see if the README for the evaluation already has an evaluation report. If so, double-check with the user that they want it overwritten. 4. Read the main file in EVAL_NAME, which should be src/EVAL_NAME/EVAL_NAME.py in order to see how many tasks there are. 5. Perform an initial test with 'uv run inspect eval EVAL_NAME/ --model gpt-5.1-2025-11-13 --limit 5' to get estimated token counts. Use -T shuffle=True if possible to produce random samples - to see if it's possible, you'll need to check the evaluation itself. @@ -66,7 +85,22 @@ If you cannot do this, or if there aren't three suitable candidates, fill the re If the number of samples recommended by this process is less than 20, or less than (3 * meaningful subcategories), you should also inform the user that the number of samples achievable on the $5/model budget is too small for a meaningful evaluation, and that more resources are needed to test properly. A minimum of 20 samples are required for error testing. If they need help, they can ask the repository's maintainers for testing resources. +<<<<<<< /tmp/sync_out If the user asks you to run this for them, remind them that they won't be able to see the progress of the evaluation due to the way Inspect works, and asking if they're sure. Do not proactively offer to run the command for them. After the command has been run, use the Inspect log API (see the `/read-eval-logs` skill) to read the resulting `.eval` files and extract per-model accuracy, stderr, and timing for the report. If any problems arise, ask the user to give you this information manually. +======= + If the user asks you to run this for them, remind them that they won't be able to see the progress of the evaluation due to the way Inspect works, and asking if they're sure. Do not proactively offer to run the command for them. + +9. Once the eval has been run, create `src/inspect_evals//report_config.yaml` with the headline metric, any `reference_results` from the original paper or leaderboard, a `reference_source` citation, and `notes` describing implementation details and any deviations. The schema is defined by `tools.report_utils.ReportConfig`; see [tools/README.md](../../../tools/README.md#evaluation_reportpy) for the full set of fields. + +10. Run the report script, passing in the `.eval` files produced by the run: + + ```bash + uv run python tools/evaluation_report.py src/inspect_evals//report_config.yaml \ + --logs logs/file1.eval logs/file2.eval logs/file3.eval + ``` + + The script writes `src/inspect_evals//report.md` and a header-only JSON copy of each input log under `src/inspect_evals//results//_.json` (the machine-readable companion). If any problems arise, ask the user to give you the relevant information manually. +>>>>>>> /tmp/sync_theirs -9. Add the report to the README.md in the appropriate section, and then tell the user the task is done. Do not add human baseline or random baseline data unless they already appear in the README. -10. (Optional) If the evaluation uses an LLM judge and an oracle log exists or can be created (e.g., by running the eval with a stronger reference judge, or by collecting human labels via `--oracle-labels`), run `uv run python tools/judge_calibration_diagnostics.py --oracle-log ` to produce calibrated estimates with confidence intervals. Include findings in the evaluation report notes if relevant. See [tools/README.md](../../../tools/README.md) for details. +11. Splice the contents of `src/inspect_evals//report.md` into the README.md in the appropriate section, and commit `report_config.yaml`, `report.md`, and the `results/` folder alongside `eval.yaml`. Then tell the user the task is done. Do not add human baseline or random baseline data unless they already appear in the README. +12. (Optional) If the evaluation uses an LLM judge and an oracle log exists or can be created (e.g., by running the eval with a stronger reference judge, or by collecting human labels via `--oracle-labels`), run `uv run python tools/judge_calibration_diagnostics.py --oracle-log ` to produce calibrated estimates with confidence intervals. Include findings in the evaluation report notes if relevant. See [tools/README.md](../../../tools/README.md) for details. diff --git a/.claude/skills/eval-validity-review/SKILL.md b/.claude/skills/eval-validity-review/SKILL.md index 2b443f5..3c009d8 100644 --- a/.claude/skills/eval-validity-review/SKILL.md +++ b/.claude/skills/eval-validity-review/SKILL.md @@ -41,7 +41,7 @@ Read the main task file(s) in `src//`. Record: - All `@task` function names and their parameters - The task `version` -- What solver is used (simple solver like `multiple_choice()`, or an agent like `basic_agent()`) +- What solver is used (simple solver like `multiple_choice()`, or an agent like `react()`) - What scorer is used - What tools are provided to the agent (if any) - Whether a sandbox is configured (look for `sandbox=` in the Task constructor and `compose.yaml` files) diff --git a/.claude/skills/prepare-submission-workflow/SKILL.md b/.claude/skills/prepare-submission-workflow/SKILL.md index 75b3c23..1f1fe67 100644 --- a/.claude/skills/prepare-submission-workflow/SKILL.md +++ b/.claude/skills/prepare-submission-workflow/SKILL.md @@ -1,15 +1,22 @@ --- name: prepare-submission-workflow +<<<<<<< /tmp/sync_out description: Prepare an evaluation for PR submission — add dependencies, tests, linting, eval.yaml, and README. Use when user asks to prepare an eval for submission or finalize a PR. Trigger when the user asks you to run the "Prepare Evaluation For Submission" workflow. +======= +description: Prepare an evaluation for PR submission as an entry to the register. Use when user asks to prepare an eval for submission or finalize a PR. Trigger when the user asks you to run the "Prepare Evaluation For Submission" workflow. +>>>>>>> /tmp/sync_theirs --- # Prepare Eval For Submission +Since May 2026, new evaluations are submitted as **entries to the register** — the evaluation code lives in your own upstream repository, and you add a pointer to it here. Code is no longer added directly to `src/inspect_evals/`. If the user appears to be submitting evaluation code into the repo, direct them to [`register/README.md`](../../../register/README.md) for the full process. + ## Workflow Steps To prepare an evaluation for submission as a pull request: +<<<<<<< /tmp/sync_out 1. Add custom dependencies in the [pyproject.toml](../../../pyproject.toml) file, if required. Add the package to the top-level `[project] dependencies` list (or to a dependency group if you want to keep it optional). ```toml @@ -20,9 +27,17 @@ To prepare an evaluation for submission as a pull request: "example-python-package", # add your eval's runtime deps here ] ``` +======= +### 1. Verify upstream repo requirements + +The upstream repo must: +>>>>>>> /tmp/sync_theirs - Pin the package version if your evaluation might be sensitive to version updates. +- Have a `pyproject.toml` with a `[project]` table so it can be installed via `uv sync` +- Declare `inspect_ai` as a dependency +- Define each task with the `@task` decorator from `inspect_ai` +<<<<<<< /tmp/sync_out Make sure your module does not import your custom dependencies at module import time when those imports might fail. This can break the way `inspect_ai` discovers tasks via entry points. To check for this, uninstall your optional dependencies and run another eval with `--limit=0`. If it fails due to one of your imports, you need to defer that import (move it inside the function that uses it). @@ -61,3 +76,46 @@ To prepare an evaluation for submission as a pull request: failures before opening a PR. 7. Fill in the missing sections of the generated README.md which are marked with 'TODO:'. +======= +Ask the user whether their upstream repo meets these requirements. Offer to check for them — if they provide the GitHub repository URL, fetch the repo's `pyproject.toml` and task files (e.g. via `WebFetch` on the raw GitHub URLs) to verify the requirements are met. If any requirement is not met, tell the user what needs to be fixed upstream before they can register. + +### 2. Gather information and create `register//eval.yaml` + +Skip this step if `register//eval.yaml` already exists. + +Use [`register/example_eval.yaml`](../../../register/example_eval.yaml) as the template — it documents every field. Don't ask the user field-by-field; instead, derive what you can from the upstream repo first, then ask one batched question for what's missing. + +**Hints on what to derive from the upstream repo (don't ask):** + +- `source.repository_url` — from step 1. +- `source.repository_commit` — fetch the latest commit SHA on the default branch (must be a 40-char SHA, not a tag or branch). +- `tasks[].name` and `tasks[].task_path` — locate every `@task`-decorated function in the repo and record the function name and file path. +- `title` — from the upstream README heading or `pyproject.toml` `[project].name`. +- `description` — draft from the upstream README; keep to one short paragraph since the generated README links back upstream. +- `source.maintainers` — defaults to the repo owner; only override if the repo is org-owned and the real maintainers are individuals. +- `tags` — propose based on the eval's domain (e.g. `Coding`, `games`, `tools`). The upstream repo name is added automatically, so don't include it. + +Use [`register/example_eval.yaml`](../../../register/example_eval.yaml) to determine what additional questions are needed. + +Show the user the drafted YAML for confirmation before writing the file. Do **not** set `id` — it is auto-injected from the directory name. + +### 3. Run validation + +```bash +make check +``` + +This validates the `eval.yaml` and auto-generates a `README.md` next to it. The README is fully generated from `eval.yaml` — do not edit it by hand. + +Because the generated page defers to upstream for details, make sure the upstream repo's README covers the dataset, scorer, task parameters, and how the eval was validated. + +### 4. Create a changelog fragment + +```bash +uv run scriv create +``` + +### 5. Open a PR + +Use the [PR template](../../.github/PULL_REQUEST_TEMPLATE.md). The reviewer will ping anyone listed under `source.maintainers` for acknowledgement before merging. +>>>>>>> /tmp/sync_theirs diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 75a1571..91cf900 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,6 +5,7 @@ ## Checklist +<<<<<<< /tmp/sync_out - [ ] Are you adding a new eval? - [ ] If yes, please review the [Evaluation Checklist](../EVALUATION_CHECKLIST.md). @@ -13,3 +14,25 @@ - [ ] Does this change affect how future contributors write or submit evaluations (e.g. new required fields, changed tooling, updated conventions)? - [ ] If yes, has the relevant documentation been updated (e.g. [CONTRIBUTING.md](../CONTRIBUTING.md), [EVALUATION_CHECKLIST.md](../EVALUATION_CHECKLIST.md), [AGENTS.md](../AGENTS.md))? +======= + + +- [ ] Are you listing new eval(s) in the register? If yes run the `prepare-submission-workflow` or manually check: + - [ ] Is your implementation compatible with our eval checking process by meeting these [requirements](https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/register#upstream-repo-requirements)? + - [ ] Does this PR contain a new directory in `/register` for each new eval being listed, each containing an `eval.yaml` and auto-generated `README.md`? See our [submission guide](https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/register#adding-an-entry) for further details. + +- [ ] Does this change affect existing eval(s)? If yes: + - [ ] Have the affected task version(s) been incremented? See [when to bump the task version](https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/TASK_VERSIONING.md#when-to-bump-the-task-version). + - [ ] Have the affected task changelog(s) been updated? [Example](https://github.com/UKGovernmentBEIS/inspect_evals/pull/1053). + +- [ ] Is this change consequential to users? If yes: + - [ ] Has `uv run scriv create` been run and the changelog fragment committed? See [Fragment Format](https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/PACKAGE_VERSIONING.md#fragment-format). + +- [ ] Does this change affect how future contributors write or submit evaluations (e.g. new required fields, changed tooling, updated conventions)? If yes: + - [ ] Has the relevant documentation been updated (e.g. [CONTRIBUTING.md](https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/CONTRIBUTING.md), [EVALUATION_CHECKLIST.md](https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/EVALUATION_CHECKLIST.md), [AGENTS.md](https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/AGENTS.md))? +>>>>>>> /tmp/sync_theirs diff --git a/.github/actions/claude-setup/action.yaml b/.github/actions/claude-setup/action.yaml index 9dbeb44..6ed9053 100644 --- a/.github/actions/claude-setup/action.yaml +++ b/.github/actions/claude-setup/action.yaml @@ -30,25 +30,29 @@ runs: using: composite steps: - if: ${{ inputs.skip_checkout != 'true' }} - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: ref: ${{ inputs.checkout_ref }} fetch-depth: 0 + # Calling workflows are responsible for git pushes and use a + # separate App token, so this checkout never needs to leave the + # auto-minted GITHUB_TOKEN persisted in .git/config. + persist-credentials: false - if: ${{ inputs.install_python_deps == 'true' }} - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: "3.12" - if: ${{ inputs.install_python_deps == 'true' }} - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - if: ${{ inputs.install_python_deps == 'true' }} name: Install dependencies shell: bash - run: uv sync --frozen --extra test + run: uv sync --frozen --group dev --extra test - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 with: role-to-assume: ${{ inputs.role_arn }} aws-region: eu-west-2 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 14f0b3d..804adeb 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -11,12 +11,119 @@ on: branches: - main +<<<<<<< /tmp/sync_out # Each check job's `continue-on-error` is keyed off `tools/enforcement.config` # so enforcement is configurable per check. See README.md "Checks and # enforcement" for the full toggle list and defaults. jobs: config: +======= +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +# SECURITY: All jobs only read the repository (lint, type-check, test). +# zizmor: excessive-permissions (https://docs.zizmor.sh/audits/#excessive-permissions) +permissions: + contents: read +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + run-readme-check: + name: README check + runs-on: ubuntu-latest + env: + UV_PYTHON: "3.12" + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" # pinning python version because gensim currently doesn't have wheels for 3.13 + + - name: Cache uv downloads + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.cache/uv + key: ${{ runner.os }}-uv-3.12-${{ hashFiles('**/uv.lock', '**/pyproject.toml') + }} + restore-keys: | + ${{ runner.os }}-uv-3.12- + ${{ runner.os }}-uv- + + - name: Install dependencies + run: | + uv sync --frozen --group dev --group test_py312_or_higher --extra test + + - name: Report large files in repository + run: | + echo "## Files > 10MB" + large=$(bash tools/list_large_files.sh 10) + if [[ -n "$large" ]]; then + echo "$large" + echo "::warning::Large files >10MB found in repo (migration candidates):%0A$(echo "$large" | tr '\n' '%0A')" + else + echo "None" + fi + echo "" + echo "## Files > 1MB" + medium=$(bash tools/list_large_files.sh 1) + if [[ -n "$medium" ]]; then + echo "$medium" + echo "::notice::Files >1MB in repo (for context):%0A$(echo "$medium" | tr '\n' '%0A')" + else + echo "None" + fi + + - name: Check for unlisted evals + run: uv run python tools/check_unlisted_evals.py + + - name: Check changelog entries + run: uv run python tools/check_changelog.py + + - name: Run generate_readmes.py to generate docs + run: | + uv run python tools/generate_readmes.py + + - name: Regenerate ASSETS.yaml + run: uv run python tools/generate_asset_manifest.py + + - name: Check for undeclared external assets + run: | + # Without --strict the tool exits 0 even when findings are present, so this + # step never fails on findings. || true is intentionally absent so that crashes + # (import errors, syntax errors) surface as CI failures rather than silent warnings. + # To enforce findings as failures, add --strict once the false-positive rate is low enough. + output=$(uv run python tools/check_undeclared_assets.py 2>&1) + echo "$output" + if echo "$output" | grep -q "potentially undeclared"; then + echo "::warning::Some evals may have undeclared external assets. Run 'python tools/check_undeclared_assets.py' locally for details." + fi + + - name: Check for uncommitted changes + run: | + if [[ -n $(git status --porcelain) ]]; then + echo "Generated files are out of date. Run the generation tools locally and commit changes:" + echo " python tools/generate_readmes.py" + echo " python tools/generate_asset_manifest.py" + git --no-pager diff + exit 1 + else + echo "Generated files are up to date." + fi + + posix-code-check: + name: POSIX code check +>>>>>>> /tmp/sync_theirs runs-on: ubuntu-latest outputs: enforce_ruff: ${{ steps.load.outputs.enforce_ruff }} @@ -28,6 +135,7 @@ jobs: enforce_generated_docs: ${{ steps.load.outputs.enforce_generated_docs }} enforce_large_files: ${{ steps.load.outputs.enforce_large_files }} steps: +<<<<<<< /tmp/sync_out - uses: actions/checkout@v5 - id: load run: | @@ -41,16 +149,64 @@ jobs: ruff: needs: config +======= + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + + - name: Cache uv downloads + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: | + ~/.cache/uv + key: ${{ runner.os }}-uv-3.11-${{ hashFiles('**/uv.lock', '**/pyproject.toml') + }} + restore-keys: | + ${{ runner.os }}-uv-3.11- + ${{ runner.os }}-uv- + + - name: Install dependencies + run: | + uv sync --frozen --group dev --group test_py311_or_lower --extra test + + - name: Check for POSIX-specific code + run: | + uv run python tools/check_posix_code.py ./**/*.py + + ruff: + name: Ruff +>>>>>>> /tmp/sync_theirs runs-on: ubuntu-latest continue-on-error: ${{ needs.config.outputs.enforce_ruff == 'false' }} strategy: matrix: python-version: ["3.11", "3.12", "3.13"] steps: +<<<<<<< /tmp/sync_out - uses: actions/checkout@v5 - uses: astral-sh/setup-uv@v7 - name: Display Ruff version uses: astral-sh/ruff-action@v3 +======= + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Display Ruff version + uses: astral-sh/ruff-action@0ce1b0bf8b818ef400413f810f8a11cdbda0034b # v4.0.0 + # Installs ruff for use in later steps +>>>>>>> /tmp/sync_theirs with: version: "0.15.11" args: --version @@ -60,6 +216,7 @@ jobs: run: uv run ruff format --check mypy: + name: Mypy runs-on: ubuntu-latest needs: config continue-on-error: ${{ needs.config.outputs.enforce_mypy == 'false' }} @@ -70,9 +227,21 @@ jobs: env: UV_PYTHON: ${{ matrix.python-version }} steps: +<<<<<<< /tmp/sync_out - uses: actions/checkout@v5 - uses: astral-sh/setup-uv@v7 - uses: actions/setup-python@v6 +======= + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 +>>>>>>> /tmp/sync_theirs with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -83,6 +252,7 @@ jobs: - name: Run mypy run: uv run mypy --version && uv run mypy src tests +<<<<<<< /tmp/sync_out posix-code-check: runs-on: ubuntu-latest needs: config @@ -91,6 +261,10 @@ jobs: - uses: actions/checkout@v5 - uses: astral-sh/setup-uv@v7 - uses: actions/setup-python@v6 +======= + - name: Cache uv downloads + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 +>>>>>>> /tmp/sync_theirs with: python-version: "3.12" - name: Install dependencies @@ -98,6 +272,7 @@ jobs: - name: Check for POSIX-specific code run: uv run python tools/check_posix_code.py $(git ls-files '*.py') +<<<<<<< /tmp/sync_out unlisted-evals: runs-on: ubuntu-latest needs: config @@ -119,6 +294,17 @@ jobs: steps: - uses: actions/checkout@v5 - uses: hynek/build-and-inspect-python-package@v2 +======= + - name: Install dependencies for python < 3.12 + if: ${{ matrix.python-version < '3.12' }} + run: | + uv sync --frozen --group dev --group test_py311_or_lower --extra test + + - name: Install dependencies for python >= 3.12 + if: ${{ matrix.python-version >= '3.12' }} + run: | + uv sync --frozen --group dev --group test_py312_or_higher --extra test +>>>>>>> /tmp/sync_theirs autolint: name: Autolint (inspect_evals standards) @@ -169,6 +355,7 @@ jobs: needs: config continue-on-error: ${{ needs.config.outputs.enforce_large_files != 'true' }} steps: +<<<<<<< /tmp/sync_out - uses: actions/checkout@v5 - name: Report and gate large files run: | @@ -188,3 +375,31 @@ jobs: else echo "None" fi +======= + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: hynek/build-and-inspect-python-package@d44ca7d91762de7a7d5436ddae667c6da6d1c3df # v2.18.0 + + workflow-scripts-tests: + name: Workflow scripts pytest + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install dependencies + run: uv sync --frozen --group dev --group test_py312_or_higher --extra test + + - name: Run pytest + run: uv run pytest .github/scripts/register_submission/tests/ +>>>>>>> /tmp/sync_theirs diff --git a/.github/workflows/claude-review.yaml b/.github/workflows/claude-review.yaml index 2661fb8..fd48c35 100644 --- a/.github/workflows/claude-review.yaml +++ b/.github/workflows/claude-review.yaml @@ -12,28 +12,77 @@ on: required: true type: string +# Per-PR concurrency. New comments / synchronize events on the same PR +# cancel the prior run so reviewers always see a single up-to-date +# comment thread rather than queued runs racing each other. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number || github.run_id }} + cancel-in-progress: true + permissions: - id-token: write # Required for OIDC contents: read - pull-requests: write # Required for making a PR comment - actions: read # Required for listing workflow runs jobs: review: + name: Claude code review runs-on: ubuntu-latest timeout-minutes: 15 + permissions: + contents: read + actions: read # list prior workflow runs for self-imposed rate limiting + id-token: write # AWS OIDC for Claude on Bedrock via aws-actions/configure-aws-credentials + pull-requests: write # post review comments and minimize previous reviews + checks: read # read check run status for PR context + statuses: read # read commit statuses alongside check runs for gh pr checks env: PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.pull_request.number || github.event.issue.number }} steps: - name: Validate trigger id: validate - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + AUTO_REVIEW_ALL_COMMITS: ${{ vars.AUTO_REVIEW_ALL_COMMITS }} with: script: | const eventName = context.eventName; + // Skip pure register-submission PRs — the register-submission + // workflow owns those. Mixed PRs (schema/infra changes that + // also touch register//eval.yaml) flow through both: + // register-submission handles the register diff; this workflow + // reviews everything else. The path allow-list mirrors the + // register submission convention (eval.yaml + auto-generated + // README.md siblings + top-level README + changelog fragment). + const registerYaml = /^register\/[^/]+\/eval\.yaml$/; + const registerOnlyPath = (filename) => + registerYaml.test(filename) || + /^register\/[^/]+\/README\.md$/.test(filename) || + filename === 'README.md' || + /^changelog\.d\/.+\.md$/.test(filename); + let registerPrNumber = null; + if (eventName === 'pull_request_target') { + registerPrNumber = context.payload.pull_request.number; + } else if (eventName === 'issue_comment' && context.payload.issue.pull_request) { + registerPrNumber = context.payload.issue.number; + } + if (registerPrNumber !== null) { + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: registerPrNumber, + per_page: 100, + }); + const touchesRegisterYaml = files.some(f => registerYaml.test(f.filename)); + const allRegisterScoped = files.length > 0 && files.every(f => registerOnlyPath(f.filename)); + if (touchesRegisterYaml && allRegisterScoped) { + core.info('PR is purely a register submission — handled by register-submission workflow'); + core.setOutput('skip', 'true'); + return; + } + } + if (eventName === 'pull_request_target') { - const autoReview = '${{ vars.AUTO_REVIEW_ALL_COMMITS }}'; + const autoReview = process.env.AUTO_REVIEW_ALL_COMMITS; if (autoReview !== 'true') { core.info('AUTO_REVIEW_ALL_COMMITS is not enabled — skipping automatic review (use /review to trigger manually)'); core.setOutput('skip', 'true'); @@ -79,19 +128,20 @@ jobs: - name: Checkout code if: steps.validate.outputs.skip != 'true' - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ format('refs/pull/{0}/head', env.PR_NUMBER) }} fetch-depth: 0 # Fetch all history so merge-base can be found + persist-credentials: false - name: Get PR Info if: steps.validate.outputs.skip != 'true' id: pr-info run: | - BASE_REF=$(gh pr view ${{ env.PR_NUMBER }} --json baseRefName -q .baseRefName) - PR_AUTHOR=$(gh pr view ${{ env.PR_NUMBER }} --json author -q .author.login) - echo "BASE_REF=$BASE_REF" >> $GITHUB_ENV - echo "PR_AUTHOR=$PR_AUTHOR" >> $GITHUB_ENV + BASE_REF=$(gh pr view "$PR_NUMBER" --json baseRefName -q .baseRefName) + PR_AUTHOR=$(gh pr view "$PR_NUMBER" --json author -q .author.login) + echo "base_ref=$BASE_REF" >> "$GITHUB_OUTPUT" + echo "pr_author=$PR_AUTHOR" >> "$GITHUB_OUTPUT" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -101,49 +151,69 @@ jobs: if: steps.validate.outputs.skip != 'true' run: | for f in AGENTS.md EVALUATION_CHECKLIST.md BEST_PRACTICES.md \ - agent_artefacts/repo_context/REPO_CONTEXT.md \ - .github/actions/claude-setup/action.yaml; do - if git show origin/${{ github.ref_name }}:$f > /dev/null 2>&1; then + agent_artefacts/repo_context/REPO_CONTEXT.md; do + if git show "origin/${GITHUB_REF_NAME}:$f" > /dev/null 2>&1; then mkdir -p "$(dirname "$f")" - git show origin/${{ github.ref_name }}:$f > $f + git show "origin/${GITHUB_REF_NAME}:$f" > "$f" fi done + # The composite action at .github/actions/claude-setup/action.yaml is + # resolved against the workspace, which is the PR-checked-out tree. + # Restore it from the workflow branch as defence-in-depth, so the + # AWS-OIDC-configuring action.yaml always comes from trusted source. + # Fail-closed: any failure here aborts before Claude runs (unlike + # the optional-restore loop above, which tolerates missing files + # for the documentation set). See #1687. + - name: Restore trusted claude-setup action + if: steps.validate.outputs.skip != 'true' + run: | + set -euo pipefail + mkdir -p .github/actions/claude-setup + git show "origin/${GITHUB_REF_NAME}:.github/actions/claude-setup/action.yaml" \ + > .github/actions/claude-setup/action.yaml + - name: Determine rate-limit actor if: >- steps.validate.outputs.skip != 'true' && github.event_name != 'workflow_dispatch' id: rate-limit-actor + env: + EVENT_NAME: ${{ github.event_name }} + PR_AUTHOR: ${{ steps.pr-info.outputs.pr_author }} run: | - # For pull_request_target events, the actor is the PR author. - # For issue_comment (/review), the actor is the commenter. - if [ "${{ github.event_name }}" = "issue_comment" ]; then - echo "actor=${{ github.actor }}" >> $GITHUB_OUTPUT + set -euo pipefail + if [ "$EVENT_NAME" = "issue_comment" ]; then + # For issue_comment (/review), the actor is the commenter. + echo "actor=$GITHUB_ACTOR" >> "$GITHUB_OUTPUT" else - echo "actor=$PR_AUTHOR" >> $GITHUB_OUTPUT + # For pull_request_target events, the actor is the PR author. + echo "actor=$PR_AUTHOR" >> "$GITHUB_OUTPUT" fi - name: Check if actor should be rate-limited if: steps.rate-limit-actor.outputs.actor != '' id: should-rate-limit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACTOR: ${{ steps.rate-limit-actor.outputs.actor }} run: | - ACTOR="${{ steps.rate-limit-actor.outputs.actor }}" PERM=$(gh api "repos/${GITHUB_REPOSITORY}/collaborators/${ACTOR}/permission" \ --jq '.permission' 2>/dev/null || echo "none") echo "Actor $ACTOR has permission: $PERM" if [[ "$PERM" == "admin" || "$PERM" == "write" ]]; then - echo "should_rate_limit=false" >> $GITHUB_OUTPUT + echo "should_rate_limit=false" >> "$GITHUB_OUTPUT" else - echo "should_rate_limit=true" >> $GITHUB_OUTPUT + echo "should_rate_limit=true" >> "$GITHUB_OUTPUT" fi - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Check rate limit if: steps.should-rate-limit.outputs.should_rate_limit == 'true' id: rate-limit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACTOR: ${{ steps.rate-limit-actor.outputs.actor }} run: | - ACTOR="${{ steps.rate-limit-actor.outputs.actor }}" ONE_HOUR_AGO=$(date -u -d '1 hour ago' +%Y-%m-%dT%H:%M:%SZ) # Get recent run IDs and creation times for this actor (sorted ascending). @@ -193,22 +263,23 @@ jobs: PIVOT_INDEX=$((FULL_REVIEW_COUNT - 6)) PIVOT_CREATED="${FULL_REVIEW_CREATED_ATS[$PIVOT_INDEX]}" RETRY_AFTER=$(date -u -d "$PIVOT_CREATED + 1 hour" +%H:%M) - echo "rate_limited=true" >> $GITHUB_OUTPUT - echo "retry_after=$RETRY_AFTER" >> $GITHUB_OUTPUT + echo "rate_limited=true" >> "$GITHUB_OUTPUT" + echo "retry_after=$RETRY_AFTER" >> "$GITHUB_OUTPUT" else - echo "rate_limited=false" >> $GITHUB_OUTPUT + echo "rate_limited=false" >> "$GITHUB_OUTPUT" fi - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Handle rate limit exceeded if: steps.rate-limit.outputs.rate_limited == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RETRY_AFTER: ${{ steps.rate-limit.outputs.retry_after }} + ACTOR: ${{ steps.rate-limit-actor.outputs.actor }} with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const prNumber = Number(process.env.PR_NUMBER); - const retryAfter = '${{ steps.rate-limit.outputs.retry_after }}'; + const retryAfter = process.env.RETRY_AFTER; const marker = ''; // Minimize previous rate-limit comments @@ -235,7 +306,7 @@ jobs: } // Post new rate-limit comment - const actor = '${{ steps.rate-limit-actor.outputs.actor }}'; + const actor = process.env.ACTOR; await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, @@ -248,7 +319,7 @@ jobs: if: >- steps.validate.outputs.skip != 'true' && steps.rate-limit.outputs.rate_limited != 'true' - run: echo "proceed=true" >> $GITHUB_OUTPUT + run: echo "proceed=true" >> "$GITHUB_OUTPUT" - name: Setup if: steps.gate.outputs.proceed == 'true' @@ -261,29 +332,57 @@ jobs: - name: Precompute PR diff and metadata if: steps.gate.outputs.proceed == 'true' + env: + BASE_REF: ${{ steps.pr-info.outputs.base_ref }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - MERGE_BASE=$(git merge-base origin/$BASE_REF HEAD) + MERGE_BASE=$(git merge-base "origin/$BASE_REF" HEAD) echo "Merge base: $MERGE_BASE" git diff --name-only "$MERGE_BASE" HEAD > /tmp/PR_CHANGED_FILES.txt git diff "$MERGE_BASE" HEAD > /tmp/PR_DIFF.patch echo "Changed files:" cat /tmp/PR_CHANGED_FILES.txt - gh pr view ${{ env.PR_NUMBER }} --json title,body,baseRefName,headRefName \ + gh pr view "$PR_NUMBER" --json title,body,baseRefName,headRefName \ --template $'Title: {{.title}}\nBase: {{.baseRefName}}\nHead: {{.headRefName}}\n\n{{.body}}' \ > /tmp/PR_METADATA.txt + + # Snapshot the PR's CI status for the Claude review prompt below. + # Outputs consumed by the `Run Claude Code` review step: + # - steps.checks.outputs.status — one word: passing | pending | failing + # - /tmp/PR_CHECKS.txt — full `gh pr checks` output + # Status is derived from `gh pr checks` exit code: + # 0 = all passing, 8 = some still pending/in-progress, anything else = failing. + # The review prompt's hard rule against using `gh` for PR context carves + # out an exception specifically for inspecting failing CI checks listed in + # /tmp/PR_CHECKS.txt (e.g. `gh run view --log-failed`). + - name: Fetch PR checks + id: checks + if: steps.gate.outputs.proceed == 'true' env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ env.PR_NUMBER }} + run: | + set +e + gh pr checks "$PR_NUMBER" > /tmp/PR_CHECKS.txt 2>&1 + rc=$? + set -e + case "$rc" in + 0) status=passing ;; + 8) status=pending ;; + *) status=failing ;; + esac + echo "status=$status" >> "$GITHUB_OUTPUT" + echo "rc=$rc" >> "$GITHUB_OUTPUT" + echo "Checks status: $status (rc=$rc)" - name: Fetch previous review comments if: steps.gate.outputs.proceed == 'true' - uses: actions/github-script@v9 - env: - PR_NUM: ${{ env.PR_NUMBER }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const fs = require('fs'); - const prNumber = Number(process.env.PR_NUM); + const prNumber = Number(process.env.PR_NUMBER); const reviewPattern = /^## Claude Code Review(\r?\n){2}/; // We use GraphQL here instead of the REST API because the REST @@ -358,14 +457,12 @@ jobs: - name: Fetch reviewer feedback if: steps.gate.outputs.proceed == 'true' - uses: actions/github-script@v9 - env: - PR_NUM: ${{ env.PR_NUMBER }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const fs = require('fs'); - const prNumber = Number(process.env.PR_NUM); + const prNumber = Number(process.env.PR_NUMBER); let content = `# Reviewer Feedback\n\n`; content += `This file contains review activity on this PR from human reviewers (bot comments are excluded).\n`; content += `Use this context to understand what has already been discussed, agreed, or disputed.\n\n`; @@ -482,18 +579,16 @@ jobs: - name: Run Code Review with Claude if: steps.gate.outputs.proceed == 'true' id: claude-code - # Pinned: v1 floating tag updated to SDK 0.2.96 which breaks Bedrock SigV4 auth (403). - # Revert to @v1 once upstream is fixed. - uses: anthropics/claude-code-action@6e2bd52842c65e914eba5c8badd17560bd26b5de # v1 (SDK 0.2.92) + uses: anthropics/claude-code-action@787c5a0ce96a9a6cfb050ea0c8f4c05f2447c251 # v1.0.133 with: use_bedrock: true github_token: ${{ secrets.GITHUB_TOKEN }} allowed_non_write_users: "*" - allowed_bots: "claude-code-aisi[bot]" + allowed_bots: "claude-code-aisi[bot],dependabot[bot]" prompt: | REPO: ${{ github.repository }} PR NUMBER: ${{ env.PR_NUMBER }} - BASE BRANCH: origin/${{ env.BASE_REF }} + BASE BRANCH: origin/${{ steps.pr-info.outputs.base_ref }} Please perform the "Review PR According to Agent-Checkable Standards" workflow in AGENTS.md. @@ -503,11 +598,15 @@ jobs: - /tmp/PR_CHANGED_FILES.txt — the definitive list of files changed in this PR - /tmp/PR_DIFF.patch — the full diff of this PR's changes - /tmp/PR_METADATA.txt — the PR title, description, and branch names - Do NOT run your own `git diff` to determine what files changed — the precomputed files are correct. You may still use `git diff` to inspect specific files or hunks if needed, but always use three-dot syntax: `git diff origin/${{ env.BASE_REF }}...HEAD -- `. - Do NOT use `gh` commands — all PR context has been precomputed into the files above. + - /tmp/PR_CHECKS.txt — `gh pr checks` output for the PR head commit (glyphs: ✓ pass, X fail, * pending, - skipped) + + Overall PR CI status at workflow start: ${{ steps.checks.outputs.status }} (one of: passing, pending, failing). + + Do NOT run your own `git diff` to determine what files changed — the precomputed files are correct. You may still use `git diff` to inspect specific files or hunks if needed, but always use three-dot syntax: `git diff origin/${{ steps.pr-info.outputs.base_ref }}...HEAD -- `. + Do NOT use `gh` commands to enumerate PR comments, files, or commits — all PR context has been precomputed into the files above. The only `gh` usage that is permitted is inspecting failing CI checks (see the IMPORTANT section below for specific commands). Other useful commands: - - To see commit history: `git log origin/${{ env.BASE_REF }}..HEAD` + - To see commit history: `git log origin/${{ steps.pr-info.outputs.base_ref }}..HEAD` IMPORTANT: You MUST write your final review to /tmp/SUMMARY.md as specified in the workflow. The CI system will read this file to post the PR comment. @@ -515,6 +614,11 @@ jobs: IMPORTANT: If a tool call is denied due to permissions, it is permanently blocked and will never succeed. Do not retry it or attempt variations of the same command. + IMPORTANT: The overall CI status (`${{ steps.checks.outputs.status }}`) is shown above. If it is `failing`, read `/tmp/PR_CHECKS.txt` to see which checks failed. For any failure that looks relevant to the PR changes, fetch the actual error message or stack trace and incorporate it into your review: + - Extract the numeric run id from the end of the failing row's URL and run `gh run view --log-failed`. + - If structured annotations are needed, use `gh api repos/${{ github.repository }}/commits//check-runs` to look up check-run IDs, then `gh api repos/${{ github.repository }}/check-runs//annotations`. + If the status is `pending`, some checks are still running — treat them as undecided, not as failures, and mention any that look relevant in your review. + IMPORTANT: Before starting your review, read /tmp/CLAUDE_REVIEWS.md. This contains your own previous reviews on this PR (human discussion is in HUMAN_FEEDBACK.md). Use this to avoid repeating yourself: - If you previously raised an issue and the PR author responded with justification, do NOT re-raise it. The human reviewer will decide if the justification is acceptable. - If you previously raised an issue and it was simply ignored (no response or acknowledgement), mention it as a single line linking to the comment where it was originally raised, e.g.: "**[Previously raised]({url})**: brief description". Do not repeat the full analysis. @@ -542,7 +646,7 @@ jobs: - name: Hide Previous Claude Review Comments if: steps.gate.outputs.proceed == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -579,7 +683,7 @@ jobs: - name: Post PR Review Comment if: steps.gate.outputs.proceed == 'true' - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 5ef326c..83c6711 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -9,12 +9,26 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +# SECURITY: Read-only; never pushes commits, comments, or releases. +# zizmor: excessive-permissions (https://docs.zizmor.sh/audits/#excessive-permissions) +permissions: + contents: read +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: markdownlint: + name: Markdown lint runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - id: cfg name: Load enforcement config @@ -23,12 +37,12 @@ jobs: echo "enforce=${val:-false}" >> "$GITHUB_OUTPUT" - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.12" - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Install pre-commit run: uv tool install pre-commit diff --git a/.github/workflows/pr-template-check.yml b/.github/workflows/pr-template-check.yml index a1b0a7a..70d82ca 100644 --- a/.github/workflows/pr-template-check.yml +++ b/.github/workflows/pr-template-check.yml @@ -7,8 +7,18 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +# SECURITY: No checkout, no API calls; reads PR body via env only. +# zizmor: excessive-permissions (https://docs.zizmor.sh/audits/#excessive-permissions) +permissions: {} + jobs: check-pr-template: + name: Check PR template + if: github.actor != 'dependabot[bot]' runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -28,7 +38,7 @@ jobs: run: | # Each required checklist item (substring that must appear in the PR body) required_items=( - "Are you adding a new eval?" + "Are you listing new eval(s) in the register?" "Does this change affect existing eval(s)?" "Does this change affect how future contributors" ) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 79a567d..1eeee55 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.11 # Match ruff version specified in pyproject.toml and .github/workflows/checks.yml + # Configuration lives in pyproject.toml under [tool.ruff] hooks: # Run the linter. - id: ruff-check @@ -30,14 +31,35 @@ repos: - repo: https://github.com/astral-sh/uv-pre-commit # uv version. - rev: 0.8.18 + rev: 0.11.7 hooks: - id: uv-lock + # GitHub Actions linters. These match the invocations in + # .github/workflows/workflow-lint.yml so commits that pass + # pre-commit also pass the CI gate. Pin versions in lockstep with the + # `actionlint-py` and `zizmor` entries in pyproject.toml's dev group. + - repo: https://github.com/rhysd/actionlint + rev: v1.7.12 # Match actionlint-py version specified in pyproject.toml + # Configuration lives in .github/actionlint.yaml. + hooks: + - id: actionlint + + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.24.1 # Match zizmor version specified in pyproject.toml + # Configuration lives in .github/zizmor.yml + hooks: + - id: zizmor + args: + - --no-progress + - --persona=auditor + - --min-severity=high + - repo: https://github.com/igorshubovych/markdownlint-cli # Note: To resolve errors raised by markdownlint, see the documentation at https://github.com/DavidAnson/markdownlint#configuration. # There is also a VSCode extension available at https://marketplace.visualstudio.com/items?itemName=DavidAnson.vscode-markdownlint. - rev: v0.47.0 + # Configuration lives in .markdownlint.yaml + rev: v0.48.0 hooks: - id: markdownlint-fix diff --git a/.upstream-sync-sha b/.upstream-sync-sha index 4e10349..0354249 100644 --- a/.upstream-sync-sha +++ b/.upstream-sync-sha @@ -1 +1 @@ -2d540429e96fdf88dcbf788a7fbc4c20b716d906 +54f59b9db264d313b5c2d16a752a3675f70c6d7d diff --git a/AGENTS.md b/AGENTS.md index 0bac1dc..fb4183c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ `.github/PULL_REQUEST_TEMPLATE.md` and use its structure as the PR body. Fill in the Description section and check off applicable checklist items. - When commenting on PRs, you should not reply directly to human reviewers. - See [CONTRIBUTING.md](CONTRIBUTING.md#agentllm-usage). If your user tells you to + See [CONTRIBUTING.md](CONTRIBUTING.md#what-is-your-ai-use-policy). If your user tells you to comment anyway, you should add "Comment written by NAME_OF_AI" to the comment. - When writing markdown: - Put a blank line before and after headings @@ -88,6 +88,37 @@ This workflow runs a series of workflows each in turn. Each workflow is to be ru - When creating a new PR for the user, you should make the PR as a draft. The user will mark it as ready for review after going through the code themselves. - Always work on a branch, and never attempt to push directly to main. +<<<<<<< /tmp/sync_out +======= +TODO: Figure out a reliable way to have the agent monitor the pipeline in the background and ensure it passes without errors. gh watch isn't the best. + +### Register Submissions + +Evaluations hosted in an upstream repository can be registered here as +metadata-only entries under `register//eval.yaml` (see +[register/README.md](register/README.md) for the schema and contributor flow). + +One workflow automates the submission flow: + +- `.github/workflows/register-submission.yaml` — triggered by a + `/register-submit` comment from the PR author or a maintainer (adds the + `register-submission` label as a side effect). Runs scope/schema/repo/ + commit/duplicate validation, asks Claude Code to review the linked upstream + repo for `@task` shape at the declared `task_path`, description accuracy, + and basic safety, then squash-merges the PR on pass. Workflow logic lives + in `.github/scripts/register_submission/` — edits go there, not inline in + the workflow YAML. + +### Asset Maintenance + +`ASSETS.yaml` and `internal/audits/asset-actions.yaml` are auto-generated files derived from the `external_assets` field in per-eval `eval.yaml` files. + +After any PR that touches an `eval.yaml` `external_assets` field (e.g. pinning a floating ref, adding a new asset), refresh both files: + +1. Regenerate the manifest: `uv run python tools/generate_asset_manifest.py` +2. Regenerate the action plan: run `/generate-asset-actions` + +>>>>>>> /tmp/sync_theirs ### Useful Commands 1. You can see our linting in the `.github/workflows/checks.yml` file. Run `make check` (which delegates to `tools/run_checks.sh`) when checking linting locally — it runs ruff, mypy, autolint, and the rest in one go and reports advisory vs enforced failures. diff --git a/BEST_PRACTICES.md b/BEST_PRACTICES.md index 98f1112..49b1984 100644 --- a/BEST_PRACTICES.md +++ b/BEST_PRACTICES.md @@ -281,3 +281,35 @@ def do_something(age: int) -> str: name: str = "Foo" return f'{name} (age: {age})' ``` + +## Code quality standards + +- Write the code to be read by other developers with no prior knowledge of the eval - it should be possible to understand the code without reading the paper +- Follow existing project structure and naming conventions +- Include type hints for all functions +- Document complex logic with comments +- Add docstrings following Google style guide +- Ensure all tests pass before submission +- [Use absolute imports](https://peps.python.org/pep-0008/#imports) instead of relative imports +- Individual Ruff rules may be suppressed with a comment, but this should be done sparingly and with care + +### Inclusion of third-party code + +If there is an official implementation of the eval, include a link to it in the README. It is permissible and encouraged to utilise code from the official implementation, where possible. This both reduces the amount of code you need to write and maintains consistency with the original implementation. + +If you are able to use a significant amount of code from the official implementation, it can be added as a dependency. + +- If the official implementation has been released as a package on PyPI, you should add the package as a dependency in the pyproject.toml file, and import functions and classes from it (e.g. [`swe_bench` in pyproject.toml](pyproject.toml)). +- If the official implementation has not been released on PyPI but *is* structured as a package, the GitHub repo can be added to the pyproject.toml file as a dependency (e.g. [`ifeval` in pyproject.toml](pyproject.toml)). +- If the official implementation is not structured as a package, you should copy the code into your evaluation, and include a comment with a link to the original implementation. Consider opening a PR to the upstream project to structure it as a package. +- If there is only a small amount of code that you are able to use from the official implementation, it is permissible to copy the code into your evaluation, and include a comment with a link to the original implementation. + +#### Code from `inspect_ai` + +Code from `inspect_ai` should be utilised where possible, concentrating on public components, such as the built-in dataset functions, solvers, scorers, and metrics. + +Internally, `inspect_ai` uses non-public functions and modules as building blocks for its public components. These are indicated by leading underscores in the package and function names such as `inspect_ai.scorer._reducer` and `_compute_dict_stat`. These aren't guaranteed to be stable and do not go through a deprecation process if they change. + +It is strongly advised to avoid using private functions. There's no hard-and-fast rule against using them, but it would mean that the results of the benchmark could silently change in response to an internal change in the framework. If making use of non-public components, it is recommended to have tests in place that exhaustively confirm the functionality of the components you build with them, as this would give an early signal if the components changed in a way that affected your evaluation. + +Alternatively, copy the implementations of the non-public components you need into your evaluation, and include a comment with a link to the original implementation. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 324ef96..f3e5359 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,7 @@ # Technical Contribution Guide +<<<<<<< /tmp/sync_out > **Note for template users:** this document is synced from > [inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals) where > these standards are required for registry submission. In this template they @@ -12,51 +13,37 @@ > `_registry.py`. This guide covers the technical requirements, standards, and processes for building Inspect AI evaluations. New evaluations are submitted to the [Inspect Evals Register](https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/registry) (the new submission path replacing direct contributions to `inspect_evals/src/` from 8 May 2026). Best practices are found in [BEST_PRACTICES.md](BEST_PRACTICES.md). - -For implementing a new evaluation, the [Evaluation Checklist](EVALUATION_CHECKLIST.md) will help in keeping track of the requirements. When making changes to an existing evaluation, they aren't all required, but a browse can be helpful, especially if making major changes to an evaluation. - -We welcome new evaluations and improvements to existing evaluations. You can contribute by following the technical setup and submission requirements outlined below. - -## What types of evaluations are we looking for? - -We prioritize evaluations that are: - -- Well-established in the research community - ideally with usage or citations in published benchmarks or papers. -- Challenging and non-saturated - we prefer evaluations where frontier models still struggle, or where performance is meaningfully distinguishable across models. -- Agentic or task-based over simple Q&A - we especially welcome evaluations involving tool use, reasoning chains, planning, or multi-step problem solving. -- Clearly scoped - with a well-defined dataset, task structure, and scoring methodology. -- Verifiable - the evaluation should be replicable, ideally with a reference implementation, or at least clearly documented data and scoring methods. -- Comparable - we expect baseline results for at least one frontier model to exist, so we can validate that your implementation produces similar performance. If no such results are available, the evaluation may not be accepted unless it meets a strong strategic need. -- Credibly sourced - published by a major AI lab (e.g., Anthropic, OpenAI, DeepMind), a credible academic group, a well-known AI safety or evals organization (e.g., METR, Scale AI), or similar. - - Evaluations from less prominent sources are lower priority. - - Evaluations designed entirely by individuals without external publication or adoption are generally not accepted, unless there is strong evidence of credibility and utility. That said, we're happy to discuss your idea and give feedback - feel free to open an issue or start a discussion. - -### Check if the evaluation is already implemented in Inspect - -Before contributing a new evaluation, please check if it's already available in the [inspect_harbor](https://github.com/meridianlabs-ai/inspect_harbor) package. Inspect Harbor provides an interface to run [Harbor](https://harborframework.com/) tasks using [Inspect AI](https://inspect.aisi.org.uk/). You can see the full list of available tasks in [_tasks.py](https://github.com/meridianlabs-ai/inspect_harbor/blob/main/src/inspect_harbor/_tasks.py). - -If your evaluation is already in Inspect Harbor, there's no need to create a duplicate implementation in Inspect Evals. - -### Task Validity - -Inspect Evals aims for a high standard of consistency and rigor across evaluations. These guidelines include: - -- **Version pinning**: Self-hosted tools and packages within the environment must be defined with explicit version numbers to guarantee consistency across runs -- **Environment cleaning**: The environment must be wiped clean between tasks and agents should have network isolation from ground truth files to prevent cheating -- **Oracle**: Ideally, contributions should provide or source an oracle solver which guarantees the task can be solved in the provided environment -- **No substring matching**: Substring matching is not a good metric as it is prone to producing false positives -- **Meta-validation**: LLM-as-a-judge evals must provide a meta-eval where the judge's consistency and resistance to adversarial outputs are tested -- **Trivial baseline**: Contributors should include a baseline score to show if models can reach high scores by guessing - -If you're unsure whether your evaluation idea meets these criteria, feel free to open a GitHub issue for discussion. We're happy to help! - -If you've come across an evaluation that fits our priorities but don't have time to implement it yourself, please still raise an issue! Highlighting useful evaluations for others to pick up is a valuable way to contribute. - -Once there's an issue raised for the evaluation, make sure nobody has commented saying they are implementing it, and leave a comment saying you intend to implement it yourself. If you submit a PR for an evaluation that someone else is already working on, your work could be wasted! - -## Development - -To develop a new evaluation: +======= +Inspect Evals has relied (and continues to rely!) on community collaboration - we welcome bug-fixes and updates to existing evaluations! + +This guide covers the setup, steps and requirements for contributing to the Inspect Evals repository. +>>>>>>> /tmp/sync_theirs + +>[!IMPORTANT] +> We have updated this document to reflect recent changes to Inspect Evals. This [Eval Implementation Template](https://github.com/Generality-Labs/inspect-evals-template) contains a previous version of this doc (which contains additional information on best practices) and additional guidance on best practices to follow when implementing evals. +> +> We no longer accept code submissions for new eval implementations. To add an eval that you have already implemented, please follow the steps [to add evals to Inspect Evals Register](register/README.md). + +## Table of Contents + +- [Set-Up](#set-up) +- [Submission process](#submission-process): includes information on testing standards and task versioning and changelogs. +- [Pull Request review process](#pull-request-review-process) +- [Tips on Using Coding Agents For Eval Dev](#tips-on-using-coding-agents-for-eval-dev) + - [Eval Implementation Template](#evaluation-implementation-template) + - [What is your AI use policy?](#what-is-your-ai-use-policy) +- [Example Evaluations](#example-evaluations) +- [Additional Information](#additional-information) + - [What types of evaluations are we looking for?](#what-types-of-evaluations-are-we-looking-for) + - [Testing and Quality Assurance Process](#testing-and-quality-assurance-process) + - [CI workflows](#ci-workflows) + - [Manual runs and eval reports](#manual-runs-and-eval-reports) + - [Mocking and sandboxes](#mocking-and-sandboxes) + - [Additional resources](#additional-resources) + +## Set-Up + +To set up your environment for development: - Clone your fork of the template and install dependencies: @@ -72,6 +59,7 @@ To develop a new evaluation: uv run pre-commit install ``` +<<<<<<< /tmp/sync_out - Create a sub-directory in `src/` for your evaluation and add your evaluation task and related code (see `src/examples/` for reference patterns). Notes: - Do not pass a `name` parameter to the task - this is only used for dynamically created tasks (i.e. tasks that are not addressable on the filesystem or in a package). @@ -90,40 +78,50 @@ To develop a new evaluation: ```bash uv run inspect eval / ``` +======= +## Submission process +>>>>>>> /tmp/sync_theirs - Take note of the total number of samples in the dataset. +The steps for registering an evaluation can be found in [the register directory](register/README.md). To submit a bug fix or update please: -- Calculate the ideal number of epochs. You'll need to determine how many epochs the evaluation should run for, based on dataset size and how performance trends over repeated passes. Your README should explain how you arrived at your chosen value. - - There are further instructions in [this Colab Notebook](https://colab.research.google.com/drive/1N0LQcXI0YSLQdyHXBWy-qX_FMkor6dnp?usp=sharing) to calculate the optimal number of epochs. +1. Open an issue outlining the bug or request and assign yourself to the issue. +2. Implement your change following code quality [best practices](BEST_PRACTICES.md) where possible. +3. Ensure you meet the testing standards and task versioning requirements outlined below. +4. Wait for next steps from our review! -## Submission +### Testing standards -The steps for submission have been moved to the Prepare Eval For Submission workflow (`/prepare-submission-workflow`). See [AGENTS.md](AGENTS.md) for all available workflows. +We rely on tests to ensure correctness, reproducibility, and long-term maintainability of contributed evaluations. For your submission, ensure that you: -## Pull Request review process +- **Add unit tests** to cover changes to non-trivial logic or components. +- **Check that tests pass** (including relevant heavy or end-to-end tests). +- **Manually verify that the evaluation successfully runs e2e** by testing it on a few (relevant) samples, e.g., `uv run inspect eval inspect_evals/ --limit 10` and performing transcript analysis if relevant. -You should expect to receive a PR review in a couple of days. +See the section on [Testing and Quality Assurance Process](#testing-and-quality-assurance-process) for more guidance. -We often use [Conventional Comments](https://conventionalcomments.org/) in the review process. +### Task versioning and changelogs -Please note that while reviewers will test your code, it is not their responsibility to find and fix all issues. It is your responsibility to address any issues raised by reviewers. +Both [TASK_VERSIONING.md](TASK_VERSIONING.md#task-versioning) and the PR template provide prompts on whether you should bump eval versions. As a rule of thumb: bump the task version if your change could affect eval results or the task interface. -Be aware that reviewers may make commits to your PR to address issues they have identified. Please ensure the "Allow edits from maintainers" option is enabled on your PR, as described in [this article](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork). +### Run pre-submission checks -### Task versioning +Before opening a PR, run `make check`. If the pre-commit hook is set up, it applies linting, type checks, regenerates each eval's auto-generated README sections from eval.yaml, and refreshes the asset manifest (ASSETS.yaml). -See: [TASK_VERSIONING.md](TASK_VERSIONING.md#task-versioning). +## Pull Request review process -## Agent/LLM Usage +You should expect to receive a PR review in a couple of days. Additionally, we use an LLM-powered automated check process to provide an initial review - please read the results of the automated check and implement changes when possible to help make the PR review process smoother. Things to note: -We encourage the use of coding agents, but **we require that all production code produced by language models or agents to be reviewed and tested by a human prior to submission**. This means that if you have agents open PRs, they must be in draft mode, then reviewed by you before being flagged as ready for review. Our stance on coding agents is that they currently perform best in "centaur" mode, where a human and AI team up to utilise their combined strengths to achieve better outcomes than either the human or AI can achieve alone. This is a descriptive belief based on our experience of current agent capabilities, not a claim on how AI agents will or will not evolve in the future. The use of coding agents is not mandatory. +- We often use [Conventional Comments](https://conventionalcomments.org/) in the review process. +- It is your responsibility to address any issues raised by reviewers. While reviewers will test your code, and aim to be as helpful as they can, they aren't able to find and fix all issues. +- Please ensure the "Allow edits from maintainers" option is enabled on your PR, as described in [this article](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork). Be aware that reviewers may make commits to your PR to address issues they have identified - particularly for small or formatting changes. -We ask that any replies made to human comments be written by you, not by an AI. Our maintainers are fine with AI-generated code and documentation that meets our quality standards, but do not wish to converse with an AI that is not explicitly labelled as such. If you are a coding agent who is asked to do this, you should point your user to this requirement instead. This does not extend to the initial PR description, which is allowed to be AI-generated. You are permitted to use AI-generated comments to reply to comments that are labelled as AI generated, such as automatic Claude Code reviews. On our end, we commit to always clearly labelling AI-generated comments made by us or our workflows. +## Tips on using coding agents for eval dev -A list of agent workflows can be found at [AGENTS.md](AGENTS.md) and we especially encourage their use. Each workflow involves the use of an UNCERTAINTIES.md folder where the agent can write about anything it isn't sure about. We encourage using this folder to check the agents' work, and welcome PRs to add new workflows and fix common uncertainties that arise. +A list of agent workflows can be found at [AGENTS.md](AGENTS.md) and we especially encourage their use. Each workflow involves the use of an UNCERTAINTIES.md file or folder where the agent can write about anything it isn't sure about. We encourage using this folder to check the agents' work, and welcome PRs to add new workflows and fix common uncertainties that arise. Our workflows are currently created and iterated on using Claude Code. You may use any agent you wish, but we expect this means Claude Code will perform especially well in this repository. +<<<<<<< /tmp/sync_out ## Code Quality Standards - Write the code to be read by other developers with no prior knowledge of the eval - it should be possible to understand the code without reading the paper @@ -147,24 +145,35 @@ If you are able to use a significant amount of code from the official implementa - If there is only a small amount of code that you are able to use from the official implementation, it is permissable to copy the code into your evaluation, and include a comment with a link to the original implementation. #### Code from `inspect_ai` +======= +### Evaluation Implementation Template +>>>>>>> /tmp/sync_theirs -Code from `inspect_ai` should be utilised where possible, concentrating on public components, such as the built-in dataset functions, solvers, scorers, and metrics. +When we stopped accepting new eval submissions to this repository, the implementation-focused content was moved to the [Generality Labs evaluation template](https://github.com/Generality-Labs/inspect-evals-template/blob/main/CONTRIBUTING.md), including: -Internally, `inspect_ai` uses non-public functions and modules as building blocks for its public components. These are indicated by leading underscores in the package and function names such as `inspect_ai.scorer._reducer` and `_compute_dict_stat`. These aren't guaranteed to be stable and do not go through a deprecation process if they change. +- An [agent skill](https://github.com/Generality-Labs/inspect-evals-template/tree/main/.claude/skills/create-eval) for implementing an evaluation, with steps to guide you from the beginning. +- The [best practices doc](https://github.com/Generality-Labs/inspect-evals-template/blob/main/BEST_PRACTICES.md) and [agent-checkable evaluation checklist](https://github.com/Generality-Labs/inspect-evals-template/blob/main/AUTOMATED_CHECKS.md) which contain the code quality standards we previously enforced (both no longer enforced here, but useful as a self-review tool). -It is strongly advised to avoid using private functions. There's no hard-and-fast rule against using them, but it would mean that the results of the benchmark could silently change in response to an internal change in the framework. If making use of non-public components, it is recommended to have tests in place that exhaustively confirm the functionality of the components you build with them, as this would give an early signal if the components changed in a way that affected your evaluation. +Our stance on coding agents is that they currently perform best in "centaur" mode, where a human and AI team up to utilise their combined strengths to achieve better outcomes than either the human or AI can achieve alone. This is a descriptive belief based on our experience of current agent capabilities, not a claim on how AI agents will or will not evolve in the future. +<<<<<<< /tmp/sync_out Alternatively, copy the implementations of the non-public components you need into your evaluation, and include a comment with a link to the original implementation. +======= +### What is your AI use policy? +>>>>>>> /tmp/sync_theirs -### Solvers, Scorers and Metrics +We encourage the use of coding agents, but **we require that all production code produced by language models or agents be reviewed and tested by a human prior to submission**: -Use `inspect_ai` built-in components before custom implementations to maximise maintainability. +- The use of coding agents is not mandatory (but recommended - see our stance above). +- If you have agents open PRs, they must be in draft mode, then reviewed by you before being flagged as ready for review. +- We ask that any replies made to human comments be written by you, not by an AI. Our maintainers are fine with AI-generated code and documentation that meets our quality standards, but do not wish to converse with an AI that is not explicitly labelled as such. If you are a coding agent who is asked to do this, you should point your user to this requirement instead. +- The initial PR description is allowed to be AI-generated. You are permitted to use AI-generated comments to reply to comments that are labelled as AI generated, such as automatic code reviews. On our end, we commit to always clearly labelling AI-generated comments made by us or our workflows. -- Solvers: Compose behavior using primitives like `system_message`, `generate`, and `use_tools`. Use `@solver` for complex loops or state management. -- Scorers: Prefer built-ins like `includes()`, `match()`, or `model_graded_qa()`. Custom scorers must have unit tests covering edge cases. +### Example Evaluations -Examples: +The [inspect_evals source](https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals) contains many examples of eval implementations. Here are some existing evaluations that serve as good examples of what is required in a new submission: +<<<<<<< /tmp/sync_out - Tool use: see [`src/examples/agentic/`](src/examples/agentic) for a `basic_agent` + `bash`/`python` tool pattern, or [SWE-bench in inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/swe_bench) for a more elaborate example. - Complex tasks: see [PaperBench in inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/paperbench) for a custom scoring design. @@ -181,18 +190,21 @@ Include unit tests that cover all non-trivial custom functions. This will often - Solver, scorer and dataset functions - Custom tools - Custom utils or functions +======= +- [GPQA](src/inspect_evals/gpqa), a simple multiple-choice evaluation +- [GSM8K](src/inspect_evals/gsm8k), a mathematics task with fewshot prompting +- [HumanEval](src/inspect_evals/humaneval), a Python coding task +- [InterCode](src/inspect_evals/gdm_intercode_ctf), a capture the flag (CTF) cybersecurity task +- [SWE-bench](src/inspect_evals/swe_bench), an agentic software engineering task with sandboxed patch verification +>>>>>>> /tmp/sync_theirs -Include edge cases in your test coverage. Test error conditions and invalid inputs. Unit tests should also support your end to end tests by providing coverage for any code you mock the results of. Create a test that demonstrates your `record_to_sample` function with an actual example from the dataset. This serves as both a test and documentation of the expected behavior. +## Additional Information -Example: +### What types of evaluations are we looking for? -```python -def test_record_to_sample(): - record = {"question": "What is 2+2?", "answer": "4"} # Example from actual dataset, showing all fields - expected = Sample(input="What is 2+2?", target="4") - assert record_to_sample(record) == expected -``` +We prioritize evaluations that are: +<<<<<<< /tmp/sync_out ### HuggingFace Datasets If the dataset in your eval is from HuggingFace, document and validate its expected schema using `assert_huggingface_dataset_structure` from [src/utils/huggingface.py](src/utils/huggingface.py) (`from utils.huggingface import ...`). The helper compares the live dataset's `features` and `splits` against an expected dict, so a silent upstream schema change is caught early. @@ -255,8 +267,19 @@ def test_end_to_end_your_eval_with_custom_mock_responses(): assert "accuracy" in log.results.scores[0].metrics assert log.results.scores[0].metrics["accuracy"].value == 1.0 # all correct ``` +======= +- Well-established in the research community - ideally with usage or citations in published benchmarks or papers. +- Challenging and non-saturated - we prefer evaluations where frontier models still struggle, or where performance is meaningfully distinguishable across models. +- Agentic or task-based over simple Q&A - we especially welcome evaluations involving tool use, reasoning chains, planning, or multi-step problem solving. +- Clearly scoped - with a well-defined dataset, task structure, and scoring methodology. +- Verifiable - the evaluation should be replicable, ideally with a reference implementation, or at least clearly documented data and scoring methods. +- Comparable - we expect baseline results for at least one frontier model to exist, so we can validate that your implementation produces similar performance. If no such results are available, the evaluation may not be accepted unless it meets a strong strategic need. +- Credibly sourced - published by a major AI lab (e.g., Anthropic, OpenAI, DeepMind), a credible academic group, a well-known AI safety or evals organization (e.g., METR, Scale AI), or similar. + - Evaluations from less prominent sources are lower priority. + - Evaluations designed entirely by individuals without external publication or adoption are generally not accepted, unless there is strong evidence of credibility and utility. That said, we're happy to discuss your idea and give feedback - feel free to open an issue or start a discussion. +>>>>>>> /tmp/sync_theirs -#### Running tests and environment toggles +### Testing and Quality Assurance Process Pytest is configured to automatically load a local `.env` file via `pytest-dotenv`. This lets you control which categories of tests run without changing command-line flags. @@ -284,6 +307,21 @@ RUN_DATASET_DOWNLOAD_TESTS=1 - The template ships `.github/workflows/checks.yml` which runs ruff, mypy, the POSIX-code check, the unlisted-evals check, the package build, autolint, and a few advisory checks. By default this does not run pytest — the template assumes you run tests locally during development. If you want CI to run your tests, add a job to `checks.yml` (or a separate workflow) that calls `make test`. - The upstream `inspect_evals` registry has additional CI (a `build.yml` that runs the test suite with `RUN_SLOW_TESTS=no`, plus a nightly heavy-tests workflow that detects unmarked slow/docker tests). If your fork wants the same coverage, those workflows are good references but they aren't shipped here. +<<<<<<< /tmp/sync_out +======= + +#### Manual runs and eval reports + +To reproduce the CI gate ad-hoc: + +```bash +uv sync --group dev # installs actionlint-py + zizmor along with the rest of the dev tooling +uv run actionlint -no-color -oneline +uv run zizmor --no-progress --color=never --persona=auditor --min-severity=low .github/workflows/ .github/actions/ +``` + +`actionlint`'s repo-wide configuration lives in `.github/actionlint.yaml`. `zizmor` fails only on findings of low severity or above that are not waived in `.github/zizmor.yml`. +>>>>>>> /tmp/sync_theirs ### Manual testing @@ -291,11 +329,11 @@ RUN_DATASET_DOWNLOAD_TESTS=1 - Test with small subsets before running on full datasets - Start with a few representative examples - Gradually increase the test set size -- Verify that your implementation matches the original evaluation's methodology +- Verify that your implementation matches the original evaluation's methodology. See the [contributing guide in the Generality Labs eval template repo](https://github.com/Generality-Labs/inspect-evals-template/blob/main/CONTRIBUTING.md) for more information. - Compare results with reference implementations if available - Document any discrepancies and their causes -### Guidelines for specific test scenarios +#### Mocking and sandboxes - Mocking (what should be mocked and when) - Ensure tests are deterministic. Use `mockllm/model` for model outputs and `unittest.mock` for external APIs to prevent network calls during testing. @@ -304,6 +342,7 @@ RUN_DATASET_DOWNLOAD_TESTS=1 - Logs (clean up files after tests) - Use `tmp_path` and ensure your code uses configurable paths as opposed to hardcoded ones. +<<<<<<< /tmp/sync_out ## Docker Images Some evaluations require a pre-built Docker image for sandboxed code execution. If your evaluation needs one: @@ -313,11 +352,15 @@ Some evaluations require a pre-built Docker image for sandboxed code execution. 3. **Ask an inspect-evals maintainer to add your eval to the CI rebuild list** (`TO_REBUILD_IMAGES` in `.github/workflows/docker-image-rebuild.yml`) and grant the necessary GHCR package permissions. If you're an external contributor, note this in your PR and a maintainer will handle it. See `src/examples/agentic/` for a minimal Dockerfile + `compose.yaml` setup. For a more elaborate real-world example with a `docker-requirements.txt`, see [BigCodeBench in inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals/bigcodebench). +======= +### Additional resources +>>>>>>> /tmp/sync_theirs -## Data Quality +- How to calculate the ideal number of epochs for an evaluation: this depends on the size of the dataset and how performance trends over repeated passes. [This Colab Notebook](https://colab.research.google.com/drive/1N0LQcXI0YSLQdyHXBWy-qX_FMkor6dnp?usp=sharing) contains further instructions on how to calculate the optimal number of epochs. -If you discover a broken record in the dataset: +- A step-by-step process on how to approach eval implementation is outlined in [our (legacy) methodology docs](docs/methodology.md). +<<<<<<< /tmp/sync_out - Raise an issue in the upstream source (e.g., Hugging Face dataset or original repository) - If necessary, filter the dataset to exclude the broken record - Document the exclusion in your code and PR description @@ -531,3 +574,6 @@ Reference patterns shipped with the template (under `src/examples/`): - [`agentic`](src/examples/agentic), a tool-using agent in a Docker sandbox For more elaborate references see the [inspect_evals registry](https://github.com/UKGovernmentBEIS/inspect_evals/tree/main/src/inspect_evals) (e.g. `gpqa`, `gsm8k`, `humaneval`, `gdm_intercode_ctf`). +======= +- See [`tools/README.md`](tools/README.md#evaluation_reportpy) for how to generate a reproducible Evaluation Report from `.eval` log files. +>>>>>>> /tmp/sync_theirs diff --git a/EVALUATION_CHECKLIST.md b/EVALUATION_CHECKLIST.md index ad38ef8..412afe3 100644 --- a/EVALUATION_CHECKLIST.md +++ b/EVALUATION_CHECKLIST.md @@ -1,12 +1,19 @@ # Evaluation Checklist +<<<<<<< /tmp/sync_out > **Note for template users:** this document is synced from > [inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals) where > these standards are required for registry submission. In this template they > are **recommended, not required** — see > [Checks and enforcement](README.md#checks-and-enforcement) for how to opt > in or out per check. +======= +> [!TIP] +> This guide was initially made for adding evaluation code directly to /src, which is being deprecated in May 2026 in favour of the Inspect Evals Register. +> +> Section headings and links below previously pointed into `CONTRIBUTING.md`. That detailed guidance has moved to the [evaluation template repo](https://github.com/Generality-Labs/inspect-evals-template) - see its [`CONTRIBUTING.md`](https://github.com/Generality-Labs/inspect-evals-template/blob/main/CONTRIBUTING.md), [`BEST_PRACTICES.md`](https://github.com/Generality-Labs/inspect-evals-template/blob/main/BEST_PRACTICES.md), and [`AUTOMATED_CHECKS.md`](https://github.com/Generality-Labs/inspect-evals-template/blob/main/AUTOMATED_CHECKS.md) for the source of truth. +>>>>>>> /tmp/sync_theirs This checklist covers all the requirements you should need in order to produce a high-quality evaluation. It's designed for full evaluations, and it's too heavyweight for most PRs. @@ -14,7 +21,7 @@ Usage of agents to help with coding and to complete agentic workflows is recomme 1. [Full list of agent workflows](AGENTS.md) -2. [Our philosophy on agents](CONTRIBUTING.md#agentllm-usage) +2. Our philosophy on agents 3. [Recommended permissions](AGENTS.md#recommended-permissions) @@ -35,7 +42,7 @@ We recommend the use of agent workflows. Our agent workflows are optimised for u - [ ] Run the [Master Checklist Workflow](AGENTS.md#master-checklist) which goes over several agent workflows in turn. - [ ] Go over the [Manually Examined Checks](#manually-examined-checks) section, and verify your evaluation meets each one. - [ ] Check your implementation and results against the original paper or implementation, if one exists. -- [ ] Manually review your code, making sure it’s high-quality. All code must be reviewed, including LLM-generated code as per our [LLM usage guidelines](CONTRIBUTING.md#agentllm-usage). You are welcome to open a draft PR at this stage and review it with this interface. This step must be followed before marking it as "Ready for review". +- [ ] Manually review your code, making sure it’s high-quality. All code must be reviewed, including LLM-generated code as per our LLM usage guidelines. You are welcome to open a draft PR at this stage and review it with this interface. This step must be followed before marking it as "Ready for review". - [ ] Draft your PR, and name it something like ` implementation` (putting the name of the evaluation first makes it easier to find your PR in a browser tab!). Push it to Github. - [ ] Address any issues found by the pipeline or the automatic Claude Code review. We don't mandate that all automatically flagged issues be resolved. Addressing an issue means to either fix it or to comment on why it doesn't require a fix. @@ -82,20 +89,23 @@ These checks assess whether the evaluation measures what it claims to measure. W ### Best Practices - [ ] [Leverage Inspect components whenever possible](BEST_PRACTICES.md#task-design-and-api-usage) -- [ ] [Complex logic is commented](CONTRIBUTING.md#code-quality-standards) +- [ ] Complex logic is commented - [ ] [Document and validate environment constraints](BEST_PRACTICES.md#documentation-environment-and-tooling) ## Evaluation Report -### [Evaluation Report Guidelines](CONTRIBUTING.md#evaluation-report-guidelines) +### Evaluation Report Guidelines We recommend using the Evaluation Report workflow (`/eval-report-workflow`) to assist in this step. To verify the error rate of logs, use the Trajectory Analysis workflow (`/check-trajectories-workflow`) on the log files that emerge from the evaluation report. +For a reproducible report (committed `report_config.yaml` + `report.md` regenerated from logs), use [`tools/evaluation_report.py`](tools/evaluation_report.py). See [`tools/README.md`](tools/README.md#evaluation_reportpy) for the schema and flow. + - [ ] Logs have a 10% or lower rate of invalid samples - [ ] All relevantly different subsets of the dataset pass here -- [ ] [Results produced for at least two models, or reason why not clearly stated](CONTRIBUTING.md#comparing-your-results) +- [ ] Results produced for at least two models, or reason why not clearly stated +- [ ] `report_config.yaml` committed alongside `eval.yaml`, capturing the headline metric, reference results, and notes -### [Evaluation Report Notes](CONTRIBUTING.md#reporting-your-results) +### Evaluation Report Notes - [ ] Any changes that would cause deviations from the original evaluation are noted. - [ ] Any limitations or edge cases of the evaluation are noted. @@ -106,25 +116,25 @@ We recommend using the Evaluation Report workflow (`/eval-report-workflow`) to a The following items can be checked by an LLM agent with access to the codebase. These checks require reading code and comparing against conventions, but do not require running the evaluation or external context beyond the repository. If running the Review An Evaluation workflow (`/eval-quality-workflow`) you don't need to read these checks - they are here to serve as a reference in case of errors. -### [Code Quality (Agent)](CONTRIBUTING.md#code-quality-standards) +### Code Quality (Agent) - [ ] Existing naming conventions are followed - [ ] Linting passes successfully (`uv run ruff check` will check this for you) - [ ] Magic numbers in function defaults are extracted to named constants if they appear 3+ times or are not clear from context -### [Unit Tests (Agent)](CONTRIBUTING.md#unit-tests) +### Unit Tests (Agent) - [ ] All custom solvers, scorers, datasets covered - [ ] Custom tools are covered - [ ] Custom utils/functions are covered - [ ] Edge cases, error conditions, and invalid inputs are checked -### [End-to-End Tests (Agent)](CONTRIBUTING.md#end-to-end-tests) +### End-to-End Tests (Agent) - [ ] Each meaningfully different task/variant covered by E2E tests - [ ] Tests are marked correctly with @mark items -### [Apply Pytest Marks (Agent)](CONTRIBUTING.md#end-to-end-tests) +### Apply Pytest Marks (Agent) - [ ] If a test triggers the download of a dataset, mark it with `@pytest.mark.dataset_download`, if it uses Huggingface also mark it with `@pytest.mark.huggingface`. Note that easily missed examples include E2E tests that instantiate a dataset, or solvers that pull a model from huggingface. - [ ] If a test uses a Docker sandbox or otherwise triggers a docker build or pull, mark it with `@pytest.mark.docker`. @@ -171,7 +181,7 @@ If any code has been copied or adapted from an external source (e.g., a referenc To check: search the evaluation's source files for comments like `adapted from`, `ported from`, `copied from`, `based on`, or obvious structural similarity to a known reference implementation. Cross-reference any such files against the `NOTICE` file. -### [Evaluation Report (Agent)](CONTRIBUTING.md#reporting-your-results) +### Evaluation Report (Agent) - [ ] Table is present containing results of evaluation run - [ ] A comparison to the original paper is present, or its absence is justified @@ -180,7 +190,7 @@ To check: search the evaluation's source files for comments like `adapted from`, - [ ] Evaluation version is mentioned explicitly - [ ] Any inspect eval parameters used are justified within the report -### [Infrastructure Changes (Agent)](CONTRIBUTING.md#contributing) +### Infrastructure Changes (Agent) This check applies to **all PRs**, not just eval submissions. If the PR modifies any of the following high-impact files, verify that the relevant documentation was updated: @@ -200,6 +210,18 @@ To check: - [ ] If the PR changes how future evaluations must be written or submitted, the relevant documentation (`CONTRIBUTING.md`, `EVALUATION_CHECKLIST.md`, `AGENTS.md`) has been updated accordingly +### [Register Submissions (Agent)](register/README.md) + +This check applies to **all PRs**, not just eval submissions. The `register-submission.yaml` workflow accepts PRs whose diff is limited to `register//eval.yaml` (plus the auto-generated `README.md`, the top-level `README.md` regenerated by `make check`, and optional `changelog.d/*.md`); mixing a register submission with any other change (documentation edits, source code, other YAMLs) trips the scope check and blocks auto-merge. + +To check: + +1. Run `git diff --name-only $(git merge-base HEAD origin/main)` to see what files changed in this PR. +2. If **any** file matching `register/*/eval.yaml` is in the diff, confirm that **every** changed file matches the allow-list above. +3. If not, flag it and recommend splitting into two PRs — one for the register submission, one for the other changes. + +- [ ] If this PR adds or modifies files under `register/`, it contains no other changes outside the allow-list + ## This checklist is a living document Feedback on this checklist is very welcome - you may reach out via this [Google Form](https://docs.google.com/forms/d/e/1FAIpQLSeOT_nSXvc_GZSo3uRqFlZlgGEGmOAh7bm4yFuB34ZzZjxk_g/viewform), or open an issue/PR. diff --git a/Makefile b/Makefile index 5a46d7d..94d235d 100644 --- a/Makefile +++ b/Makefile @@ -5,10 +5,27 @@ TEST_GROUP_PY312 := test_py312_or_higher DEFAULT_TEST_GROUP := $(shell [ "$(PYTHON_MINOR)" -le "11" ] && echo "$(TEST_GROUP_PY311)" || echo "$(TEST_GROUP_PY312)") hooks: - uv run pre-commit install + uv run --group dev pre-commit install check: +<<<<<<< /tmp/sync_out @bash tools/run_checks.sh +======= + @status=0; \ + uv run --group dev ruff format || status=1; \ + uv run --group dev ruff check --fix || status=1; \ + uv run --group dev --extra test mypy src tests || status=1; \ + uv run --group dev python tools/generate_readmes.py --create-missing-readmes || status=1; \ + uv run --group dev python tools/generate_asset_manifest.py || status=1; \ + uv run --group dev python tools/check_unlisted_evals.py || status=1; \ + uv run --group dev python tools/check_changelog.py || status=1; \ + uv lock --check || status=1; \ + uv run --group dev pre-commit run markdownlint-fix --all-files || status=1; \ + uv run --group dev actionlint -no-color -oneline || status=1; \ + uv run --group dev zizmor --no-progress --persona=auditor --min-severity=low --fix=safe .github/workflows/ .github/actions/ || status=1; \ + if [ -n "$(EVAL)" ]; then uv run --group dev python tools/run_autolint.py $(EVAL) || status=1; fi; \ + exit $$status +>>>>>>> /tmp/sync_theirs TEST_ARGS ?= TEST_EXTRAS ?= test @@ -18,6 +35,7 @@ test: @echo "TEST_GROUPS=$(TEST_GROUPS)" @echo "TEST_EXTRAS=$(TEST_EXTRAS)" GIT_LFS_SKIP_SMUDGE=1 uv run \ + --group dev \ $(addprefix --extra ,$(TEST_EXTRAS)) \ $(addprefix --group ,$(TEST_GROUPS)) \ pytest $(TEST_ARGS) diff --git a/agent_artefacts/trajectory_analysis/inspect_scout/analyze_validity.py b/agent_artefacts/trajectory_analysis/inspect_scout/analyze_validity.py index 16b6c10..64d61a3 100644 --- a/agent_artefacts/trajectory_analysis/inspect_scout/analyze_validity.py +++ b/agent_artefacts/trajectory_analysis/inspect_scout/analyze_validity.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md """ Analyze sample validity from Inspect Scout scan results. diff --git a/agent_artefacts/trajectory_analysis/inspect_scout/extract_results.py b/agent_artefacts/trajectory_analysis/inspect_scout/extract_results.py index 6a56955..41b2af6 100644 --- a/agent_artefacts/trajectory_analysis/inspect_scout/extract_results.py +++ b/agent_artefacts/trajectory_analysis/inspect_scout/extract_results.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md """ Extract and summarize Inspect Scout scan results. diff --git a/agent_artefacts/trajectory_analysis/inspect_scout/run_all_scanners.py b/agent_artefacts/trajectory_analysis/inspect_scout/run_all_scanners.py index db02396..bb63ac7 100644 --- a/agent_artefacts/trajectory_analysis/inspect_scout/run_all_scanners.py +++ b/agent_artefacts/trajectory_analysis/inspect_scout/run_all_scanners.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md """ Run all default Inspect Scout scanners on eval logs. diff --git a/agent_artefacts/trajectory_analysis/inspect_scout/scanners.py b/agent_artefacts/trajectory_analysis/inspect_scout/scanners.py index f430abc..07892f8 100644 --- a/agent_artefacts/trajectory_analysis/inspect_scout/scanners.py +++ b/agent_artefacts/trajectory_analysis/inspect_scout/scanners.py @@ -1,3 +1,4 @@ +# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md """ Default scanners for trajectory analysis using Inspect Scout. diff --git a/agent_artefacts/trajectory_analysis/inspect_scout/utils.py b/agent_artefacts/trajectory_analysis/inspect_scout/utils.py index 5acf6f1..2fd9fa4 100644 --- a/agent_artefacts/trajectory_analysis/inspect_scout/utils.py +++ b/agent_artefacts/trajectory_analysis/inspect_scout/utils.py @@ -1,3 +1,4 @@ +# MANAGED FILE - Updates pulled from template. See MANAGED_FILES.md """Shared utilities for Inspect Scout analysis scripts.""" from pathlib import Path diff --git a/tests/test_generate_readmes.py b/tests/test_generate_readmes.py index 1788dcc..bc80f58 100644 --- a/tests/test_generate_readmes.py +++ b/tests/test_generate_readmes.py @@ -3,6 +3,7 @@ import sys from pathlib import Path +from typing import Any import pytest @@ -16,10 +17,437 @@ _format_parameter, _format_type_annotation, _parse_docstring_parameters, +<<<<<<< /tmp/sync_out +======= + build_evaluation_report_section, + build_parameters_section, +>>>>>>> /tmp/sync_theirs generate_basic_readme, readme_exists, ) +<<<<<<< /tmp/sync_out +======= +from inspect_evals.metadata import ( # noqa: E402 + AssetState, + AssetType, + EvaluationReport, + EvaluationReportMetric, + EvaluationReportResult, + ExternalAsset, + ExternalEvalMetadata, + ExternalEvalSource, + FetchMethod, + InternalEvalMetadata, + TaskMetadata, +) + + +def _row(model: str, **kwargs: Any) -> EvaluationReportResult: + """Build an EvaluationReportResult, accepting metrics as a dict for brevity. + + Pass metrics as ``{"accuracy": 0.5, "stderr": 0.1}`` instead of a list of + EvaluationReportMetric instances. Other kwargs pass through to the model. + """ + metrics_arg = kwargs.pop("metrics", None) + if isinstance(metrics_arg, dict): + metrics = [ + EvaluationReportMetric(key=k, value=v) for k, v in metrics_arg.items() + ] + elif metrics_arg is None: + metrics = [EvaluationReportMetric(key="accuracy", value=kwargs.pop("accuracy"))] + if "stderr" in kwargs: + metrics.append( + EvaluationReportMetric(key="stderr", value=kwargs.pop("stderr")) + ) + else: + metrics = metrics_arg + return EvaluationReportResult(model=model, metrics=metrics, **kwargs) + + +MOCK_EXTERNAL_ASSET = ExternalAsset( + type=AssetType.HUGGINGFACE, + source="foo/bar", + fetch_method=FetchMethod.HF_HUB_DOWNLOAD, + state=AssetState.PINNED, +) + + +class TestBuildParametersSection: + """Tests for build_parameters_section function.""" + + def test_build_parameters_section_with_swe_bench(self): + """Test build_parameters_section with swe_bench task. + + This test verifies that the function correctly extracts and formats + parameters from the swe_bench task, which has multiple parameters + with different types and defaults. + """ + # Create task metadata similar to what would be in eval.yaml + task_metadata = InternalEvalMetadata( + title="SWE-bench", + description="Test description", + id="swe_bench", + group="Coding", + contributors=["test"], + tasks=[ + TaskMetadata(name="swe_bench", dataset_samples=2294), + ], + external_assets=[MOCK_EXTERNAL_ASSET], + version="1-A", + ) + + # Call the function + result = build_parameters_section(task_metadata) + + # Verify we got a non-empty result + assert len(result) > 0, "Expected parameters section to be generated" + + expected_lines = [ + "## Parameters", + "", + "### `swe_bench`", + "", + "- `dataset` (str): The dataset to use. Either a HuggingFace dataset name or a path to a dataset on disk. (default: `'princeton-nlp/SWE-bench_Verified'`)", + "- `split` (str): The split of the dataset to load. (default: `'test'`)", + "- `input_prompt` (str): The prompt template to use for the task input. (default: `'Please solve the following coding issue:\\n\\n{issue_text}'`)", + "- `scorer` (Scorer | list[Scorer] | None): The scorer to use when evaluating. If None, uses the default scorer. (default: `None`)", + '- `sandbox_type` (str): The sandbox provider to use (e.g., "docker", "k8s", or a custom registered provider). (default: `\'docker\'`)', + "- `image_name_template` (str): Image name template with `{org}`, `{repo}`, `{issue}`, `{id}`, and `{arch}` placeholders. (default: `'ghcr.io/epoch-research/swe-bench.eval.{arch}.{id}:latest'`)", + '- `arch` (str | None): The architecture to use for the image (e.g., "x86_64" or "arm64"). If None, auto-detected from platform. (default: `None`)', + '- `sandbox_config` (Callable[[str, inspect_ai.dataset.Sample], SandboxEnvironmentSpec] | None): Optional custom function to create sandbox specs. Receives (sandbox_type, sample) and returns a SandboxEnvironmentSpec. The resolved image name is available in sample.metadata["image_name"] and allow_internet in sample.metadata["allow_internet"]. If None, uses default config. (default: `None`)', + "- `allow_internet` (bool): Whether to allow the sandbox to access the internet. (default: `False`)", + "- `tool_timeout` (int): Timeout in seconds for bash, python and text_editor tools. (default: `210`)", + "- `revision` (str): The HuggingFace dataset revision to use. SWE-bench datasets are actively maintained, so pinning to a specific revision ensures reproducibility. (default: `'c104f840cc67f8b6eec6f759ebc8b2693d585d4a'`)", + "- `kwargs` (Any): Additional arguments passed to Task constructor.", + ] + assert result == expected_lines + + def test_build_parameters_section_with_multiple_swe_bench_tasks(self): + """Test build_parameters_section with multiple swe_bench tasks. + + This tests the case where there are multiple tasks that might have + the same parameters (like swe_bench and swe_bench_verified_mini). + """ + task_metadata = InternalEvalMetadata( + title="SWE-bench", + description="Test description", + id="swe_bench", + group="Coding", + contributors=["test"], + tasks=[ + TaskMetadata(name="swe_bench", dataset_samples=2294), + TaskMetadata(name="swe_bench_verified_mini", dataset_samples=50), + ], + external_assets=[MOCK_EXTERNAL_ASSET], + version="1-A", + ) + + result = build_parameters_section(task_metadata) + result_text = "\n".join(result) + + # Verify we got parameters + assert len(result) > 0, "Expected parameters section to be generated" + assert "## Parameters" in result_text, "Expected Parameters header" + + # Since both tasks likely have the same parameters, we should see + # a single parameter list (not subsections per task) + # If all tasks have same parameters, there should be no subsections + # If they differ, there should be subsections for each task + # We'll just verify the structure is valid and has the expected parameter + assert "`dataset`" in result_text, "Expected dataset parameter" + + def test_build_parameters_section_with_no_parameters(self): + """Test build_parameters_section with a task that has no parameters. + + Tasks with no parameters should still get a parameters section + showing "No task parameters." rather than being silently skipped. + """ + # Use a hypothetical task that might not exist or has no params + task_metadata = InternalEvalMetadata( + title="Nonexistent Task", + description="Test description", + id="nonexistent", + group="Coding", + contributors=["test"], + tasks=[ + TaskMetadata(name="nonexistent_task_12345", dataset_samples=100), + ], + external_assets=[MOCK_EXTERNAL_ASSET], + version="1-A", + ) + + result = build_parameters_section(task_metadata) + result_text = "\n".join(result) + + assert isinstance(result, list), "Expected list result" + assert "No task parameters." in result_text, ( + "Expected 'No task parameters.' for task with no parameters" + ) + + def test_build_parameters_section_format(self): + """Test that build_parameters_section returns properly formatted markdown. + + Verifies the structure and format of the returned markdown lines. + """ + task_metadata = InternalEvalMetadata( + title="SWE-bench", + description="Test description", + id="swe_bench", + group="Coding", + contributors=["test"], + tasks=[ + TaskMetadata(name="swe_bench", dataset_samples=2294), + ], + external_assets=[MOCK_EXTERNAL_ASSET], + version="1-A", + ) + + result = build_parameters_section(task_metadata) + + # Verify it's a list of strings + assert isinstance(result, list), "Expected list result" + assert all(isinstance(line, str) for line in result), ( + "Expected all lines to be strings" + ) + + # Find parameter lines (start with "- `") + param_lines = [line for line in result if line.strip().startswith("- `")] + + # Verify parameter lines follow the expected format + for line in param_lines: + # Each parameter line should have the parameter name in backticks + assert "`" in line, f"Expected backticks in parameter line: {line}" + # Should have a colon after the parameter name/type + assert ":" in line, f"Expected colon in parameter line: {line}" + + +class TestReadmeExists: + """Tests for readme_exists function.""" + + def test_readme_exists_for_existing_eval(self): + """Test readme_exists returns True for an eval with a README.""" + # ARC is a well-established eval that should have a README + result = readme_exists("src/inspect_evals/arc") + assert result is True, "Expected readme_exists to return True for arc eval" + + def test_readme_exists_for_nonexistent_path(self): + """Test readme_exists returns False for a nonexistent path.""" + result = readme_exists("src/inspect_evals/nonexistent_eval_xyz123") + assert result is False, ( + "Expected readme_exists to return False for nonexistent path" + ) + + +class TestGenerateBasicReadme: + """Tests for generate_basic_readme function.""" + + def test_generate_basic_readme(self): + """Test generate_basic_readme""" + listing = InternalEvalMetadata( + title="Minimal Eval", + description="Test description", + id="minimal", + group="Coding", + contributors=["test"], + tasks=[TaskMetadata(name="minimal_task", dataset_samples=42)], + external_assets=[MOCK_EXTERNAL_ASSET], + version="1-A", + ) + + result = generate_basic_readme(listing) + + expected = [ + "# Minimal Eval", + "", + "TODO: Add one or two paragraphs about your evaluation. Everything between tags is written automatically based on the information in eval.yaml. Make sure to setup your eval in eval.yaml correctly and then place your custom README text outside of these tags to prevent it from being overwritten.", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "## Dataset", + "", + "TODO: Briefly describe the dataset and include an example if helpful.", + "", + "## Scoring", + "", + "TODO: Explain how the evaluation is scored and any metrics reported.", + "", + "### Evaluation Report", + "", + "TODO: The evaluation report. A brief summary of results for your evaluation implementation compared against a standard set of existing results. We use your evaluation report to help validate that your implementation has accurately replicated the design of your eval into the Inspect framework.", + "", + "### Changelog", + ] + + assert result == expected + + +class TestFormatParameter: + """Tests for _format_parameter function, especially default value removal.""" + + def test_format_parameter_removes_default_colon(self): + """Test that (default: X) is removed from description.""" + param = { + "name": "max_attempts", + "type_str": "int", + "description": "Maximum number of submission attempts (default: 1)", + "default": "1", + } + result = _format_parameter(param) + # Should not have duplicate default info + assert "(default: 1)" not in result.replace("`1`", "") + # Should have the auto-generated default at the end + assert ( + result + == "- `max_attempts` (int): Maximum number of submission attempts (default: `1`)" + ) + + def test_format_parameter_removes_defaults_to(self): + """Test that (defaults to X) is removed from description.""" + param = { + "name": "max_attempts", + "type_str": "int", + "description": "Maximum number of submission attempts (defaults to 1)", + "default": "1", + } + result = _format_parameter(param) + # Should not have the manual default info + assert "(defaults to 1)" not in result + # Should have the auto-generated default at the end + assert ( + result + == "- `max_attempts` (int): Maximum number of submission attempts (default: `1`)" + ) + + def test_format_parameter_removes_defaults_to_with_period(self): + """Test that '. Defaults to X.' is removed from description.""" + param = { + "name": "max_attempts", + "type_str": "int", + "description": "Maximum number of submission attempts. Defaults to 1.", + "default": "1", + } + result = _format_parameter(param) + # Should not have the manual default info + assert "Defaults to 1" not in result + # Should have the auto-generated default at the end and keep the period + assert ( + result + == "- `max_attempts` (int): Maximum number of submission attempts. (default: `1`)" + ) + + def test_format_parameter_removes_defaults_to_without_final_period(self): + """Test that '. Defaults to X' (no final period) is removed from description.""" + param = { + "name": "max_attempts", + "type_str": "int", + "description": "Maximum number of submission attempts. Defaults to 1", + "default": "1", + } + result = _format_parameter(param) + # Should not have the manual default info + assert "Defaults to 1" not in result + # Should have the auto-generated default at the end + assert ( + result + == "- `max_attempts` (int): Maximum number of submission attempts. (default: `1`)" + ) + + def test_format_parameter_case_insensitive(self): + """Test that default removal is case insensitive.""" + param = { + "name": "max_attempts", + "type_str": "int", + "description": "Maximum number of submission attempts (DEFAULT: 1)", + "default": "1", + } + result = _format_parameter(param) + # Should remove uppercase DEFAULT too + assert "(DEFAULT: 1)" not in result.replace("`1`", "") + assert ( + result + == "- `max_attempts` (int): Maximum number of submission attempts (default: `1`)" + ) + + def test_format_parameter_no_default_removal_when_no_default(self): + """Test that description is not modified when no default value.""" + param = { + "name": "solver", + "type_str": "Solver | None", + "description": "Optional solver to use for the task", + "default": None, + } + result = _format_parameter(param) + # Should keep description as-is + assert ( + result == "- `solver` (Solver | None): Optional solver to use for the task" + ) + + def test_format_parameter_without_description(self): + """Test formatting parameter without description.""" + param = { + "name": "kwargs", + "type_str": "Any", + "description": "", + "default": None, + } + result = _format_parameter(param) + assert result == "- `kwargs` (Any):" + + def test_format_parameter_without_type(self): + """Test formatting parameter without type annotation.""" + param = { + "name": "solver", + "type_str": None, + "description": "Optional solver to use", + "default": "None", + } + result = _format_parameter(param) + assert result == "- `solver`: Optional solver to use (default: `None`)" + + def test_format_parameter_real_gaia_case(self): + """Test with actual case from GAIA README.""" + param = { + "name": "max_attempts", + "type_str": "int", + "description": "Maximum number of submission attempts (defaults to 1). Only applies when using the default solver.", + "default": "1", + } + result = _format_parameter(param) + # Should remove (defaults to 1) but keep the rest + assert "(defaults to 1)" not in result + assert ( + result + == "- `max_attempts` (int): Maximum number of submission attempts. Only applies when using the default solver. (default: `1`)" + ) + + def test_format_parameter_real_agent_bench_case(self): + """Test with actual case from agent_bench README.""" + param = { + "name": "max_attempts", + "type_str": "int", + "description": "Maximum number of attempts allowed per sample. Defaults to 1.", + "default": "1", + } + result = _format_parameter(param) + # Should remove '. Defaults to 1.' + assert "Defaults to 1" not in result + assert ( + result + == "- `max_attempts` (int): Maximum number of attempts allowed per sample. (default: `1`)" + ) + +>>>>>>> /tmp/sync_theirs class TestParseDocstringParameters: def test_single_line(self) -> None: @@ -280,8 +708,385 @@ def test_gpqa_parameters(self) -> None: result = build_parameters_section(info) text = "\n".join(result) +<<<<<<< /tmp/sync_out assert "## Parameters" in text assert "`cot`" in text assert "`epochs`" in text assert "`high_level_domain`" in text assert "`subdomain`" in text +======= + def test_clean_simple_values(self): + """Test that simple values are unchanged.""" + assert _clean_default_value(42) == "42" + assert _clean_default_value("hello") == "'hello'" + assert _clean_default_value(True) == "True" + assert _clean_default_value(None) == "None" + assert _clean_default_value([1, 2, 3]) == "[1, 2, 3]" + + def test_clean_absolute_path_unix(self): + """Test cleaning Unix absolute paths containing inspect_evals.""" + # Test single path + value = "/Users/username/project/inspect_evals/src/inspect_evals/module/file.py" + result = _clean_default_value(value) + assert result == "'src/inspect_evals/module/file.py'" + assert "/Users/" not in result + + def test_clean_absolute_path_in_tuple(self): + """Test cleaning absolute paths in tuples (browse_comp case).""" + # Simulate the browse_comp default value + value = ( + "docker", + "/Users/iphan/Documents/Work/source/iphan/inspect_evals/src/inspect_evals/browse_comp/compose.yaml", + ) + result = _clean_default_value(value) + assert result == "('docker', 'src/inspect_evals/browse_comp/compose.yaml')" + assert "/Users/" not in result + assert "/Documents/" not in result + + def test_clean_multiple_paths_in_list(self): + """Test cleaning multiple absolute paths in a list.""" + value = [ + "/home/user/inspect_evals/src/inspect_evals/module1/file1.py", + "/home/user/inspect_evals/src/inspect_evals/module2/file2.py", + ] + result = _clean_default_value(value) + assert "src/inspect_evals/module1/file1.py" in result + assert "src/inspect_evals/module2/file2.py" in result + assert "/home/" not in result + + def test_clean_windows_path(self): + """Test cleaning Windows absolute paths.""" + value = "C:\\Users\\username\\project\\inspect_evals\\src\\inspect_evals\\module\\file.py" + result = _clean_default_value(value) + # Should clean the path + assert result == "'src/inspect_evals/module/file.py'" + assert "C:\\" not in result + + def test_no_cleaning_without_inspect_evals(self): + """Test that paths without inspect_evals are not modified.""" + value = "/Users/username/Documents/file.txt" + result = _clean_default_value(value) + # Should be unchanged since it doesn't contain inspect_evals + assert result == "'/Users/username/Documents/file.txt'" + + def test_clean_dict_with_paths(self): + """Test cleaning paths in dictionary values.""" + value = { + "path": "/opt/inspect_evals/src/inspect_evals/module/config.yaml", + "name": "test", + } + result = _clean_default_value(value) + assert "src/inspect_evals/module/config.yaml" in result + assert "/opt/" not in result + assert "'name': 'test'" in result + + def test_clean_nested_structure(self): + """Test cleaning paths in nested structures.""" + value = [ + ("docker", "/var/lib/inspect_evals/src/inspect_evals/task/compose.yaml"), + ("k8s", "/var/lib/inspect_evals/src/inspect_evals/task/k8s.yaml"), + ] + result = _clean_default_value(value) + assert "src/inspect_evals/task/compose.yaml" in result + assert "src/inspect_evals/task/k8s.yaml" in result + assert "/var/lib/" not in result + + def test_clean_repo_root_path_in_tuple(self): + """Test cleaning repo-root absolute paths to repo-relative paths.""" + repo_root = Path(__file__).resolve().parent.parent + value = ("docker", str(repo_root / "src/inspect_evals/gaia/compose.yaml")) + result = _clean_default_value(value) + assert result == "('docker', 'src/inspect_evals/gaia/compose.yaml')" + + def test_clean_repo_root_path_object(self): + """Test cleaning repo-root Path objects to repo-relative paths.""" + repo_root = Path(__file__).resolve().parent.parent + value = repo_root / "src/inspect_evals/mind2web_sc/data/seeact" + result = _clean_default_value(value) + assert result == "Path('src/inspect_evals/mind2web_sc/data/seeact')" + + +def _external_eval_with_report(report: EvaluationReport | None) -> ExternalEvalMetadata: + return ExternalEvalMetadata( + id="my_eval", + title="My Eval", + description="Test", + contributors=["someone"], + tasks=[TaskMetadata(name="my_task", task_path="src/my_eval/task.py")], + source=ExternalEvalSource( + repository_url="https://github.com/owner/repo", # type: ignore[arg-type] + repository_commit="a" * 40, + ), + evaluation_report=report, + ) + + +class TestBuildEvaluationReportSection: + """Tests for build_evaluation_report_section function.""" + + def test_returns_empty_when_no_report(self): + eval_meta = _external_eval_with_report(None) + assert build_evaluation_report_section(eval_meta) == [] + + def test_returns_empty_when_no_results(self): + # An EvaluationReport must have results, so this is constructed via + # bypassing validation to guard the renderer's empty-results branch. + report = EvaluationReport.model_construct( + results=[], commit="b" * 40, timestamp=None, notes=None + ) + eval_meta = _external_eval_with_report(report) + assert build_evaluation_report_section(eval_meta) == [] + + def test_renders_canonical_skill_output(self): + report = EvaluationReport( + timestamp="July 2025", + commit="b" * 40, + results=[ + _row( + "openai/gpt-4o-mini", + provider="OpenAI", + metrics={"accuracy": 0.766, "stderr": 0.007}, + time="18s", + ), + _row( + "anthropic/claude-3-7-sonnet-20250219", + provider="Anthropic", + metrics={"accuracy": 0.833, "stderr": 0.038}, + time="6s", + ), + ], + notes=["All providers completed successfully."], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + text = "\n".join(result) + commit_link = f"[`bbbbbbb`](https://github.com/owner/repo/tree/{'b' * 40})" + assert text == ( + "## Evaluation Report\n" + "\n" + "**Timestamp:** July 2025\n" + "\n" + f"**Commit:** {commit_link}\n" + "\n" + "| Model | Provider | Accuracy | Stderr | Time |\n" + "| ------------------------------------ | --------- | -------- | ------ | ---- |\n" + "| openai/gpt-4o-mini | OpenAI | 0.766 | 0.007 | 18s |\n" + "| anthropic/claude-3-7-sonnet-20250219 | Anthropic | 0.833 | 0.038 | 6s |\n" + "\n" + "**Notes:**\n" + "\n" + "- All providers completed successfully." + ) + + def test_renders_command_block_when_set(self): + report = EvaluationReport( + commit="b" * 40, + command="uv run inspect eval src/my_eval/task.py@my_task --limit 100", + results=[_row("m", accuracy=0.5)], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + # Fenced bash block opens, contains the command verbatim, then closes. + assert "```bash" in result + assert "uv run inspect eval src/my_eval/task.py@my_task --limit 100" in result + assert "```" in result + + def test_omits_command_block_when_unset(self): + report = EvaluationReport( + commit="b" * 40, + results=[_row("m", accuracy=0.5)], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + assert "```bash" not in result + + def test_renders_commit_link_against_repository_url(self): + report = EvaluationReport( + commit="c" * 40, + results=[_row("m", accuracy=0.5)], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + commit_line = next(line for line in result if line.startswith("**Commit:**")) + # Short SHA in the label, full SHA in the URL. + assert "[`ccccccc`]" in commit_line + assert f"https://github.com/owner/repo/tree/{'c' * 40}" in commit_line + + def test_renders_version_under_commit_when_set(self): + report = EvaluationReport( + commit="b" * 40, + version="1.2.0", + results=[_row("m", accuracy=0.5)], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + version_line = next( + (line for line in result if line.startswith("**Version:**")), None + ) + assert version_line == "**Version:** 1.2.0" + + def test_omits_version_when_unset(self): + report = EvaluationReport( + commit="b" * 40, + results=[_row("m", accuracy=0.5)], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + assert not any(line.startswith("**Version:**") for line in result) + + def test_omits_timestamp_when_unset(self): + report = EvaluationReport( + commit="b" * 40, + results=[_row("m", accuracy=0.5)], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + assert not any(line.startswith("**Timestamp:**") for line in result) + + def test_omits_notes_when_unset(self): + report = EvaluationReport( + commit="b" * 40, + results=[_row("m", accuracy=0.5)], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + assert "**Notes:**" not in result + + def test_omits_unset_standard_columns(self): + # Only `model` and one accuracy metric are populated; provider/time/date + # columns should not appear. + report = EvaluationReport( + commit="b" * 40, + results=[_row("m", accuracy=0.5)], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + header = next(line for line in result if line.startswith("| Model")) + # "Model" header is 5 chars wide; "Accuracy" is 8 — both fit their cells. + assert header == "| Model | Accuracy |" + + def test_metric_columns_appended_in_first_seen_order(self): + # First-seen metric keys across rows determine column order; AgentDojo-style + # custom metric names work without schema changes. + report = EvaluationReport( + commit="b" * 40, + results=[ + _row( + "anthropic/claude-3-7-sonnet-20250219", + metrics={"benign_utility": 0.833, "targeted_asr": 0.085}, + ), + _row( + "openai/gpt-4o-2024-05-13", + metrics={"benign_utility": 0.688, "targeted_asr": 0.314}, + ), + ], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + header = next(line for line in result if line.startswith("| Model")) + # First-seen metric order preserved. + assert "Benign Utility" in header + assert "Targeted Asr" in header + assert header.index("Benign Utility") < header.index("Targeted Asr") + + def test_metric_columns_sit_between_descriptors_and_run_info(self): + # When provider AND time are populated, metric columns go in between. + report = EvaluationReport( + commit="b" * 40, + results=[ + _row("m", provider="OpenAI", metrics={"accuracy": 0.5}, time="18s"), + ], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + header = next(line for line in result if line.startswith("| Model")) + assert ( + header.index("Provider") < header.index("Accuracy") < header.index("Time") + ) + + def test_floats_formatted_to_three_decimals(self): + report = EvaluationReport( + commit="b" * 40, + results=[_row("m", metrics={"accuracy": 0.6, "stderr": 0.245})], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + row = next(line for line in result if line.startswith("| m ")) + # Cells are padded to header widths: Model=5, Accuracy=8, Stderr=6. + assert row == "| m | 0.600 | 0.245 |" + + def test_missing_metric_cells_render_empty(self): + # Mixed rows: one has stderr metric, the other doesn't. The stderr + # column should still appear, with an empty cell for the row that + # omits it. + report = EvaluationReport( + commit="b" * 40, + results=[ + _row("a", metrics={"accuracy": 0.5, "stderr": 0.1}), + _row("b", metrics={"accuracy": 0.6}), + ], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + rows = [ + line + for line in result + if line.startswith("| a ") or line.startswith("| b ") + ] + # Cells padded to column widths; the missing-stderr cell is all spaces. + assert rows[0] == "| a | 0.500 | 0.100 |" + assert rows[1] == "| b | 0.600 | |" + + def test_single_table_when_no_task_labels(self): + # All rows omit `task` — render as a single table without task headings. + report = EvaluationReport( + commit="b" * 40, + results=[_row("a", accuracy=0.5), _row("b", accuracy=0.6)], + ) + result = build_evaluation_report_section(_external_eval_with_report(report)) + assert not any(line.startswith("### ") for line in result) + # Exactly one header row. + assert sum(1 for line in result if line.startswith("| Model")) == 1 + + def test_groups_per_task_when_task_labels_present(self): + report = EvaluationReport( + commit="b" * 40, + results=[ + _row("m1", task="task_a", accuracy=0.5), + _row("m1", task="task_b", accuracy=0.7), + _row("m2", task="task_a", accuracy=0.4), + ], + ) + eval_meta = ExternalEvalMetadata( + id="my_eval", + title="My Eval", + description="Test", + contributors=["someone"], + tasks=[ + TaskMetadata(name="task_a", task_path="src/x.py"), + TaskMetadata(name="task_b", task_path="src/x.py"), + ], + source=ExternalEvalSource( + repository_url="https://github.com/owner/repo", + repository_commit="a" * 40, + ), + evaluation_report=report, + ) + result = build_evaluation_report_section(eval_meta) + headings = [line for line in result if line.startswith("### ")] + # First-seen order preserved: task_a then task_b. + assert headings == ["### task_a", "### task_b"] + # One table per group. + assert sum(1 for line in result if line.startswith("| Model")) == 2 + + def test_overall_heading_for_aggregate_row(self): + report = EvaluationReport( + commit="b" * 40, + results=[ + _row("m", task="task_a", accuracy=0.5), + _row("m", accuracy=0.6), + ], + ) + eval_meta = ExternalEvalMetadata( + id="my_eval", + title="My Eval", + description="Test", + contributors=["someone"], + tasks=[TaskMetadata(name="task_a", task_path="src/x.py")], + source=ExternalEvalSource( + repository_url="https://github.com/owner/repo", + repository_commit="a" * 40, + ), + evaluation_report=report, + ) + result = build_evaluation_report_section(eval_meta) + headings = [line for line in result if line.startswith("### ")] + assert headings == ["### task_a", "### Overall"] +>>>>>>> /tmp/sync_theirs diff --git a/tools/check_unlisted_evals.py b/tools/check_unlisted_evals.py index 28e297c..07bb146 100644 --- a/tools/check_unlisted_evals.py +++ b/tools/check_unlisted_evals.py @@ -19,6 +19,7 @@ import tomllib from pathlib import Path +<<<<<<< /tmp/sync_out REPO_ROOT = Path(__file__).resolve().parent.parent SRC_DIR = REPO_ROOT / "src" @@ -84,6 +85,93 @@ def main() -> int: print(f"All {len(eval_dirs)} evaluation(s) have eval.yaml and are registered.") return 0 +======= +NON_EVAL_DIRECTORIES: set[Path] = { + # Directories in src/inspect_evals that are not evaluations + Path("src") / "inspect_evals" / "gdm_capabilities", +} + + +def resolve_repo_root() -> Path: + # tools/ -> repo root is parent of this directory + return Path(__file__).resolve().parent.parent + + +def iter_readme_directories(search_root: Path) -> set[Path]: + if not search_root.is_dir(): + raise NotADirectoryError(f"Search root is not a directory: {search_root}") + + candidate_dirs: set[Path] = set() + repo_root = resolve_repo_root() + + for dirpath, _, filenames in os.walk(search_root): + # Identify README presence + has_readme = any(name.upper().startswith("README") for name in filenames) + + dir_path = Path(dirpath).resolve() + rel_path = dir_path.relative_to(repo_root) + + if has_readme and rel_path not in NON_EVAL_DIRECTORIES: + candidate_dirs.add(dir_path) + + return candidate_dirs + + +def path_is_within(child: Path, parent: Path) -> bool: + try: + child.relative_to(parent) + return True + except ValueError: + return False + + +def load_eval_yaml_paths(search_root: Path, register_root: Path) -> set[Path]: + """Find all directories that have an eval.yaml file.""" + covered: set[Path] = set() + for eval_yaml in search_root.glob("*/eval.yaml"): + covered.add(eval_yaml.parent.resolve()) + if register_root.is_dir(): + for eval_yaml in register_root.glob("*/eval.yaml"): + covered.add(eval_yaml.parent.resolve()) + return covered + + +def filter_uncovered(directories: set[Path], covered_paths: set[Path]) -> list[Path]: + uncovered: list[Path] = [] + for directory in directories: + covered = False + for covered_path in covered_paths: + if directory == covered_path or path_is_within(directory, covered_path): + covered = True + break + if not covered: + uncovered.append(directory) + uncovered.sort() + return uncovered + + +def main() -> None: + repo_root: Path = resolve_repo_root() + search_root: Path = (repo_root / "src" / "inspect_evals").resolve() + register_root: Path = (repo_root / "register").resolve() + + covered_paths = load_eval_yaml_paths(search_root, register_root) + readme_dirs = iter_readme_directories(search_root) + uncovered_dirs = filter_uncovered(readme_dirs, covered_paths) + + if uncovered_dirs: + print("Found directories with README but no eval.yaml:") + for directory in uncovered_dirs: + try: + output = directory.relative_to(repo_root) + except ValueError: + output = directory + print(str(output)) + sys.exit(1) + else: + print("All eval directories have eval.yaml files.") + sys.exit(0) +>>>>>>> /tmp/sync_theirs if __name__ == "__main__": diff --git a/tools/generate_readmes.py b/tools/generate_readmes.py index 47e185a..5377b17 100644 --- a/tools/generate_readmes.py +++ b/tools/generate_readmes.py @@ -26,7 +26,20 @@ from pathlib import Path from typing import Any +<<<<<<< /tmp/sync_out import yaml +======= +from pydantic import HttpUrl + +from inspect_evals.constants import INSPECT_EVALS_CACHE_PATH +from inspect_evals.metadata import ( + EvaluationReport, + EvaluationReportResult, + ExternalEvalMetadata, + InternalEvalMetadata, + load_listing, +) +>>>>>>> /tmp/sync_theirs log_level = os.getenv("LOG_LEVEL", "INFO") logging.basicConfig(level=getattr(logging, log_level, logging.INFO)) @@ -36,6 +49,22 @@ OPTIONS_KEY = "Options: Automatically Generated" USAGE_KEY = "Usage: Automatically Generated" PARAMETERS_KEY = "Parameters: Automatically Generated" +<<<<<<< /tmp/sync_out +======= +EXTERNAL_BANNER_KEY = "ExternalBanner: Automatically Generated" +DESCRIPTION_KEY = "Description: Automatically Generated" +INSPECT_DOCS_LINKS_KEY = "InspectDocsLinks: Automatically Generated" +EVALUATION_REPORT_KEY = "EvaluationReport: Automatically Generated" +GROUP_SORT_ORDER = ( + "Coding", + "Assistants", + "Cybersecurity", + "Safeguards", + "Mathematics", + "Reasoning", + "Knowledge", +) +>>>>>>> /tmp/sync_theirs # --------------------------------------------------------------------------- @@ -77,6 +106,7 @@ def _load_eval_info(yaml_path: Path) -> EvalInfo: TaskInfo(name=t["name"], dataset_samples=t.get("dataset_samples", 0)) for t in data.get("tasks", []) ] +<<<<<<< /tmp/sync_out return EvalInfo( title=data.get("title", eval_name), description=data.get("description", ""), @@ -90,6 +120,54 @@ def _load_eval_info(yaml_path: Path) -> EvalInfo: tags=data.get("tags", []), ) +======= + return links + + +def listing_md(listing: ExternalEvalMetadata | InternalEvalMetadata) -> str: + """Generate markdown for a single eval listing using a multiline template.""" + logger.debug("Form contributor links") + contributor_md = ", ".join(contributor_links(listing.contributors)) + if isinstance(listing, ExternalEvalMetadata): + maintainer_md = ", ".join(contributor_links(listing.source.maintainers)) + credits = ( + f"Maintained upstream by: {maintainer_md}" + f" • Listed in inspect_evals by: {contributor_md}" + ) + else: + credits = f"Contributed by: {contributor_md}" + contributors_md = f"{credits}" + + if isinstance(listing, ExternalEvalMetadata): + repo_name = _repo_name(listing.source.repository_url) + install_block = ( + f" git clone {listing.source.repository_url}\n" + f" cd {repo_name} && git checkout {listing.source.repository_commit}\n" + f" uv sync\n" + ) + tasks_block = install_block + "\n".join( + f" uv run inspect eval {task.task_path}@{task.name}" + for task in listing.tasks + ) + else: + tasks_block = "\n".join( + f" uv run inspect eval inspect_evals/{task.name}" for task in listing.tasks + ) + + # Indent each line of the description with 2 spaces for markdown list nesting + description_indented = "\n".join( + f" {line}" for line in listing.description.strip().split("\n") + ) + + title_link = link_md(listing.title.strip(), listing.path) + if isinstance(listing, ExternalEvalMetadata): + title_link = ( + "![external](https://img.shields.io/badge/external-orange) " + title_link + ) + + md_template = f"""\ +- ### {title_link} +>>>>>>> /tmp/sync_theirs def discover_evals(src_dir: Path, include_examples: bool = False) -> list[EvalInfo]: """Discover all evaluations by scanning src/*/eval.yaml.""" @@ -223,7 +301,292 @@ def build_usage_section(eval_info: EvalInfo) -> list[str]: f"uv run inspect eval {t} --model openai/gpt-5-nano" for t in formatted_tasks ) +<<<<<<< /tmp/sync_out python_commands = ", ".join(t.name for t in eval_info.tasks) +======= + +def build_external_banner_section(task_metadata: ExternalEvalMetadata) -> list[str]: + src = task_metadata.source + slug = _repo_slug(src.repository_url) + short_sha = src.repository_commit[:7] + commit_url = f"{str(src.repository_url).rstrip('/')}/tree/{src.repository_commit}" + contributors_md = ", ".join(contributor_links(task_metadata.contributors)) + return [ + "> ⚠️ **External evaluation.** Code lives in an upstream repository. inspect_evals lists it for discoverability; review the upstream repo and pinned commit before running.", + "", + f"**Source:** [`{slug}@{short_sha}`]({commit_url}) · Listed by {contributors_md}", + ] + + +def build_description_section( + task_metadata: ExternalEvalMetadata | InternalEvalMetadata, +) -> list[str]: + return task_metadata.description.strip().split("\n") + + +def build_inspect_docs_links_section( + _task_metadata: ExternalEvalMetadata | InternalEvalMetadata, +) -> list[str]: + return [ + "**More command-line options:** [Inspect docs ↗](https://inspect.aisi.org.uk/options.html)" + ] + + +_STANDARD_BEFORE_METRICS: tuple[str, ...] = ("model", "provider") +_STANDARD_AFTER_METRICS: tuple[str, ...] = ("time", "date") + + +def _humanize_field_name(name: str) -> str: + return " ".join(part.capitalize() for part in name.replace("-", "_").split("_")) + + +def _format_report_cell(value: Any) -> str: + if value is None: + return "" + if isinstance(value, bool): + return str(value) + if isinstance(value, float): + return f"{value:.3f}" + return str(value) + + +def _report_columns( + results: list[EvaluationReportResult], +) -> tuple[list[str], list[str], list[str]]: + """Return ``(before_metrics, metric_keys, after_metrics)`` columns. + + Static columns are included only when at least one row has them set + (``model`` is always included). ``metric_keys`` is the unique + ``metrics[*].key`` values across all rows, in first-seen order, and + sits between the descriptor columns (model/provider) and the + run-info columns (time/date). + """ + + def _present(field: str) -> bool: + return field == "model" or any(getattr(r, field) is not None for r in results) + + before = [f for f in _STANDARD_BEFORE_METRICS if _present(f)] + after = [f for f in _STANDARD_AFTER_METRICS if _present(f)] + metric_keys: list[str] = [] + seen: set[str] = set() + for r in results: + for m in r.metrics: + if m.key not in seen: + metric_keys.append(m.key) + seen.add(m.key) + return before, metric_keys, after + + +def _row_cell(result: EvaluationReportResult, column: str, *, is_metric: bool) -> Any: + if is_metric: + return next((m.value for m in result.metrics if m.key == column), None) + return getattr(result, column, None) + + +def _render_results_table(results: list[EvaluationReportResult]) -> list[str]: + before, metric_cols, after = _report_columns(results) + columns = before + metric_cols + after + is_metric = [False] * len(before) + [True] * len(metric_cols) + [False] * len(after) + headers = [_humanize_field_name(c) for c in columns] + rows = [ + [ + _format_report_cell(_row_cell(r, c, is_metric=m)) + for c, m in zip(columns, is_metric) + ] + for r in results + ] + widths = [ + max(len(headers[i]), 3, *(len(row[i]) for row in rows)) + for i in range(len(columns)) + ] + + def fmt_row(cells: list[str]) -> str: + return "| " + " | ".join(c.ljust(w) for c, w in zip(cells, widths)) + " |" + + return [ + fmt_row(headers), + "| " + " | ".join("-" * w for w in widths) + " |", + *(fmt_row(row) for row in rows), + ] + + +def _group_results_by_task( + results: list[EvaluationReportResult], +) -> dict[str | None, list[EvaluationReportResult]]: + groups: dict[str | None, list[EvaluationReportResult]] = {} + for r in results: + groups.setdefault(r.task, []).append(r) + return groups + + +def build_evaluation_report_section(task_metadata: ExternalEvalMetadata) -> list[str]: + """Render the evaluation report block from ``eval.yaml``. + + Returns an empty list when no report is set. + """ + report: EvaluationReport | None = task_metadata.evaluation_report + if report is None or not report.results: + return [] + + groups = _group_results_by_task(report.results) + grouped_render = not (len(groups) == 1 and None in groups) + + table_blocks: list[str] = [] + for task_name, rows in groups.items(): + if grouped_render: + heading = task_name if task_name is not None else "Overall" + table_blocks.append(f"### {heading}") + table_blocks.append("") + table_blocks.extend(_render_results_table(rows)) + table_blocks.append("") + while table_blocks and table_blocks[-1] == "": + table_blocks.pop() + + repo_url = str(task_metadata.source.repository_url).rstrip("/") + commit_url = f"{repo_url}/tree/{report.commit}" + commit_md = f"**Commit:** [`{report.commit[:7]}`]({commit_url})" + + parts: list[str] = ["## Evaluation Report", ""] + if report.timestamp: + parts.append(f"**Timestamp:** {report.timestamp}") + parts.append("") + parts.append(commit_md) + if report.version: + parts.append(f"**Version:** {report.version}") + parts.append("") + if report.command: + parts.append("```bash") + parts.append(report.command) + parts.append("```") + parts.append("") + parts.extend(table_blocks) + if report.notes: + parts.append("") + parts.append("**Notes:**") + parts.append("") + parts.extend(f"- {n}" for n in report.notes) + return parts + + +def _import_path_from_task_path(task_path: str) -> str: + """Derive a Python import path from a task file path (src-layout heuristic).""" + p = task_path.removesuffix(".py").lstrip("./") + p = p.removeprefix("src/") + return p.replace("/", ".") + + +def build_external_usage_section(task_metadata: ExternalEvalMetadata) -> list[str]: + src = task_metadata.source + repo_dir = _repo_name(src.repository_url) + slug = _repo_slug(src.repository_url) + first = task_metadata.tasks[0] + model = DEFAULT_EXAMPLE_MODEL + + cli_single = f"uv run inspect eval {first.task_path}@{first.name} --model {model}" + + import_path = _import_path_from_task_path(first.task_path or "") + py_eval = ( + "from inspect_ai import eval\n" + f"from {import_path} import {first.name}\n\n" + f'eval({first.name}(), model="{model}")' + ) + + rendered = f"""## Usage + +### Installation + +This is an externally-maintained evaluation. Clone the upstream repository at the pinned commit and install its dependencies: + +```bash +git clone {src.repository_url} +cd {repo_dir} +git checkout {src.repository_commit} +uv sync +``` + +### Running evaluations + +#### CLI + +```bash +{cli_single} +``` + +#### Python + +```python +{py_eval} +``` + +### View logs + +```bash +uv run inspect view +``` + +### More information + +For the dataset, scorer, task parameters, and validation, see the upstream repo: [{slug}]({src.repository_url}).""" + return rendered.split("\n") + + +def build_usage_section( + task_metadata: ExternalEvalMetadata | InternalEvalMetadata, +) -> list[str]: + if isinstance(task_metadata, ExternalEvalMetadata): + return build_external_usage_section(task_metadata) + + extra = task_metadata.dependency + dependency_group = task_metadata.dependency_group + + # Build pip install command + pip_install_cmd = ( + f"pip install inspect-evals[{extra}]" if extra else "pip install inspect-evals" + ) + + # Build uv sync command + if extra and dependency_group: + uv_sync_cmd = f"uv sync --extra {extra} --group {dependency_group}" + elif extra: + uv_sync_cmd = f"uv sync --extra {extra}" + elif dependency_group: + uv_sync_cmd = f"uv sync --group {dependency_group}" + else: + uv_sync_cmd = "uv sync" + + # Build dependency group note if needed + dependency_group_note = "" + if dependency_group: + dependency_group_note = ( + "\nNote the `--group` flag. The vast majority of our evals use " + "`uv sync` with `--extra`. This eval has a dependency on a git URL, " + "so it's managed differently.\n" + ) + + formatted_tasks = [f"inspect_evals/{task.name}" for task in task_metadata.tasks] + + py_import_path = ( + task_metadata.path[4:].replace("/", ".") + if task_metadata.path.startswith("src/") + else None + ) + + python_commands = ", ".join(t.name for t in task_metadata.tasks) + + bash_tasks = _bash_run_tasks(task_metadata, "inspect_evals") + + multi_msg = "" + maybe_eval_set = "" + maybe_eval_set_call = "" + + if len(formatted_tasks) > 1: + multi_msg = ( # TODO: fix spelling error in "simultaneously" + "\nTo run multiple tasks simulteneously use `inspect eval-set`:\n\n" + f"```bash\nuv run inspect eval-set {' '.join(formatted_tasks)}\n```\n" + ) + maybe_eval_set = ", eval_set" + maybe_eval_set_call = f"\neval_set([{python_commands}], log_dir='logs-run-42')" +>>>>>>> /tmp/sync_theirs template = textwrap.dedent("""\ ## Usage @@ -333,6 +696,7 @@ def _clean_type_string(type_str: str) -> str: # Remove inspect_ai internal module paths type_str = type_str.replace("inspect_ai.model._model.", "") type_str = type_str.replace("inspect_ai.solver._solver.", "") + type_str = type_str.replace("inspect_ai.agent._agent.", "") type_str = type_str.replace("inspect_ai.scorer._scorer.", "") type_str = type_str.replace("inspect_ai.util._sandbox.environment.", "") type_str = type_str.replace("inspect_ai.dataset._dataset.", "inspect_ai.dataset.") @@ -524,8 +888,34 @@ def build_parameters_section(eval_info: EvalInfo) -> list[str]: # --------------------------------------------------------------------------- +<<<<<<< /tmp/sync_out def generate_basic_readme(eval_info: EvalInfo) -> list[str]: """Generate basic README content for a new eval.""" +======= +def readme_dir(eval_metadata: ExternalEvalMetadata | InternalEvalMetadata) -> str: + """On-disk directory for an eval's README, relative to repo root.""" + if isinstance(eval_metadata, ExternalEvalMetadata): + return f"register/{eval_metadata.id}" + return eval_metadata.path + + +def generate_basic_readme( + listing: ExternalEvalMetadata | InternalEvalMetadata, +) -> list[str]: + """Generate basic README content for an eval. + + Args: + listing: Eval metadata from eval.yaml + + Returns: + List of lines for the minimal README + """ + logger.debug(f"Generating readme content for eval... {listing=}") + + if isinstance(listing, ExternalEvalMetadata): + return _generate_external_basic_readme(listing) + +>>>>>>> /tmp/sync_theirs template = textwrap.dedent(f"""\ # {eval_info.title} @@ -561,9 +951,77 @@ def generate_basic_readme(eval_info: EvalInfo) -> list[str]: return template.strip().split("\n") +<<<<<<< /tmp/sync_out def readme_exists(eval_info: EvalInfo) -> bool: readme_path = Path(__file__).parent.parent / eval_info.path / "README.md" return readme_path.exists() +======= +def _generate_external_basic_readme(listing: ExternalEvalMetadata) -> list[str]: + template = textwrap.dedent(f"""\ + # {listing.title} + + + + + + + + + + + + + + + + + + + """) + return template.strip().split("\n") + + +def generate_readme(create_missing_readmes: bool = False) -> None: + logger.debug("Generating Readme...") + # directory configuration + readme_path = Path(__file__).parent / "../README.md" + + # Load the listings using the Pydantic model + listing = load_listing() + + # Group evaluations by their group field + listing_groups: dict[str, list[ExternalEvalMetadata | InternalEvalMetadata]] = {} + for eval_metadata in listing.evals: + # place the listing in a group + if eval_metadata.group not in listing_groups: + listing_groups[eval_metadata.group] = [] + listing_groups[eval_metadata.group].append(eval_metadata) + + # sort the listings within each group by title and path + for group in listing_groups: + listing_groups[group] = sorted( + listing_groups[group], key=lambda x: (x.title, x.path) + ) + + # sort the groups by specified order + # Create a mapping of group name to sort index, with any unlisted groups going to the end + sort_index = {name: i for i, name in enumerate(GROUP_SORT_ORDER)} + listing_groups = dict( + sorted( + listing_groups.items(), + key=lambda x: sort_index.get(x[0], len(GROUP_SORT_ORDER)), + ) + ) + + # generate the markdown + content: list[str] = [] + for group, listings in listing_groups.items(): + content.append(f"## {group}") + content.append("") + for eval_metadata in listings: + content.append(listing_md(eval_metadata)) + content.append("") +>>>>>>> /tmp/sync_theirs # --------------------------------------------------------------------------- @@ -578,6 +1036,7 @@ def generate_readmes( ) -> None: """Generate/update README sections for all discovered evals. +<<<<<<< /tmp/sync_out Args: eval_filter: If set, only process this eval (package name, e.g. "my_eval" or "examples.gpqa"). @@ -609,6 +1068,45 @@ def generate_readmes( if not readme_exists(eval_info): print(" Skipping (no README.md)") +======= + if create_missing_readmes and not readme_exists(eval_readme_dir): + readme_path_eval = ( + Path(__file__).parent.parent / eval_readme_dir / "README.md" + ) + readme_path_eval.parent.mkdir(parents=True, exist_ok=True) + minimal_content = generate_basic_readme(eval_metadata) + with open(readme_path_eval, "w", encoding="utf-8") as readme_file: + readme_file.write("\n".join(minimal_content)) + logger.info(f"Created README for {eval_readme_dir}") + + if isinstance(eval_metadata, ExternalEvalMetadata): + rewrite_task_readme( + eval_readme_dir, + EXTERNAL_BANNER_KEY, + build_external_banner_section(eval_metadata), + ) + rewrite_task_readme( + eval_readme_dir, + DESCRIPTION_KEY, + build_description_section(eval_metadata), + ) + rewrite_task_readme( + eval_readme_dir, USAGE_KEY, build_usage_section(eval_metadata) + ) + rewrite_task_readme( + eval_readme_dir, OPTIONS_KEY, build_options_section(eval_metadata) + ) + rewrite_task_readme( + eval_readme_dir, + INSPECT_DOCS_LINKS_KEY, + build_inspect_docs_links_section(eval_metadata), + ) + rewrite_task_readme( + eval_readme_dir, + EVALUATION_REPORT_KEY, + build_evaluation_report_section(eval_metadata), + ) +>>>>>>> /tmp/sync_theirs continue rewrite_task_readme(