From 1efdabe2f4da19da482b3e6682e9e9d531889174 Mon Sep 17 00:00:00 2001 From: Jason Adams Date: Thu, 22 Oct 2026 20:26:19 -0600 Subject: [PATCH 1/7] feat: define one public benchmark release contract --- .../skills/wp-bench-execution-tests/SKILL.md | 2 +- .github/workflows/ci.yml | 11 +- AGENTS.md | 6 + README.md | 33 +- benchmark/README.md | 151 ++ benchmark/definition.json | 125 ++ benchmark/releases.json | 11 + benchmark/releases/v1.json | 1723 +++++++++++++++++ datasets/README.md | 16 +- .../execution/cli-content.json | 14 +- .../execution/cli-maintenance.json | 26 +- .../execution/cli-migration.json | 10 +- .../execution/cli-options.json | 18 +- .../execution/cli-reporting.json | 14 +- .../execution/cli-taxonomy-meta.json | 18 +- .../execution/cli-users.json | 22 +- .../execution/abilities-api.json | 13 +- .../execution/admin.json | 3 +- .../execution/ai-client.json | 20 +- .../execution/auth.json | 3 +- .../execution/block-patterns-styles.json | 3 +- .../execution/caching.json | 3 +- .../execution/comments-users.json | 3 +- .../execution/connectors-api.json | 7 +- .../execution/cron.json | 3 +- .../execution/database.json | 3 +- .../execution/dates-time.json | 3 +- .../execution/gb-block-api.json | 3 +- .../execution/gb-block-bindings.json | 3 +- .../execution/gb-block-editor.json | 3 +- .../execution/gb-block-hooks.json | 3 +- .../execution/gb-block-markup.json | 3 +- .../execution/gb-font-library.json | 3 +- .../execution/gb-interactivity-api.json | 3 +- .../execution/gb-templates-navigation.json | 3 +- .../execution/hooks.json | 3 +- .../execution/html-api.json | 3 +- .../execution/http-api.json | 3 +- .../execution/internationalization.json | 3 +- .../execution/mail-url.json | 3 +- .../execution/media.json | 3 +- .../execution/performance.json | 3 +- .../execution/plugin-lifecycle.json | 3 +- .../execution/post-meta.json | 3 +- .../execution/post-types-taxonomy.json | 3 +- .../execution/posts-lifecycle.json | 3 +- .../execution/queries.json | 3 +- .../execution/rest-api.json | 3 +- .../execution/rewrite-permalinks.json | 3 +- .../execution/roles-caps.json | 3 +- .../execution/scripts-styles.json | 3 +- .../execution/security.json | 3 +- .../execution/settings-options.json | 3 +- .../execution/shortcodes.json | 3 +- .../execution/theme-templates.json | 3 +- python/tests/test_artifacts.py | 4 +- python/tests/test_benchmark_releases.py | 347 ++++ python/tests/test_config_behavior.py | 6 +- python/tests/test_continue_on_error.py | 4 +- python/tests/test_dataset_filtering.py | 6 +- python/tests/test_dataset_metadata.py | 14 +- python/tests/test_execution_dataset.py | 2 +- python/tests/test_execution_isolation.py | 6 +- python/tests/test_execution_prompt.py | 2 +- python/tests/test_exploit_audit.py | 4 +- python/tests/test_reference_solution_mode.py | 4 +- python/tests/test_result_schema.py | 12 +- python/tests/test_result_streaming.py | 4 +- python/tests/test_scoring.py | 2 +- python/tests/test_selection.py | 6 +- .../tests/test_shell_runtime_integration.py | 6 +- python/tests/test_skill_variants.py | 4 +- python/tests/test_usage_capture.py | 4 +- python/tests/test_wp_cli_shell.py | 42 +- python/wp_bench/benchmark.py | 298 +++ python/wp_bench/benchmark_cli.py | 185 ++ python/wp_bench/cli.py | 2 + python/wp_bench/config.py | 22 +- python/wp_bench/core.py | 23 + python/wp_bench/records.py | 5 +- wp-bench.example.yaml | 13 +- 81 files changed, 3134 insertions(+), 241 deletions(-) create mode 100644 benchmark/README.md create mode 100644 benchmark/definition.json create mode 100644 benchmark/releases.json create mode 100644 benchmark/releases/v1.json rename datasets/suites/{wp-cli-v1 => wp-cli}/execution/cli-content.json (95%) rename datasets/suites/{wp-cli-v1 => wp-cli}/execution/cli-maintenance.json (96%) rename datasets/suites/{wp-cli-v1 => wp-cli}/execution/cli-migration.json (97%) rename datasets/suites/{wp-cli-v1 => wp-cli}/execution/cli-options.json (95%) rename datasets/suites/{wp-cli-v1 => wp-cli}/execution/cli-reporting.json (96%) rename datasets/suites/{wp-cli-v1 => wp-cli}/execution/cli-taxonomy-meta.json (95%) rename datasets/suites/{wp-cli-v1 => wp-cli}/execution/cli-users.json (96%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/abilities-api.json (97%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/admin.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/ai-client.json (93%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/auth.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/block-patterns-styles.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/caching.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/comments-users.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/connectors-api.json (98%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/cron.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/database.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/dates-time.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/gb-block-api.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/gb-block-bindings.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/gb-block-editor.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/gb-block-hooks.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/gb-block-markup.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/gb-font-library.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/gb-interactivity-api.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/gb-templates-navigation.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/hooks.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/html-api.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/http-api.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/internationalization.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/mail-url.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/media.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/performance.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/plugin-lifecycle.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/post-meta.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/post-types-taxonomy.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/posts-lifecycle.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/queries.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/rest-api.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/rewrite-permalinks.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/roles-caps.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/scripts-styles.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/security.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/settings-options.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/shortcodes.json (99%) rename datasets/suites/{wp-core-v1 => wp-core}/execution/theme-templates.json (99%) create mode 100644 python/tests/test_benchmark_releases.py create mode 100644 python/wp_bench/benchmark.py create mode 100644 python/wp_bench/benchmark_cli.py diff --git a/.agents/skills/wp-bench-execution-tests/SKILL.md b/.agents/skills/wp-bench-execution-tests/SKILL.md index 35d9581..fc5544b 100644 --- a/.agents/skills/wp-bench-execution-tests/SKILL.md +++ b/.agents/skills/wp-bench-execution-tests/SKILL.md @@ -151,7 +151,7 @@ The default `run.execution_isolation: reset_per_test` gives every candidate a fr ## WP-CLI Workflow Tests -Use `wp-cli-v1` for realistic operational tasks that combine discovery, selection, +Use `wp-cli` for realistic operational tasks that combine discovery, selection, changes, reports, and preservation. Prioritize model capability over testing tiny WordPress units. This suite targets single-site WordPress 7.1 with SQLite; avoid unsupported MySQL operations, network installs, or multisite instead of diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3124cf..f815c77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-python@v5 with: @@ -33,6 +35,11 @@ jobs: - name: Type check (mypy) run: mypy python + - name: Benchmark release integrity + env: + BASE_REF: ${{ github.event.pull_request.base.sha || github.event.before }} + run: wp-bench benchmark check --base-ref "$BASE_REF" + - name: Unit tests (pytest) run: pytest python -q @@ -97,12 +104,12 @@ jobs: - name: WP-CLI workflow reference solutions run: >- - wp-bench run --config wp-bench.example.yaml --suite wp-cli-v1 + wp-bench run --config wp-bench.example.yaml --suite wp-cli --check-reference-solution --execution-concurrency 4 - name: WP-CLI workflow exploit audit run: >- - wp-bench run --config wp-bench.example.yaml --suite wp-cli-v1 + wp-bench run --config wp-bench.example.yaml --suite wp-cli --check-exploits --execution-concurrency 4 - name: SQLite database reference solutions diff --git a/AGENTS.md b/AGENTS.md index 66cf111..2b3250b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,3 +36,9 @@ This guide is the fastest path to contributing to WP-Bench. Keep changes minimal - Follow Conventional Commits (`feat:`, `refactor:`, etc.) as seen in git history. Scope prefixes are optional. - PRs should state intent, key commands run (`ruff`, `mypy`, `pytest`, runtime smoke test), and config changes. Link related issues. Include logs or screenshots for runtime/verifier output when relevant. - Keep changes small and focused; update README/AGENTS.md when workflows or flags change. Avoid touching `legacy/` unless explicitly required. + +## Benchmark Releases +- Public versions (`v1`, `v2`, …) identify the entire evaluation; `wp-core` and `wp-cli` are stable component IDs, never independent public versions. +- Read `benchmark/README.md` before changing the release protocol, category mapping, result provenance, or website export contract. +- Published `benchmark/releases/v*.json` files are immutable. Batch evaluation changes during development, then freeze the next release. Adding model results and changing presentation do not create benchmark versions. +- Run `wp-bench benchmark check` and `pytest python/tests/test_benchmark_releases.py -q` after changes to versioning. Dataset/runtime/harness changes intentionally produce unreleased results until a new release is frozen. diff --git a/README.md b/README.md index 1a0f32e..b164877 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,19 @@ The official WordPress AI benchmark. Evaluate how well language models understand WordPress development—from core APIs and coding standards to plugin architecture and security best practices. +## Public benchmark version + +One public version identifies the complete evaluation: **WP-Bench v1**. The +`wp-core` and `wp-cli` datasets are stable components of that release. Changes to +any dataset or score-affecting rules require the next benchmark version; adding +model results updates the same version. Release manifests, run provenance and +validated website exports are documented in [benchmark/README.md](benchmark/README.md). + +```sh +wp-bench benchmark show --version v1 +wp-bench benchmark check +``` + ## Overview WP-Bench measures AI model capabilities on **execution**: code generation tasks graded by static checks and runtime assertions in a real WordPress environment. @@ -148,7 +161,7 @@ Copy `wp-bench.example.yaml` and customize: ```yaml dataset: source: local # 'local' or 'huggingface' - name: wp-core-v1 # suite name + name: wp-core # suite name models: - name: gpt-4o @@ -160,7 +173,7 @@ grader: setup_timeout_seconds: 600 # hard cap for environment setup run: - suite: wp-core-v1 + suite: wp-core limit: 10 # limit tests (null = all); seeded stratified selection seed: 1337 # selection seed (same seed = same subset) test_ids: [] # optional explicit test IDs to run @@ -260,12 +273,12 @@ Test suites live in `datasets/suites//`: - `execution/` — Code generation tasks with assertions (one JSON file per category) -The default suite `wp-core-v1` contains 350 tasks covering WordPress core APIs, +The default suite `wp-core` contains 350 tasks covering WordPress core APIs, hooks, database operations, and security patterns. ### WP-CLI workflows -`wp-cli-v1` adds 27 operational tasks across content, taxonomy and metadata, +`wp-cli` adds 27 operational tasks across content, taxonomy and metadata, users and capabilities, options and transients, reporting, maintenance, and migration. Tasks combine selection, changes, and preservation: examples include retiring an author while reassigning content, consolidating categories, repairing a @@ -286,10 +299,10 @@ its research basis and source links in metadata. Prompts describe the desired WordPress outcome, and equivalent native command workflows are accepted. ```bash -wp-bench run --config wp-bench.example.yaml --suite wp-cli-v1 --execution-concurrency 4 +wp-bench run --config wp-bench.example.yaml --suite wp-cli --execution-concurrency 4 # Maintainer validation without model calls: -wp-bench run --config wp-bench.example.yaml --suite wp-cli-v1 --check-reference-solution -wp-bench run --config wp-bench.example.yaml --suite wp-cli-v1 --check-exploits +wp-bench run --config wp-bench.example.yaml --suite wp-cli --check-reference-solution +wp-bench run --config wp-bench.example.yaml --suite wp-cli --check-exploits ``` The model returns a **plain command or Bash script string**, optionally inside @@ -324,7 +337,7 @@ and repeat runs under `grader.raw.command`. Each output stream is capped at output is still recorded. Private container limits also apply, and timeouts or output overflow score zero. -The WP-CLI suite is scored separately from `wp-core-v1`; existing PHP task +The WP-CLI suite is scored separately from `wp-core`; existing PHP task contracts and scoring remain unchanged. Compare scores only within the same suite and execution profile. @@ -378,11 +391,11 @@ tasks retain specialized assertions for their distinct contracts. dataset: source: huggingface name: WordPress/wp-bench-v1 - suite: wp-core-v1 # filter a multi-suite export; defaults to run.suite + suite: wp-core # filter a multi-suite export; defaults to run.suite ``` An explicit `dataset.suite` selects the rows and their result-suite label; -`--suite` overrides both. `--suite wp-cli-v1` selects CLI rows without changing +`--suite` overrides both. `--suite wp-cli` selects CLI rows without changing the Hugging Face repository name. The new suite is available locally immediately; Hub users need an export that includes it. diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..2597810 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,151 @@ +# Public benchmark releases + +**WP-Bench v1** identifies one complete, frozen evaluation. `wp-core` and `wp-cli` +are stable component IDs. They have no independent public versions. The Python +package, result JSON schema, and scoring formula have internal version numbers; +those are reproducibility details, not additional leaderboard versions. + +`definition.json` is the working evaluation recipe. `releases.json` indexes the +append-only `releases/vN.json` manifests. A release freezes every dataset file, +complete test membership, runtime and relevant harness source, WordPress target, +scoring rules, broad category mapping, approved runtime images, generation +settings, and required repeats. The manifest fingerprint covers the entire +recipe. Changing any dataset changes the overall evaluation fingerprint. + +| Change | Release handling | +| --- | --- | +| Add/remove/edit a dataset task, including a corrected assertion | Next benchmark version | +| Change grading, task weighting, prompting, or the evaluation protocol | Next benchmark version | +| Change runtime behavior or the approved runtime images | Next benchmark version | +| Evaluate another model on the same recipe | Updated results under the same version | +| Re-run the same models under that recipe | Another results snapshot under the same version | +| Edit documentation, website layout, or result serialization | Same benchmark version | + +Source fingerprinting is deliberately conservative. Even a dataset formatting or +editorial-only edit changes its frozen file hash. Batch source changes during +development rather than publishing a version for every commit. A changed checkout +can run normally, but records `status: unreleased` and no public version. A new +release is an explicit maintainer action; it is never inferred from directory +names, a date, or a model list. + +## Running a release + +Inspect the complete protocol before spending on providers: + +```sh +wp-bench benchmark show --version v1 +wp-bench benchmark check +``` + +An explicit assertion in the run config refuses mismatching sources, loaded +local/Hub tasks, model settings, or runtime settings before model generation: + +```yaml +benchmark: + version: v1 + +dataset: + source: local + name: wp-core + +model: + name: YOUR_MODEL_ID + temperature: 1 + max_tokens: 16384 + +grader: + kind: docker + # Use one of runtime_image_ids from the frozen manifest, not an unverified tag. + image: sha256:6dcab391357137b2ce52d99ef15f342499b6b9f3287e5b96946036628a7dfb1e + +run: + suite: wp-core + execution_isolation: reset_per_test +``` + +Run both components and the required independent repeats. A subset remains +useful for diagnostics but cannot enter the overall leaderboard. With Hugging +Face, pin `dataset.revision` to the published export commit and select the stable +suite ID. Loaded task fingerprints must match the frozen release regardless of +transport. The historical Hub repository name `WordPress/wp-bench-v1` is an +address, not a separate dataset-version authority; publish the new stable suite +IDs there before using that export. + +Every completed payload carries `metadata.benchmark`; each streamed record also +carries the benchmark identity, task fingerprint and `run_id`. Multi-model +payloads retain a separate identity and run ID in each `models` entry. These +fields survive partial runs, but publication requires actual complete records. +A `full_suite` flag alone never authorizes a score. Source identity is captured +before generation, so an edit during a run cannot retroactively relabel it. + +## Publishing a new benchmark version + +1. Batch intended evaluation changes in `definition.json`, datasets, and code. +2. Run the normal Python/dataset checks and applicable reference/exploit audits. +3. Freeze the next version, then inspect the generated manifest: + + ```sh + wp-bench benchmark freeze v2 + wp-bench benchmark check --base-ref origin/trunk + ``` + +4. Commit the new manifest and catalog in the release PR. Existing manifests + cannot be edited, removed, or reassigned. CI compares their bytes with the base + commit, even if someone recomputes the embedded hashes. +5. After merging, tag the release commit `benchmark-v2` and retain its approved + runtime images. To reproduce an old version, check out its tag; selecting an + old version alone does not restore its source into a changed checkout. + +If a changed file does not affect evaluation behavior, review that deliberately. +The default fingerprint policy favors refusing an accidental comparison over +silently declaring two evaluations equivalent. No file change should be hidden +by modifying an old manifest. + +## Website data contract + +Use the harness exporter, rather than recreating release numbers in the website: + +```sh +wp-bench benchmark export --version v1 --output-dir public-data \ + --input core-model-a-repeat-1.json --input core-model-a-repeat-2.json \ + --input cli-model-a-repeat-1.json --input cli-model-a-repeat-2.json +``` + +Supply the complete set of models desired in the new snapshot. Single- and +multi-model result JSONs are supported. The exporter checks the release identity +on both envelopes and individual records, task fingerprints, exact suite +membership, approved runtime image, model/grader settings, baseline policy, +unique run IDs, and the required independent repeats. Reference solutions, +skills variants, errored/partial runs, duplicate repeats and mixed versions fail +before output is created. A legitimate failed model answer is still a scored +answer and remains in the denominator. + +The output is a small, stable ingestion contract, not a website theme: + +```text +public-data/ + results.json # mutable index, grouped by benchmark version + v1/.json # immutable scored-answer snapshots + v2/.json +``` + +Snapshot schema `1.0` contains `benchmark.version`, `benchmark.fingerprint`, the +release-owned broad category labels, model IDs, per-answer pass/usage data, run +IDs, input hashes, and an update date. It excludes generated code and raw provider +responses. Every answer has a stable component ID, test ID and broad category. +Aggregate pass rates weight every task equally and average the same number of +repeats per task. Use task-paired uncertainty estimates when comparing models. + +The website should expose **one benchmark-version selector** and an **updated +date**. Snapshot IDs and internal schemas belong in downloads/details. A model +addition creates a new immutable snapshot under v1; changing any dataset creates +v2 and its own leaderboard. Preserve v1 scores and files. Never fill missing v2 +rows with v1 scores. Cross-version charts must disclose test changes and cannot +establish model improvement by themselves. + +The first release includes the already-audited four core assertion corrections +and native read-only `wp db columns` allowance behind the initial public scores. +Renaming component IDs and removing per-file versions changes no prompts or +behavioral checks beyond those recorded corrections. Historical files without +the new identity must be explicitly audited/imported for the first release; +the exporter never guesses that an old `*-v1` suite name proves compatibility. diff --git a/benchmark/definition.json b/benchmark/definition.json new file mode 100644 index 0000000..3c17323 --- /dev/null +++ b/benchmark/definition.json @@ -0,0 +1,125 @@ +{ + "schema_version": "1.0", + "wordpress_version": "7.1", + "scoring_version": "3.0", + "suites": [ + "wp-core", + "wp-cli" + ], + "categories": [ + { + "id": "core", + "label": "Core development", + "description": "Hooks, content, queries, settings, admin, internationalization, dates, plugins, and shortcodes." + }, + { + "id": "blocks", + "label": "Block editor", + "description": "Blocks, patterns, bindings, interactivity, editor customization, and block markup." + }, + { + "id": "themes-media", + "label": "Themes & media", + "description": "Theme templates, scripts, styles, and media handling." + }, + { + "id": "apis", + "label": "APIs & integrations", + "description": "REST endpoints, HTTP, email, URLs, and scheduled tasks." + }, + { + "id": "security", + "label": "Security & access", + "description": "Authentication, permissions, roles, capabilities, and safe handling of input and output." + }, + { + "id": "modern", + "label": "New WordPress APIs", + "description": "Abilities, the AI Client, and Connectors APIs in the WordPress 7.1 runtime." + }, + { + "id": "performance", + "label": "Data & performance", + "description": "Database access, caching, and performance tasks." + }, + { + "id": "wp-cli", + "label": "WP-CLI", + "description": "Content, users, options, taxonomy, metadata, reporting, migration, and maintenance with WP-CLI." + } + ], + "category_mapping": { + "wp-core": { + "abilities-api": "modern", + "admin": "core", + "ai-client": "modern", + "auth": "security", + "block-patterns-styles": "blocks", + "caching": "performance", + "comments-users": "core", + "connectors-api": "modern", + "cron": "apis", + "database": "performance", + "dates-time": "core", + "gb-block-api": "blocks", + "gb-block-bindings": "blocks", + "gb-block-editor": "blocks", + "gb-block-hooks": "blocks", + "gb-block-markup": "blocks", + "gb-font-library": "blocks", + "gb-interactivity-api": "blocks", + "gb-templates-navigation": "blocks", + "hooks": "core", + "html-api": "core", + "http-api": "apis", + "internationalization": "core", + "mail-url": "apis", + "media": "themes-media", + "performance": "performance", + "plugin-lifecycle": "core", + "post-meta": "core", + "post-types-taxonomy": "core", + "posts-lifecycle": "core", + "queries": "core", + "rest-api": "apis", + "rewrite-permalinks": "core", + "roles-caps": "security", + "scripts-styles": "themes-media", + "security": "security", + "settings-options": "core", + "shortcodes": "core", + "theme-templates": "themes-media" + }, + "wp-cli": { + "cli-content": "wp-cli", + "cli-maintenance": "wp-cli", + "cli-migration": "wp-cli", + "cli-options": "wp-cli", + "cli-reporting": "wp-cli", + "cli-taxonomy-meta": "wp-cli", + "cli-users": "wp-cli" + } + }, + "aggregation": "strict pass rate; every task weighted equally; average independent repeats per task", + "protocol": { + "model": { + "temperature": 1.0, + "top_p": null, + "max_tokens": 16384 + }, + "grader": { + "kind": "docker", + "timeout_seconds": 90, + "container_cpus": 1.0, + "container_memory_mb": 512, + "container_filesystem_mb": 256, + "container_pids_limit": 64 + }, + "repeats": 2, + "reasoning_effort": "provider default", + "answer_policy": "first response; baseline without skills; transient provider errors retried; no scored-answer regeneration" + }, + "runtime_image_ids": [ + "sha256:6dcab391357137b2ce52d99ef15f342499b6b9f3287e5b96946036628a7dfb1e" + ] +} diff --git a/benchmark/releases.json b/benchmark/releases.json new file mode 100644 index 0000000..d546de3 --- /dev/null +++ b/benchmark/releases.json @@ -0,0 +1,11 @@ +{ + "schema_version": "1.0", + "releases": [ + { + "version": "v1", + "path": "releases/v1.json", + "fingerprint": "83e81aa6ebc36f6516d0b38655f4818d556039a3507c03a029aef868c4171220" + } + ], + "current": "v1" +} diff --git a/benchmark/releases/v1.json b/benchmark/releases/v1.json new file mode 100644 index 0000000..47283aa --- /dev/null +++ b/benchmark/releases/v1.json @@ -0,0 +1,1723 @@ +{ + "schema_version": "1.0", + "version": "v1", + "fingerprint": "83e81aa6ebc36f6516d0b38655f4818d556039a3507c03a029aef868c4171220", + "evaluation": { + "definition": { + "schema_version": "1.0", + "wordpress_version": "7.1", + "scoring_version": "3.0", + "suites": [ + "wp-core", + "wp-cli" + ], + "categories": [ + { + "id": "core", + "label": "Core development", + "description": "Hooks, content, queries, settings, admin, internationalization, dates, plugins, and shortcodes." + }, + { + "id": "blocks", + "label": "Block editor", + "description": "Blocks, patterns, bindings, interactivity, editor customization, and block markup." + }, + { + "id": "themes-media", + "label": "Themes & media", + "description": "Theme templates, scripts, styles, and media handling." + }, + { + "id": "apis", + "label": "APIs & integrations", + "description": "REST endpoints, HTTP, email, URLs, and scheduled tasks." + }, + { + "id": "security", + "label": "Security & access", + "description": "Authentication, permissions, roles, capabilities, and safe handling of input and output." + }, + { + "id": "modern", + "label": "New WordPress APIs", + "description": "Abilities, the AI Client, and Connectors APIs in the WordPress 7.1 runtime." + }, + { + "id": "performance", + "label": "Data & performance", + "description": "Database access, caching, and performance tasks." + }, + { + "id": "wp-cli", + "label": "WP-CLI", + "description": "Content, users, options, taxonomy, metadata, reporting, migration, and maintenance with WP-CLI." + } + ], + "category_mapping": { + "wp-core": { + "abilities-api": "modern", + "admin": "core", + "ai-client": "modern", + "auth": "security", + "block-patterns-styles": "blocks", + "caching": "performance", + "comments-users": "core", + "connectors-api": "modern", + "cron": "apis", + "database": "performance", + "dates-time": "core", + "gb-block-api": "blocks", + "gb-block-bindings": "blocks", + "gb-block-editor": "blocks", + "gb-block-hooks": "blocks", + "gb-block-markup": "blocks", + "gb-font-library": "blocks", + "gb-interactivity-api": "blocks", + "gb-templates-navigation": "blocks", + "hooks": "core", + "html-api": "core", + "http-api": "apis", + "internationalization": "core", + "mail-url": "apis", + "media": "themes-media", + "performance": "performance", + "plugin-lifecycle": "core", + "post-meta": "core", + "post-types-taxonomy": "core", + "posts-lifecycle": "core", + "queries": "core", + "rest-api": "apis", + "rewrite-permalinks": "core", + "roles-caps": "security", + "scripts-styles": "themes-media", + "security": "security", + "settings-options": "core", + "shortcodes": "core", + "theme-templates": "themes-media" + }, + "wp-cli": { + "cli-content": "wp-cli", + "cli-maintenance": "wp-cli", + "cli-migration": "wp-cli", + "cli-options": "wp-cli", + "cli-reporting": "wp-cli", + "cli-taxonomy-meta": "wp-cli", + "cli-users": "wp-cli" + } + }, + "aggregation": "strict pass rate; every task weighted equally; average independent repeats per task", + "protocol": { + "model": { + "temperature": 1.0, + "top_p": null, + "max_tokens": 16384 + }, + "grader": { + "kind": "docker", + "timeout_seconds": 90, + "container_cpus": 1.0, + "container_memory_mb": 512, + "container_filesystem_mb": 256, + "container_pids_limit": 64 + }, + "repeats": 2, + "reasoning_effort": "provider default", + "answer_policy": "first response; baseline without skills; transient provider errors retried; no scored-answer regeneration" + }, + "runtime_image_ids": [ + "sha256:6dcab391357137b2ce52d99ef15f342499b6b9f3287e5b96946036628a7dfb1e" + ] + }, + "files": { + "datasets/suites/wp-core/execution/abilities-api.json": "c449748609ef87f348eb2a57b1aaa1f9f52dae7d0e9e03e62a832cceccbe6708", + "datasets/suites/wp-core/execution/admin.json": "e494e305051950d5feb2e6a573685548ac105cfaf382cb84cef43151384ff759", + "datasets/suites/wp-core/execution/ai-client.json": "0f78a74c65ee3ab14d90f1841bb1ce6561b2dbf241dba5493399928b49593bf0", + "datasets/suites/wp-core/execution/auth.json": "fa876f3321aa30eac364c577ca60929851cde80e50dab5a505f909f64520539d", + "datasets/suites/wp-core/execution/block-patterns-styles.json": "a8ab2f6ab6f00d0e5abb8ad93bceafe604c6785d1fe81169a60417e28d74f5fc", + "datasets/suites/wp-core/execution/caching.json": "977ef8f4ce85c411b545acc42f1df774700668ca34e546fa66f0947e446f9777", + "datasets/suites/wp-core/execution/comments-users.json": "8f95820da861c60f49246bb49470edf12fa2adc3c9c2ebf4dbbee922aac3ced6", + "datasets/suites/wp-core/execution/connectors-api.json": "00244363a5ca965d2e8fa41e26e846f7d81f96636b2911a5c366f7d689019d83", + "datasets/suites/wp-core/execution/cron.json": "d80ee06220559f75e3c06cc960a540c37130bb6e128346e0e938a4b8470ff877", + "datasets/suites/wp-core/execution/database.json": "ccc32e70707ab93296d7759907a7eb824fbf60d6dc102692f66b797bd7c09c30", + "datasets/suites/wp-core/execution/dates-time.json": "fd4d2c7bebe9c746de66a50105422533bf0d21b084e967f73ce336395ca78f36", + "datasets/suites/wp-core/execution/gb-block-api.json": "1bd71b882a108baf44cba039782f8b993bcf168eede297e24b12f776c88f1626", + "datasets/suites/wp-core/execution/gb-block-bindings.json": "2cc4011318e56b543a0d9f65979b5fd709f13756a948bc7ea463b1168d65e57e", + "datasets/suites/wp-core/execution/gb-block-editor.json": "87d732f2693bb7beeeacff7bfc901bfc24e8acf827709eb4fc7fbabcd945ab73", + "datasets/suites/wp-core/execution/gb-block-hooks.json": "69daec04a64ccfad6911c9160c7b2f20ae6103d386fd42bf6d93c8f6edc8b43c", + "datasets/suites/wp-core/execution/gb-block-markup.json": "d49e2b15e2159b2f6e5e39784fc503dbaf6f1de909ae7f963c69dde77176dcb3", + "datasets/suites/wp-core/execution/gb-font-library.json": "9fbb0f37c11e7362bcd1670c10a7f49e24899525704d8641c327cecdf7bb624f", + "datasets/suites/wp-core/execution/gb-interactivity-api.json": "0c7e24c0c099ee545e8722a4d886a7b164983e53091c97071b5bfd32f273baa1", + "datasets/suites/wp-core/execution/gb-templates-navigation.json": "a044bc7506b1c5f7ae69f659435a995b3092aa6a2c1d806d015930bae3c6a965", + "datasets/suites/wp-core/execution/hooks.json": "f2e431af3a8c0b753f9e6cc93ab83e3d17264044f4ea44ec7b332578868e053a", + "datasets/suites/wp-core/execution/html-api.json": "4948bf9fbbbef6c7c486ddaef4668e22bcddb6aec778ad879c0afdd3991b7dfb", + "datasets/suites/wp-core/execution/http-api.json": "37a09fbb73837d456aa1942c6b3dd0b59a8c68daddca4dde14cc4d43e80308ca", + "datasets/suites/wp-core/execution/internationalization.json": "9f4cc060f6981504b3e9b7bacc53b7be9254ae9062635c1774c9b896db8c79ab", + "datasets/suites/wp-core/execution/mail-url.json": "2bae7cbbb3b794cac2d265ef61859e18aea06139b3efcd1c6238946a698514de", + "datasets/suites/wp-core/execution/media.json": "2427ab4935ba3a210c9724d07e4e54f796adc62ef016bf1039d408db6deca990", + "datasets/suites/wp-core/execution/performance.json": "b105c8e25adc825039d166df85fb9f433aeb28551e8624d441b830e67c3b51e5", + "datasets/suites/wp-core/execution/plugin-lifecycle.json": "98bd5bd0abd8b2944066b70c0f18d7a26ac7ae4bc457672360c3bd1b217d284a", + "datasets/suites/wp-core/execution/post-meta.json": "2b483afe5921f024bf753978ef013a9cf27591391ce3b5f81d929e4bdbd1cd44", + "datasets/suites/wp-core/execution/post-types-taxonomy.json": "ef207f34592bbf0780f575a815d44f92033f7a02b5861f43b40d2aaec9a0e98b", + "datasets/suites/wp-core/execution/posts-lifecycle.json": "03a5998f33d4b769b775c515ba56d3cd2fe9f861862eede8b43ad94ee7552561", + "datasets/suites/wp-core/execution/queries.json": "95f0a93e048d8d591c82aefaad0e7b12484be0c37eb3d7dc578c89adfb8f4d9b", + "datasets/suites/wp-core/execution/rest-api.json": "a948d0807fbdf3f45e3642531b1e732e4511d8d318362026dbd11b4ef3f440e2", + "datasets/suites/wp-core/execution/rewrite-permalinks.json": "7e41feda35a88954f107f47b0f94775d158e5b8ab9428af8bb17ee702549fa25", + "datasets/suites/wp-core/execution/roles-caps.json": "309323bfbe2001bc31e6e316d59e63024f562916dbad57d070d78af0b9739d18", + "datasets/suites/wp-core/execution/scripts-styles.json": "5f73409944d39813f4208de4b5c118ce6e6aa9580fb384cb0d8c15de7c6816f1", + "datasets/suites/wp-core/execution/security.json": "6ac099254eec381715b60c799ebc691e7ac02c0f9da496bc1788bc720972f66c", + "datasets/suites/wp-core/execution/settings-options.json": "9094bb1ff9e0570c878ff97bfe312dbd6dff19f06745f453bcb27a08056b7d2a", + "datasets/suites/wp-core/execution/shortcodes.json": "75601615067205696d0ac2feb59170ea73cc0d7b0aadffed6547254670f39e94", + "datasets/suites/wp-core/execution/theme-templates.json": "fc6b591a3b0b69b714645b04c50b29d2c533fc26c3a494eebe77b49269b7208e", + "datasets/suites/wp-cli/execution/cli-content.json": "e965ac79c813024a576d12a92f1be9d713bfc186b06e26518328a3df78308568", + "datasets/suites/wp-cli/execution/cli-maintenance.json": "873dfb05e7fe86df5d77b7a88408721fa223bf3aab97f3a9a4428319f1fabe1f", + "datasets/suites/wp-cli/execution/cli-migration.json": "25b178f3dde386acc62e990f2c28d5ac6d80c78099ad4a87f0bcd59d5284c1b7", + "datasets/suites/wp-cli/execution/cli-options.json": "ae573e7be2e23d12803cd43c0d98a7ac5bd7289386aa088e1780c6191092ae94", + "datasets/suites/wp-cli/execution/cli-reporting.json": "399cb90846e969ecea88b2a15d16189a7bc5c9e0201a5596e430cf3917f14af5", + "datasets/suites/wp-cli/execution/cli-taxonomy-meta.json": "5fe9afcd957fdd66448d85dcd3354ccd38f3dd1b9890c7f2d3b5a6f2cfdd44ee", + "datasets/suites/wp-cli/execution/cli-users.json": "e12f6331139a114606d59dbe9ed6b2e11857fef034140b8ad54f05b24424e583", + "runtime/Dockerfile": "b504512a5f989e38f1d44208f32b52581742b0ce5df34d72f3fabbf2fd98ffc7", + "runtime/docker-entrypoint.sh": "42a788d594951a5adb24f39da74c1f026d9d3ed92b5cc0cb26773290c484c317", + "runtime/record-loaded-files.php": "e65576fa8a4c32af088773793a0a67ee4fc1c11e166703f8d7f279b1c311c497", + "runtime/sqlite-snapshot.php": "3f69e126ed71d03e3840b62c08b72e8ba2b04ea133fe2eb69fc5713aad04dc13", + "runtime/src/class-artifact-installer.php": "cbee15169ea378ccf86e88e72a666f66563289432e1487fe16a54d50bbebecfa", + "runtime/src/class-preservation.php": "e24b9ba72f45008a00b9eac0dafa853cae864cd8c2a669c1f29b07c53c6f4861", + "runtime/src/class-sandbox.php": "ec5c1d95d511447ea25acb170efde1e838571062b02a412fc204fd6f3a7e331d", + "runtime/src/class-shell-verifier.php": "8f001f4cda8ff687b616c26139f1c6de7be6d83674395f09e7141dda0e5df17c", + "runtime/src/class-static-analysis.php": "368c5d17ee94401449237a7d37fa4ed47213e2b0979d56b9afc1eb00170da887", + "runtime/src/class-verifier.php": "f9962fd3f71becbf7951fe9ae23370aa0fdb5e46bc1fbd0d41392bcd79834e72", + "runtime/verify-runtime.php": "4fc6aad058e75f35b87bf236b9abb96c230998e820fa8bd3f61768e43d80362e", + "runtime/wp-bench-runtime.php": "2048b872b1d44cf94a9e710b847ad2242d5410597c66d33f533bb5f29985161a", + "python/wp_bench/artifacts.py": "e2d542a485ae14f5ad2c71681925d76ae2a374da526fa8ffdffc6786c1ea4179", + "python/wp_bench/config.py": "6ebcc19a98d58353baf7c8380ed9576a8cfcba97bef1fbb7688b364550a15ff5", + "python/wp_bench/utils.py": "53902e5ae682d0b4bfe8e9b031fd988374f4dd08ab25a976e08b3bf70c8aa30e", + "python/wp_bench/core.py": "30e9e518d24c94b114180bdc8a134e6d35bf17a2c4b3e7a7c7aa64524937bf65", + "python/wp_bench/datasets.py": "3d90bbc1fc6b96d739f8d8363ab69f31a3002856b4d9e3c11b8d4e3bfef9cca9", + "python/wp_bench/environment.py": "aad79cb091ccce35ebfc8c8f4f5b7bfd2ac12539742401d59b864256f2b72630", + "python/wp_bench/models.py": "f3a788544ffe8d23c0f3eec57abade81cd5b44b8b9d9ec6a2e48a73878568ada", + "python/wp_bench/scoring.py": "3d2b4664fe38741235960824a26156bcbbe32b046b39ae921d89b04e5761e91e", + "python/wp_bench/selection.py": "f486d2d960c3d159ce61b71024ea9487d52eff4dd7be0ebe1a441baf005ba16a", + "python/wp_bench/skills.py": "5d99dd457b45a6ee836f8ab46fcbd0693f89cfef14aafff027bd23b8252e548d", + "python/wp_bench/exploits.py": "f21b46043669b9f1d7ed1c78258a329c19f46803799f09ed1e9dcd73e2e391be" + }, + "components": { + "wp-core": { + "task_count": 350, + "tasks": { + "e-abilities-api-001": { + "sha256": "f0b7a9198a50e40f08cdc9d4579e2614b4e4fe693df10153f9144cf9678b19ba", + "category": "abilities-api" + }, + "e-abilities-api-002": { + "sha256": "7c7000f6e3128de66686acd22ec4b0b8adb42fca61fe0048509421ae225178c5", + "category": "abilities-api" + }, + "e-abilities-api-003": { + "sha256": "93a5235867aac483660d7c98e4994b4191e53f2728bd061fc62f620b92077b62", + "category": "abilities-api" + }, + "e-abilities-api-004": { + "sha256": "57723bc14ef830e6f5ed54f1ec0b46bef81e29482c4d2972dc068334e8e651d5", + "category": "abilities-api" + }, + "e-abilities-api-005": { + "sha256": "53f74d879b76745863690468106cc6478d2daba6fa884b02a7c6c6b2298086c7", + "category": "abilities-api" + }, + "e-abilities-api-006": { + "sha256": "d79dea07707e4ffef59a3528f922a30645e80c0e5cc19f7272367a051a1e655c", + "category": "abilities-api" + }, + "e-abilities-api-007": { + "sha256": "838b3d8f115776492e6d0c8c533d24bb2c03bf6f62f53b1ea9797d0eea071f48", + "category": "abilities-api" + }, + "e-abilities-api-008": { + "sha256": "f463f52c66092c9ba3bca32bdd106c5fc63fde8accdc7c16904cd2cf68f260f9", + "category": "abilities-api" + }, + "e-abilities-api-009": { + "sha256": "d55f97d676ea098877c434af8417116537d174ff59e46aa2a94d2c0bf2e5938f", + "category": "abilities-api" + }, + "e-abilities-api-011": { + "sha256": "5305117e02aaddab1eac19a1d424b0661e129df77ac52f8d1318b687c7f98898", + "category": "abilities-api" + }, + "e-abilities-api-012": { + "sha256": "383291a210449db4cf968df2cd2c08cbd4817a522ed4a8c78b1bb0712305e774", + "category": "abilities-api" + }, + "e-abilities-api-013": { + "sha256": "95ffe433eeafa9525634725bbe996f5626f70dd1a6a56de26ce93eda07aaa6a2", + "category": "abilities-api" + }, + "e-abilities-api-014": { + "sha256": "78152b41ccc721cf7548e4927bdc99464404ca9f866f4b7852113cc8f9dd6904", + "category": "abilities-api" + }, + "e-admin-001": { + "sha256": "7203857ddc667f16fb49cb1e1dd1e0c6655e31c0be5953c679bd02a3f5e17dff", + "category": "admin" + }, + "e-admin-002": { + "sha256": "994939f9924b848eb6e11434c5901d0920d5b2ea345e7d3b61a207a84cdc3851", + "category": "admin" + }, + "e-admin-003": { + "sha256": "1c731cb639b260d88120b1a26355bba92d324d67db82bfda4875a46112109244", + "category": "admin" + }, + "e-admin-004": { + "sha256": "af8cbed7c0da0a61e6e96b96be157cbff334a25d4663d6141518e9aedbf457de", + "category": "admin" + }, + "e-admin-005": { + "sha256": "e4cce76e32367a47c4e70d90cf2999b48ef3fd0b73927a5266ea67b747e18e4f", + "category": "admin" + }, + "e-admin-006": { + "sha256": "61603606f30834c7bf33281e23ad843d44847b7011da3ae85e8eddb6e33483e5", + "category": "admin" + }, + "e-admin-007": { + "sha256": "cccbb0d5270de0f7e0393856a39aeba33f5a54fc5fad30d024a816428138f9d6", + "category": "admin" + }, + "e-admin-008": { + "sha256": "f491ed15036fa07a3382eb0eb2434935ba05232cca2513bc80c53d175e6c37ac", + "category": "admin" + }, + "e-admin-009": { + "sha256": "bd41372ffe3fb8fbdb3cecaae6038e5e7377e9bcb9d7a8635d5d1b920862caff", + "category": "admin" + }, + "e-admin-010": { + "sha256": "8c9476ffc99f2aaf38a766a58cdc34653117a203c3112fd879443163c1533fb0", + "category": "admin" + }, + "e-admin-011": { + "sha256": "d5b5f2dd67df9a9e2bb3dd3df92810a278eed357d2be483b0970fb5d5eb359df", + "category": "admin" + }, + "e-admin-012": { + "sha256": "f3d4220faf96d22292538b5faee3c37fc8f0146dd24391d20e4a44b487538fb8", + "category": "admin" + }, + "e-ai-client-001": { + "sha256": "88c89c7471e9bf1383de5b6da6dc4f991ec2737a462a67df305a68f4f5744396", + "category": "ai-client" + }, + "e-ai-client-002": { + "sha256": "ac0bd966fd50e8b245d11b59b3ba5238bb7a5c5313ea22b8cf8b4d4d4f3091bc", + "category": "ai-client" + }, + "e-ai-client-003": { + "sha256": "0ed0faa53756de8c653c4a47f4efdf9c219c3b19b167267524b1458a35b1b08a", + "category": "ai-client" + }, + "e-ai-client-004": { + "sha256": "f07d6d691cb9004d6fc1ace903e593fe264e28a11ff56c21b55be8489487a26a", + "category": "ai-client" + }, + "e-ai-client-005": { + "sha256": "3d2aa8c9dca48fb7fa2dc0ab27293b94fb8ae67f7b919c41e3c9e5d3ec40edd9", + "category": "ai-client" + }, + "e-ai-client-006": { + "sha256": "f20296b043005e0f823b346a9d4ee080dbd520060628bd2718c1439e9ae669c7", + "category": "ai-client" + }, + "e-ai-client-007": { + "sha256": "8cd6f23f2b7d68c723e2fc03cac0d61d61889b2e72b5972a7f51638e932d8deb", + "category": "ai-client" + }, + "e-ai-client-010": { + "sha256": "afc2060fa491db3e90f98e84088fd6e689db960076edb1236d71af36159d3221", + "category": "ai-client" + }, + "e-ai-client-011": { + "sha256": "25c73d85aca695b84b640ac4f84d69f41a95e4cea51568cca6687473960456d5", + "category": "ai-client" + }, + "e-ai-client-015": { + "sha256": "6a48b0855bcee04dd5ff45652a1e43fc5180314e431028fdedbc2aa7a5259f7b", + "category": "ai-client" + }, + "e-auth-001": { + "sha256": "7837ee77326929ed4bf7df3d8bed18d66b8dd282ff6fc1d8e079d3a7a078e375", + "category": "auth" + }, + "e-auth-002": { + "sha256": "4c0951721844d88afb0888a945f51a5a907bac797c53a998b3d51dae221be698", + "category": "auth" + }, + "e-auth-003": { + "sha256": "010ff53f3ccc8b50dc239438392618ec655147adc2b44f008ce1a3e3d12a81b6", + "category": "auth" + }, + "e-auth-004": { + "sha256": "7bd468058467747b724d63242b4c1499ef1137862c6b2d2d7572f3c7d1c7debe", + "category": "auth" + }, + "e-auth-005": { + "sha256": "86e0d5b68673e096569948d20f8d27eaaa7662d84e526e38c9ce88d0995a1997", + "category": "auth" + }, + "e-auth-006": { + "sha256": "7ec0252336430b9479eb1d07fdbb63070d2353e6b3e5074fb1aedc2ba9203f57", + "category": "auth" + }, + "e-auth-007": { + "sha256": "b8995c7530bb9d834a645383b95ad98a8770ec15e5d490968ddfd0bf3cd4cb99", + "category": "auth" + }, + "e-auth-008": { + "sha256": "91362bfd1d22f006bad8e564a61d829feb8d4a24614ac5e237cb35b88531bbf2", + "category": "auth" + }, + "e-auth-009": { + "sha256": "a3ad8a2ada30bdc6d9a2dac2c1b61ccf9ea3ccecddb99edc060645695f3835fe", + "category": "auth" + }, + "e-block-patterns-styles-001": { + "sha256": "2eddc347c6538eba9279f9993b3478f056fed959e98fd64db9215d7bdeca5a05", + "category": "block-patterns-styles" + }, + "e-block-patterns-styles-002": { + "sha256": "57b1b86b5e6d06fad04d22581a1ec553258cd111099f14ba3240e90e3943c5de", + "category": "block-patterns-styles" + }, + "e-block-patterns-styles-003": { + "sha256": "20f7d5387aeaa5803b6ae68fa5c23b9de380b0a20021648a445993f812489ace", + "category": "block-patterns-styles" + }, + "e-block-patterns-styles-004": { + "sha256": "1130fd349afce650967f7afc9c680a0bdec0930cb28cfe860e2b797bfeb18410", + "category": "block-patterns-styles" + }, + "e-block-patterns-styles-005": { + "sha256": "629d28a49250a1317fcdd5554be90f021f0a7ff00b6d941d43adbbb447dcf32c", + "category": "block-patterns-styles" + }, + "e-caching-001": { + "sha256": "49d71affc4ec9f5e6f8eb7c073d161f044baef314dd3f443bc9f3c2e822acf83", + "category": "caching" + }, + "e-caching-002": { + "sha256": "a7b6db8f1a76080b7d0f38176fea00818775e863598c966fa162ff2adcbcf371", + "category": "caching" + }, + "e-caching-003": { + "sha256": "c2cf29995f4dbff7e5c9c931fca1148c0734911399fa0ca1936668e2fba1226a", + "category": "caching" + }, + "e-caching-004": { + "sha256": "eedc7d787133d75a4d8da992214ddc2378d13327213eba75b9760083cdca6596", + "category": "caching" + }, + "e-caching-005": { + "sha256": "5d3d11a21ca6a094e6d31e4269cba3dc628863851d1fd1022363d1c73154c373", + "category": "caching" + }, + "e-caching-006": { + "sha256": "2e6bebb63568258f46ae4545910b3e39ee13cedaae1e558b724484b461ee6c0b", + "category": "caching" + }, + "e-caching-007": { + "sha256": "a41138cbb365bc9db191c6d265a870658d18c71b2cabe1b065cec26c9c8e0ad1", + "category": "caching" + }, + "e-caching-008": { + "sha256": "651fe2824b0787c16393887daa1725fced185285dc94c8b7d7c5bddac804f50b", + "category": "caching" + }, + "e-caching-009": { + "sha256": "d3faec335333f47daa23c54af87ec84066c359caa3d3f3f4a88d69fde6b53458", + "category": "caching" + }, + "e-comments-users-001": { + "sha256": "219bf20b686beca2c214d37c529c527839862702b7f3609b21e2cbb3c60a30b8", + "category": "comments-users" + }, + "e-comments-users-002": { + "sha256": "eb2e78afa2f91932f0d2579881a88155fdcd4310902dcccc2d1df86324492cf8", + "category": "comments-users" + }, + "e-comments-users-003": { + "sha256": "636280a7a3a7f13aeca5121dceb60623431411647a56ecbe3757ba5e7264dfd0", + "category": "comments-users" + }, + "e-comments-users-004": { + "sha256": "04edd076018b7593e14088eba8eb2b781cd193eddb3140cb2642c090fd59b7cb", + "category": "comments-users" + }, + "e-comments-users-005": { + "sha256": "570ed94a9de1ff0d4d0b3b21e3d9d52069c17b8dc8878dab9cfc7ec0dff93f16", + "category": "comments-users" + }, + "e-comments-users-006": { + "sha256": "b628458ae28d6d318956bea170cd40c0a152a09a8fcb82f34a3a7d8e0cbbcffe", + "category": "comments-users" + }, + "e-comments-users-007": { + "sha256": "b258ac238c14b48efc113637c3270f0b1c3b0ffd7e871dd2704a97c6dfd3f848", + "category": "comments-users" + }, + "e-connectors-api-001": { + "sha256": "88fd0d229ed06764856c25e4d801cb82a465f5632579448f5dfb156d764e39be", + "category": "connectors-api" + }, + "e-connectors-api-002": { + "sha256": "c221ff980bb918b39317952da4531ec56c589a67daa005c79e022a3291cd08b0", + "category": "connectors-api" + }, + "e-connectors-api-003": { + "sha256": "59451f5a1626677285561a62f6ad4c76ccd491312b7bb93e75d94d40418fb45e", + "category": "connectors-api" + }, + "e-connectors-api-004": { + "sha256": "502098d053a8963a5d97f29216984fb60a2b688644593423a14f9569c9a99640", + "category": "connectors-api" + }, + "e-connectors-api-006": { + "sha256": "13e4dc178119c96ca0adb363176c285a43080e04b247902b74f0883198936646", + "category": "connectors-api" + }, + "e-connectors-api-008": { + "sha256": "cdf52c5948b7d069ff1ad49837923cc924ee45a82611711fb4d22c28e364399d", + "category": "connectors-api" + }, + "e-connectors-api-009": { + "sha256": "03095a89a152296925172198d6e318be8ab0b2186145f11fdd85d929c4b567e3", + "category": "connectors-api" + }, + "e-connectors-api-010": { + "sha256": "f45d3683d7bf4889fcaeb4251807c5a52e281640d67862b339ff429c7b4f9335", + "category": "connectors-api" + }, + "e-connectors-api-011": { + "sha256": "4ccfccfc73e9868a47b67e8da7439b900ff6192f715a5d8b89ef22cb0fbbb733", + "category": "connectors-api" + }, + "e-connectors-api-013": { + "sha256": "122adca07ee95b93c5631dd7432a0db15cf51824c7b9922b22ea46ff02fd9476", + "category": "connectors-api" + }, + "e-cron-001": { + "sha256": "82a89b9a2bf017d4c6b2a48c756132d4729f7de927c6e274ec7f45b8474ed866", + "category": "cron" + }, + "e-cron-002": { + "sha256": "8c2a06c38db536f5c4cf6d2c4e365524b52f2af95c0d93bcc6ea67673d275986", + "category": "cron" + }, + "e-cron-003": { + "sha256": "621f52a29071e0a174f03d811ed599a7a33e26bb7b66c1d77fd5db5544ef9aac", + "category": "cron" + }, + "e-cron-004": { + "sha256": "3895a945375beec935af78cf04e90ecf14ceaeb245e26c54649989513d16cfe0", + "category": "cron" + }, + "e-cron-005": { + "sha256": "e4144556b3a6f2ef6b1cd763abcab119185949f67442919058a436ace15a9038", + "category": "cron" + }, + "e-cron-006": { + "sha256": "ac12653ac263b523ed47838479b37b761ec265d99de8adf395454b24a789bbcb", + "category": "cron" + }, + "e-cron-007": { + "sha256": "2bdbaa37576948c06cb3469c940ba4e6f4bbfdbcb68291d4333528d224772f0a", + "category": "cron" + }, + "e-database-001": { + "sha256": "1929351d46953ab765200bc69bc23ef9228e3057e5c1b01f965eddeb32e0736b", + "category": "database" + }, + "e-database-002": { + "sha256": "b3c3c0ac052266855abc0a2ab2de2c812700d1e66d532d087a159ac6e7d6a78d", + "category": "database" + }, + "e-database-003": { + "sha256": "ce954e9a38701dc7380f7b236788662bf6f90606667953e28452d1eb5495d739", + "category": "database" + }, + "e-database-004": { + "sha256": "909b3094a2dcd7ff3f3c5f3ffadfb52fd87635d0b393dd7517b7a55921b06461", + "category": "database" + }, + "e-database-005": { + "sha256": "b6b9f91556141857c2b8ebfc216889a39f9fe255f66832a3039ada6cb63af45f", + "category": "database" + }, + "e-database-006": { + "sha256": "958b6e1f580b162f28eca2101fd2eba1135daaaa95105f467c99d9c5c7c07f2f", + "category": "database" + }, + "e-database-007": { + "sha256": "317bb3729838c0236b16b43be51d3d28d149ae954f1a4bb05505b97e937aea82", + "category": "database" + }, + "e-database-008": { + "sha256": "a0dc921ae2547c1d6f47ec16ad33267613c1f0cf9745323e4711f18d620ba057", + "category": "database" + }, + "e-dates-time-001": { + "sha256": "43999e8178476f3c57456297ea96312dd4a87f14b57f3ab50b056b780265e1a3", + "category": "dates-time" + }, + "e-dates-time-002": { + "sha256": "87299679beb13389977e537916e02d60d1f00e95a236c5d8c8ef4f52996b5d57", + "category": "dates-time" + }, + "e-dates-time-003": { + "sha256": "836783c1acc9dc94a368e6fd9c99d8b522f8d50bc3414016294e6f426b1bea96", + "category": "dates-time" + }, + "e-dates-time-004": { + "sha256": "1d253a7717cda0bbcf4c23c5a866aa0dbdcbb0267a572955d3f6b3bc187f26fa", + "category": "dates-time" + }, + "e-dates-time-005": { + "sha256": "3140e1e7fadae051333d3f221aaa3333cdbc01a4dcb099f05ede42efab2ee946", + "category": "dates-time" + }, + "e-dates-time-006": { + "sha256": "438859ac8d91bae827f0f1d6a4ee1a66fbaa7b0c006ce9c50535dfbb13cff3eb", + "category": "dates-time" + }, + "e-gb-block-api-001": { + "sha256": "f8cc073f8aa62de612315caa5e54be8e9ccd3ca28d863deacf5f1576a019a2b9", + "category": "gb-block-api" + }, + "e-gb-block-api-002": { + "sha256": "9dc48929edd30170f825169d24eac96583a5a923be0d744fe73a34fab8333944", + "category": "gb-block-api" + }, + "e-gb-block-api-003": { + "sha256": "3a4ef7a068da883da2788cdc3b592688377c7e392a8501844f0f959f1908e99a", + "category": "gb-block-api" + }, + "e-gb-block-api-004": { + "sha256": "14f557553bee50df87a3dc6c1123607beb8ed4334320449890ef830dc3faed6c", + "category": "gb-block-api" + }, + "e-gb-block-api-005": { + "sha256": "986ef4300e03f106a70971ad89da4ae676b6b662f61d597d22a24aa70ecf4dbd", + "category": "gb-block-api" + }, + "e-gb-block-api-006": { + "sha256": "65f8fb0ec0c414cd8b308e0dc489d97083ac15f504d8a038bea6ffbe7069244f", + "category": "gb-block-api" + }, + "e-gb-block-api-007": { + "sha256": "01e9c174ff82fddf84ff0833614315f12697b793438cc2ef087a60b36b11cc4c", + "category": "gb-block-api" + }, + "e-gb-block-api-008": { + "sha256": "883c5405a16809d76ed38505bfa1fc585b6409d0aeaf07745ecf0f364486768c", + "category": "gb-block-api" + }, + "e-gb-block-api-009": { + "sha256": "e2befec1a0315aed852dc30f7da1f7b3d04d1d883b1b7cd9a4b1dd299e4c6c62", + "category": "gb-block-api" + }, + "e-gb-block-api-010": { + "sha256": "9393f872e6d44b78b80287bbaf2dac961a7bf75d993d02090e6c082900eafb13", + "category": "gb-block-api" + }, + "e-gb-block-api-011": { + "sha256": "4e1e1140ccf77a6d7924ed0dcb17241db2c2065e7aa4c77236fa7a78673d4713", + "category": "gb-block-api" + }, + "e-gb-block-api-012": { + "sha256": "a1e2a8e73e1ed4919c9b5249f5af61702cc995ca4b1184911d0f1fc1111e41b7", + "category": "gb-block-api" + }, + "e-gb-block-api-013": { + "sha256": "09becd2a1b96c8f37e8abfca018ae495bb2e4ab2aa9387f8a87263488c9970d3", + "category": "gb-block-api" + }, + "e-gb-block-api-014": { + "sha256": "c580ab75fa0f372728e1f7a76a6a8883d074c7730dcf05aa14b06ce7ce38cb71", + "category": "gb-block-api" + }, + "e-gb-block-api-015": { + "sha256": "1b0c1fe194e8576ad75298c50e0b91f6e8d7ac706716b3fcff9e34753f64256f", + "category": "gb-block-api" + }, + "e-gb-block-api-016": { + "sha256": "ff7ed858de6b8e34016db615c43e3176b65ba9f72825ab18fd173d56432aadb6", + "category": "gb-block-api" + }, + "e-gb-block-bindings-001": { + "sha256": "8b17e7def95f2f262f93aee13979b9c155e456080c102ad90bf49be9f3f2fd81", + "category": "gb-block-bindings" + }, + "e-gb-block-bindings-002": { + "sha256": "aacc2324a42de95830730f8f49867fc66ba72b6a803f38559aa95bf158e2f059", + "category": "gb-block-bindings" + }, + "e-gb-block-bindings-003": { + "sha256": "670af9861734083a5d6790b433d4faf4991cf1c4ff0e91fc7e2123c18be86ef9", + "category": "gb-block-bindings" + }, + "e-gb-block-bindings-004": { + "sha256": "50b172d1a9736c4b94b73678eea6be96ef117d2491cd492735ee26fa20534b13", + "category": "gb-block-bindings" + }, + "e-gb-block-bindings-005": { + "sha256": "88ea6c95db288f14ee8f7f3b159ac9259025622e28e839c3c8b62bce0fdea8a7", + "category": "gb-block-bindings" + }, + "e-gb-block-bindings-006": { + "sha256": "44dda7169244841de5c7112cf72fa9cfb56491cb1745813a58e52b91a2cd24d4", + "category": "gb-block-bindings" + }, + "e-gb-block-editor-001": { + "sha256": "52fd7ef7abf4f9ca41aa64dafcdbe5f32f5d66323b37108e35aa7d32e4c44413", + "category": "gb-block-editor" + }, + "e-gb-block-editor-002": { + "sha256": "ca0ca4d30110404a3d27bdd26f07680337c675522c9d1cf40c06d7d312611783", + "category": "gb-block-editor" + }, + "e-gb-block-editor-003": { + "sha256": "f1e7c6a7a1f71e89bc3000cc02201194090a9f22019d11f4483490a5f73d2006", + "category": "gb-block-editor" + }, + "e-gb-block-editor-004": { + "sha256": "23ce260c07bf0a9cfe1c4e42d3d04281bd6c8d0fd46c84800100c5d9173d4277", + "category": "gb-block-editor" + }, + "e-gb-block-editor-005": { + "sha256": "620dfcd38648df06ed8b647c3bda9be5b0b0bbe0709de0a755b739a34ce4de76", + "category": "gb-block-editor" + }, + "e-gb-block-editor-006": { + "sha256": "c2d1b57cc8c7ceba601daf9870d8b590a10ae86209d3bc84db16cf8a4776d8b3", + "category": "gb-block-editor" + }, + "e-gb-block-hooks-001": { + "sha256": "43c1e0819ae9004562d58b9c5fce134fd9712b1b6da6ab215fcef64421346e13", + "category": "gb-block-hooks" + }, + "e-gb-block-hooks-002": { + "sha256": "388caece059cb61f5f9a7404e68ab21ff4195bd1cd7032402bbd89b0ea9e0b6c", + "category": "gb-block-hooks" + }, + "e-gb-block-hooks-003": { + "sha256": "d42f4ffb4cfc32f0901cbeac425e7c733ac7fb40355d337f026249dae0e4a3e1", + "category": "gb-block-hooks" + }, + "e-gb-block-hooks-004": { + "sha256": "a16c2f05f81a804c4d53a6569c9d543600495afd53182d43a4e450db96a7ce9e", + "category": "gb-block-hooks" + }, + "e-gb-block-markup-001": { + "sha256": "a6a087d6cad8728c4e431abde258fec264cab808123b0fa51610b9c77546ebaf", + "category": "gb-block-markup" + }, + "e-gb-block-markup-002": { + "sha256": "a0a33425bee8e1bf3b305afdb56a85739d222d05a547a2c394db7b211d44e9f5", + "category": "gb-block-markup" + }, + "e-gb-block-markup-003": { + "sha256": "2ec07b89781c30a113828aa453a5273472336d9be9ca7414215190837842f87a", + "category": "gb-block-markup" + }, + "e-gb-block-markup-004": { + "sha256": "a773d8a648960ead43335b020c9d75032bfd0a099b0831ab11b8642c11216723", + "category": "gb-block-markup" + }, + "e-gb-block-markup-005": { + "sha256": "e0b12fcb692a98e6faad5edfc14d01cc27467d44e19eaebcabe2177ac474f623", + "category": "gb-block-markup" + }, + "e-gb-block-markup-006": { + "sha256": "0d0ec7e514e5e7bda7519122d3dde216c03cb066e727c208165d8d16049b5079", + "category": "gb-block-markup" + }, + "e-gb-block-markup-007": { + "sha256": "71868a9e886c2ab762546f9c035dcedb0e7238a80f14c6381a3ff2282f161d5c", + "category": "gb-block-markup" + }, + "e-gb-block-markup-008": { + "sha256": "0aa49fdbdaed84971da2e44d9c245db6804b638f3cb6988c99fb0de7cbd62dde", + "category": "gb-block-markup" + }, + "e-gb-block-markup-009": { + "sha256": "3e5992b6e09887835b750897bfc5841927746996b49addc5d01d412e73b20892", + "category": "gb-block-markup" + }, + "e-gb-block-markup-010": { + "sha256": "4898a12dcba55520d69e294a8c50e0e881f8707d8cdac588af6db025260db91a", + "category": "gb-block-markup" + }, + "e-gb-block-markup-011": { + "sha256": "5456bb751eb09a10df2d26422d51bf22596abb6720cbda86f9216fec180be1c1", + "category": "gb-block-markup" + }, + "e-gb-block-markup-012": { + "sha256": "e20e5f76d9b596af066a1bd530c9343215168b2bbaad9842f872ecf1a954343c", + "category": "gb-block-markup" + }, + "e-gb-block-markup-013": { + "sha256": "e01bf1e499efc7b5cc258ac56b38e7998936080dbeb15ad457a9e6461260a1dd", + "category": "gb-block-markup" + }, + "e-gb-block-markup-014": { + "sha256": "4359cb583990d903259cc4de0d1520563869d1cff5bad1ff78bc7d8dbcd8c204", + "category": "gb-block-markup" + }, + "e-gb-font-library-001": { + "sha256": "9175bb47f6e4006883e8748165500b488e14a722736fdd1ec5d4c3763b810514", + "category": "gb-font-library" + }, + "e-gb-font-library-002": { + "sha256": "2ec520094b91ead39b3caf5a91487997dfeb0fbe76c20918ffe88c90a76ebdec", + "category": "gb-font-library" + }, + "e-gb-font-library-003": { + "sha256": "fb1c4dfe6f2f865f3f6e455cd37e86136bc3d7b9bd9c7b3608c1a4cc3a83f45b", + "category": "gb-font-library" + }, + "e-gb-interactivity-api-001": { + "sha256": "ce22414ee3041ca7f7c9d1af6fbd9d09fe091c34ca9af912d92e6588792f727d", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-002": { + "sha256": "8c82e5ba14644f0e892fc8c1766c0bc5ff3cc1fefb317cf0cef5fad23bfe6923", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-003": { + "sha256": "8821e5c9c939ba4f284e9072560c1c49630439980c033aca6866837bd93cd1bc", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-004": { + "sha256": "843dc6df92a7b393322a74c91a67b7a64d60a74d054d09050e204494ac3757b2", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-005": { + "sha256": "045ffd36a945c144e8c0e2a97393c23dd570175b4f291b173b99514ae9ce011a", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-006": { + "sha256": "85f7002f86c0f4817b37c97af69786c7f860903e1a857ceda5482c3f62f394a1", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-007": { + "sha256": "96ac27aae295d6532afd0e8ca9cf0992b0ed28f4f70e0aa716328e475e039c09", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-008": { + "sha256": "1b833832d8316f2addb03379b2813b132ed078a99cea38589ec9ad18deacd683", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-009": { + "sha256": "9862f1b67050de54ca0d7dd63647740aa6b14fb6318d017b0cf4b7131d6fe1d7", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-010": { + "sha256": "5c820b7c83da82e8e359610f2fec0230ff53d2565a695866509bd230f2a89231", + "category": "gb-interactivity-api" + }, + "e-gb-interactivity-api-011": { + "sha256": "c4fa4334b2725facb89094579edebc5cd53b81c4874ad04a5bc3626ebc6cd162", + "category": "gb-interactivity-api" + }, + "e-gb-templates-navigation-001": { + "sha256": "4e7224549b6f17f82fcece5205fe12cd7942a2551673dee88147a7acac8eb086", + "category": "gb-templates-navigation" + }, + "e-gb-templates-navigation-002": { + "sha256": "4ee8e5f44e8d26cd69eb946def6afe7c0ebe2c19f1dcb56a7bc56f86f305e8b0", + "category": "gb-templates-navigation" + }, + "e-gb-templates-navigation-003": { + "sha256": "dbc706ada6e54f7a5f8c4a6465a13b3c507b08b75090e49bc616b3d6476f80d3", + "category": "gb-templates-navigation" + }, + "e-hooks-001": { + "sha256": "fdbf40ed5e90a1fd1f4c0b821d1ac51a04bfaef1b754cc8b3770ada3ebdc4788", + "category": "hooks" + }, + "e-hooks-002": { + "sha256": "64bd91b63cfef0fc2c8abc19641d6220e4d8912911b903c5b2cf2d388a110832", + "category": "hooks" + }, + "e-hooks-003": { + "sha256": "5acd616f402032b24cb50a743a0d8737f4c4ef84ecc7026cec3bb0728124f445", + "category": "hooks" + }, + "e-hooks-004": { + "sha256": "705e6d00a48aae394fd11a555c69aa0a42a50275757bcdf3f8c52f3cf0043d70", + "category": "hooks" + }, + "e-hooks-005": { + "sha256": "3d74ab83fad0fc386b2d8f4041532f79fc9393cb05726f9fe66855b653ed5b0e", + "category": "hooks" + }, + "e-hooks-006": { + "sha256": "4023ac4c5912361768e09f92d3f1b916267943657b7975af765035e8103dbee8", + "category": "hooks" + }, + "e-hooks-007": { + "sha256": "d35bc795ba08b31b8c10c08daf96ea4208cc0f58297b362054878a726cb1db9f", + "category": "hooks" + }, + "e-hooks-008": { + "sha256": "7e3a695ee050f9e7033df7a123da977db3a006ebc23f3e4da445c4e69f96e324", + "category": "hooks" + }, + "e-html-api-001": { + "sha256": "ae9a3dc35798c8d81cff6a5dc3971cc45627eed3180b2cd864d76307d9e0ca73", + "category": "html-api" + }, + "e-html-api-002": { + "sha256": "03d2fabd0e5a8b0d94ebc190a147702f26dcc83722adb127c277d2d6493d875f", + "category": "html-api" + }, + "e-html-api-003": { + "sha256": "c2634aeb795c1c773abfcb85e1be3b5fbda946778de7c493e9cec82c570ca8d5", + "category": "html-api" + }, + "e-html-api-004": { + "sha256": "f3d13a19a7641ad242bdbacdff85a2cff2ab2403f8008af144dcdffef69ee5ef", + "category": "html-api" + }, + "e-html-api-005": { + "sha256": "a6d57b8bf95098f9b91f31c86c49aba2771fabbbf5753ace5d1768a892c2012d", + "category": "html-api" + }, + "e-html-api-006": { + "sha256": "e81b0d271f68e0a556003d6698e0e3b918aa9684d2308d08540084fafb30f53e", + "category": "html-api" + }, + "e-html-api-007": { + "sha256": "6e967b5d727f905b26be73d80c65dc29ee6c8ab7e5bcbdbb6ca9dabfecbff8bb", + "category": "html-api" + }, + "e-html-api-008": { + "sha256": "9234ae62d5a760450268c0a02e56c0de92b308717bf19103d2a09f84a204a17d", + "category": "html-api" + }, + "e-html-api-009": { + "sha256": "a023ab121f752644970242994d11b85964b7d12eec1a4a861337b7d2551e537d", + "category": "html-api" + }, + "e-html-api-010": { + "sha256": "50936d425764a304104d07d59bc4eeb3a6c82fd4ff7c82a66953dc1d6cbfc275", + "category": "html-api" + }, + "e-http-api-001": { + "sha256": "afce26b1848b5fe254a3d2f914c3ffdd4bf0966d9036c76697f1a0f27e05119f", + "category": "http-api" + }, + "e-http-api-002": { + "sha256": "14433b68d51b7d1787b707ec037204e92380ad93885e06948cab2c6e8e803150", + "category": "http-api" + }, + "e-http-api-003": { + "sha256": "67a702839a3eaa90b0a9cfbb060860f114d83de2692ecb033fb540f07d6157ed", + "category": "http-api" + }, + "e-http-api-004": { + "sha256": "dabd6ed0f97fa023cbe360f7564959dad8b4d03728eb0e283ec4b6ccdb1df8fb", + "category": "http-api" + }, + "e-http-api-005": { + "sha256": "34a1e6ef947fd50f824822576dac133b090d14e8f2fcfb53085da378c76a7a8c", + "category": "http-api" + }, + "e-http-api-006": { + "sha256": "ab668a3240c865b9b54a4e47c4e6c683eed2704634acf5eddc41cad3d29a76af", + "category": "http-api" + }, + "e-http-api-007": { + "sha256": "e98ecc5a61dad8c7bd57edd469cd3b44f6817e08edf58b7b74d036f68f114fc9", + "category": "http-api" + }, + "e-http-api-008": { + "sha256": "a49228cbca205c971a6ffe5ed7be99186546338dbe5adaa2973dd0f3963fda5b", + "category": "http-api" + }, + "e-internationalization-001": { + "sha256": "43dd4f043e2d8f15b121aaa5f64f38b7897e3b5c16351b589026083504cbcede", + "category": "internationalization" + }, + "e-internationalization-002": { + "sha256": "7390f00dbb1a67b43c4f8fa9d739f986ea1a548f7757c19aba5a333e8caa44ae", + "category": "internationalization" + }, + "e-internationalization-003": { + "sha256": "3116acfe7e6ec848726842132db3ef914d5aa19f17cedadc9f784096ea9fc792", + "category": "internationalization" + }, + "e-internationalization-004": { + "sha256": "b256532c7a2ab8b4ef2f467eed36bbe241353b9309fad348c65f6a2705ad17f7", + "category": "internationalization" + }, + "e-internationalization-005": { + "sha256": "5e57ad18f4697a53082ff0a425ad9837053d9513d5ce70d88a9dbe1d3f39fca1", + "category": "internationalization" + }, + "e-internationalization-006": { + "sha256": "96e73daef4f570f54cf972df42cd2c732302e5d81fd87359664a86decbeb2763", + "category": "internationalization" + }, + "e-mail-url-001": { + "sha256": "6070e535c2423236048ba66440e83460b1e50d977de7686bea68d8a245846968", + "category": "mail-url" + }, + "e-mail-url-002": { + "sha256": "a00c33f3f27577341cc7983f079bc20c6b63ab29715da9194ff4be3ab5a61c69", + "category": "mail-url" + }, + "e-mail-url-003": { + "sha256": "80679c4904ddcd6276fbee74aa4785d66a6519681cdd8f6ded6a6d29f74d2393", + "category": "mail-url" + }, + "e-mail-url-004": { + "sha256": "6a158d379a6bbfd102d690e02108ca36b7dee3ee65abd9642a6ef10c55e5617d", + "category": "mail-url" + }, + "e-mail-url-005": { + "sha256": "3ae117eff7944c6a3d5c34c766b8f91f4960de3836c83edb3c4becec32ad4e78", + "category": "mail-url" + }, + "e-mail-url-006": { + "sha256": "9d1ec712b75acc806813860191dc2d3242b56ff663f58ec78127553711090080", + "category": "mail-url" + }, + "e-mail-url-007": { + "sha256": "2fbdce59645583ee09957d9d962b1aec8aab480a7ed2aca07a155428477c54cc", + "category": "mail-url" + }, + "e-media-001": { + "sha256": "4fb806f46b50758eb1137dea81b97bfd77f7d10e0003522c015394bb17bcaf1f", + "category": "media" + }, + "e-media-002": { + "sha256": "bd21ad7ea4b84f2397040a0966ad599c28a94044b92eb72725d70ba78d0a6a5f", + "category": "media" + }, + "e-media-003": { + "sha256": "168bb3b888a4a96d740cb8279864998c79f997665f4e390bb317e3d528f5683c", + "category": "media" + }, + "e-media-004": { + "sha256": "2ba990a143425af283b469b988312d9581ac9b2ec6b9b470a40a32ea94b139fa", + "category": "media" + }, + "e-media-005": { + "sha256": "869db99c30f136709639ec9a5b5ed684f22e5aefc0d0206ff24959e5f023e141", + "category": "media" + }, + "e-media-006": { + "sha256": "bf71a04216505f83cca58994b9eed57182bc38c6275c01962510cb0ef7ea458b", + "category": "media" + }, + "e-media-007": { + "sha256": "aed5365ebb743da8929f505a4486757e04e30183112d4f39520e8cf21bd4cfb7", + "category": "media" + }, + "e-media-008": { + "sha256": "cc3a680bffaae639ec90e13427b3e84798e317bd64d391d355ed1a605323298d", + "category": "media" + }, + "e-media-009": { + "sha256": "ded6188407772a7bfe46459823a84dca614b6887a0bd1329b4a89d8361540f78", + "category": "media" + }, + "e-performance-001": { + "sha256": "d1ce8cd064c32f4408dd0b692fcd6346b421178309bd4cbcb6ff7a50c9222b6f", + "category": "performance" + }, + "e-performance-002": { + "sha256": "17db6298c2df5368025c9b3488cafbb9887a646d9d86c58845b8d61e535962d3", + "category": "performance" + }, + "e-performance-003": { + "sha256": "baff3d17e02ecc59526a4ba2c5cf19f487ccdfb3a8af1a04005a161ca2664d69", + "category": "performance" + }, + "e-performance-004": { + "sha256": "3a3d8cb7e1e30f54f74699a256c814c9a8bf2c6561a0a9401b4e038f2dee9a53", + "category": "performance" + }, + "e-performance-005": { + "sha256": "239863a1f7659bd6a795adda79b678c369db0d2128520de3df2505f5945a2946", + "category": "performance" + }, + "e-performance-006": { + "sha256": "54a336e213f64e3f5ff4967a1f60e2349dd1d450b31e8f8f2750ee745a2c4b75", + "category": "performance" + }, + "e-performance-007": { + "sha256": "15365a1ba03131d1a5f8dcebc46e91794c82bced99c71377e88d82f87f187f9c", + "category": "performance" + }, + "e-plugin-lifecycle-001": { + "sha256": "6fa5fb59a3765613f0fb901eb2e172ff0d4b55b398996cc3faa21704761646d1", + "category": "plugin-lifecycle" + }, + "e-plugin-lifecycle-002": { + "sha256": "58680765787719aa96c45252635ca8858d8a8bf3d426555132e89b19f9503f77", + "category": "plugin-lifecycle" + }, + "e-post-meta-001": { + "sha256": "3983010e8dfdc201b8326da32e7beedb0361f5be4cf5b01c7450c28a192d9986", + "category": "post-meta" + }, + "e-post-meta-002": { + "sha256": "258f03374a98fddc518fbc2efa8f781abb6754ea774a561ac615c1babedcf2ca", + "category": "post-meta" + }, + "e-post-meta-003": { + "sha256": "a8df5bef82fd40590704c2520ffbeaf76495f7d6c8d4fa5cdfec1427ecd0ceea", + "category": "post-meta" + }, + "e-post-meta-004": { + "sha256": "7a81322e17df7cd22de11c825bf731dbd8e094e4b35cc2361e3b678bc684bcd8", + "category": "post-meta" + }, + "e-post-meta-005": { + "sha256": "ae0a06a7bd81822f12bd290e4f4e267ead53831a5514e6aa528689060af5bcde", + "category": "post-meta" + }, + "e-post-meta-006": { + "sha256": "9011ab8c64911e800539ab8a07105338ea8fac3d599623afcd90b9edd6fd7529", + "category": "post-meta" + }, + "e-post-types-taxonomy-001": { + "sha256": "4d99b0c3bcc10bc1d1789c7f1b2fc3f6f7524d62f9d3fc2eb734cb88ff167dd7", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-002": { + "sha256": "a1485fa4e681046a33129d36896165f784fa21a168b13d32d419f4987035ec7b", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-003": { + "sha256": "dced4db39bce3c84e8fc167a342674fb4d611e756a463f7da4f72647a8794e94", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-004": { + "sha256": "c769d66fa33a988abc0ee833c528d336f11744c09bf00da981175561193b8272", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-005": { + "sha256": "ebaf3b98cf1c2ce4da008bbbe98cfe229980d34841a4bc80f9b45e0fa238e2e2", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-006": { + "sha256": "4fcdc928ffff98b12f291ee5b353b65ef76363875ddd3b95f0b690189406d822", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-007": { + "sha256": "73c2c17ac717f7eeaa2ef876d3136e04ebc0d25797a617810c52f384c15ec4da", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-008": { + "sha256": "2d6a7f5333fb14dbdb5672fa7ad9cdc938cb388f8eceddea7464adae53b9d6b9", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-009": { + "sha256": "c1a519195596b17d8e06775b2d68c75b3d97ebe298d6ff009d8afc66b0f533c4", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-010": { + "sha256": "85900e1f4c9a6a8c16d1785ed8888a8528326f86556105d7f064d2b3ec098c98", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-011": { + "sha256": "c025e824eb5bd38c42007769833ea4ee548c2eef6fbdfeb8efbaa22d73bb80e3", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-012": { + "sha256": "376429fcdf62f8f4c4f7d3469fcd44908919e45116513075e4362aa0f0d7f195", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-013": { + "sha256": "19b1c7683b5b9eb8e2a8823461f99211942dea4d7a67cde71f657545a6234388", + "category": "post-types-taxonomy" + }, + "e-post-types-taxonomy-014": { + "sha256": "3526f93d417807469fefa062f86e495faa551e5ef1f798cb69508559d89870d9", + "category": "post-types-taxonomy" + }, + "e-posts-lifecycle-001": { + "sha256": "1f0455301f64c69b5fd9a207309dde7c4b4bd4848316181a37df18a3bd4487bb", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-002": { + "sha256": "944c08bd5e48a82ca0f269294b025dcacdcc7ceba7aef39f77c5d8ab01d46f3c", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-003": { + "sha256": "220f446a4a020fb8812d6393623f6d3d7bf0b923fc6d0ff72d20a0a0fc5855e2", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-004": { + "sha256": "a659a204417357319be42748c3a936e03d7600e7a800fd9d243f10d068abdb9a", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-005": { + "sha256": "dcce8d2921a1ffc8ad7ced4f640462da9195450a0b6b29ee7b410571bcb757eb", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-006": { + "sha256": "df9ece1797a791a8327be7dcd67ca6048039aafcd0ccee44446dd83e4561ed90", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-007": { + "sha256": "2e4c89e23b522ac055f48f2df5db2c6f9af8be3620afbed1e01328cda851b3cb", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-008": { + "sha256": "1c23c811d11997994a26ce31cd2e75f8531d8422ad4a35666bcb5983c9f1e789", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-009": { + "sha256": "7dfcd742bede510bb6ee08dad9cb159fb0305c06cf0ff2aed340f6adfd909045", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-010": { + "sha256": "20350eb1b1ca1aa1cfe3c0c6a70677d7d8b8903a6fade9a5133a98d8f9e7e078", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-011": { + "sha256": "b955bb2a19947150de620f523f130e946852907f586946d98b51fc7826ec4bbc", + "category": "posts-lifecycle" + }, + "e-posts-lifecycle-012": { + "sha256": "c7908eacff97783281c4d9b463e1639f5d73d206868fea72649414f2594a2e7a", + "category": "posts-lifecycle" + }, + "e-queries-001": { + "sha256": "9fb5e4d6469ffe0b21b14b55c003db44e0ed85cf7b3d866f2a64d02e02c3f34e", + "category": "queries" + }, + "e-queries-002": { + "sha256": "e07f4289c72f0d4e865565780d93df9a21623336ad3bd70d10a248e602bbbbae", + "category": "queries" + }, + "e-queries-003": { + "sha256": "e4375af0b046bc708a3664640317504032570ad22a7ff443053886d6a72144ba", + "category": "queries" + }, + "e-queries-004": { + "sha256": "05124f74f45817617418ace69b2a06e70ee7e7d81d5cb9ff176fbf6ad0c6352b", + "category": "queries" + }, + "e-queries-005": { + "sha256": "260f491ac70491e05d5b2a9f48a3f3d8818669180a12ef320913a1e018a9207e", + "category": "queries" + }, + "e-queries-006": { + "sha256": "f6355351c542979390d32d8997196925b61cb4cc8cfffa388c906f9df0aea2ba", + "category": "queries" + }, + "e-queries-007": { + "sha256": "af79700b283644a6fb6797b7b7c9d6d0012b3db0b3f95afb51d026a723797cf5", + "category": "queries" + }, + "e-queries-008": { + "sha256": "bc7b0cc2c250960d0df86a5fb32bb77be72055f1f42c6b3ebacb29720e60d239", + "category": "queries" + }, + "e-queries-009": { + "sha256": "0ddf4d9667e223dd086eeda0e16e73bc1edf8a9300a850fe7121b86cfe72db19", + "category": "queries" + }, + "e-queries-010": { + "sha256": "7ca667fe4bb0ac74c0cb19f4a180d52fbe8dda920f0ba35a224f7ac6b7f44589", + "category": "queries" + }, + "e-queries-011": { + "sha256": "ac0b01ee3be8f93dbccba584ac974756177618d59b71b46c2fc27f6ff369f901", + "category": "queries" + }, + "e-queries-012": { + "sha256": "d65a677d03c6b58ceba2bbb63b8835c587a58f57772a17147a885ffb2ac5d0f9", + "category": "queries" + }, + "e-queries-013": { + "sha256": "1178cf0ea0fe1c2a3733197707d54403130ca0ae73297e8c70752652255e3fdc", + "category": "queries" + }, + "e-queries-014": { + "sha256": "1ee2b028a8e262af90b165ea479075352ea1c6ef8c48b72afbada4627d02a01e", + "category": "queries" + }, + "e-queries-015": { + "sha256": "7875fad01cee86dbbdac9a719689675f8dd3d167a2c68ad090df1c373d10922f", + "category": "queries" + }, + "e-queries-016": { + "sha256": "3d6140e0a78dcc8e90c330af84115befa0c9e0fa1b523458d64d919854a77371", + "category": "queries" + }, + "e-queries-017": { + "sha256": "052483fee2ef13adbad2b973bd87527116c9b6fc7cc022480ad865f5476a0583", + "category": "queries" + }, + "e-queries-018": { + "sha256": "dab7fe80b114e044be14736674519830dcebacf6e88411eec1788c01a280dd6e", + "category": "queries" + }, + "e-queries-019": { + "sha256": "f333263e10c27578ebac3299a87748a2db881642214623766230b7f624e5d1a3", + "category": "queries" + }, + "e-rest-api-001": { + "sha256": "d9ab7281153d491f55d242812a50e484ac2155888460978e009356dc5d0ba6ce", + "category": "rest-api" + }, + "e-rest-api-002": { + "sha256": "678fad7743c4383f96042e2da3c30b5bb0ee07c9c4d39da9535e01e8bd16c775", + "category": "rest-api" + }, + "e-rest-api-003": { + "sha256": "acf6a6fbbd88124e3518f81da30f422adea5e5405d6629189981e2f36fe4056d", + "category": "rest-api" + }, + "e-rest-api-004": { + "sha256": "1b85069a8c618abc32ead24e0bb8c38be98ce7430808d051247f289918b7f6e8", + "category": "rest-api" + }, + "e-rest-api-005": { + "sha256": "cfdfe1a8f821cda07290df45f7e0211117ef2c4b1701cf5275cd2806ce403105", + "category": "rest-api" + }, + "e-rest-api-006": { + "sha256": "b13c89de43f7bb07696f916d06006a4d6fafdc6fb7bc82c2aa262a2ebce6c9c4", + "category": "rest-api" + }, + "e-rest-api-007": { + "sha256": "94aa4df70916f8e031a7332766e7b13da4121130d16b0718c69fed464b8c6fb0", + "category": "rest-api" + }, + "e-rest-api-008": { + "sha256": "6205bab9df485c5d56c4a088571e1fd227a32927ed798a472e4e0625e8ba4b96", + "category": "rest-api" + }, + "e-rest-api-009": { + "sha256": "fa0741fc19b9506c31be12b677d232b8a9d95ab3e9b01df2e866f7d9513e7f94", + "category": "rest-api" + }, + "e-rest-api-010": { + "sha256": "75a5f72ce329b45bb10bc67103b9a50f671e129fa40154851858c2b27a5d78d4", + "category": "rest-api" + }, + "e-rest-api-011": { + "sha256": "e9a2be427963b1946ebfb0551c244d2c0e77e45c8efc7a6be26d9a5f9ad372e9", + "category": "rest-api" + }, + "e-rest-api-012": { + "sha256": "40d2e9cca4197e66ea651eda4d3c551ad7fd1ab2b64d91d5d00e1b1d56b35148", + "category": "rest-api" + }, + "e-rest-api-013": { + "sha256": "be8290358600946c4b7bc7d36b8b35489babb289ad95d7a5d735e39de8eb48c0", + "category": "rest-api" + }, + "e-rest-api-014": { + "sha256": "89cdc01d63ed44a5b510d119be7aaf7fe6d98968a834181d26ef3eca8d10cf2d", + "category": "rest-api" + }, + "e-rest-api-015": { + "sha256": "3dac803bcfad05c094e4f660881d53cf559a2738bf44b33a4c5a2fc427bb9f15", + "category": "rest-api" + }, + "e-rest-api-016": { + "sha256": "66c2c8c612a840e90ea8ce963ecfda7a157f22d8275a662fa18cc9d78b6bae3c", + "category": "rest-api" + }, + "e-rest-api-017": { + "sha256": "39373a52cd01fabcf4a44079cd4633e095e746f455b375c1dd47582ea1ebf991", + "category": "rest-api" + }, + "e-rest-api-018": { + "sha256": "f068b574b3acb0aba5d976dd2156d7bf5543c2111476f507dd3beac5d640f0bf", + "category": "rest-api" + }, + "e-rest-api-019": { + "sha256": "c18b595ee04bf69354252b9f759999d7e5883cc5017be77d38ff656c1cf4e9dc", + "category": "rest-api" + }, + "e-rest-api-020": { + "sha256": "6931931c82563174c7eff4de42249ec8488ad4e8c4c389fabdcc711ebd2cde94", + "category": "rest-api" + }, + "e-rest-api-021": { + "sha256": "306d551d11b6cb8707b2aa454670d3d371f86ddce54dd580d9da77b7d433546c", + "category": "rest-api" + }, + "e-rest-api-022": { + "sha256": "77dd3a9b379e8c3e581cd8f089812dfb7c774ec1ce3607340d6eaeecf6ed4465", + "category": "rest-api" + }, + "e-rest-api-023": { + "sha256": "9e65be64350f1d0b2b200f6c1c87154fee3666e44b2cc4641797ab85b965886b", + "category": "rest-api" + }, + "e-rest-api-024": { + "sha256": "5f334566cd588cc64e98ad582a6c53c7328f58cdfa898a435bf30948205bc75c", + "category": "rest-api" + }, + "e-rewrite-permalinks-001": { + "sha256": "967e5d11339c294adcb1b138baeedd15db74c638f28a7a2b355d28ab179dbc8c", + "category": "rewrite-permalinks" + }, + "e-rewrite-permalinks-002": { + "sha256": "00493c6b5c4b04476faa710856a23e078225b4ef3ccb0bc2e11e51dae8ff402e", + "category": "rewrite-permalinks" + }, + "e-rewrite-permalinks-003": { + "sha256": "a22ca57706ccc418f27f6a8aa8d7b4760a9837afb62e900f884b8f94a4663e4c", + "category": "rewrite-permalinks" + }, + "e-rewrite-permalinks-004": { + "sha256": "12594a5f3f92a120ccff4d7644d968e985d8052497ad19d4b2abf5632066a89a", + "category": "rewrite-permalinks" + }, + "e-rewrite-permalinks-005": { + "sha256": "539fa30daaaa0e97d4787d9a7507ce666b426370d2b614346dc8272fb7b83fd4", + "category": "rewrite-permalinks" + }, + "e-rewrite-permalinks-006": { + "sha256": "e6bbb4bbf43c9736838ad28a51a294b260a62189af3fe7b25c998a558b878887", + "category": "rewrite-permalinks" + }, + "e-roles-caps-001": { + "sha256": "ce22427aea55b9b8d506345bf6cb8ef55be98353962dc18336353acc2f415f38", + "category": "roles-caps" + }, + "e-roles-caps-002": { + "sha256": "f3894de0a9e966015bf62e561ee21fdc6201574fee789540bc3595ccd32925d2", + "category": "roles-caps" + }, + "e-roles-caps-003": { + "sha256": "96c78acc40f95f56906ec8d1aac3badafd2e48765f9e20bd1901f7ecdd3fc18a", + "category": "roles-caps" + }, + "e-roles-caps-004": { + "sha256": "12f4243f1a1b36c994e8bbb145b0d3fb89f7c1dd92fb3680f8a3fedc82024218", + "category": "roles-caps" + }, + "e-roles-caps-005": { + "sha256": "68d1db17fe8392e8f24aeb6b9f05040d958441f6d6479b5bcf5a454c301f2704", + "category": "roles-caps" + }, + "e-roles-caps-006": { + "sha256": "56b90d0703a89189b4431f2f9e9defff28622b41d3cc00ca7817cc8a545ab5f3", + "category": "roles-caps" + }, + "e-roles-caps-007": { + "sha256": "b1f63108faa5014bb1c5038c1574bef3b24909df63a4cd5c342d9dc5d5c27dea", + "category": "roles-caps" + }, + "e-roles-caps-008": { + "sha256": "bba927d3ab0c8fd72cba425337880abe168e55a5e3175d25196902fe273dc810", + "category": "roles-caps" + }, + "e-roles-caps-009": { + "sha256": "03b876ca6cf578cd1311b6e308cba5b0a3483ca6bd2beb917679c6b544838ae6", + "category": "roles-caps" + }, + "e-roles-caps-010": { + "sha256": "edf2f8034f77a01d5c2783312b2ce17d71481e992bc5360d3534728ed5943801", + "category": "roles-caps" + }, + "e-scripts-styles-001": { + "sha256": "02d1a6a3acbf61bbb13e7b2585fdc8300af480da163457a4f138ad8d07cf2a31", + "category": "scripts-styles" + }, + "e-scripts-styles-002": { + "sha256": "f6dcd1bb1bf6fce42faaecec76f7830efa4b73309deebf69285bca11c6f4fd5b", + "category": "scripts-styles" + }, + "e-scripts-styles-003": { + "sha256": "635392e52f1808bf220d3983f9ae1bbcccd2600727f56b2ab6a6aa832f489a4d", + "category": "scripts-styles" + }, + "e-scripts-styles-004": { + "sha256": "b7dee4b4d1c733fd664ac95c0655e1387c0234aff0dfbc8eab70ec2ce4451dc9", + "category": "scripts-styles" + }, + "e-scripts-styles-005": { + "sha256": "46bb678738ef7d67117c2ffb50b855ffd45656bdc161c429fac7a2a52729a6e0", + "category": "scripts-styles" + }, + "e-security-001": { + "sha256": "285de0012b04f21480aa2051ccf887e94e1c47984ee02373319f7deaa8ed7914", + "category": "security" + }, + "e-security-002": { + "sha256": "f9636bdb437b24ab2362a006e65f6d059b989debcd6e145998dc432fbe5f376f", + "category": "security" + }, + "e-security-003": { + "sha256": "edaec4580f5d7a34e733ca6a41b3ccd0c19a587cfdf07c34a96ed4ccddc678ec", + "category": "security" + }, + "e-security-004": { + "sha256": "414d6048edc7c00d6ff9c5bc4d17dd4fc31afdd55ca12a5fa0d31d73b4ad3564", + "category": "security" + }, + "e-security-005": { + "sha256": "5b922b78eb03e1ca575c67cc96cd6333d9da74e7df92c13c6894286631a9f822", + "category": "security" + }, + "e-security-006": { + "sha256": "e6a92248230e78c500ef2f712c020a7191d3d7dbe951378070884b554b875113", + "category": "security" + }, + "e-security-007": { + "sha256": "123416ad09b0dbf6b225116f9d66dc8a9a180b774ad44d90dbddafc0027ebd71", + "category": "security" + }, + "e-security-008": { + "sha256": "52acb723640e4657bbbc06c1462ccf1a717cb84bba3262a91d6c6f979c55a6ff", + "category": "security" + }, + "e-security-009": { + "sha256": "2bda33ac1a5e942a599ad3d3439c5182280e5d41f525ca51ed2bf56dfe95489a", + "category": "security" + }, + "e-security-010": { + "sha256": "3d53dc1d5c95981893bb8e2f0aaf7b39ed399fc5349c1e3e92e4378df4895e12", + "category": "security" + }, + "e-security-011": { + "sha256": "e0c1e469f0b9415539dd9c560e83353222cb1e3d0ea35d8a1ed825866aa6bf22", + "category": "security" + }, + "e-security-012": { + "sha256": "2926fdae0890db235fdecf3ce8d87d848900b4692e11e4868de9e9b5e9ef5073", + "category": "security" + }, + "e-settings-options-001": { + "sha256": "681b37ae6f71f58b765807677b54402632e9db08e349a396cd69b6f7c43b4abf", + "category": "settings-options" + }, + "e-settings-options-002": { + "sha256": "6f9c271418f996fa73090baf334af94a2d94ca2486bc45e463edc3a20c1dac9c", + "category": "settings-options" + }, + "e-settings-options-003": { + "sha256": "ddf187cd0c4f2194ac09c32e5e47dbe46c35f027e2bfb490d6dc20367d2e67a4", + "category": "settings-options" + }, + "e-settings-options-004": { + "sha256": "eacce56a125c4a843c1eff2739bcecdef9c9d1b67a64c344ae376c339ce8f250", + "category": "settings-options" + }, + "e-settings-options-005": { + "sha256": "776ee219e8ec63bdd16942cd1443fe9c528fe7cb6dac797b16e6182169abb6be", + "category": "settings-options" + }, + "e-settings-options-006": { + "sha256": "76ddf34e691490a5fa9e2130794811f6aa086f35b8a1ea8cf4e6476cfebb10e3", + "category": "settings-options" + }, + "e-settings-options-007": { + "sha256": "60df513615bb66508df1f012cbccfdcace3bc0a7de02433623c67dbfdd532dcd", + "category": "settings-options" + }, + "e-settings-options-008": { + "sha256": "3118ed256024157f2b6e0b9f01d58c5329f5a4e4833f3a8c8ee1e8955ee83cf1", + "category": "settings-options" + }, + "e-settings-options-009": { + "sha256": "2a344efe63a2f37169dc0c828f1a69afedde1d3d41f6d82af315a76c0f898be9", + "category": "settings-options" + }, + "e-settings-options-010": { + "sha256": "30bb54deb2fc8ec09d35099c2da387e1689235d392458807f955e2e005cd51d3", + "category": "settings-options" + }, + "e-shortcodes-001": { + "sha256": "963c6a5792a9328d3b43c0992d6cde74110ecb518900b0ab1d8030023a8ae5ca", + "category": "shortcodes" + }, + "e-shortcodes-002": { + "sha256": "954f03cd431638b955572c7cfe8c1614c977755870668cbfda9c17226acf06e9", + "category": "shortcodes" + }, + "e-shortcodes-003": { + "sha256": "9826d11f94d99da14778eee80277a3ed8fb1243246f72e01de2cfd4e0cfc9a8d", + "category": "shortcodes" + }, + "e-shortcodes-004": { + "sha256": "6c17c64b8a6466079412ab1be477be29c561cc4ebc5d51b0a201138b193582af", + "category": "shortcodes" + }, + "e-shortcodes-005": { + "sha256": "3beb0bee322c5588898cad5fc93478152251e0eb4afac41558cbf4aa8f36abb1", + "category": "shortcodes" + }, + "e-theme-templates-001": { + "sha256": "f8527a5b5382c9e43cca62f58b39b34a514138c84754b5157b49c492456b9aa6", + "category": "theme-templates" + }, + "e-theme-templates-002": { + "sha256": "8b99b0d9833a35fcd6cb56cf3f3358dc1a4674889cb20073d57d22f28419d1bc", + "category": "theme-templates" + }, + "e-theme-templates-003": { + "sha256": "bff6717888e27dc7fc3cee96747d4bbb8a174d3d1adbb0a0ee2b5a48b5e690f5", + "category": "theme-templates" + }, + "e-theme-templates-004": { + "sha256": "294179329405945a06017cf043a4b267c7c19d3b99fd34f4cfdd321a90938a17", + "category": "theme-templates" + }, + "e-theme-templates-005": { + "sha256": "688904294299c05b1c24522aca8f234c76a19284f2c5c3e3ba94217d5b2ad17b", + "category": "theme-templates" + }, + "e-theme-templates-006": { + "sha256": "0027fb8c80a348315af6de675103d803d0759641808dde54c3409aa254d488e0", + "category": "theme-templates" + }, + "e-theme-templates-007": { + "sha256": "ffb04ff7f8e838a0ce015f32c55c49ac2ed29037d5c4ebb180620d5b784fe7d7", + "category": "theme-templates" + }, + "e-theme-templates-008": { + "sha256": "0c4ba4bb9a755a79f32564671902a3e6d25cf2ed8682406fe6713365931647f0", + "category": "theme-templates" + }, + "e-theme-templates-009": { + "sha256": "d8ecf1b85393292a7b1d03eeae6f333125874d2046e42f0f00c7ef47e0f5b0a3", + "category": "theme-templates" + }, + "e-theme-templates-010": { + "sha256": "37af351e0d0e19e7973c019f247ccade7b757ccb2848f0b1217763155ca7887b", + "category": "theme-templates" + }, + "e-theme-templates-011": { + "sha256": "bfa08ffa6a37632638c43c6a653010f8fd458c5bdea1d64e09f6e0d160e78080", + "category": "theme-templates" + } + } + }, + "wp-cli": { + "task_count": 27, + "tasks": { + "e-cli-content-002": { + "sha256": "ce2add7db4e00b84ffddadcb45d67947d7de1bc0844a4b9b3b34b8a058b213a4", + "category": "cli-content" + }, + "e-cli-content-003": { + "sha256": "6416b2553ce430bf69bea9776cfe252fc2a328727c873f0d5d2f98b35725014e", + "category": "cli-content" + }, + "e-cli-content-004": { + "sha256": "8213ace8a9fdf1476f8c713c65e7d805f0096402849cf20b878725abe8aea3ec", + "category": "cli-content" + }, + "e-cli-maintenance-001": { + "sha256": "8d37474d528f1164fd360d61ba45f06ebd215eb959cddc0efc93bc455d3812a0", + "category": "cli-maintenance" + }, + "e-cli-maintenance-002": { + "sha256": "b8d0878edfeec1b99d813f6e420e3c6ab059690fe88c28c85d275509f9d83c92", + "category": "cli-maintenance" + }, + "e-cli-maintenance-003": { + "sha256": "f2a799c438434c1ddec3f9d98222d1c17420fcbae82d8de37b8d3f9fdfa40949", + "category": "cli-maintenance" + }, + "e-cli-maintenance-004": { + "sha256": "c01e091c69db679099543b7c65e059d72ec960bae5cf25b9a489d1e630b0a5ed", + "category": "cli-maintenance" + }, + "e-cli-maintenance-005": { + "sha256": "ac0fb3d555ff811ae334c8ac31cd044b52ca825072a4ac4ef826c0e149b50e63", + "category": "cli-maintenance" + }, + "e-cli-maintenance-006": { + "sha256": "dff9bf6f0c132fc19cb333f8a84058efda41cf39279c0521514ce8eafcc1c5ee", + "category": "cli-maintenance" + }, + "e-cli-migration-001": { + "sha256": "07381136df2708db5193dc44e6cad520c711f5f83845cdd3acf04c421dec657a", + "category": "cli-migration" + }, + "e-cli-migration-002": { + "sha256": "9bcaf8f60dde80d8e01e26c1822450a126056486c411159e22bbfb8aa368b6f6", + "category": "cli-migration" + }, + "e-cli-options-001": { + "sha256": "8070e8f55e35e9f526ba029f380db7ef65b6b389300aba24702ac8c904760809", + "category": "cli-options" + }, + "e-cli-options-003": { + "sha256": "5a2c65a56a8c0adecf6b1e0701d64b2819edb279093a9b0521ca4b89bf633996", + "category": "cli-options" + }, + "e-cli-options-004": { + "sha256": "25e34b1d0a08d1d4ce72d53a0619b0850502ca305326b93f8536ba582af6082c", + "category": "cli-options" + }, + "e-cli-options-005": { + "sha256": "d8ceb48dcfe168a620cc9790fc2ae3e80cfea1f6ea33fba2d52da12c2dc1468a", + "category": "cli-options" + }, + "e-cli-reporting-001": { + "sha256": "a033c28e62bd407bd58728f9c035012dcea4dd3d6c9dd8224f59ce4b31f8ad7d", + "category": "cli-reporting" + }, + "e-cli-reporting-002": { + "sha256": "4ec28641a37806b31463352fc4572d3d7fd3c30894fb682eedb56c1a0c863935", + "category": "cli-reporting" + }, + "e-cli-reporting-004": { + "sha256": "05262f1ab6cb1e75a0790c175be00f016bb3e7514ef5a800405128ba0f2928ed", + "category": "cli-reporting" + }, + "e-cli-taxonomy-meta-001": { + "sha256": "38f02e100479909fce654594550d09a5975bc307c8e8ffe0b2ab009edba12d67", + "category": "cli-taxonomy-meta" + }, + "e-cli-taxonomy-meta-002": { + "sha256": "bc8489f9891de385bcfd47e12af2cbb1e44f965df57d557438e2912485932de7", + "category": "cli-taxonomy-meta" + }, + "e-cli-taxonomy-meta-004": { + "sha256": "0dee15444c2719cce56539b0f3af59b6ab5bef73683ed39919ca5b3773ce3ac5", + "category": "cli-taxonomy-meta" + }, + "e-cli-taxonomy-meta-005": { + "sha256": "b23366af2c9fb965e16435d7980f34b5b67f0773252094d1339cc4760e2b978a", + "category": "cli-taxonomy-meta" + }, + "e-cli-users-002": { + "sha256": "14b54832a4d080b0fb08c040a083d376eb310e2cf8c7c6ea5019e6ff88a0d491", + "category": "cli-users" + }, + "e-cli-users-003": { + "sha256": "b5cfd991e7e1c9a2c1dfe37fc86c434a85e1d5f85bfdf4d6939b22cab6d471c6", + "category": "cli-users" + }, + "e-cli-users-004": { + "sha256": "2924923d621a991251f8a7d3edf1a7823ddd66df41602e161065e55745dfedaf", + "category": "cli-users" + }, + "e-cli-users-005": { + "sha256": "b6496d115bee0f083ce2fbe9983b64bdb12f7449ee90d8a7de530fbdc13b6dc5", + "category": "cli-users" + }, + "e-cli-users-006": { + "sha256": "caf1a3204c54b3d88491495ad99c3874265b4ea114969ea5555fb33009914055", + "category": "cli-users" + } + } + } + } + } +} diff --git a/datasets/README.md b/datasets/README.md index 61d59d7..9018d11 100644 --- a/datasets/README.md +++ b/datasets/README.md @@ -7,12 +7,12 @@ This directory contains the benchmark test suites and tooling for publishing to ``` datasets/ ├── suites/ # Source of truth (human-editable JSON) -│ ├── wp-core-v1/ +│ ├── wp-core/ │ │ └── execution/ # Code generation tests (one file per category) │ │ ├── hooks.json │ │ ├── rest-api.json │ │ └── ... -│ └── wp-cli-v1/ +│ └── wp-cli/ │ └── execution/ # 27 WP-CLI workflows, seven categories ├── data/ # Generated Parquet for HF (gitignored) │ └── test.parquet @@ -28,7 +28,7 @@ The harness loads directly from `suites/` JSON files: # wp-bench.yaml dataset: source: local - name: wp-core-v1 + name: wp-core ``` ## Publishing to Hugging Face @@ -108,7 +108,7 @@ The local loader selects `dataset.name`; the Hub loader uses `dataset.name` as the repository and `dataset.suite` to filter the combined export, defaulting to `run.suite`. An explicit Hub `dataset.suite` also sets the result-suite label. `--suite` handles both sources. Exports include both suites, and omit authored -exploit candidates. CLI scores should be reported separately from core scores. +exploit candidates. Report component/category scores alongside the task-weighted overall score defined by the public benchmark release. ### Per-Test Metadata @@ -132,3 +132,11 @@ field named `suite_metadata` would be shadowed — avoid that name). All fields are optional; unknown fields are preserved as-is. Prefer adding new fields here before inventing per-suite conventions. + +## Release identity + +`wp-core` and `wp-cli` are stable component IDs. Category JSON files carry no +independent version. The complete public release and all source fingerprints +are defined in [../benchmark/README.md](../benchmark/README.md). Any dataset +change creates an unreleased evaluation until the next overall release is frozen. +Exports use the stable IDs; publish a pinned Hub revision for each public release. diff --git a/datasets/suites/wp-cli-v1/execution/cli-content.json b/datasets/suites/wp-cli/execution/cli-content.json similarity index 95% rename from datasets/suites/wp-cli-v1/execution/cli-content.json rename to datasets/suites/wp-cli/execution/cli-content.json index d8e58cc..673cbfe 100644 --- a/datasets/suites/wp-cli-v1/execution/cli-content.json +++ b/datasets/suites/wp-cli/execution/cli-content.json @@ -1,5 +1,5 @@ { - "id": "wp-cli-v1", + "id": "wp-cli", "name": "WP-CLI operational workflows: cli-content", "metadata": { "wp_version": "7.1", @@ -24,8 +24,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -82,8 +82,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -139,8 +139,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] diff --git a/datasets/suites/wp-cli-v1/execution/cli-maintenance.json b/datasets/suites/wp-cli/execution/cli-maintenance.json similarity index 96% rename from datasets/suites/wp-cli-v1/execution/cli-maintenance.json rename to datasets/suites/wp-cli/execution/cli-maintenance.json index 4aa388d..e99303c 100644 --- a/datasets/suites/wp-cli-v1/execution/cli-maintenance.json +++ b/datasets/suites/wp-cli/execution/cli-maintenance.json @@ -1,5 +1,5 @@ { - "id": "wp-cli-v1", + "id": "wp-cli", "name": "WP-CLI operational workflows: cli-maintenance", "metadata": { "wp_version": "7.1", @@ -24,8 +24,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -81,8 +81,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -137,8 +137,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -196,8 +196,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -258,8 +258,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -317,8 +317,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] diff --git a/datasets/suites/wp-cli-v1/execution/cli-migration.json b/datasets/suites/wp-cli/execution/cli-migration.json similarity index 97% rename from datasets/suites/wp-cli-v1/execution/cli-migration.json rename to datasets/suites/wp-cli/execution/cli-migration.json index eac7360..41848db 100644 --- a/datasets/suites/wp-cli-v1/execution/cli-migration.json +++ b/datasets/suites/wp-cli/execution/cli-migration.json @@ -1,5 +1,5 @@ { - "id": "wp-cli-v1", + "id": "wp-cli", "name": "WP-CLI operational workflows: cli-migration", "metadata": { "wp_version": "7.1", @@ -26,8 +26,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -89,8 +89,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] diff --git a/datasets/suites/wp-cli-v1/execution/cli-options.json b/datasets/suites/wp-cli/execution/cli-options.json similarity index 95% rename from datasets/suites/wp-cli-v1/execution/cli-options.json rename to datasets/suites/wp-cli/execution/cli-options.json index 8c23edd..fb318a0 100644 --- a/datasets/suites/wp-cli-v1/execution/cli-options.json +++ b/datasets/suites/wp-cli/execution/cli-options.json @@ -1,5 +1,5 @@ { - "id": "wp-cli-v1", + "id": "wp-cli", "name": "WP-CLI operational workflows: cli-options", "metadata": { "wp_version": "7.1", @@ -24,8 +24,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -80,8 +80,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -137,8 +137,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -195,8 +195,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] diff --git a/datasets/suites/wp-cli-v1/execution/cli-reporting.json b/datasets/suites/wp-cli/execution/cli-reporting.json similarity index 96% rename from datasets/suites/wp-cli-v1/execution/cli-reporting.json rename to datasets/suites/wp-cli/execution/cli-reporting.json index babf5c7..2fca47f 100644 --- a/datasets/suites/wp-cli-v1/execution/cli-reporting.json +++ b/datasets/suites/wp-cli/execution/cli-reporting.json @@ -1,5 +1,5 @@ { - "id": "wp-cli-v1", + "id": "wp-cli", "name": "WP-CLI operational workflows: cli-reporting", "metadata": { "wp_version": "7.1", @@ -24,8 +24,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -89,8 +89,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -146,8 +146,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] diff --git a/datasets/suites/wp-cli-v1/execution/cli-taxonomy-meta.json b/datasets/suites/wp-cli/execution/cli-taxonomy-meta.json similarity index 95% rename from datasets/suites/wp-cli-v1/execution/cli-taxonomy-meta.json rename to datasets/suites/wp-cli/execution/cli-taxonomy-meta.json index 50bea88..903fa73 100644 --- a/datasets/suites/wp-cli-v1/execution/cli-taxonomy-meta.json +++ b/datasets/suites/wp-cli/execution/cli-taxonomy-meta.json @@ -1,5 +1,5 @@ { - "id": "wp-cli-v1", + "id": "wp-cli", "name": "WP-CLI operational workflows: cli-taxonomy-meta", "metadata": { "wp_version": "7.1", @@ -23,8 +23,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -80,8 +80,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -137,8 +137,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -195,8 +195,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] diff --git a/datasets/suites/wp-cli-v1/execution/cli-users.json b/datasets/suites/wp-cli/execution/cli-users.json similarity index 96% rename from datasets/suites/wp-cli-v1/execution/cli-users.json rename to datasets/suites/wp-cli/execution/cli-users.json index b7ee476..058af25 100644 --- a/datasets/suites/wp-cli-v1/execution/cli-users.json +++ b/datasets/suites/wp-cli/execution/cli-users.json @@ -1,5 +1,5 @@ { - "id": "wp-cli-v1", + "id": "wp-cli", "name": "WP-CLI operational workflows: cli-users", "metadata": { "wp_version": "7.1", @@ -23,8 +23,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -86,8 +86,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -143,8 +143,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -201,8 +201,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] @@ -262,8 +262,8 @@ "static_checks": { "forbidden_patterns": [ { - "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", - "description": "Use native WordPress operations; read-only table/prefix discovery is allowed, but PHP evaluation and direct SQL are forbidden", + "pattern": "/(?:\\bwp\\s+(?:--[^\\s]+\\s+)*(?:eval(?:-file)?|db(?!\\s+(?:--[^\\s]+\\s+)*(?:prefix|tables|columns)\\b)|shell)\\b|--(?:exec|require)(?:=|\\s)|(?:^|[\\s;&|])(?:php|sqlite3|mysql)\\s)/m", + "description": "Use native WordPress operations; read-only table, prefix, and column discovery is allowed, but PHP evaluation and direct SQL are forbidden", "severity": "error" } ] diff --git a/datasets/suites/wp-core-v1/execution/abilities-api.json b/datasets/suites/wp-core/execution/abilities-api.json similarity index 97% rename from datasets/suites/wp-core-v1/execution/abilities-api.json rename to datasets/suites/wp-core/execution/abilities-api.json index 56c9339..f2b5c89 100644 --- a/datasets/suites/wp-core-v1/execution/abilities-api.json +++ b/datasets/suites/wp-core/execution/abilities-api.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-abilities_api", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Abilities API", "description": "Code generation tasks for WordPress Abilities API APIs", @@ -182,7 +181,7 @@ "id": "e-abilities-api-004", "category": "abilities-api", "prompt": "Register an Abilities API ability named 'wpbp/rest-visible' in the 'wpbp-rest-abilities' category and configure the ability to be available through the Abilities REST API.", - "expected_behavior": "Registers the `wpbp-rest-abilities` category and the `wpbp/rest-visible` ability through the Abilities API init hooks, with `meta.show_in_rest` set to true on the ability. Runtime validation inspects the registered ability metadata and executes its callback; REST visibility should come from Abilities API metadata, not a custom REST route.", + "expected_behavior": "The ability is registered in the requested category with REST visibility metadata. Its callback result is unconstrained because the prompt does not specify a return payload.", "requirements": [ "Use WordPress Abilities API", "Configure the ability for Abilities REST API availability", @@ -233,7 +232,7 @@ "assertions": [ { "type": "custom_assertion", - "code": "WP_Abilities_Registry::get_instance(); $ability = wp_get_ability( 'wpbp/rest-visible' ); if ( ! $ability instanceof WP_Ability ) { return false; } $meta = $ability->get_meta(); return 'wpbp-rest-abilities' === $ability->get_category() && true === ( $meta['show_in_rest'] ?? false ) && array( 'ok' => true ) === $ability->execute();", + "code": "WP_Abilities_Registry::get_instance(); $ability = wp_get_ability( 'wpbp/rest-visible' ); if ( ! $ability instanceof WP_Ability ) { return false; } $meta = $ability->get_meta(); return 'wpbp-rest-abilities' === $ability->get_category() && true === ( $meta['show_in_rest'] ?? false );", "description": "The ability is registered with REST visibility metadata", "weight": 1 } @@ -248,7 +247,11 @@ "src/wp-includes/abilities-api/class-wp-ability-categories-registry.php" ], "release_focus": "6.9" - } + }, + "exploit_solutions": [ + "add_action( 'wp_abilities_api_categories_init', function () { wp_register_ability_category( 'wpbp-rest-abilities', array( 'label' => 'REST', 'description' => 'REST abilities.' ) ); } ); add_action( 'wp_abilities_api_init', function () { wp_register_ability( 'wpbp/rest-visible', array( 'label' => 'Probe', 'description' => 'Probe', 'category' => 'wpbp-rest-abilities', 'execute_callback' => '__return_true', 'permission_callback' => '__return_true' ) ); } );", + "add_action( 'wp_abilities_api_categories_init', function () { wp_register_ability_category( 'wpbp-wrong', array( 'label' => 'Wrong', 'description' => 'Wrong category.' ) ); } ); add_action( 'wp_abilities_api_init', function () { wp_register_ability( 'wpbp/rest-visible', array( 'label' => 'Probe', 'description' => 'Probe', 'category' => 'wpbp-wrong', 'execute_callback' => '__return_true', 'permission_callback' => '__return_true', 'meta' => array( 'show_in_rest' => true ) ) ); } );" + ] }, { "id": "e-abilities-api-005", diff --git a/datasets/suites/wp-core-v1/execution/admin.json b/datasets/suites/wp-core/execution/admin.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/admin.json rename to datasets/suites/wp-core/execution/admin.json index 2bb2f42..0c26ba1 100644 --- a/datasets/suites/wp-core-v1/execution/admin.json +++ b/datasets/suites/wp-core/execution/admin.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-admin", - "version": "1.0.1", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Admin", "description": "Code generation tasks for the WordPress admin: menus, list tables, settings, AJAX and notices", diff --git a/datasets/suites/wp-core-v1/execution/ai-client.json b/datasets/suites/wp-core/execution/ai-client.json similarity index 93% rename from datasets/suites/wp-core-v1/execution/ai-client.json rename to datasets/suites/wp-core/execution/ai-client.json index 79295ba..b5bbdf0 100644 --- a/datasets/suites/wp-core-v1/execution/ai-client.json +++ b/datasets/suites/wp-core/execution/ai-client.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-ai_client", - "version": "2.4.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - AI Client", "description": "Code generation tasks for WordPress AI Client APIs", @@ -60,7 +59,7 @@ "id": "e-ai-client-002", "category": "ai-client", "prompt": "Implement a function that creates a text prompt builder with a concise system instruction, temperature 0.2, and a model preference order of 'gpt-4o-mini' then 'gpt-4o'. Return the builder without generating content.", - "expected_behavior": "`wpbp_ai_client_002()` returns a `WP_AI_Client_Prompt_Builder` from `wp_ai_client_prompt()` after configuring text-generation options with snake_case AI Client methods. It must not call a generating method; runtime validation later prevents generation and confirms the returned builder is not already in an error state.", + "expected_behavior": "The returned builder must contain the requested temperature, a nonempty system instruction, and the requested model preference order, and must remain ready for a later prevented generation. Runtime inspection uses the pinned WordPress 7.1 SDK fields without making a provider call.", "requirements": [ "Use wp_ai_client_prompt()", "Configure temperature, system instruction, and model preference", @@ -102,7 +101,7 @@ "assertions": [ { "type": "custom_assertion", - "code": "if ( ! function_exists( 'wpbp_ai_client_002' ) ) { return false; } $builder = wpbp_ai_client_002(); if ( ! $builder instanceof WP_AI_Client_Prompt_Builder ) { return false; } add_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); $result = $builder->generate_text(); remove_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); return is_wp_error( $result ) && 'prompt_prevented' === $result->get_error_code();", + "code": "if ( ! function_exists( 'wpbp_ai_client_002' ) ) { return false; } $builder = wpbp_ai_client_002(); if ( ! $builder instanceof WP_AI_Client_Prompt_Builder ) { return false; } $wrapped = ( new ReflectionProperty( WP_AI_Client_Prompt_Builder::class, 'builder' ) )->getValue( $builder ); $config = ( new ReflectionProperty( $wrapped, 'modelConfig' ) )->getValue( $wrapped ); $preferences = ( new ReflectionProperty( $wrapped, 'modelPreferenceKeys' ) )->getValue( $wrapped ); if ( 0.2 !== $config->getTemperature() || ! is_string( $config->getSystemInstruction() ) || '' === trim( $config->getSystemInstruction() ) || array( 'model::gpt-4o-mini', 'model::gpt-4o' ) !== $preferences ) { return false; } add_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); $result = $builder->generate_text(); remove_filter( 'wp_ai_client_prevent_prompt', '__return_true' ); return is_wp_error( $result ) && 'prompt_prevented' === $result->get_error_code();", "description": "The helper returns a usable builder without calling a provider", "weight": 1 } @@ -113,10 +112,15 @@ "source_refs": [ "src/wp-includes/ai-client.php", "src/wp-includes/ai-client/class-wp-ai-client-prompt-builder.php", - "src/wp-includes/php-ai-client/src/Builders/PromptBuilder.php" + "src/wp-includes/php-ai-client/src/Builders/PromptBuilder.php", + "src/wp-includes/php-ai-client/src/Providers/Models/DTO/ModelConfig.php" ], "release_focus": "7.0" - } + }, + "exploit_solutions": [ + "function wpbp_ai_client_002(): WP_AI_Client_Prompt_Builder { return wp_ai_client_prompt( 'Summarize WordPress.' ); }", + "function wpbp_ai_client_002(): WP_AI_Client_Prompt_Builder { return wp_ai_client_prompt( 'Summarize WordPress.' )->using_system_instruction( 'Be concise.' )->using_temperature( 0.2 )->using_model_preference( 'gpt-4o', 'gpt-4o-mini' ); }" + ] }, { "id": "e-ai-client-003", @@ -176,7 +180,7 @@ "id": "e-ai-client-004", "category": "ai-client", "prompt": "Implement a function that returns the AI Client function-call name for a supplied ability name and the ability name obtained by converting it back.", - "expected_behavior": "`wpbp_ai_client_004()` uses `WP_AI_Client_Ability_Function_Resolver` conversion helpers to return `array( $function_name, $round_trip_ability_name )` for any supplied ability name. Runtime validation checks multiple ability names so hard-coded output does not pass.", + "expected_behavior": "The helper returns both the converted function name and the round-tripped ability name for multiple inputs. Numeric and associative array keys are both accepted because the prompt specifies values, not array keys.", "requirements": [ "Use WP_AI_Client_Ability_Function_Resolver", "Use both conversion directions", @@ -206,7 +210,7 @@ "assertions": [ { "type": "custom_assertion", - "code": "if ( ! function_exists( 'wpbp_ai_client_004' ) ) { return false; } $first = wpbp_ai_client_004( 'wpbp/do-thing' ); $second = wpbp_ai_client_004( 'my-plugin/export_data' ); return is_array( $first ) && is_array( $second ) && 'wpab__wpbp__do-thing' === ( $first[0] ?? null ) && 'wpbp/do-thing' === ( $first[1] ?? null ) && 'wpab__my-plugin__export_data' === ( $second[0] ?? null ) && 'my-plugin/export_data' === ( $second[1] ?? null );", + "code": "if ( ! function_exists( 'wpbp_ai_client_004' ) ) { return false; } $first = wpbp_ai_client_004( 'wpbp/do-thing' ); $second = wpbp_ai_client_004( 'my-plugin/export_data' ); return is_array( $first ) && is_array( $second ) && in_array( 'wpab__wpbp__do-thing', $first, true ) && in_array( 'wpbp/do-thing', $first, true ) && in_array( 'wpab__my-plugin__export_data', $second, true ) && in_array( 'my-plugin/export_data', $second, true );", "description": "Ability names round-trip through AI function-call names", "weight": 1 } diff --git a/datasets/suites/wp-core-v1/execution/auth.json b/datasets/suites/wp-core/execution/auth.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/auth.json rename to datasets/suites/wp-core/execution/auth.json index 07ee515..d53cbaa 100644 --- a/datasets/suites/wp-core-v1/execution/auth.json +++ b/datasets/suites/wp-core/execution/auth.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-auth", - "version": "1.0.1", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Authentication", "description": "Code generation tasks for WordPress authentication, password and session APIs", diff --git a/datasets/suites/wp-core-v1/execution/block-patterns-styles.json b/datasets/suites/wp-core/execution/block-patterns-styles.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/block-patterns-styles.json rename to datasets/suites/wp-core/execution/block-patterns-styles.json index 5ec25e3..6f52299 100644 --- a/datasets/suites/wp-core-v1/execution/block-patterns-styles.json +++ b/datasets/suites/wp-core/execution/block-patterns-styles.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-block_patterns_styles", - "version": "1.0.1", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Block Patterns and Styles", "description": "Code generation tasks for WordPress block pattern, block style and block template registration APIs", diff --git a/datasets/suites/wp-core-v1/execution/caching.json b/datasets/suites/wp-core/execution/caching.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/caching.json rename to datasets/suites/wp-core/execution/caching.json index 85ee08e..661ecaa 100644 --- a/datasets/suites/wp-core-v1/execution/caching.json +++ b/datasets/suites/wp-core/execution/caching.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-caching", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Caching", "description": "Code generation tasks for WordPress Caching APIs", diff --git a/datasets/suites/wp-core-v1/execution/comments-users.json b/datasets/suites/wp-core/execution/comments-users.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/comments-users.json rename to datasets/suites/wp-core/execution/comments-users.json index de900da..4ae2055 100644 --- a/datasets/suites/wp-core-v1/execution/comments-users.json +++ b/datasets/suites/wp-core/execution/comments-users.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-comments_users", - "version": "2.4.1", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Comments and Users", "description": "Code generation tasks for WordPress Comments and Users APIs", diff --git a/datasets/suites/wp-core-v1/execution/connectors-api.json b/datasets/suites/wp-core/execution/connectors-api.json similarity index 98% rename from datasets/suites/wp-core-v1/execution/connectors-api.json rename to datasets/suites/wp-core/execution/connectors-api.json index 3544ef0..2b2001d 100644 --- a/datasets/suites/wp-core-v1/execution/connectors-api.json +++ b/datasets/suites/wp-core/execution/connectors-api.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-connectors_api", - "version": "2.4.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Connectors API", "description": "Code generation tasks for WordPress Connectors API APIs", @@ -183,7 +182,7 @@ "id": "e-connectors-api-004", "category": "connectors-api", "prompt": "Register a connector with the ID 'wpbp_crm' through the Connectors API extension action: name 'WPBP CRM', type 'crm', API-key authentication with no explicit setting name.", - "expected_behavior": "The connector registers from a `wp_connectors_init` callback with `authentication => array( 'method' => 'api_key' )`. The runtime re-fires the action and verifies the stored connector, including the registry's auto-generated setting name `connectors_crm_wpbp_crm_api_key` and the default `plugin.is_active` callback.", + "expected_behavior": "The connector has the requested ID, name, type, and authentication method, and receives the registry-generated setting name. An incidental plugin callback representation is not constrained by the prompt.", "requirements": [ "Register from the wp_connectors_init action", "Use api_key authentication without a setting name" @@ -211,7 +210,7 @@ "assertions": [ { "type": "custom_assertion", - "code": "$registry = WP_Connector_Registry::get_instance(); if ( ! $registry instanceof WP_Connector_Registry ) { return false; } do_action( 'wp_connectors_init', $registry ); if ( ! wp_is_connector_registered( 'wpbp_crm' ) ) { return false; } $connector = wp_get_connector( 'wpbp_crm' ); return 'WPBP CRM' === $connector['name'] && 'crm' === $connector['type'] && 'api_key' === $connector['authentication']['method'] && 'connectors_crm_wpbp_crm_api_key' === $connector['authentication']['setting_name'] && '__return_true' === $connector['plugin']['is_active'];", + "code": "$registry = WP_Connector_Registry::get_instance(); if ( ! $registry instanceof WP_Connector_Registry ) { return false; } do_action( 'wp_connectors_init', $registry ); if ( ! wp_is_connector_registered( 'wpbp_crm' ) ) { return false; } $connector = wp_get_connector( 'wpbp_crm' ); return 'WPBP CRM' === $connector['name'] && 'crm' === $connector['type'] && 'api_key' === $connector['authentication']['method'] && 'connectors_crm_wpbp_crm_api_key' === $connector['authentication']['setting_name'];", "description": "The connector registers with the auto-generated setting name and default plugin callback", "weight": 1 } diff --git a/datasets/suites/wp-core-v1/execution/cron.json b/datasets/suites/wp-core/execution/cron.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/cron.json rename to datasets/suites/wp-core/execution/cron.json index 9b24e66..8881f7e 100644 --- a/datasets/suites/wp-core-v1/execution/cron.json +++ b/datasets/suites/wp-core/execution/cron.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-cron", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Cron", "description": "Code generation tasks for WordPress Cron APIs", diff --git a/datasets/suites/wp-core-v1/execution/database.json b/datasets/suites/wp-core/execution/database.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/database.json rename to datasets/suites/wp-core/execution/database.json index 176d4f7..e0f8112 100644 --- a/datasets/suites/wp-core-v1/execution/database.json +++ b/datasets/suites/wp-core/execution/database.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-database", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Database", "description": "Code generation tasks for WordPress Database APIs", diff --git a/datasets/suites/wp-core-v1/execution/dates-time.json b/datasets/suites/wp-core/execution/dates-time.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/dates-time.json rename to datasets/suites/wp-core/execution/dates-time.json index 23f81ed..4c99914 100644 --- a/datasets/suites/wp-core-v1/execution/dates-time.json +++ b/datasets/suites/wp-core/execution/dates-time.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-dates_time", - "version": "1.0.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Dates and Time", "description": "Timezone-aware date handling: site timezone rendering, UTC round trips, relative ages and local calendar-day queries.", diff --git a/datasets/suites/wp-core-v1/execution/gb-block-api.json b/datasets/suites/wp-core/execution/gb-block-api.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/gb-block-api.json rename to datasets/suites/wp-core/execution/gb-block-api.json index f3831ca..491cc54 100644 --- a/datasets/suites/wp-core-v1/execution/gb-block-api.json +++ b/datasets/suites/wp-core/execution/gb-block-api.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-gb_block_api", - "version": "2.3.1", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Block API", "description": "Code generation tasks for WordPress Block API APIs", diff --git a/datasets/suites/wp-core-v1/execution/gb-block-bindings.json b/datasets/suites/wp-core/execution/gb-block-bindings.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/gb-block-bindings.json rename to datasets/suites/wp-core/execution/gb-block-bindings.json index 9c62eb8..4fdeee7 100644 --- a/datasets/suites/wp-core-v1/execution/gb-block-bindings.json +++ b/datasets/suites/wp-core/execution/gb-block-bindings.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-gb_block_bindings", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Block Bindings", "description": "Code generation tasks for WordPress Block Bindings APIs", diff --git a/datasets/suites/wp-core-v1/execution/gb-block-editor.json b/datasets/suites/wp-core/execution/gb-block-editor.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/gb-block-editor.json rename to datasets/suites/wp-core/execution/gb-block-editor.json index 75aa635..86bd8aa 100644 --- a/datasets/suites/wp-core-v1/execution/gb-block-editor.json +++ b/datasets/suites/wp-core/execution/gb-block-editor.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-gb_block_editor", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Block Editor", "description": "Code generation tasks for WordPress Block Editor APIs", diff --git a/datasets/suites/wp-core-v1/execution/gb-block-hooks.json b/datasets/suites/wp-core/execution/gb-block-hooks.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/gb-block-hooks.json rename to datasets/suites/wp-core/execution/gb-block-hooks.json index 79bbd8d..5e30b1c 100644 --- a/datasets/suites/wp-core-v1/execution/gb-block-hooks.json +++ b/datasets/suites/wp-core/execution/gb-block-hooks.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-gb_block_hooks", - "version": "2.2.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Block Hooks", "description": "Code generation tasks for WordPress Block Hooks APIs", diff --git a/datasets/suites/wp-core-v1/execution/gb-block-markup.json b/datasets/suites/wp-core/execution/gb-block-markup.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/gb-block-markup.json rename to datasets/suites/wp-core/execution/gb-block-markup.json index 2ff2da8..cad454c 100644 --- a/datasets/suites/wp-core-v1/execution/gb-block-markup.json +++ b/datasets/suites/wp-core/execution/gb-block-markup.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-gb_block_markup", - "version": "1.2.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Block Markup Authoring", "description": "Code generation tasks that require authoring serialized Gutenberg block markup as stored in post_content", diff --git a/datasets/suites/wp-core-v1/execution/gb-font-library.json b/datasets/suites/wp-core/execution/gb-font-library.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/gb-font-library.json rename to datasets/suites/wp-core/execution/gb-font-library.json index 48e8f46..686c15c 100644 --- a/datasets/suites/wp-core-v1/execution/gb-font-library.json +++ b/datasets/suites/wp-core/execution/gb-font-library.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-gb_font_library", - "version": "2.2.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Font Library", "description": "Code generation tasks for WordPress Font Library APIs", diff --git a/datasets/suites/wp-core-v1/execution/gb-interactivity-api.json b/datasets/suites/wp-core/execution/gb-interactivity-api.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/gb-interactivity-api.json rename to datasets/suites/wp-core/execution/gb-interactivity-api.json index 5196103..e4a4fa7 100644 --- a/datasets/suites/wp-core-v1/execution/gb-interactivity-api.json +++ b/datasets/suites/wp-core/execution/gb-interactivity-api.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-gb_interactivity_api", - "version": "2.3.1", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Interactivity API", "description": "Code generation tasks for WordPress Interactivity API APIs", diff --git a/datasets/suites/wp-core-v1/execution/gb-templates-navigation.json b/datasets/suites/wp-core/execution/gb-templates-navigation.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/gb-templates-navigation.json rename to datasets/suites/wp-core/execution/gb-templates-navigation.json index 0de679b..6592f6a 100644 --- a/datasets/suites/wp-core-v1/execution/gb-templates-navigation.json +++ b/datasets/suites/wp-core/execution/gb-templates-navigation.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-gb_templates_navigation", - "version": "2.2.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Templates and Navigation", "description": "Code generation tasks for WordPress Templates and Navigation APIs", diff --git a/datasets/suites/wp-core-v1/execution/hooks.json b/datasets/suites/wp-core/execution/hooks.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/hooks.json rename to datasets/suites/wp-core/execution/hooks.json index 83b577b..4a14e84 100644 --- a/datasets/suites/wp-core-v1/execution/hooks.json +++ b/datasets/suites/wp-core/execution/hooks.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-hooks", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Hooks", "description": "Code generation tasks for WordPress Hooks APIs", diff --git a/datasets/suites/wp-core-v1/execution/html-api.json b/datasets/suites/wp-core/execution/html-api.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/html-api.json rename to datasets/suites/wp-core/execution/html-api.json index 133b683..f062f4e 100644 --- a/datasets/suites/wp-core-v1/execution/html-api.json +++ b/datasets/suites/wp-core/execution/html-api.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-html_api", - "version": "1.0.1", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - HTML API", "description": "Code generation tasks for the WordPress HTML API (Tag Processor, HTML Processor, and decoder)", diff --git a/datasets/suites/wp-core-v1/execution/http-api.json b/datasets/suites/wp-core/execution/http-api.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/http-api.json rename to datasets/suites/wp-core/execution/http-api.json index 2ac1ad0..2c667d8 100644 --- a/datasets/suites/wp-core-v1/execution/http-api.json +++ b/datasets/suites/wp-core/execution/http-api.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-http_api", - "version": "1.0.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - HTTP API", "description": "Code generation tasks for the WordPress HTTP API", diff --git a/datasets/suites/wp-core-v1/execution/internationalization.json b/datasets/suites/wp-core/execution/internationalization.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/internationalization.json rename to datasets/suites/wp-core/execution/internationalization.json index 3076f8d..9992391 100644 --- a/datasets/suites/wp-core-v1/execution/internationalization.json +++ b/datasets/suites/wp-core/execution/internationalization.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-internationalization", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Internationalization", "description": "Code generation tasks for WordPress Internationalization APIs", diff --git a/datasets/suites/wp-core-v1/execution/mail-url.json b/datasets/suites/wp-core/execution/mail-url.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/mail-url.json rename to datasets/suites/wp-core/execution/mail-url.json index 082cf7c..42aa5c0 100644 --- a/datasets/suites/wp-core-v1/execution/mail-url.json +++ b/datasets/suites/wp-core/execution/mail-url.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-mail_url", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Mail and URL Handling", "description": "Code generation tasks for WordPress Mail and URL Handling APIs", diff --git a/datasets/suites/wp-core-v1/execution/media.json b/datasets/suites/wp-core/execution/media.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/media.json rename to datasets/suites/wp-core/execution/media.json index 64001df..72eacfd 100644 --- a/datasets/suites/wp-core-v1/execution/media.json +++ b/datasets/suites/wp-core/execution/media.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-media", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Media", "description": "Code generation tasks for WordPress Media APIs", diff --git a/datasets/suites/wp-core-v1/execution/performance.json b/datasets/suites/wp-core/execution/performance.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/performance.json rename to datasets/suites/wp-core/execution/performance.json index 6f7abed..34aa341 100644 --- a/datasets/suites/wp-core-v1/execution/performance.json +++ b/datasets/suites/wp-core/execution/performance.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-performance", - "version": "1.0.1", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Performance", "description": "Code generation tasks for WordPress asset loading, script modules, speculative loading, and media performance APIs", diff --git a/datasets/suites/wp-core-v1/execution/plugin-lifecycle.json b/datasets/suites/wp-core/execution/plugin-lifecycle.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/plugin-lifecycle.json rename to datasets/suites/wp-core/execution/plugin-lifecycle.json index 219ca87..c253a8b 100644 --- a/datasets/suites/wp-core-v1/execution/plugin-lifecycle.json +++ b/datasets/suites/wp-core/execution/plugin-lifecycle.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-plugin_lifecycle", - "version": "1.0.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Plugin Lifecycle", "description": "Plugin artifact tasks for WordPress plugin activation, upgrade and plugin screen integration", diff --git a/datasets/suites/wp-core-v1/execution/post-meta.json b/datasets/suites/wp-core/execution/post-meta.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/post-meta.json rename to datasets/suites/wp-core/execution/post-meta.json index c13341e..2cf21f4 100644 --- a/datasets/suites/wp-core-v1/execution/post-meta.json +++ b/datasets/suites/wp-core/execution/post-meta.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-post_meta", - "version": "2.2.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Post Meta", "description": "Code generation tasks for WordPress Post Meta APIs", diff --git a/datasets/suites/wp-core-v1/execution/post-types-taxonomy.json b/datasets/suites/wp-core/execution/post-types-taxonomy.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/post-types-taxonomy.json rename to datasets/suites/wp-core/execution/post-types-taxonomy.json index 9a19c8a..ddac1b3 100644 --- a/datasets/suites/wp-core-v1/execution/post-types-taxonomy.json +++ b/datasets/suites/wp-core/execution/post-types-taxonomy.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-post_types_taxonomy", - "version": "2.4.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Post Types and Taxonomy", "description": "Code generation tasks for WordPress Post Types and Taxonomy APIs", diff --git a/datasets/suites/wp-core-v1/execution/posts-lifecycle.json b/datasets/suites/wp-core/execution/posts-lifecycle.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/posts-lifecycle.json rename to datasets/suites/wp-core/execution/posts-lifecycle.json index dbcb69c..a6bf0f7 100644 --- a/datasets/suites/wp-core-v1/execution/posts-lifecycle.json +++ b/datasets/suites/wp-core/execution/posts-lifecycle.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-posts_lifecycle", - "version": "1.0.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Posts Lifecycle", "description": "Code generation tasks for the WordPress post creation, publication, revision and deletion lifecycle", diff --git a/datasets/suites/wp-core-v1/execution/queries.json b/datasets/suites/wp-core/execution/queries.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/queries.json rename to datasets/suites/wp-core/execution/queries.json index e0b9709..ea2b82f 100644 --- a/datasets/suites/wp-core-v1/execution/queries.json +++ b/datasets/suites/wp-core/execution/queries.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-queries", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Queries", "description": "Code generation tasks for WordPress Queries APIs", diff --git a/datasets/suites/wp-core-v1/execution/rest-api.json b/datasets/suites/wp-core/execution/rest-api.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/rest-api.json rename to datasets/suites/wp-core/execution/rest-api.json index bc0c868..84f54bf 100644 --- a/datasets/suites/wp-core-v1/execution/rest-api.json +++ b/datasets/suites/wp-core/execution/rest-api.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-rest_api", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - REST API", "description": "Code generation tasks for WordPress REST API APIs", diff --git a/datasets/suites/wp-core-v1/execution/rewrite-permalinks.json b/datasets/suites/wp-core/execution/rewrite-permalinks.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/rewrite-permalinks.json rename to datasets/suites/wp-core/execution/rewrite-permalinks.json index b91d03e..fd14923 100644 --- a/datasets/suites/wp-core-v1/execution/rewrite-permalinks.json +++ b/datasets/suites/wp-core/execution/rewrite-permalinks.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-rewrite_permalinks", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Rewrite and Permalinks", "description": "Code generation tasks for WordPress Rewrite and Permalinks APIs", diff --git a/datasets/suites/wp-core-v1/execution/roles-caps.json b/datasets/suites/wp-core/execution/roles-caps.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/roles-caps.json rename to datasets/suites/wp-core/execution/roles-caps.json index 3b3d5b7..63057eb 100644 --- a/datasets/suites/wp-core-v1/execution/roles-caps.json +++ b/datasets/suites/wp-core/execution/roles-caps.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-roles_caps", - "version": "2.4.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Roles and Capabilities", "description": "Code generation tasks for WordPress Roles and Capabilities APIs", diff --git a/datasets/suites/wp-core-v1/execution/scripts-styles.json b/datasets/suites/wp-core/execution/scripts-styles.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/scripts-styles.json rename to datasets/suites/wp-core/execution/scripts-styles.json index 0c80c36..f103a27 100644 --- a/datasets/suites/wp-core-v1/execution/scripts-styles.json +++ b/datasets/suites/wp-core/execution/scripts-styles.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-scripts_styles", - "version": "2.2.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Scripts and Styles", "description": "Code generation tasks for WordPress Scripts and Styles APIs", diff --git a/datasets/suites/wp-core-v1/execution/security.json b/datasets/suites/wp-core/execution/security.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/security.json rename to datasets/suites/wp-core/execution/security.json index c62e499..dcaa037 100644 --- a/datasets/suites/wp-core-v1/execution/security.json +++ b/datasets/suites/wp-core/execution/security.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-security", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Security", "description": "Code generation tasks for WordPress Security APIs", diff --git a/datasets/suites/wp-core-v1/execution/settings-options.json b/datasets/suites/wp-core/execution/settings-options.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/settings-options.json rename to datasets/suites/wp-core/execution/settings-options.json index 03a0e6d..e14c198 100644 --- a/datasets/suites/wp-core-v1/execution/settings-options.json +++ b/datasets/suites/wp-core/execution/settings-options.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-settings_options", - "version": "2.3.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Settings and Options", "description": "Code generation tasks for WordPress Settings and Options APIs", diff --git a/datasets/suites/wp-core-v1/execution/shortcodes.json b/datasets/suites/wp-core/execution/shortcodes.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/shortcodes.json rename to datasets/suites/wp-core/execution/shortcodes.json index f05f35e..7584476 100644 --- a/datasets/suites/wp-core-v1/execution/shortcodes.json +++ b/datasets/suites/wp-core/execution/shortcodes.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-shortcodes", - "version": "2.2.0", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Shortcodes", "description": "Code generation tasks for WordPress Shortcodes APIs", diff --git a/datasets/suites/wp-core-v1/execution/theme-templates.json b/datasets/suites/wp-core/execution/theme-templates.json similarity index 99% rename from datasets/suites/wp-core-v1/execution/theme-templates.json rename to datasets/suites/wp-core/execution/theme-templates.json index 67defdc..d208f60 100644 --- a/datasets/suites/wp-core-v1/execution/theme-templates.json +++ b/datasets/suites/wp-core/execution/theme-templates.json @@ -1,6 +1,5 @@ { - "id": "wp-core-execution-v1-theme_templates", - "version": "1.0.1", + "id": "wp-core", "metadata": { "name": "WordPress Core Execution Tests - Theme Templates", "description": "Code generation tasks for WordPress theme, block template, global styles, and template rendering APIs", diff --git a/python/tests/test_artifacts.py b/python/tests/test_artifacts.py index 24df833..53cfdc5 100644 --- a/python/tests/test_artifacts.py +++ b/python/tests/test_artifacts.py @@ -39,7 +39,7 @@ def _plugin_test(test_id: str = "e-plugin-001") -> ExecutionTest: return ExecutionTest( id=test_id, - suite="wp-core-v1", + suite="wp-core", prompt="Build a plugin.", expected_behavior="expected", category="plugins", @@ -143,7 +143,7 @@ def test_prompt_instructions_differ_by_kind() -> None: def _config(tmp_path: Path, **run_overrides: object) -> HarnessConfig: return HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig.model_validate(dict(run_overrides)), diff --git a/python/tests/test_benchmark_releases.py b/python/tests/test_benchmark_releases.py new file mode 100644 index 0000000..5a7cdea --- /dev/null +++ b/python/tests/test_benchmark_releases.py @@ -0,0 +1,347 @@ +"""Release identity, immutable history, and safe website ingestion.""" +from __future__ import annotations + +import copy +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +from wp_bench.benchmark import ( + HARNESS_INPUTS, + capture_identity, + digest, + evaluation_snapshot, + freeze_release, + load_release, + publication_issues, + read_json, +) +from wp_bench.benchmark_cli import app, check_releases, export_results +from wp_bench.config import ( + BenchmarkConfig, + DatasetConfig, + HarnessConfig, + ModelConfig, + RunConfig, +) +from wp_bench.datasets import _parse_execution_suite + + +@pytest.fixture +def release_root(tmp_path: Path) -> Path: + root = tmp_path / "repo" + definition = { + "schema_version": "1.0", "wordpress_version": "7.1", "scoring_version": "3.0", + "suites": ["wp-core", "wp-cli"], + "categories": [{"id": "core", "label": "Core"}, {"id": "wp-cli", "label": "WP-CLI"}], + "category_mapping": {"wp-core": {"hooks": "core"}, "wp-cli": {"cli": "wp-cli"}}, + "protocol": {"model": {"temperature": 1.0, "top_p": None, "max_tokens": 16384}, + "grader": {"kind": "docker", "timeout_seconds": 90}, "repeats": 2}, + "runtime_image_ids": ["sha256:approved"], + } + (root / "benchmark").mkdir(parents=True) + (root / "benchmark/definition.json").write_text(json.dumps(definition)) + for suite, category in [("wp-core", "hooks"), ("wp-cli", "cli")]: + directory = root / "datasets/suites" / suite / "execution" + directory.mkdir(parents=True) + (directory / "tasks.json").write_text(json.dumps({"id": suite, "tests": [{ + "id": f"e-{category}-001", "prompt": "Implement the requested WordPress behavior.", + "requirements": [], "category": category, "runtime_checks": {"assertions": []}, + }]})) + (root / "runtime").mkdir() + (root / "runtime/verify-runtime.php").write_text(" HarnessConfig: + return HarnessConfig( + benchmark=BenchmarkConfig(version="v1" if pinned else None), + dataset=DatasetConfig(source="local", name=suite), + model=ModelConfig(name="model-a", temperature=1, max_tokens=16384), + run=RunConfig(suite=suite), + ) + + +def result(root: Path, suite: str, run_id: str, name: str = "model-a") -> dict[str, Any]: + cfg = config(suite) + tests = _parse_execution_suite(root / "datasets/suites" / suite / "execution/tasks.json") + identity = capture_identity(cfg, tests, root=root) + identity.bind_runtime("sha256:approved") + records = [{"test_id": test.id, "category": test.category, "mode": "model", + "model": {"name": name, "temperature": 1.0, "top_p": None, "max_tokens": 16384}, + "variant": {"kind": "none"}, "scores": {"execution_pass": True}, + "error": None, "run_id": run_id, + "benchmark": {**identity.record_info(), "task_sha256": identity.tasks[test.id]}} + for test in tests] + return {"metadata": {"suite": suite, "run_id": run_id, "benchmark": identity.metadata(records), + "grader": cfg.grader.model_dump(mode="json"), "scoring_version": "3.0", + "runtime_isolation": "reset_per_test", "runtime_image_id": "sha256:approved"}, + "results": records} + + +def inputs(root: Path, tmp_path: Path, name: str = "model-a") -> list[Path]: + paths = [] + for suite in ["wp-core", "wp-cli"]: + for repeat in [1, 2]: + path = tmp_path / f"{name}-{suite}-{repeat}.json" + path.write_text(json.dumps(result(root, suite, f"{name}-{suite}-{repeat}", name))) + paths.append(path) + return paths + + +def test_one_version_freezes_every_dataset_and_the_runtime(release_root: Path) -> None: + manifest = load_release(root=release_root) + assert manifest["version"] == "v1" + assert set(manifest["evaluation"]["components"]) == {"wp-core", "wp-cli"} + assert manifest["fingerprint"] == digest(evaluation_snapshot(release_root)) + assert check_releases(release_root)["working_tree"] == "released" + + +@pytest.mark.parametrize("relative", [ + "datasets/suites/wp-core/execution/tasks.json", "datasets/suites/wp-cli/execution/tasks.json", + "runtime/verify-runtime.php", "python/wp_bench/scoring.py", "benchmark/definition.json", +]) +def test_any_component_change_requires_a_new_release(release_root: Path, relative: str) -> None: + path = release_root / relative + if relative == "benchmark/definition.json": + definition = read_json(path) + definition["protocol"]["grader"]["timeout_seconds"] = 120 + path.write_text(json.dumps(definition)) + else: + path.write_text(path.read_text() + "\n") + # JSON formatting is conservatively part of a frozen source snapshot too. + tests = _parse_execution_suite(release_root / "datasets/suites/wp-core/execution/tasks.json") + identity = capture_identity(config(), tests, root=release_root) + assert identity.record_info()["version"] is None + assert identity.record_info()["status"] == "unreleased" + with pytest.raises(ValueError, match="Cannot run as v1"): + capture_identity(config(pinned=True), tests, root=release_root) + old = load_release(root=release_root) + new = freeze_release("v2", release_root) + assert old["fingerprint"] != new["fingerprint"] + assert load_release("v1", release_root) == old + + +def test_model_additions_do_not_create_a_benchmark_version(release_root: Path) -> None: + before = load_release(root=release_root) + cfg = config() + cfg.model = ModelConfig(name="another-model", temperature=1, max_tokens=16384) + tests = _parse_execution_suite(release_root / "datasets/suites/wp-core/execution/tasks.json") + assert capture_identity(cfg, tests, root=release_root).record_info()["version"] == "v1" + assert load_release(root=release_root) == before + with pytest.raises(ValueError, match="unchanged"): + freeze_release("v2", release_root) + + +def test_changed_loaded_hub_task_cannot_claim_a_release(release_root: Path) -> None: + tests = _parse_execution_suite(release_root / "datasets/suites/wp-core/execution/tasks.json") + tests[0].prompt = "A different prompt, from an unpinned Hub revision" + with pytest.raises(ValueError, match="Loaded tasks differ"): + capture_identity(config(pinned=True), tests, root=release_root) + + +def test_model_protocol_change_is_not_v1(release_root: Path) -> None: + cfg = config(pinned=True) + assert cfg.model is not None + cfg.model.temperature = 0 + tests = _parse_execution_suite(release_root / "datasets/suites/wp-core/execution/tasks.json") + with pytest.raises(ValueError, match="Model settings"): + capture_identity(cfg, tests, root=release_root) + + +def test_runtime_is_verified_before_pinned_generation(release_root: Path) -> None: + tests = _parse_execution_suite(release_root / "datasets/suites/wp-core/execution/tasks.json") + identity = capture_identity(config(pinned=True), tests, root=release_root) + with pytest.raises(ValueError, match="Runtime image"): + identity.bind_runtime("sha256:unapproved", required=True) + + +def test_frozen_releases_cannot_be_overwritten_or_relabelled(release_root: Path) -> None: + path = release_root / "benchmark/releases/v1.json" + original = path.read_bytes() + with pytest.raises(ValueError, match="immutable"): + freeze_release("v1", release_root) + with pytest.raises(ValueError, match="Next benchmark release must be v2"): + freeze_release("v3", release_root) + assert path.read_bytes() == original + changed = json.loads(original) + changed["evaluation"]["definition"]["wordpress_version"] = "another-version" + path.write_text(json.dumps(changed)) + with pytest.raises(ValueError, match="Invalid benchmark release"): + load_release(root=release_root) + + +@pytest.mark.parametrize("change", ["partial", "duplicate", "error", "reference", "skills", "mixed-version", + "wrong-image", "wrong-grader", "wrong-task", "wrong-run", "ungraded"]) +def test_unsafe_results_cannot_enter_a_leaderboard(release_root: Path, change: str) -> None: + payload = result(release_root, "wp-core", "run-one") + assert publication_issues(payload, load_release(root=release_root)) == [] + record = payload["results"][0] + if change == "partial": + payload["results"] = [] + elif change == "duplicate": + payload["results"].append(copy.deepcopy(record)) + elif change == "error": + record["error"] = {"type": "provider_error"} + elif change == "reference": + record["mode"] = "reference_solution" + elif change == "skills": + record["variant"]["kind"] = "skills" + elif change == "mixed-version": + record["benchmark"]["version"] = "v2" + elif change == "wrong-image": + payload["metadata"]["runtime_image_id"] = "sha256:another" + elif change == "wrong-grader": + payload["metadata"]["grader"]["timeout_seconds"] = 120 + elif change == "wrong-task": + record["benchmark"]["task_sha256"] = "different" + elif change == "wrong-run": + record["run_id"] = "another-run" + elif change == "ungraded": + record["scores"]["execution_pass"] = None + assert publication_issues(payload, load_release(root=release_root)) + + +def test_export_preserves_old_snapshots_and_is_idempotent(release_root: Path, tmp_path: Path) -> None: + paths = inputs(release_root, tmp_path) + output = tmp_path / "public" + first = export_results(paths, output, root=release_root) + original = first.read_bytes() + assert export_results(paths, output, root=release_root) == first + assert first.read_bytes() == original + bundle = json.loads(original) + assert bundle["benchmark"]["version"] == "v1" + assert len(bundle["models"][0]["answers"]) == 4 + second = export_results(paths + inputs(release_root, tmp_path, "model-b"), output, root=release_root) + assert second != first + assert first.read_bytes() == original + catalog = read_json(output / "results.json") + assert len(catalog["benchmarks"]["v1"]["snapshots"]) == 2 + + +def test_export_requires_both_complete_suites_and_independent_repeats(release_root: Path, tmp_path: Path) -> None: + paths = inputs(release_root, tmp_path) + with pytest.raises(ValueError, match="independent complete runs"): + export_results(paths[:-1], tmp_path / "public", root=release_root) + with pytest.raises(ValueError, match="Duplicate run identity"): + export_results(paths + [paths[0]], tmp_path / "public", root=release_root) + assert not (tmp_path / "public").exists() + + +def test_export_supports_multi_model_result_envelopes(release_root: Path, tmp_path: Path) -> None: + paths = inputs(release_root, tmp_path) + for path in paths: + payload = read_json(path) + metadata = payload["metadata"] + wrapped = {"metadata": metadata, "models": {"model-a": { + "run_id": metadata["run_id"], "benchmark": metadata["benchmark"], "results": payload["results"], + }}} + path.write_text(json.dumps(wrapped)) + destination = export_results(paths, tmp_path / "public", root=release_root) + assert len(read_json(destination)["models"]) == 1 + + +def test_ci_detects_changes_even_when_manifest_hashes_are_recomputed(release_root: Path) -> None: + def git(*args: str) -> str: + return subprocess.run(["git", *args], cwd=release_root, check=True, + capture_output=True, text=True).stdout.strip() + git("init") + git("add", ".") + git("-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-m", "initial") + base = git("rev-parse", "HEAD") + assert check_releases(release_root, base)["working_tree"] == "released" + path = release_root / "benchmark/releases/v1.json" + manifest = read_json(path) + manifest["evaluation"]["definition"]["wordpress_version"] = "changed" + manifest["fingerprint"] = digest(manifest["evaluation"]) + path.write_text(json.dumps(manifest)) + catalog_path = release_root / "benchmark/releases.json" + catalog = read_json(catalog_path) + catalog["releases"][0]["fingerprint"] = manifest["fingerprint"] + catalog_path.write_text(json.dumps(catalog)) + with pytest.raises(ValueError, match="Published benchmark manifest changed"): + check_releases(release_root, base) + + +def test_cli_exposes_json_contract_without_a_model_call(release_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("wp_bench.benchmark_cli.load_release", lambda version=None: load_release(version, release_root)) + response = CliRunner().invoke(app, ["show", "--version", "v1"]) + assert response.exit_code == 0 + assert json.loads(response.stdout)["version"] == "v1" + + + +def test_export_refuses_a_corrupted_existing_snapshot(release_root: Path, tmp_path: Path) -> None: + paths = inputs(release_root, tmp_path) + output = tmp_path / "public" + destination = export_results(paths, output, root=release_root) + snapshot = read_json(destination) + snapshot["models"][0]["answers"][0]["passed"] = False + destination.write_text(json.dumps(snapshot)) + with pytest.raises(ValueError, match="different fingerprint"): + export_results(paths, output, root=release_root) + + +@pytest.mark.parametrize("field", ["repeats", "images", "categories", "mapping", "temperature"]) +def test_freeze_rejects_an_invalid_recipe(release_root: Path, field: str) -> None: + path = release_root / "benchmark/definition.json" + definition = read_json(path) + if field == "repeats": + definition["protocol"]["repeats"] = 0 + elif field == "images": + definition["runtime_image_ids"] = [] + elif field == "categories": + definition["categories"].append(definition["categories"][0]) + elif field == "mapping": + del definition["category_mapping"]["wp-cli"] + elif field == "temperature": + definition["protocol"]["model"]["temperature"] = 20 + path.write_text(json.dumps(definition)) + with pytest.raises(ValueError): + freeze_release("v2", release_root) + assert not (release_root / "benchmark/releases/v2.json").exists() + + +def test_harness_streams_the_same_verified_identity_as_the_completed_payload( + release_root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: + from conftest import fake_generation + + from wp_bench.config import OutputConfig + from wp_bench.core import BenchmarkRunner + from wp_bench.environment import ExecutionResult + + cfg = config(pinned=True) + cfg.output = OutputConfig(path=tmp_path / "complete.json", jsonl_path=tmp_path / "stream.jsonl") + tests = _parse_execution_suite(release_root / "datasets/suites/wp-core/execution/tasks.json") + monkeypatch.setattr("wp_bench.core.load_tests", lambda config: tests) + monkeypatch.setattr("wp_bench.core.capture_identity", lambda config, tests: capture_identity(config, tests, root=release_root)) + + class Environment: + image_id = "sha256:approved" + + def setup(self, **kwargs: Any) -> None: ... + def reset(self) -> None: ... + def release(self) -> None: ... + def close(self) -> None: ... + def execute_artifact(self, artifact: Any, spec: Any) -> ExecutionResult: + return ExecutionResult(success=True, raw={"success": True, "runtime": {"score": 1.0}, "static": {"score": 1.0}}, stdout="", stderr="") + + runner = BenchmarkRunner(cfg) + runner.environment = Environment() # type: ignore[assignment] + monkeypatch.setattr(runner.model, "generate_with_metadata", lambda prompt: fake_generation(" None: """The shipped example must only use supported fields.""" example = PROJECT_ROOT / "wp-bench.example.yaml" config = HarnessConfig.from_file(example) - assert config.run.suite == "wp-core-v1" + assert config.run.suite == "wp-core" assert config.grader.kind in ("docker", "cli") @@ -67,7 +67,7 @@ def test_skip_runtime_controls_verification_payload() -> None: test = ExecutionTest( id="e-one", - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="general", @@ -101,7 +101,7 @@ def test_skipped_dimension_not_scored() -> None: test = ExecutionTest( id="e-one", - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="general", diff --git a/python/tests/test_continue_on_error.py b/python/tests/test_continue_on_error.py index 105cffd..ba97527 100644 --- a/python/tests/test_continue_on_error.py +++ b/python/tests/test_continue_on_error.py @@ -23,7 +23,7 @@ def _execution_test(test_id: str) -> ExecutionTest: return ExecutionTest( id=test_id, - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="general", @@ -47,7 +47,7 @@ def _passing_result() -> ExecutionResult: def _config(tmp_path: Path, **run_overrides: Any) -> HarnessConfig: return HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig(**run_overrides), diff --git a/python/tests/test_dataset_filtering.py b/python/tests/test_dataset_filtering.py index 488435e..7b5c713 100644 --- a/python/tests/test_dataset_filtering.py +++ b/python/tests/test_dataset_filtering.py @@ -13,7 +13,7 @@ def _execution_test(test_id: str, category: str = "general") -> ExecutionTest: return ExecutionTest( id=test_id, - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category=category, @@ -58,7 +58,7 @@ def test_filter_tests_by_ids_rejects_unknown_ids() -> None: def test_zero_selected_tests_fails_loudly() -> None: """An empty selection (missing suite, execution-less dataset) must not produce a vacuous successful run.""" - config = HarnessConfig.model_validate({"dataset": {"source": "local", "name": "wp-core-v1"}}) + config = HarnessConfig.model_validate({"dataset": {"source": "local", "name": "wp-core"}}) with pytest.raises(ValueError, match="No execution tests selected"): select_run_tests([], config) @@ -137,7 +137,7 @@ def test_dry_run_category_filters_selected_test_ids(monkeypatch: pytest.MonkeyPa def test_category_filter_rejects_mixed_known_and_unknown_categories() -> None: config = HarnessConfig.model_validate( { - "dataset": {"source": "local", "name": "wp-core-v1"}, + "dataset": {"source": "local", "name": "wp-core"}, "run": {"categories": ["hooks", "not-a-category"]}, } ) diff --git a/python/tests/test_dataset_metadata.py b/python/tests/test_dataset_metadata.py index 0199fb2..300e231 100644 --- a/python/tests/test_dataset_metadata.py +++ b/python/tests/test_dataset_metadata.py @@ -30,7 +30,7 @@ def test_local_parser_preserves_execution_task_metadata() -> None: """Per-task provenance fields survive local loading at the top level.""" - tests = load_tests(DatasetConfig(source="local", name="wp-core-v1")) + tests = load_tests(DatasetConfig(source="local", name="wp-core")) with_refs = [t for t in tests if t.metadata.get("source_refs")] assert with_refs, "expected execution tasks with source_refs metadata" @@ -73,7 +73,7 @@ def test_huggingface_loader_preserves_metadata(monkeypatch: pytest.MonkeyPatch) rows = [ { "id": "e-one", - "suite": "wp-core-v1", + "suite": "wp-core", "test_kind": "execution", "prompt": "Prompt", "requirements": "[]", @@ -84,7 +84,7 @@ def test_huggingface_loader_preserves_metadata(monkeypatch: pytest.MonkeyPatch) }, { "id": "k-one", - "suite": "wp-core-v1", + "suite": "wp-core", "test_kind": "knowledge", "prompt": "Prompt", "requirements": "[]", @@ -114,9 +114,9 @@ def test_export_dataset_writes_metadata_column() -> None: finally: sys.path.pop(0) - rows = load_suite("wp-core-v1") + rows = load_suite("wp-core") - assert rows, "expected exported rows for wp-core-v1" + assert rows, "expected exported rows for wp-core" assert all("metadata" in row for row in rows) execution_rows = [r for r in rows if r["test_kind"] == "execution"] assert any('"source_refs"' in r["metadata"] for r in execution_rows) @@ -129,7 +129,7 @@ def test_result_record_includes_task_metadata( """Task provenance is auditable from every result record.""" test = ExecutionTest( id="e-one", - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="general", @@ -141,7 +141,7 @@ def test_result_record_includes_task_metadata( metadata={"source_refs": ["plugin.php"], "release_focus": "6.9", "suite_metadata": {}}, ) config = HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig(), diff --git a/python/tests/test_execution_dataset.py b/python/tests/test_execution_dataset.py index 3c54163..37996a6 100644 --- a/python/tests/test_execution_dataset.py +++ b/python/tests/test_execution_dataset.py @@ -7,7 +7,7 @@ import orjson PROJECT_ROOT = Path(__file__).resolve().parents[2] -EXECUTION_DIR = PROJECT_ROOT / "datasets" / "suites" / "wp-core-v1" / "execution" +EXECUTION_DIR = PROJECT_ROOT / "datasets" / "suites" / "wp-core" / "execution" SUPPORTED_ASSERTIONS = { "class_exists", diff --git a/python/tests/test_execution_isolation.py b/python/tests/test_execution_isolation.py index dd28914..b18090d 100644 --- a/python/tests/test_execution_isolation.py +++ b/python/tests/test_execution_isolation.py @@ -23,7 +23,7 @@ def _execution_test(test_id: str) -> ExecutionTest: return ExecutionTest( id=test_id, - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="general", @@ -47,7 +47,7 @@ def _passing_result() -> ExecutionResult: def _config(tmp_path: Path, **run_overrides: Any) -> HarnessConfig: return HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig(**run_overrides), @@ -136,7 +136,7 @@ def test_multi_model_runner_resets_between_models( lambda dataset: tests, ) config = HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), models=[ModelConfig(name="model-a"), ModelConfig(name="model-b")], grader=GraderConfig(kind="cli"), run=RunConfig(), diff --git a/python/tests/test_execution_prompt.py b/python/tests/test_execution_prompt.py index 8eacd64..412430c 100644 --- a/python/tests/test_execution_prompt.py +++ b/python/tests/test_execution_prompt.py @@ -12,7 +12,7 @@ def _test(artifact_kind: str = "php_snippet") -> ExecutionTest: return ExecutionTest( id="e-x-001", - suite="wp-core-v1", + suite="wp-core", prompt="Register a thing.", expected_behavior="expected", category="hooks", diff --git a/python/tests/test_exploit_audit.py b/python/tests/test_exploit_audit.py index a29b185..98f2d14 100644 --- a/python/tests/test_exploit_audit.py +++ b/python/tests/test_exploit_audit.py @@ -23,7 +23,7 @@ def _execution_test(test_id: str, test_function: str | None = "wpbp_x( int $n ): int") -> ExecutionTest: return ExecutionTest( id=test_id, - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="general", @@ -94,7 +94,7 @@ def execute_artifact(self, artifact: Any, verification_spec: dict) -> ExecutionR def _config(tmp_path: Path, **run_overrides: Any) -> HarnessConfig: return HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig(check_exploits=True, **run_overrides), diff --git a/python/tests/test_reference_solution_mode.py b/python/tests/test_reference_solution_mode.py index d9c14a6..d02a7ea 100644 --- a/python/tests/test_reference_solution_mode.py +++ b/python/tests/test_reference_solution_mode.py @@ -21,7 +21,7 @@ def _config(tmp_path: Path, test_ids: list[str] | None = None) -> HarnessConfig: return HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig( @@ -38,7 +38,7 @@ def _config(tmp_path: Path, test_ids: list[str] | None = None) -> HarnessConfig: def _execution_test(test_id: str = "e-one") -> ExecutionTest: return ExecutionTest( id=test_id, - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="Reviewer contract: expected", category="general", diff --git a/python/tests/test_result_schema.py b/python/tests/test_result_schema.py index a2eb890..1266e5c 100644 --- a/python/tests/test_result_schema.py +++ b/python/tests/test_result_schema.py @@ -25,7 +25,7 @@ def _execution_test(test_id: str = "e-one") -> ExecutionTest: return ExecutionTest( id=test_id, - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="hooks", @@ -60,7 +60,7 @@ def _key_paths(record: dict[str, Any], prefix: str = "") -> set[str]: def _single_model_records(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> list: config = HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig(), @@ -81,7 +81,7 @@ def _single_model_records(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> li def _multi_model_records(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> list: config = HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), models=[ModelConfig(name="model-a")], grader=GraderConfig(kind="cli"), run=RunConfig(), @@ -154,7 +154,7 @@ def test_reference_solution_record_uses_canonical_schema( tmp_path: Path, ) -> None: config = HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig(check_reference_solution=True), @@ -191,7 +191,7 @@ def test_jsonl_records_match_json_results( tmp_path: Path, ) -> None: config = HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig(), @@ -226,7 +226,7 @@ def test_records_are_sorted_for_stable_output( tmp_path: Path, ) -> None: config = HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="test-model"), grader=GraderConfig(kind="cli"), run=RunConfig(), diff --git a/python/tests/test_result_streaming.py b/python/tests/test_result_streaming.py index 43fdf70..6d26e45 100644 --- a/python/tests/test_result_streaming.py +++ b/python/tests/test_result_streaming.py @@ -102,7 +102,7 @@ def _multi_model_config(tmp_path: Path, names: list[str]): ) return HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), models=[ModelConfig(name=name) for name in names], grader=GraderConfig(kind="cli"), run=RunConfig(), @@ -121,7 +121,7 @@ def _run_multi_model(monkeypatch, tmp_path: Path, names: list[str]): test = ExecutionTest( id="e-one", - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="hooks", diff --git a/python/tests/test_scoring.py b/python/tests/test_scoring.py index fcd9c44..b3e3fe3 100644 --- a/python/tests/test_scoring.py +++ b/python/tests/test_scoring.py @@ -14,7 +14,7 @@ def _make_test( ) -> ExecutionTest: return ExecutionTest( id="e-test-001", - suite="wp-core-v1", + suite="wp-core", prompt="Do something.", expected_behavior="Reviewer contract: does something observable.", category="hooks", diff --git a/python/tests/test_selection.py b/python/tests/test_selection.py index bcfc01d..c5196cc 100644 --- a/python/tests/test_selection.py +++ b/python/tests/test_selection.py @@ -12,7 +12,7 @@ def _test(test_id: str, category: str) -> ExecutionTest: return ExecutionTest( id=test_id, - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category=category, @@ -176,7 +176,7 @@ def test_category_filtering_happens_before_limit_selection() -> None: def test_categories_flow_from_harness_config_to_run_selection() -> None: config = HarnessConfig.model_validate( { - "dataset": {"source": "local", "name": "wp-core-v1"}, + "dataset": {"source": "local", "name": "wp-core"}, "run": {"categories": ["bbb"]}, } ) @@ -189,7 +189,7 @@ def test_categories_flow_from_harness_config_to_run_selection() -> None: def test_unknown_category_raises_value_error_naming_category() -> None: config = HarnessConfig.model_validate( { - "dataset": {"source": "local", "name": "wp-core-v1"}, + "dataset": {"source": "local", "name": "wp-core"}, "run": {"categories": ["not-a-category"]}, } ) diff --git a/python/tests/test_shell_runtime_integration.py b/python/tests/test_shell_runtime_integration.py index a87751a..c09e398 100644 --- a/python/tests/test_shell_runtime_integration.py +++ b/python/tests/test_shell_runtime_integration.py @@ -74,7 +74,7 @@ def test_role_changes_are_reloaded_before_assertions(environment) -> None: def test_capability_workflow_accepts_explicit_denial_as_well_as_removal(environment) -> None: - test = next(t for t in load_tests(DatasetConfig(source="local", name="wp-cli-v1")) if t.id == "e-cli-users-002") + test = next(t for t in load_tests(DatasetConfig(source="local", name="wp-cli")) if t.id == "e-cli-users-002") assert test.reference_solution is not None script = test.reference_solution.replace( "wp cap remove cli-contributor publish_posts", "wp cap add cli-contributor publish_posts --grant=false", @@ -156,7 +156,7 @@ def test_shell_limits_remove_candidate_and_descendant_processes(environment, scr def test_slow_script_stops_inside_the_shell_and_keeps_its_output(environment, monkeypatch) -> None: grader = GraderConfig(timeout_seconds=10) monkeypatch.setattr(environment.config, "timeout_seconds", grader.timeout_seconds) - test = load_tests(DatasetConfig(source="local", name="wp-cli-v1"))[0] + test = load_tests(DatasetConfig(source="local", name="wp-cli"))[0] test.runtime_checks = {"assertions": [{"type": "custom_assertion", "code": "return true;"}]} environment.reset() result = environment.execute_artifact( @@ -201,7 +201,7 @@ def test_rewriting_a_fixture_php_file_in_the_same_second_scores_zero(environment @pytest.mark.parametrize("test_id", ["e-cli-maintenance-001", "e-cli-maintenance-002"]) def test_fixture_plugins_do_not_leak_into_the_next_test(environment, test_id) -> None: - test = next(t for t in load_tests(DatasetConfig(source="local", name="wp-cli-v1")) if t.id == test_id) + test = next(t for t in load_tests(DatasetConfig(source="local", name="wp-cli")) if t.id == test_id) artifact = Artifact(kind="wp_cli_shell", code=test.reference_solution or "") spec = _build_verification_spec(test, HarnessConfig()) payload = base64.b64encode(json.dumps({**artifact.payload_fields(), **spec}).encode()).decode() diff --git a/python/tests/test_skill_variants.py b/python/tests/test_skill_variants.py index 9815df7..512686e 100644 --- a/python/tests/test_skill_variants.py +++ b/python/tests/test_skill_variants.py @@ -29,7 +29,7 @@ def _execution_test(test_id: str = "e-one") -> ExecutionTest: return ExecutionTest( id=test_id, - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="hooks", @@ -74,7 +74,7 @@ def _skill_dir(tmp_path: Path) -> Path: def _config(tmp_path: Path, skill_path: Path) -> HarnessConfig: return HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), models=[ModelConfig(name="model-a")], grader=GraderConfig(kind="cli"), run=RunConfig(), diff --git a/python/tests/test_usage_capture.py b/python/tests/test_usage_capture.py index aacf299..90561e5 100644 --- a/python/tests/test_usage_capture.py +++ b/python/tests/test_usage_capture.py @@ -135,7 +135,7 @@ def test_result_record_contains_usage_and_model_call( ) -> None: test = ExecutionTest( id="e-one", - suite="wp-core-v1", + suite="wp-core", prompt="Prompt", expected_behavior="expected", category="hooks", @@ -147,7 +147,7 @@ def test_result_record_contains_usage_and_model_call( metadata={}, ) config = HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-core-v1"), + dataset=DatasetConfig(source="local", name="wp-core"), model=ModelConfig(name="gpt-4o-mini"), grader=GraderConfig(kind="cli"), run=RunConfig(), diff --git a/python/tests/test_wp_cli_shell.py b/python/tests/test_wp_cli_shell.py index c1d22fd..d18c3cb 100644 --- a/python/tests/test_wp_cli_shell.py +++ b/python/tests/test_wp_cli_shell.py @@ -45,21 +45,21 @@ def test_shell_artifact_rejects_invalid_size_or_content(script: str) -> None: @pytest.mark.parametrize("grader_timeout,shell_timeout", [(90, 85), (6, 1), (1, 1)]) def test_shell_budget_ends_before_the_grader_timeout(grader_timeout, shell_timeout) -> None: - test = load_tests(DatasetConfig(source="local", name="wp-cli-v1"))[0] + test = load_tests(DatasetConfig(source="local", name="wp-cli"))[0] config = HarnessConfig(grader=GraderConfig(timeout_seconds=grader_timeout)) assert _build_verification_spec(test, config)["timeout_seconds"] == shell_timeout @pytest.mark.parametrize("mode", ["model", "reference", "exploits"]) def test_shell_scripts_reach_the_runtime_in_every_mode(monkeypatch, tmp_path, mode) -> None: - test = load_tests(DatasetConfig(source="local", name="wp-cli-v1"))[0] + test = load_tests(DatasetConfig(source="local", name="wp-cli"))[0] reference = test.reference_solution assert reference is not None monkeypatch.setattr("wp_bench.core.load_tests", lambda dataset: [test]) config = HarnessConfig( - dataset=DatasetConfig(source="local", name="wp-cli-v1"), + dataset=DatasetConfig(source="local", name="wp-cli"), run=RunConfig( - suite="wp-cli-v1", check_reference_solution=mode == "reference", + suite="wp-cli", check_reference_solution=mode == "reference", check_exploits=mode == "exploits", ), output=OutputConfig(path=tmp_path / "result.json", jsonl_path=None), @@ -87,7 +87,7 @@ def execute(artifact: Artifact, spec: dict) -> ExecutionResult: def test_shell_prompt_describes_the_actual_execution_contract() -> None: - test = load_tests(DatasetConfig(source="local", name="wp-cli-v1"))[0] + test = load_tests(DatasetConfig(source="local", name="wp-cli"))[0] prompt = BenchmarkRunner._render_execution_prompt(test) assert "Bash command or script" in prompt assert "SQLite" in prompt and "no network" in prompt @@ -99,8 +99,8 @@ def test_shell_prompt_describes_the_actual_execution_contract() -> None: def test_cli_suite_flag_selects_the_local_cli_suite(tmp_path) -> None: path = tmp_path / "config.yaml" - path.write_text("dataset: {source: local, name: wp-core-v1}\n") - result = CliRunner().invoke(app, ["run", "--config", str(path), "--suite", "wp-cli-v1", "--dry-run"]) + path.write_text("dataset: {source: local, name: wp-core}\n") + result = CliRunner().invoke(app, ["run", "--config", str(path), "--suite", "wp-cli", "--dry-run"]) assert result.exit_code == 0, result.output assert "Execution tests: 27" in result.output @@ -113,30 +113,30 @@ def load(name, **kwargs): return [ {"id": suite, "suite": suite, "test_kind": "execution", "prompt": "Task", "artifact_kind": kind} - for suite, kind in [("wp-core-v1", "php_snippet"), ("wp-cli-v1", "wp_cli_shell")] + for suite, kind in [("wp-core", "php_snippet"), ("wp-cli", "wp_cli_shell")] ] monkeypatch.setattr("wp_bench.datasets.hf_load_dataset", load) - selected = load_tests(DatasetConfig(source="huggingface", name="WordPress/wp-bench-v1", suite="wp-cli-v1")) + selected = load_tests(DatasetConfig(source="huggingface", name="WordPress/wp-bench-v1", suite="wp-cli")) assert [t.artifact_kind for t in selected] == ["wp_cli_shell"] path = tmp_path / "config.yaml" path.write_text("dataset: {source: huggingface, name: WordPress/wp-bench-v1}\n") - result = CliRunner().invoke(app, ["run", "--config", str(path), "--suite", "wp-cli-v1", "--dry-run"]) + result = CliRunner().invoke(app, ["run", "--config", str(path), "--suite", "wp-cli", "--dry-run"]) assert result.exit_code == 0, result.output assert "Execution tests: 1" in result.output assert seen == ["WordPress/wp-bench-v1"] * 2 @pytest.mark.parametrize("dataset,run,suite", [ - ({}, {}, "wp-core-v1"), - ({}, {"suite": "wp-cli-v1"}, "wp-cli-v1"), - ({"suite": "wp-cli-v1"}, {}, "wp-cli-v1"), - ({"suite": "wp-cli-v1"}, {"suite": "wp-core-v1"}, "wp-cli-v1"), - ({"suite": "wp-core-v1"}, {"suite": "wp-cli-v1"}, "wp-core-v1"), + ({}, {}, "wp-core"), + ({}, {"suite": "wp-cli"}, "wp-cli"), + ({"suite": "wp-cli"}, {}, "wp-cli"), + ({"suite": "wp-cli"}, {"suite": "wp-core"}, "wp-cli"), + ({"suite": "wp-core"}, {"suite": "wp-cli"}, "wp-core"), ]) def test_huggingface_loads_only_the_run_suite_unless_overridden(monkeypatch, dataset, run, suite) -> None: rows = [{"id": suite, "suite": suite, "test_kind": "execution", "prompt": "Task"} - for suite in ["wp-core-v1", "wp-cli-v1"]] + for suite in ["wp-core", "wp-cli"]] monkeypatch.setattr("wp_bench.datasets.hf_load_dataset", lambda name, **kwargs: rows) config = HarnessConfig.model_validate({"dataset": {"source": "huggingface", **dataset}, "run": run}) assert [t.suite for t in load_tests(config.dataset)] == [suite] @@ -144,17 +144,17 @@ def test_huggingface_loads_only_the_run_suite_unless_overridden(monkeypatch, dat def test_cli_suite_has_workflows_with_behavioral_and_exploit_checks() -> None: - tests = load_tests(DatasetConfig(source="local", name="wp-cli-v1")) + tests = load_tests(DatasetConfig(source="local", name="wp-cli")) assert Counter(t.category for t in tests) == { "cli-content": 3, "cli-taxonomy-meta": 4, "cli-users": 5, "cli-options": 4, "cli-reporting": 3, "cli-maintenance": 6, "cli-migration": 2, } - core = load_tests(DatasetConfig(source="local", name="wp-core-v1")) + core = load_tests(DatasetConfig(source="local", name="wp-core")) assert len(core) == 350 assert len({t.id for t in core + tests}) == 377 for test in tests: - assert test.suite == "wp-cli-v1" and test.artifact_kind == "wp_cli_shell" + assert test.suite == "wp-cli" and test.artifact_kind == "wp_cli_shell" assert test.test_function is None assert test.reference_solution and "wp " in test.reference_solution assert test.exploit_solutions and all(s.strip() for s in test.exploit_solutions) @@ -175,10 +175,10 @@ def test_cli_export_preserves_shell_contract_and_excludes_exploits() -> None: assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - rows = module.load_suite("wp-cli-v1") + rows = module.load_suite("wp-cli") assert len(rows) == 27 assert all(r["artifact_kind"] == "wp_cli_shell" and "exploit_solutions" not in r for r in rows) - local = {t.id: t for t in load_tests(DatasetConfig(source="local", name="wp-cli-v1"))} + local = {t.id: t for t in load_tests(DatasetConfig(source="local", name="wp-cli"))} for row in rows: assert row["reference_solution"] == local[row["id"]].reference_solution assert json.loads(row["runtime_checks"]) == local[row["id"]].runtime_checks diff --git a/python/wp_bench/benchmark.py b/python/wp_bench/benchmark.py new file mode 100644 index 0000000..4aee71b --- /dev/null +++ b/python/wp_bench/benchmark.py @@ -0,0 +1,298 @@ +"""Frozen benchmark releases shared by the harness and leaderboard consumers. + +A public version identifies the entire evaluation. Suite names, result schema +versions, and result update dates are separate provenance, never substitutes. +""" +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .config import GraderConfig, HarnessConfig, ModelConfig +from .datasets import ExecutionTest +from .scoring import SCORING_VERSION + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +CONTRACT_SCHEMA_VERSION = "1.0" +VERSION_PATTERN = re.compile(r"v[1-9][0-9]*") +# Conservative: changes to these evaluation implementations need a new release. +# CLI/output/record serialization changes alone do not change the evaluation. +HARNESS_INPUTS = ( + "artifacts.py", "config.py", "utils.py", "core.py", "datasets.py", "environment.py", "models.py", + "scoring.py", "selection.py", "skills.py", "exploits.py", +) + + +def digest(value: Any) -> str: + """Hash canonical JSON, independent of object key order.""" + body = json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + return hashlib.sha256(body.encode()).hexdigest() + + +def read_json(path: Path) -> dict[str, Any]: + """Read an object with a useful error for a broken/missing manifest.""" + try: + value = json.loads(path.read_text()) + except (OSError, ValueError) as exc: + raise ValueError(f"Cannot read benchmark manifest {path}: {exc}") from exc + if not isinstance(value, dict): + raise TypeError(f"Benchmark manifest must be an object: {path}") + return value + + +def task_digest(test: ExecutionTest) -> str: + """Evaluation inputs that survive the local/Hugging Face round trip. + + ``test.suite`` is a source-file ID locally and a dataset ID on the Hub; + the enclosing run's dataset ID supplies the component identity instead. + Reference solutions, author QA and editorial metadata are not model input. + Raw dataset file hashes separately freeze *all* dataset changes. + """ + return digest({name: getattr(test, name) for name in ( + "id", "prompt", "requirements", "category", "test_function", + "static_checks", "runtime_checks", "artifact_kind", + )}) + + +def evaluation_snapshot(root: Path = PROJECT_ROOT) -> dict[str, Any]: + """Capture every component and its complete test membership.""" + from .datasets import _parse_execution_suite + + definition = read_json(root / "benchmark/definition.json") + if definition.get("schema_version") != CONTRACT_SCHEMA_VERSION: + raise ValueError("Unsupported benchmark definition schema") + if definition.get("scoring_version") != SCORING_VERSION: + raise ValueError("Working definition must use the current scoring formula") + protocol = definition.get("protocol", {}) + ModelConfig(**protocol.get("model", {})) + GraderConfig(**protocol.get("grader", {})) + repeats = protocol.get("repeats") + if not isinstance(repeats, int) or isinstance(repeats, bool) or repeats < 1: + raise ValueError("Benchmark protocol must require a positive number of repeats") + approved = definition.get("runtime_image_ids", []) + if not approved or len(approved) != len(set(approved)): + raise ValueError("Benchmark must declare distinct approved runtime images") + labels = [category["id"] for category in definition.get("categories", [])] + if not labels or len(labels) != len(set(labels)): + raise ValueError("Broad category IDs must be nonempty and distinct") + suites = definition.get("suites", []) + if not suites or len(suites) != len(set(suites)): + raise ValueError("Benchmark must list distinct dataset suites") + if set(definition.get("category_mapping", {})) != set(suites): + raise ValueError("Category mappings must cover exactly the declared suites") + files: dict[str, str] = {} + components: dict[str, Any] = {} + actual_suites = sorted(p.name for p in (root / "datasets/suites").iterdir() if p.is_dir()) + if sorted(suites) != actual_suites: + raise ValueError("Benchmark definition must include every dataset suite") + for suite in suites: + if not re.fullmatch(r"[A-Za-z0-9_-]+", suite): + raise ValueError(f"Invalid dataset suite: {suite}") + tests = [] + for path in sorted((root / "datasets/suites" / suite / "execution").glob("*.json")): + files[path.relative_to(root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + tests.extend(_parse_execution_suite(path)) + ids = [test.id for test in tests] + if not ids or len(ids) != len(set(ids)): + raise ValueError(f"Suite {suite} must contain distinct tests") + components[suite] = {"task_count": len(tests), "tasks": { + test.id: {"sha256": task_digest(test), "category": test.category} + for test in sorted(tests, key=lambda test: test.id) + }} + # Keep public category membership exhaustive and unambiguous. + mapping = definition["category_mapping"].get(suite, {}) + categories = {test.category for test in tests} + allowed_labels = set(labels) + if set(mapping) != categories or any(value not in allowed_labels for value in mapping.values()): + raise ValueError(f"Broad category mapping is incomplete for {suite}") + for path in sorted((root / "runtime").rglob("*")): + if path.is_file() and (path.suffix in {".php", ".sh"} or path.name == "Dockerfile"): + files[path.relative_to(root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + for name in HARNESS_INPUTS: + path = root / "python/wp_bench" / name + files[path.relative_to(root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + return {"definition": definition, "files": files, "components": components} + + +def load_release(version: str | None = None, root: Path = PROJECT_ROOT) -> dict[str, Any]: + """Read a frozen public release; validate its content-addressed identity.""" + catalog = read_json(root / "benchmark/releases.json") + version = version or catalog["current"] + if not isinstance(version, str) or not VERSION_PATTERN.fullmatch(version): + raise ValueError("Benchmark versions must be v1, v2, ...") + entry = next((entry for entry in catalog["releases"] if entry["version"] == version), None) + if entry is None: + raise ValueError(f"Unknown benchmark version: {version}") + manifest = read_json(root / "benchmark/releases" / f"{version}.json") + if (manifest.get("schema_version") != CONTRACT_SCHEMA_VERSION + or manifest.get("version") != version + or manifest.get("fingerprint") != digest(manifest.get("evaluation")) + or entry.get("fingerprint") != manifest["fingerprint"]): + raise ValueError(f"Invalid benchmark release manifest: {version}") + return manifest + + +def freeze_release(version: str, root: Path = PROJECT_ROOT) -> dict[str, Any]: + """Append a release, refusing to replace a previously published identity.""" + if not VERSION_PATTERN.fullmatch(version): + raise ValueError("Benchmark versions must be v1, v2, ...") + path = root / "benchmark/releases" / f"{version}.json" + if path.exists(): + raise ValueError(f"Benchmark {version} already exists; published releases are immutable") + catalog_path = root / "benchmark/releases.json" + catalog = read_json(catalog_path) if catalog_path.exists() else { + "schema_version": CONTRACT_SCHEMA_VERSION, "releases": [], + } + expected = max((int(entry["version"][1:]) for entry in catalog["releases"]), default=0) + 1 + if version != f"v{expected}": + raise ValueError(f"Next benchmark release must be v{expected}") + evaluation = evaluation_snapshot(root) + if any(entry["fingerprint"] == digest(evaluation) for entry in catalog["releases"]): + raise ValueError("Evaluation is unchanged; export updated results under the existing version") + manifest = {"schema_version": CONTRACT_SCHEMA_VERSION, "version": version, + "fingerprint": digest(evaluation), "evaluation": evaluation} + path.parent.mkdir(parents=True, exist_ok=True) + # Exclusive create protects old releases even if two release jobs race. + with path.open("x") as handle: + handle.write(json.dumps(manifest, indent=2) + "\n") + catalog["releases"].append({"version": version, "path": f"releases/{version}.json", + "fingerprint": manifest["fingerprint"]}) + catalog["current"] = version + catalog_path.write_text(json.dumps(catalog, indent=2) + "\n") + return manifest + + +@dataclass +class BenchmarkIdentity: + """Capture once before model calls so later disk edits cannot relabel a run.""" + + manifest: dict[str, Any] | None + fingerprint: str | None + issues: list[str] + suite: str + tasks: dict[str, str] + + def record_info(self) -> dict[str, Any]: + """Small identity repeated on streamed records, including partial runs.""" + return {"version": self.manifest["version"] if self.manifest and not self.issues else None, + "fingerprint": self.fingerprint, "suite": self.suite, + "status": "released" if self.manifest and not self.issues else "unreleased"} + + def bind_runtime(self, image_id: str | None, *, required: bool = False) -> None: + """Verify the actual image before generation, rather than trusting a tag.""" + if self.manifest and image_id not in self.manifest["evaluation"]["definition"]["runtime_image_ids"]: + self.issues.append("Runtime image is not approved for this release") + if required and self.issues: + raise ValueError("Cannot run as a public benchmark release: " + "; ".join(self.issues)) + + def metadata(self, records: list[dict[str, Any]]) -> dict[str, Any]: + """Report exact coverage without promoting partial or errored runs.""" + info = self.record_info() + expected = self.manifest["evaluation"]["components"].get(self.suite, {}).get("tasks", {}) if self.manifest else {} + ids = [record["test_id"] for record in records] + complete = bool(expected) and len(ids) == len(set(ids)) and set(ids) == set(expected) + return {"schema_version": CONTRACT_SCHEMA_VERSION, **info, + "based_on": self.manifest["version"] if self.manifest else None, + "issues": self.issues, "full_suite": complete, + "expected_task_count": len(expected), "observed_task_count": len(ids)} + + +def capture_identity(config: HarnessConfig, tests: list[ExecutionTest], *, + root: Path = PROJECT_ROOT) -> BenchmarkIdentity: + """Identify source/test changes; unknown inputs never inherit a public label.""" + issues: list[str] = [] + tasks = {test.id: task_digest(test) for test in tests} + manifest = None + fingerprint = None + try: + manifest = load_release(config.benchmark.version, root) + snapshot = evaluation_snapshot(root) + fingerprint = digest(snapshot) + if fingerprint != manifest["fingerprint"]: + issues.append("Evaluation sources differ from the frozen release") + component = manifest["evaluation"]["components"].get(config.run.suite, {}) + expected = component.get("tasks", {}) + if not expected or len(tasks) != len(tests) or any( + test_id not in expected or value != expected[test_id]["sha256"] + for test_id, value in tasks.items() + ): + issues.append("Loaded tasks differ from the frozen release") + protocol = manifest["evaluation"]["definition"]["protocol"] + if config.run.skip_static or config.run.skip_runtime or config.run.continue_on_error: + issues.append("Diagnostic run settings are enabled") + if config.run.execution_isolation != "reset_per_test": + issues.append("Per-test isolation is disabled") + if any(getattr(config.grader, key) != value for key, value in protocol["grader"].items()): + issues.append("Grader settings differ from the release protocol") + if not config.run.check_reference_solution and not config.run.check_exploits: + for model in config.get_models(): + if any(getattr(model, key) != value for key, value in protocol["model"].items()): + issues.append(f"Model settings differ from the release protocol: {model.name}") + if SCORING_VERSION != manifest["evaluation"]["definition"]["scoring_version"]: + issues.append("Scoring version differs from the frozen release") + except (ValueError, TypeError) as exc: + issues.append(str(exc)) + if config.benchmark.version and issues: + raise ValueError("Cannot run as " + config.benchmark.version + ": " + "; ".join(issues)) + return BenchmarkIdentity(manifest, fingerprint, issues, config.run.suite, tasks) + + +def publication_issues(payload: dict[str, Any], manifest: dict[str, Any]) -> list[str]: + """Validate a full-suite model result before any leaderboard ingestion. + + Receives a single-model payload. Multi-model callers adapt each models[] + entry into this shape. No score is recomputed from a partial result. + """ + metadata = payload.get("metadata", {}) + identity = metadata.get("benchmark") or {} + records = payload.get("results", []) + evaluation = manifest["evaluation"] + definition = evaluation["definition"] + issues = [] + if (identity.get("version") != manifest["version"] + or identity.get("fingerprint") != manifest["fingerprint"] + or identity.get("status") != "released"): + issues.append("Missing or incompatible benchmark identity") + suite = metadata.get("suite") + expected = evaluation["components"].get(suite, {}).get("tasks", {}) + ids = [record.get("test_id") for record in records] + if not expected or len(ids) != len(set(ids)) or set(ids) != set(expected): + issues.append("Incomplete, duplicate, or unknown suite tasks") + if metadata.get("scoring_version") != definition["scoring_version"]: + issues.append("Scoring version mismatch") + if metadata.get("runtime_image_id") not in definition["runtime_image_ids"]: + issues.append("Runtime image is not approved for this release") + grader = metadata.get("grader", {}) + if any(grader.get(key) != value for key, value in definition["protocol"]["grader"].items()): + issues.append("Grader settings differ from the release protocol") + if metadata.get("runtime_isolation") != "reset_per_test": + issues.append("Run did not use per-test isolation") + run_id = metadata.get("run_id") + if not isinstance(run_id, str) or not run_id: + issues.append("Missing run identity") + for record in records: + if record.get("run_id") != run_id: + issues.append("Record run identity mismatch") + task = expected.get(record.get("test_id"), {}) + if (record.get("category") != task.get("category") + or (record.get("benchmark") or {}).get("task_sha256") != task.get("sha256")): + issues.append("Record task fingerprint mismatch") + model = record.get("model") or {} + if (record.get("mode") != "model" or record.get("error") is not None + or (record.get("model_call") or {}).get("temperature_fallback", False) + or record.get("variant", {}).get("kind") != "none" + or not isinstance(record.get("scores", {}).get("execution_pass"), bool)): + issues.append("Diagnostic, errored, or ungraded record") + if any(model.get(key) != value for key, value in definition["protocol"]["model"].items()): + issues.append("Model settings differ from the release protocol") + record_identity = record.get("benchmark") or {} + if (record_identity.get("version") != manifest["version"] + or record_identity.get("fingerprint") != manifest["fingerprint"] + or record_identity.get("suite") != suite + or record_identity.get("status") != "released"): + issues.append("Record benchmark identity mismatch") + return list(dict.fromkeys(issues)) diff --git a/python/wp_bench/benchmark_cli.py b/python/wp_bench/benchmark_cli.py new file mode 100644 index 0000000..805921d --- /dev/null +++ b/python/wp_bench/benchmark_cli.py @@ -0,0 +1,185 @@ +"""Release and leaderboard commands; no provider calls or runtime startup.""" +from __future__ import annotations + +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import typer + +from .benchmark import ( + CONTRACT_SCHEMA_VERSION, + PROJECT_ROOT, + digest, + evaluation_snapshot, + freeze_release, + load_release, + publication_issues, + read_json, +) + +app = typer.Typer(help="Inspect and freeze public benchmark releases; export validated results.") + + +def check_releases(root: Path = PROJECT_ROOT, base_ref: str | None = None) -> dict[str, Any]: + """Validate the catalog and protect all releases already present upstream.""" + catalog = read_json(root / "benchmark/releases.json") + if catalog.get("schema_version") != CONTRACT_SCHEMA_VERSION: + raise ValueError("Unsupported benchmark catalog schema") + entries = catalog.get("releases", []) + versions = [entry["version"] for entry in entries] + if not versions or versions != [f"v{number}" for number in range(1, len(versions) + 1)]: + raise ValueError("Release catalog must contain v1, v2, ... exactly once, in order") + if catalog.get("current") != versions[-1]: + raise ValueError("Current benchmark must be the latest frozen release") + if {path.stem for path in (root / "benchmark/releases").glob("*.json")} != set(versions): + raise ValueError("Release files and catalog differ") + for entry in entries: + if entry.get("path") != f"releases/{entry['version']}.json": + raise ValueError("Unexpected release manifest path") + load_release(entry["version"], root) + latest = load_release(root=root) + matches = digest(evaluation_snapshot(root)) == latest["fingerprint"] + if base_ref: + def git(*args: str) -> bytes: + result = subprocess.run(["git", *args], cwd=root, capture_output=True, timeout=30, check=False) + if result.returncode: + raise ValueError("Cannot inspect benchmark release history: " + result.stderr.decode().strip()) + return result.stdout + base_files = git("ls-tree", "-r", "--name-only", base_ref, "--", "benchmark/releases").decode().splitlines() + for relative in base_files: + path = root / relative + if not path.is_file() or path.read_bytes() != git("show", f"{base_ref}:{relative}"): + raise ValueError(f"Published benchmark manifest changed or disappeared: {relative}") + if f"benchmark/releases/{latest['version']}.json" not in base_files and not matches: + raise ValueError("New release does not describe the current evaluation; freeze again before publishing") + return {"current": latest["version"], "fingerprint": latest["fingerprint"], + "working_tree": "released" if matches else "unreleased", "releases": versions} + + +def export_results(paths: list[Path], output: Path, *, version: str | None = None, + root: Path = PROJECT_ROOT) -> Path: + """Publish an immutable, version-scoped scored-answer bundle for websites. + + Require complete suites and the same number of independent runs for every + model. Preserve old snapshots. Consumers derive displays from these rows; + no provider's self-reported aggregate can replace the actual task results. + """ + manifest = load_release(version, root) + definition = manifest["evaluation"]["definition"] + models: dict[str, list[dict[str, Any]]] = {} + runs: dict[tuple[str, str], set[str]] = {} + used_runs: set[str] = set() + inputs: list[str] = [] + for path in paths: + if path.name.endswith((".partial", ".jsonl")): + raise ValueError(f"Expected completed results JSON, not a stream: {path}") + payload = read_json(path) + inputs.append(digest(payload)) + entries = [payload] + if "models" in payload: + entries = [{"metadata": {**payload["metadata"], "run_id": result["run_id"], + "benchmark": result["benchmark"]}, "results": result["results"]} + for result in payload["models"].values()] + for result in entries: + issues = publication_issues(result, manifest) + if issues: + raise ValueError(f"Cannot publish {path.name}: {'; '.join(issues)}") + records = result["results"] + names = {record["model"]["name"] for record in records} + if len(names) != 1: + raise ValueError("A model run must contain exactly one model identity") + name = names.pop() + metadata = result["metadata"] + suite, run_id = metadata["suite"], metadata["run_id"] + if run_id in used_runs: + raise ValueError("Duplicate run identity; the same run cannot count as another repeat") + used_runs.add(run_id) + runs.setdefault((name, suite), set()).add(run_id) + answers = models.setdefault(name, []) + for record in records: + task = manifest["evaluation"]["components"][suite]["tasks"][record["test_id"]] + answers.append({"suite": suite, "test_id": record["test_id"], "run_id": run_id, + "category": definition["category_mapping"][suite][task["category"]], + "passed": record["scores"]["execution_pass"], + "usage": {key: (record.get("usage") or {}).get(key) for key in ( + "prompt_tokens", "completion_tokens", "total_tokens", "cost_usd", "latency_ms")}}) + if not models: + raise ValueError("No completed model results supplied") + repeats = definition["protocol"]["repeats"] + for name in models: + for suite in definition["suites"]: + if len(runs.get((name, suite), set())) != repeats: + raise ValueError(f"{name} needs exactly {repeats} independent complete runs of {suite}") + content: dict[str, Any] = {"schema_version": CONTRACT_SCHEMA_VERSION, + "benchmark": {"version": manifest["version"], "fingerprint": manifest["fingerprint"]}, + "categories": definition["categories"], + "models": [{"id": name, "answers": sorted(answers, key=lambda answer: ( + answer["suite"], answer["test_id"], answer["run_id"]))} + for name, answers in sorted(models.items())], "input_sha256": sorted(inputs)} + fingerprint = digest(content) + snapshot_id = fingerprint[:16] + directory = output / manifest["version"] + destination = directory / f"{snapshot_id}.json" + catalog_path = output / "results.json" + catalog: dict[str, Any] = read_json(catalog_path) if catalog_path.exists() else { + "schema_version": CONTRACT_SCHEMA_VERSION, "benchmarks": {}, + } + # Same exact inputs are idempotent, including the original update date. + if destination.exists(): + existing = read_json(destination) + existing_content = {key: value for key, value in existing.items() + if key not in {"snapshot_id", "fingerprint", "updated_at"}} + if existing.get("fingerprint") != fingerprint or digest(existing_content) != fingerprint: + raise ValueError("Existing result snapshot has a different fingerprint") + else: + snapshot = {**content, "snapshot_id": snapshot_id, "fingerprint": fingerprint, + "updated_at": datetime.now(timezone.utc).isoformat()} + directory.mkdir(parents=True, exist_ok=True) + with destination.open("x") as handle: + handle.write(json.dumps(snapshot, indent=2, allow_nan=False) + "\n") + entry = catalog["benchmarks"].setdefault(manifest["version"], {"current": snapshot_id, "snapshots": []}) + if snapshot_id not in [item["id"] for item in entry["snapshots"]]: + entry["snapshots"].append({"id": snapshot_id, "path": f"{manifest['version']}/{snapshot_id}.json"}) + entry["current"] = snapshot_id + temporary = catalog_path.with_suffix(".json.tmp") + temporary.write_text(json.dumps(catalog, indent=2) + "\n") + temporary.replace(catalog_path) + return destination + + +def _show(action: Any) -> None: + try: + result = action() + except (ValueError, KeyError, TypeError) as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(1) from exc + typer.echo(json.dumps(result, indent=2)) + + +@app.command("show") +def show(version: str | None = typer.Option(None, help="Public release, such as v1")) -> None: + """Print the complete machine-readable release manifest.""" + _show(lambda: load_release(version)) + + +@app.command("check") +def check(base_ref: str | None = typer.Option(None, help="Git commit whose published releases must remain immutable")) -> None: + """Validate releases; report changed evaluation sources as unreleased.""" + _show(lambda: check_releases(base_ref=base_ref)) + + +@app.command("freeze") +def freeze(version: str = typer.Argument(..., help="Next public version, e.g. v2")) -> None: + """Freeze the entire current evaluation as the next immutable release.""" + _show(lambda: {key: value for key, value in freeze_release(version).items() if key != "evaluation"}) + + +@app.command("export") +def export(inputs: list[Path] = typer.Option(..., "--input", help="Completed result JSON; repeat for every suite and run"), + output: Path = typer.Option(..., "--output-dir", help="Archive root; keeps snapshots for every benchmark version"), + version: str | None = typer.Option(None, help="Public release, defaults to latest")) -> None: + """Validate results and write an immutable leaderboard data snapshot.""" + _show(lambda: {"path": str(export_results(inputs, output, version=version))}) diff --git a/python/wp_bench/cli.py b/python/wp_bench/cli.py index 45af891..5a226ac 100644 --- a/python/wp_bench/cli.py +++ b/python/wp_bench/cli.py @@ -9,6 +9,7 @@ from dotenv import load_dotenv from rich.console import Console +from .benchmark_cli import app as benchmark_app from .config import HarnessConfig, ModelConfig from .core import BenchmarkRunner, MultiModelRunner, select_run_tests from .datasets import ExecutionTest, filter_tests_by_ids, load_tests @@ -20,6 +21,7 @@ T = TypeVar("T") app = typer.Typer(add_completion=False) +app.add_typer(benchmark_app, name="benchmark") console = Console() diff --git a/python/wp_bench/config.py b/python/wp_bench/config.py index a0ec196..df8ffc5 100644 --- a/python/wp_bench/config.py +++ b/python/wp_bench/config.py @@ -36,8 +36,8 @@ class StrictModel(BaseModel): class DatasetConfig(StrictModel): - source: Literal["huggingface", "local"] = "huggingface" - name: str = "WordPress/wp-bench-v1" + source: Literal["huggingface", "local"] = "local" + name: str = "wp-core" revision: str | None = None split: str = "test" cache_dir: Path | None = None @@ -133,7 +133,7 @@ class RunConfig(StrictModel): #: exclusive-mode rules as a config file. model_config = ConfigDict(validate_assignment=True) - suite: str = "wp-core-v1" + suite: str = "wp-core" limit: int | None = Field(default=None, gt=0) test_ids: list[str] = Field(default_factory=list) categories: list[str] = Field(default_factory=list) @@ -219,7 +219,23 @@ def _validate_only_requires_paths(self) -> SkillsConfig: return self +class BenchmarkConfig(StrictModel): + #: Optional release assertion. When set, reject mismatched evaluation inputs + #: before spending on model calls. Unset runs still record their provenance. + version: str | None = None + + @field_validator("version") + @classmethod + def _validate_version(cls, value: str | None) -> str | None: + import re + + if value is not None and not re.fullmatch(r"v[1-9][0-9]*", value): + raise ValueError("benchmark.version must be v1, v2, ...") + return value + + class HarnessConfig(StrictModel): + benchmark: BenchmarkConfig = BenchmarkConfig() dataset: DatasetConfig = DatasetConfig() model: ModelConfig | None = None # Single model (legacy) models: list[ModelConfig] | None = None # Multiple models diff --git a/python/wp_bench/core.py b/python/wp_bench/core.py index 114052e..6d99041 100644 --- a/python/wp_bench/core.py +++ b/python/wp_bench/core.py @@ -7,6 +7,7 @@ from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed from contextlib import contextmanager, suppress from typing import Any +from uuid import uuid4 from .artifacts import ( Artifact, @@ -14,6 +15,7 @@ parse_artifact, render_artifact_instructions, ) +from .benchmark import BenchmarkIdentity, capture_identity from .config import HarnessConfig, ModelConfig from .datasets import ( ExecutionTest, @@ -88,6 +90,8 @@ class _ResultBookkeeping: """ def _init_bookkeeping(self, stream: RecordStream) -> None: + self._run_id = uuid4().hex + self._benchmark_identity: BenchmarkIdentity | None = None self.aggregator = ScoreAggregator() self.usage_aggregator = UsageAggregator() self.records: list[dict[str, Any]] = [] @@ -97,6 +101,12 @@ def _init_bookkeeping(self, stream: RecordStream) -> None: def _on_result(self, result: dict[str, Any]) -> None: """Aggregate a finished record, keep it, and stream it to disk.""" with self._lock: + result["run_id"] = self._run_id + if self._benchmark_identity is not None: + result["benchmark"] = { + **self._benchmark_identity.record_info(), + "task_sha256": self._benchmark_identity.tasks.get(result["test_id"]), + } if result.get("error") is None: self.aggregator.add_execution(result["scores"]) self.usage_aggregator.add(result.get("usage")) @@ -500,11 +510,13 @@ def run(self) -> dict[str, Any]: f"Dataset '{self.config.dataset.name}' contains no execution " "tests. Check the dataset source and suite name." ) + self._benchmark_identity = capture_identity(self.config, tests) if self.config.run.check_exploits: return self._run_exploit_audit(tests) reference_mode = self.config.run.check_reference_solution with _graded_run(self._stream, self.environment): self.environment.setup(capture_baseline=_restores_a_baseline(self.config)) + self._benchmark_identity.bind_runtime(self.environment.image_id, required=bool(self.config.benchmark.version)) if reference_mode: self._run_reference_solution_tests(tests) else: @@ -515,6 +527,8 @@ def run(self) -> dict[str, Any]: "metadata": { "suite": self.config.run.suite, "mode": "reference_solution" if reference_mode else "model", + "run_id": self._run_id, + "benchmark": self._benchmark_identity.metadata(self.records), "result_schema_version": RESULT_SCHEMA_VERSION, "model": model_config, "grader": self.config.grader.model_dump(mode="json"), @@ -696,6 +710,8 @@ def _run_exploit_audit(self, tests: list[ExecutionTest]) -> dict[str, Any]: "metadata": { "suite": self.config.run.suite, "mode": "exploit_audit", + "run_id": self._run_id, + "benchmark": self._benchmark_identity.metadata(self.records) if self._benchmark_identity else None, "result_schema_version": RESULT_SCHEMA_VERSION, "scoring_version": SCORING_VERSION, "grader": self.config.grader.model_dump(mode="json"), @@ -1079,10 +1095,13 @@ def _write_outputs(self) -> None: "runtime_image_id": self.environment.image_id, "skills": self._skills_metadata(), "variants": [variant.key for variant in self.variants], + "benchmark": next(iter(self.results.values()))["benchmark"], }, "models": { name: { "config": result["model_config"], + "run_id": result["run_id"], + "benchmark": result["benchmark"], "base_model": result["base_model"], "variant": result["variant"], "scores": result["scores"], @@ -1140,10 +1159,14 @@ def run(self) -> dict[str, Any]: Returns: Dict with model config, aggregate scores, and individual results. """ + self._benchmark_identity = capture_identity(self.config, self.tests) + self._benchmark_identity.bind_runtime(self.environment.image_id, required=bool(self.config.benchmark.version)) self._run_execution_tests(self.tests) summary = self.aggregator.finalize() return { "model_config": self.model_config.model_dump(mode="json"), + "run_id": self._run_id, + "benchmark": self._benchmark_identity.metadata(self.records), "scoring_version": SCORING_VERSION, "usage": self.usage_aggregator.summary(), "errored_test_ids": errored_test_ids(self.records), diff --git a/python/wp_bench/records.py b/python/wp_bench/records.py index 800c445..dea6253 100644 --- a/python/wp_bench/records.py +++ b/python/wp_bench/records.py @@ -16,7 +16,8 @@ #: field were removed (knowledge track removed). #: 2.1: added the "variant" block for skill-injection A/B runs. #: 2.2: dropped the "difficulty" field from every record. -RESULT_SCHEMA_VERSION = "2.2" +# 2.3: benchmark identity and run_id on every streamed result. +RESULT_SCHEMA_VERSION = "2.3" def _baseline_variant_info() -> dict[str, Any]: @@ -77,6 +78,8 @@ def _base_record( the builders from drifting (the key set is asserted identical in tests). """ return { + "run_id": None, + "benchmark": None, "test_id": test.id, "suite": test.suite, "type": "execution", diff --git a/wp-bench.example.yaml b/wp-bench.example.yaml index 619714d..680413e 100644 --- a/wp-bench.example.yaml +++ b/wp-bench.example.yaml @@ -1,11 +1,16 @@ # WP-Bench Configuration # Copy to wp-bench.yaml and customize. API keys loaded from .env file. +# Optional: assert a public release before spending on models. Use the exact +# model/grader protocol from `wp-bench benchmark show --version v1`. +# benchmark: +# version: v1 + dataset: source: local # 'local' or 'huggingface' - name: wp-core-v1 # suite name or HF dataset path - # suite: wp-core-v1 # HF only: suite rows to load; defaults to run.suite - # Use --suite wp-cli-v1 to select CLI workflows. + name: wp-core # suite name or HF dataset path + # suite: wp-core # HF only: suite rows to load; defaults to run.suite + # Use --suite wp-cli to select CLI workflows. # Multiple models to benchmark (runs sequentially, outputs comparison table) models: @@ -46,7 +51,7 @@ grader: # Run configuration run: - suite: wp-core-v1 + suite: wp-core # limit: 10 # limit tests per model (null = all); limited runs # # use seeded stratified selection across categories # seed: 1337 # selection seed (same seed = same subset) From 811f363627462f21c57b314678499d2b31f7fecf Mon Sep 17 00:00:00 2001 From: Jason Adams Date: Thu, 22 Oct 2026 20:38:04 -0600 Subject: [PATCH 2/7] feat: build leaderboard data from validated release exports --- .github/workflows/ci.yml | 6 +- benchmark/README.md | 5 + python/pyproject.toml | 1 + python/tests/test_benchmark_releases.py | 57 ++++++++ python/wp_bench/benchmark.py | 2 + web/README.md | 39 ++++++ web/build_results.py | 169 ++++++++++++++++++++++++ 7 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 web/README.md create mode 100644 web/build_results.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f815c77..04c62dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,13 +27,13 @@ jobs: cache: pip - name: Install harness with dev tooling - run: pip install -e ./python[dev] + run: pip install -e ./python[dev,leaderboard] - name: Lint (ruff) - run: ruff check python + run: ruff check python web - name: Type check (mypy) - run: mypy python + run: mypy python web - name: Benchmark release integrity env: diff --git a/benchmark/README.md b/benchmark/README.md index 2597810..0fa99ee 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -120,6 +120,11 @@ skills variants, errored/partial runs, duplicate repeats and mixed versions fail before output is created. A legitimate failed model answer is still a scored answer and remains in the denominator. +The shared [website adapter](../web/README.md) turns these validated bundles +into the Spacefast display data, including category scores and paired generation +comparisons. Model names/pairs are editorial inputs; versions and categories +come from the frozen manifest. + The output is a small, stable ingestion contract, not a website theme: ```text diff --git a/python/pyproject.toml b/python/pyproject.toml index 46dad1e..99c502d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ ] [project.optional-dependencies] +leaderboard = ["numpy>=2.0"] http = ["uvicorn>=0.30", "fastapi>=0.111"] dev = ["pytest>=8.3", "ruff>=0.16,<0.17", "mypy>=1.10", "types-PyYAML>=6.0"] diff --git a/python/tests/test_benchmark_releases.py b/python/tests/test_benchmark_releases.py index 5a7cdea..47084f8 100644 --- a/python/tests/test_benchmark_releases.py +++ b/python/tests/test_benchmark_releases.py @@ -345,3 +345,60 @@ def execute_artifact(self, artifact: Any, spec: Any) -> ExecutionResult: assert streamed == payload["results"] assert publication_issues(payload, load_release(root=release_root)) == [] assert streamed[0]["run_id"] == payload["metadata"]["run_id"] + + +def _website_adapter() -> Any: + import importlib.util + + path = Path(__file__).resolve().parents[2] / "web/build_results.py" + spec = importlib.util.spec_from_file_location("leaderboard_adapter", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_website_builds_from_the_shared_export_and_release(release_root: Path, tmp_path: Path) -> None: + exported = export_results(inputs(release_root, tmp_path), tmp_path / "exports", root=release_root) + module = _website_adapter() + bundle = read_json(exported) + data = module.build_data(bundle, load_release(root=release_root), {"models": {"model-a": {"label": "Model A", "provider": "OpenAI"}}}, "commit-sha") + result = data["results"] + assert result["benchmark"]["version"] == "v1" + assert result["task_count"] == 2 + assert result["models"][0]["label"] == "Model A" + assert result["models"][0]["overall"]["passed"] == 4 + assert result["models"][0]["overall"]["rate"] == 1 + assert result["models"][0]["overall"]["ci"] == [1, 1] + target = module.write_data(data, tmp_path / "display") + original = (target / "task-results.csv").read_bytes() + assert module.write_data(data, tmp_path / "display") == target + assert (target / "task-results.csv").read_bytes() == original + data["answers"][0]["passed"] = False + with pytest.raises(ValueError, match="immutable"): + module.write_data(data, tmp_path / "display") + assert (target / "task-results.csv").read_bytes() == original + + +@pytest.mark.parametrize("change", ["coverage", "category", "version"]) +def test_website_refuses_incompatible_display_inputs(release_root: Path, tmp_path: Path, change: str) -> None: + exported = export_results(inputs(release_root, tmp_path), tmp_path / "exports", root=release_root) + bundle = read_json(exported) + if change == "coverage": + bundle["models"][0]["answers"].pop() + elif change == "category": + bundle["models"][0]["answers"][0]["category"] = "another-category" + elif change == "version": + bundle["benchmark"]["version"] = "v2" + bundle["fingerprint"] = digest({key: value for key, value in bundle.items() + if key not in {"snapshot_id", "fingerprint", "updated_at"}}) + with pytest.raises(ValueError): + _website_adapter().build_data(bundle, load_release(root=release_root), {}, "commit-sha") + + +def test_pinned_baseline_release_rejects_skills_ab_before_generation(release_root: Path) -> None: + cfg = config(pinned=True) + cfg.skills.paths = [Path('some-skill.md')] + tests = _parse_execution_suite(release_root / 'datasets/suites/wp-core/execution/tasks.json') + with pytest.raises(ValueError, match='Skills A/B runs are diagnostic'): + capture_identity(cfg, tests, root=release_root) diff --git a/python/wp_bench/benchmark.py b/python/wp_bench/benchmark.py index 4aee71b..4f0bf86 100644 --- a/python/wp_bench/benchmark.py +++ b/python/wp_bench/benchmark.py @@ -222,6 +222,8 @@ def capture_identity(config: HarnessConfig, tests: list[ExecutionTest], *, ): issues.append("Loaded tasks differ from the frozen release") protocol = manifest["evaluation"]["definition"]["protocol"] + if config.skills.paths: + issues.append("Skills A/B runs are diagnostic; this release defines the baseline protocol") if config.run.skip_static or config.run.skip_runtime or config.run.continue_on_error: issues.append("Diagnostic run settings are enabled") if config.run.execution_isolation != "reset_per_test": diff --git a/web/README.md b/web/README.md new file mode 100644 index 0000000..3ee48f4 --- /dev/null +++ b/web/README.md @@ -0,0 +1,39 @@ +# Leaderboard data adapter + +The Spacefast frontend consumes display data built from the repository's public +benchmark releases. It does not assign benchmark versions or own category rules. + +First validate complete result JSONs with `wp-bench benchmark export` (see +[../benchmark/README.md](../benchmark/README.md)). Then build display data: + +```sh +pip install -e './python[leaderboard]' +PYTHONPATH=python python web/build_results.py \ + --export public-data/v1/SNAPSHOT.json \ + --catalog model-catalog.json --source-commit COMMIT_SHA \ + --output-dir SITE_SOURCE/data +``` + +The small editorial catalog supplies only names, provider labels, generation +labels, and explicit predecessor pairs: + +```json +{"models":{"provider/model-id":{"label":"Model name","provider":"OpenAI","generation":"Current"}},"pairs":[{"label":"Model family","previous":"older-model-id","current":"provider/model-id"}]} +``` + +New models without editorial entries still appear under their API ID. Comparison +claims are made only for explicitly paired, complete models on the same frozen +benchmark. Broad categories, task membership, protocol and the public version +come exclusively from the repository manifest. The adapter verifies the export +fingerprint and complete repeated task coverage before calculating scores. + +The generated `YYYY.MM.DD.HASH/` files are immutable display-data snapshots. The +website's catalog selects the latest snapshot within each **public benchmark +version**; visitors see WP-Bench v1/v2 and an update date, while snapshot IDs remain +in downloads. Keep prior snapshots and their benchmark manifest. Never substitute +an old-version score for a missing row in a new benchmark. + +This adapter has no campaign-specific paths, provider credentials, or model API +calls. Raw completions remain in private run artifacts. Publish larger scored +answer bundles separately from the frontend source when retention grows; the +frontend only needs compact summaries and stable links to those downloads. diff --git a/web/build_results.py b/web/build_results.py new file mode 100644 index 0000000..2bbefaa --- /dev/null +++ b/web/build_results.py @@ -0,0 +1,169 @@ +"""Adapt validated WP-Bench exports to the public site's display data. + +No model calls or campaign-specific paths. Public versions and categories come +from the repository release; labels and generation pairs are editorial inputs. +""" +from __future__ import annotations + +import argparse +import csv +import io +import json +import statistics +from datetime import datetime +from pathlib import Path +from typing import Any + +import numpy as np +from wp_bench.benchmark import digest, load_release, read_json + + +def interval(values: list[float]) -> list[float]: + array = np.array(values, dtype=float) + rng = np.random.default_rng(1337) + samples = array[rng.integers(0, len(array), size=(10000, len(array)))].mean(axis=1) + return [float(value) for value in np.quantile(samples, [0.025, 0.975])] + + +def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: + grouped: dict[tuple[str, str], list[float]] = {} + for row in rows: + grouped.setdefault((row["suite"], row["test_id"]), []).append(float(row["passed"])) + latencies = [row["usage"]["latency_ms"] / 1000 for row in rows if row["usage"].get("latency_ms") is not None] + costs = [row["usage"]["cost_usd"] for row in rows if row["usage"].get("cost_usd") is not None] + passed = sum(row["passed"] for row in rows) + return {"passed": passed, "total": len(rows), "rate": passed / len(rows), + "ci": interval([statistics.mean(values) for _, values in sorted(grouped.items())]), + "median_seconds": statistics.median(latencies) if latencies else 0, + "cost_per_100": 100 * statistics.mean(costs) if len(costs) == len(rows) else None} + + +def build_data(bundle: dict[str, Any], manifest: dict[str, Any], catalog: dict[str, Any], + source_commit: str) -> dict[str, Any]: + """Check identities and coverage before deriving any displayed scores.""" + content = {key: value for key, value in bundle.items() if key not in {"snapshot_id", "fingerprint", "updated_at"}} + if (bundle.get("schema_version") != "1.0" or digest(content) != bundle.get("fingerprint") + or bundle.get("benchmark") != {"version": manifest["version"], "fingerprint": manifest["fingerprint"]}): + raise ValueError("Export and frozen benchmark identities differ") + definition = manifest["evaluation"]["definition"] + tasks = {(suite, test_id): task for suite, component in manifest["evaluation"]["components"].items() + for test_id, task in component["tasks"].items()} + categories = [{**category, "task_count": sum( + definition["category_mapping"][suite][task["category"]] == category["id"] + for (suite, _), task in tasks.items())} for category in definition["categories"]] + categories = [category for category in categories if category["task_count"]] + models: list[dict[str, Any]] = [] + means: dict[str, dict[tuple[str, str], float]] = {} + flat: list[dict[str, Any]] = [] + seen_models = set() + for model in bundle["models"]: + name, rows = model["id"], model["answers"] + if name in seen_models: + raise ValueError("Duplicate model identity") + seen_models.add(name) + grouped: dict[tuple[str, str], list[float]] = {} + run_tasks: dict[tuple[str, str], set[str]] = {} + seen_rows = set() + for row in rows: + key = row["suite"], row["test_id"] + row_key = (*key, row["run_id"]) + if row_key in seen_rows or key not in tasks or not isinstance(row["passed"], bool): + raise ValueError("Duplicate, unknown, or ungraded task") + seen_rows.add(row_key) + expected_category = definition["category_mapping"][key[0]][tasks[key]["category"]] + if row["category"] != expected_category: + raise ValueError("Category differs from the release mapping") + grouped.setdefault(key, []).append(float(row["passed"])) + run_tasks.setdefault((row["suite"], row["run_id"]), set()).add(row["test_id"]) + repeats = definition["protocol"]["repeats"] + if set(grouped) != set(tasks) or any(len(values) != repeats for values in grouped.values()): + raise ValueError("Model lacks complete, equal repeated task coverage") + for (suite, _), ids in run_tasks.items(): + if ids != set(manifest["evaluation"]["components"][suite]["tasks"]): + raise ValueError("A repeat contains only a partial suite") + meta = catalog.get("models", {}).get(name, {}) + provider = meta.get("provider") or ("Google" if name.startswith("gemini/") else "Anthropic" if name.startswith("anthropic/") else "OpenAI" if name.startswith(("gpt-", "openai/", "o1", "o3", "o4")) else "Other") + models.append({"id": name, "label": meta.get("label", name), "provider": provider, + "generation": meta.get("generation", "Evaluated"), "repeats": repeats, + "overall": summarize(rows), "categories": { + category["id"]: summarize([row for row in rows if row["category"] == category["id"]]) + for category in categories if category["task_count"]}}) + means[name] = {key: statistics.mean(values) for key, values in grouped.items()} + flat.extend({"model": name, **row} for row in rows) + if not models: + raise ValueError("Export has no models") + comparisons = [] + for pair in catalog.get("pairs", []): + previous, current = pair["previous"], pair["current"] + if previous not in means or current not in means: + continue + differences = [means[current][key] - means[previous][key] for key in sorted(tasks)] + delta, ci = 100 * statistics.mean(differences), [100 * value for value in interval(differences)] + comparisons.append({**pair, "delta_pp": delta, "ci_pp": ci, + "meaningful_improvement": delta >= 3 and ci[0] > 0, + "interpretation": "Improvement across these tasks" if ci[0] > 0 else + "Regression across these tasks" if ci[1] < 0 else "Difference is uncertain on this task set"}) + gains = sum(pair["delta_pp"] > 0 for pair in comparisons) + meaningful = sum(pair["meaningful_improvement"] for pair in comparisons) + finding = (f"Newer models scored higher in {gains} of {len(comparisons)} matched comparisons. " + f"{meaningful} gains were at least 3 percentage points with a positive paired interval. " + "Those pairs show meaningful gains on this task set; progress varies across model families.") if meaningful else ( + "This release does not establish meaningful improvement across model generations. Use the measured category results to choose a model.") + date = datetime.fromisoformat(bundle["updated_at"]) + result = {"data_schema_version": "1.0", "display_date": date.strftime("%B %d, %Y").replace(" 0", " "), + "benchmark": bundle["benchmark"], "commit": source_commit, + "dataset_sha256": digest({path: value for path, value in manifest["evaluation"]["files"].items() if path.startswith("datasets/")}), + "wordpress_version": definition["wordpress_version"], "scoring_version": definition["scoring_version"], + "task_count": len(tasks), "core_task_count": len(manifest["evaluation"]["components"].get("wp-core", {}).get("tasks", {})), + "cli_task_count": len(manifest["evaluation"]["components"].get("wp-cli", {}).get("tasks", {})), + "categories": categories, "models": sorted(models, key=lambda model: (-model["overall"]["rate"], model["label"])), + "comparisons": comparisons, "finding": finding, + "methodology": {"settings": f"Frozen {manifest['version']} protocol. First response, baseline without skills; provider default reasoning. Generation and grader settings are recorded in the release definition.", + "limitations": "A fixed public task set cannot represent every WordPress project. Training exposure is unknown. Category sizes vary. Latency and cost estimates depend on provider behavior and API concurrency.", + "cost_note": "Estimated costs use the usage recorded by the harness. Actual billing may differ.", + "comparison_note": "Generation deltas use identical tasks and 95% paired bootstrap intervals (10,000 resamples, seed 1337). Intervals are exploratory and not adjusted for multiple comparisons. A meaningful gain is at least 3 percentage points with a positive interval."}, + "coverage_note": "Strict behavioral pass rate. Ties share a rank. Category scores use only that category’s tasks. Small gaps may not be meaningful.", + "audit_files": []} + result["data_version"] = date.strftime("%Y.%m.%d.") + digest({"results": result, "export_fingerprint": bundle["fingerprint"]})[:8] + return {"results": result, "manifest": {"benchmark": bundle["benchmark"], "export_fingerprint": bundle["fingerprint"], + "input_sha256": bundle["input_sha256"], "source_commit": source_commit}, "answers": flat} + + +def write_data(data: dict[str, Any], destination: Path) -> Path: + """Keep version-scoped display files immutable and refresh the latest alias.""" + version = data["results"]["data_version"] + release = destination / version + body = json.dumps(data["results"], indent=2, allow_nan=False) + "\n" + manifest = json.dumps({"data_version": version, **data["manifest"]}, indent=2) + "\n" + stream = io.StringIO(newline="") + writer = csv.DictWriter(stream, fieldnames=["model", "suite", "test_id", "run_id", "category", "passed", "latency_ms", "cost_usd"]) + writer.writeheader() + for answer in data["answers"]: + writer.writerow({**{key: answer[key] for key in ["model", "suite", "test_id", "run_id", "category", "passed"]}, + "latency_ms": answer["usage"].get("latency_ms"), "cost_usd": answer["usage"].get("cost_usd")}) + files = {"results.json": body, "manifest.json": manifest, "task-results.csv": stream.getvalue()} + for name, content in files.items(): + path = release / name + if path.exists() and path.read_bytes() != content.encode(): + raise ValueError("An immutable display-data release would be overwritten") + release.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + (release / name).write_bytes(content.encode()) + (destination / "results.json").write_text(body) + return release + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--export", type=Path, required=True) + parser.add_argument("--catalog", type=Path, required=True, help="Editorial model labels and generation pairs") + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--source-commit", required=True) + args = parser.parse_args() + bundle = read_json(args.export) + data = build_data(bundle, load_release(bundle["benchmark"]["version"]), read_json(args.catalog), args.source_commit) + print(json.dumps({"path": str(write_data(data, args.output_dir)), "benchmark": bundle["benchmark"]})) + + +if __name__ == "__main__": + main() From 807d0df64946fdef363c7281159dc2b764140974 Mon Sep 17 00:00:00 2001 From: Jason Adams Date: Thu, 22 Oct 2026 20:42:14 -0600 Subject: [PATCH 3/7] fix: keep unavailable response latency explicit --- python/tests/test_benchmark_releases.py | 1 + web/build_results.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python/tests/test_benchmark_releases.py b/python/tests/test_benchmark_releases.py index 47084f8..7b06812 100644 --- a/python/tests/test_benchmark_releases.py +++ b/python/tests/test_benchmark_releases.py @@ -370,6 +370,7 @@ def test_website_builds_from_the_shared_export_and_release(release_root: Path, t assert result["models"][0]["overall"]["passed"] == 4 assert result["models"][0]["overall"]["rate"] == 1 assert result["models"][0]["overall"]["ci"] == [1, 1] + assert result["models"][0]["overall"]["median_seconds"] is None target = module.write_data(data, tmp_path / "display") original = (target / "task-results.csv").read_bytes() assert module.write_data(data, tmp_path / "display") == target diff --git a/web/build_results.py b/web/build_results.py index 2bbefaa..d10a1ff 100644 --- a/web/build_results.py +++ b/web/build_results.py @@ -34,7 +34,7 @@ def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: passed = sum(row["passed"] for row in rows) return {"passed": passed, "total": len(rows), "rate": passed / len(rows), "ci": interval([statistics.mean(values) for _, values in sorted(grouped.items())]), - "median_seconds": statistics.median(latencies) if latencies else 0, + "median_seconds": statistics.median(latencies) if latencies else None, "cost_per_100": 100 * statistics.mean(costs) if len(costs) == len(rows) else None} From ed6dcfb0d6adb3731700f3c80fb7802321a12879 Mon Sep 17 00:00:00 2001 From: Jason Adams Date: Thu, 22 Oct 2026 20:46:39 -0600 Subject: [PATCH 4/7] fix: describe exported cost estimates accurately --- web/README.md | 4 ++++ web/build_results.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/web/README.md b/web/README.md index 3ee48f4..cd4d410 100644 --- a/web/README.md +++ b/web/README.md @@ -37,3 +37,7 @@ This adapter has no campaign-specific paths, provider credentials, or model API calls. Raw completions remain in private run artifacts. Publish larger scored answer bundles separately from the frontend source when retention grows; the frontend only needs compact summaries and stable links to those downloads. + +Future display data identifies cost estimates as harness-recorded usage; it does +not silently claim those values were normalized to uncached price tables. Missing +latency/cost stays unavailable rather than appearing as zero. diff --git a/web/build_results.py b/web/build_results.py index d10a1ff..37941f6 100644 --- a/web/build_results.py +++ b/web/build_results.py @@ -110,7 +110,7 @@ def build_data(bundle: dict[str, Any], manifest: dict[str, Any], catalog: dict[s "Those pairs show meaningful gains on this task set; progress varies across model families.") if meaningful else ( "This release does not establish meaningful improvement across model generations. Use the measured category results to choose a model.") date = datetime.fromisoformat(bundle["updated_at"]) - result = {"data_schema_version": "1.0", "display_date": date.strftime("%B %d, %Y").replace(" 0", " "), + result = {"data_schema_version": "1.0", "cost_estimate_basis": "recorded_usage", "display_date": date.strftime("%B %d, %Y").replace(" 0", " "), "benchmark": bundle["benchmark"], "commit": source_commit, "dataset_sha256": digest({path: value for path, value in manifest["evaluation"]["files"].items() if path.startswith("datasets/")}), "wordpress_version": definition["wordpress_version"], "scoring_version": definition["scoring_version"], From c3e901b52655e61d671047096420909778a2b96f Mon Sep 17 00:00:00 2001 From: Jason Adams Date: Thu, 22 Oct 2026 20:50:48 -0600 Subject: [PATCH 5/7] fix: type the release catalog explicitly for CI --- python/wp_bench/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/wp_bench/benchmark.py b/python/wp_bench/benchmark.py index 4f0bf86..279773a 100644 --- a/python/wp_bench/benchmark.py +++ b/python/wp_bench/benchmark.py @@ -144,7 +144,7 @@ def freeze_release(version: str, root: Path = PROJECT_ROOT) -> dict[str, Any]: if path.exists(): raise ValueError(f"Benchmark {version} already exists; published releases are immutable") catalog_path = root / "benchmark/releases.json" - catalog = read_json(catalog_path) if catalog_path.exists() else { + catalog: dict[str, Any] = read_json(catalog_path) if catalog_path.exists() else { "schema_version": CONTRACT_SCHEMA_VERSION, "releases": [], } expected = max((int(entry["version"][1:]) for entry in catalog["releases"]), default=0) + 1 From 44c810b8634c3ae48a912f484b857f3ccf513da9 Mon Sep 17 00:00:00 2001 From: Jason Adams Date: Thu, 22 Oct 2026 20:52:45 -0600 Subject: [PATCH 6/7] fix: order leaderboard updates by explicit timestamps --- web/README.md | 4 ++++ web/build_results.py | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/web/README.md b/web/README.md index cd4d410..5c81252 100644 --- a/web/README.md +++ b/web/README.md @@ -41,3 +41,7 @@ frontend only needs compact summaries and stable links to those downloads. Future display data identifies cost estimates as harness-recorded usage; it does not silently claim those values were normalized to uncached price tables. Missing latency/cost stays unavailable rather than appearing as zero. + +Order result updates by their explicit `updated_at` timestamp, never by the hash +suffix of a snapshot ID. Select the latest data within each benchmark version; +updating an archived benchmark does not replace the default newer benchmark. diff --git a/web/build_results.py b/web/build_results.py index 37941f6..0ee2c6e 100644 --- a/web/build_results.py +++ b/web/build_results.py @@ -10,7 +10,7 @@ import io import json import statistics -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -110,7 +110,10 @@ def build_data(bundle: dict[str, Any], manifest: dict[str, Any], catalog: dict[s "Those pairs show meaningful gains on this task set; progress varies across model families.") if meaningful else ( "This release does not establish meaningful improvement across model generations. Use the measured category results to choose a model.") date = datetime.fromisoformat(bundle["updated_at"]) - result = {"data_schema_version": "1.0", "cost_estimate_basis": "recorded_usage", "display_date": date.strftime("%B %d, %Y").replace(" 0", " "), + if date.tzinfo is None: + raise ValueError("Export update time must include a timezone") + date = date.astimezone(timezone.utc) + result = {"data_schema_version": "1.0", "updated_at": date.isoformat(), "cost_estimate_basis": "recorded_usage", "display_date": date.strftime("%B %d, %Y").replace(" 0", " "), "benchmark": bundle["benchmark"], "commit": source_commit, "dataset_sha256": digest({path: value for path, value in manifest["evaluation"]["files"].items() if path.startswith("datasets/")}), "wordpress_version": definition["wordpress_version"], "scoring_version": definition["scoring_version"], From 1bb190e9f88869870af418d3bd4be07cec6a003a Mon Sep 17 00:00:00 2001 From: Jason Adams Date: Thu, 22 Oct 2026 20:58:01 -0600 Subject: [PATCH 7/7] fix: keep repeat exports from reverting current results --- python/tests/test_benchmark_releases.py | 2 ++ python/wp_bench/benchmark_cli.py | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/tests/test_benchmark_releases.py b/python/tests/test_benchmark_releases.py index 7b06812..7eaef80 100644 --- a/python/tests/test_benchmark_releases.py +++ b/python/tests/test_benchmark_releases.py @@ -225,6 +225,8 @@ def test_export_preserves_old_snapshots_and_is_idempotent(release_root: Path, tm assert first.read_bytes() == original catalog = read_json(output / "results.json") assert len(catalog["benchmarks"]["v1"]["snapshots"]) == 2 + assert export_results(paths, output, root=release_root) == first + assert read_json(output / "results.json") == catalog def test_export_requires_both_complete_suites_and_independent_repeats(release_root: Path, tmp_path: Path) -> None: diff --git a/python/wp_bench/benchmark_cli.py b/python/wp_bench/benchmark_cli.py index 805921d..06e196c 100644 --- a/python/wp_bench/benchmark_cli.py +++ b/python/wp_bench/benchmark_cli.py @@ -127,8 +127,9 @@ def export_results(paths: list[Path], output: Path, *, version: str | None = Non catalog: dict[str, Any] = read_json(catalog_path) if catalog_path.exists() else { "schema_version": CONTRACT_SCHEMA_VERSION, "benchmarks": {}, } - # Same exact inputs are idempotent, including the original update date. - if destination.exists(): + # Same exact inputs preserve both their date and the latest-results pointer. + created = not destination.exists() + if not created: existing = read_json(destination) existing_content = {key: value for key, value in existing.items() if key not in {"snapshot_id", "fingerprint", "updated_at"}} @@ -143,7 +144,8 @@ def export_results(paths: list[Path], output: Path, *, version: str | None = Non entry = catalog["benchmarks"].setdefault(manifest["version"], {"current": snapshot_id, "snapshots": []}) if snapshot_id not in [item["id"] for item in entry["snapshots"]]: entry["snapshots"].append({"id": snapshot_id, "path": f"{manifest['version']}/{snapshot_id}.json"}) - entry["current"] = snapshot_id + if created: + entry["current"] = snapshot_id temporary = catalog_path.with_suffix(".json.tmp") temporary.write_text(json.dumps(catalog, indent=2) + "\n") temporary.replace(catalog_path)