discovery-tasks.json

{
  "$schema": "../schema/discovery-task.schema.json",
  "registry_version": "0.3.0",
  "last_updated": "2026-08-25",
  "registry_suites": [
    {
      "id": "arc-agi-3",
      "name": "ARC-AGI-3",
      "role": "task-source",
      "evaluated_model": null,
      "result_status": "independently-certified",
      "paper": "https://arcprize.org/media/ARC_AGI_3_Technical_Report.pdf",
      "repository": "https://github.com/arcprize/ARC-AGI",
      "task_catalogue": "https://docs.arcprize.org/available-games",
      "task_contract": "An agent receives a turn-based 64x64-or-smaller 16-color frame and available actions, but no natural-language objective or mechanics. It must discover the world rules and win condition, then complete levels efficiently.",
      "task_count": 135,
      "public_task_count": 25,
      "model_families": ["gpt", "claude", "gemini", "xai"],
      "aggregate_results": [
        {"system": "Anthropic Opus 4.6 (Max), official semi-private", "score": 0.5},
        {"system": "Gemini 3.1 Pro Preview, official semi-private", "score": 0.4},
        {"system": "OpenAI GPT-5.4 (High), official semi-private", "score": 0.2},
        {"system": "xAI Grok-4.20 Beta, official semi-private", "score": 0.1}
      ],
      "notes": "The 135 environments comprise 25 public demos, 55 semi-private environments, and 55 fully private environments. Only the three anonymous public examples are itemized here until the remaining public IDs can be authenticated through the official API."
    },
    {
      "id": "dig-bench",
      "name": "DiG-bench",
      "role": "task-source",
      "evaluated_model": null,
      "result_status": "source-reported",
      "paper": "https://arxiv.org/abs/2608.12593",
      "repository": "https://github.com/discos-research/dig-bench",
      "task_catalogue": "https://digbench.ai/",
      "task_contract": "An agent interacts with a text game through JSON states and legal actions, experiments to infer unique hidden transformation rules and win conditions, and must beat the game within its fixed step budget.",
      "task_count": 70,
      "public_task_count": 21,
      "model_families": ["gpt", "claude", "glm", "kimi", "qwen", "gemini", "deepseek"],
      "aggregate_results": [
        {"system": "Claude Opus 5 basic harness, games won out of 70", "score": 50},
        {"system": "Gemini 3.1 Pro basic harness without rules, games won out of 70", "score": 18},
        {"system": "Qwen 3.6 27B basic harness, games won out of 70", "score": 1}
      ],
      "notes": "The public set contains P-1 through P-21, three games in each of seven difficulty tiers. The remaining 49 games are private and are not itemized."
    },
    {
      "id": "simpletes",
      "name": "SimpleTES",
      "role": "task-package-and-evaluated-system",
      "evaluated_model": "gpt-oss-120b",
      "result_status": "source-reported",
      "paper": "https://arxiv.org/abs/2604.19341",
      "repository": "https://github.com/wq-will/SimpleTES",
      "task_catalogue": "https://github.com/wq-will/SimpleTES/blob/main/datasets/README.md",
      "task_contract": "Each task supplies an instruction, an editable seed program, and an executable evaluator that validates constraints and recomputes the score in an isolated subprocess.",
      "task_count": 28,
      "public_task_count": 28,
      "model_families": ["other"],
      "notes": "SimpleTES packages/adapts executable evaluator contracts and reports gpt-oss-120b results. It is not treated as the original provenance of every included task; each task has separate task_origin and task_url fields."
    }
  ],
  "tasks": [
    {
      "id": "asymmetric-matrix-multiplication",
      "name": "Asymmetric matrix multiplication",
      "domain": "ai-foundations",
      "registry_suite": "simpletes",
      "task_origin": "KernelBench L1 / SimpleTES extended-shape contract",
      "task_url": "https://github.com/ScalingIntelligence/KernelBench",
      "question": "Implement a faster asymmetric matrix-multiplication GPU kernel without changing numerical semantics.",
      "input": "A seed Triton/CUDA kernel, fixed tensor shapes and dtypes, correctness tests, and a profiling harness.",
      "output": "GPU kernel source",
      "metric": {"name": "runtime", "direction": "minimize", "unit": "ms"},
      "evaluation": "Correctness-gated execution followed by runtime profiling on the fixed benchmark workload.",
      "environment": "Triton/CUDA with a GPU profiling server; headline result uses H200.",
      "reference_result": {"system": "CUDAAgent", "score": 0.747},
      "reported_result": {"system": "SimpleTES", "score": 0.440},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Runtime depends on hardware and profiler configuration; scores are source-reported and require matched-device replay."
    },
    {
      "id": "batched-cumulative-sum",
      "name": "Batched cumulative sum",
      "domain": "ai-foundations",
      "registry_suite": "simpletes",
      "task_origin": "KernelBench L1 / SimpleTES extended-shape contract",
      "task_url": "https://github.com/ScalingIntelligence/KernelBench",
      "question": "Implement a faster batched prefix-sum GPU kernel while preserving output correctness.",
      "input": "A seed GPU kernel, fixed tensors, correctness tests, and a profiling harness.",
      "output": "GPU kernel source",
      "metric": {"name": "runtime", "direction": "minimize", "unit": "ms"},
      "evaluation": "Correctness-gated execution followed by runtime profiling on the fixed benchmark workload.",
      "environment": "Triton/CUDA with a GPU profiling server; headline result uses H200.",
      "reference_result": {"system": "NVIDIA CUB", "score": 0.147},
      "reported_result": {"system": "SimpleTES", "score": 0.104},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Runtime depends on hardware and profiler configuration; scores are source-reported and require matched-device replay."
    },
    {
      "id": "cassini-trajectory",
      "name": "Cassini gravity-assist trajectory",
      "domain": "astrodynamics",
      "registry_suite": "simpletes",
      "task_origin": "Cassini historical mission / SimpleTES trajectory evaluator",
      "task_url": "https://arxiv.org/html/2604.19341",
      "question": "Find a feasible Cassini gravity-assist trajectory with lower total propulsive cost.",
      "input": "Mission window, body sequence/search variables, orbital constraints, a seed optimizer, and NAIF ephemeris kernels.",
      "output": "Trajectory-design program and feasible mission parameters",
      "metric": {"name": "propulsive cost", "direction": "minimize", "unit": "normalized mission objective"},
      "evaluation": "The mission simulator validates feasibility and computes the fixed propulsive-cost objective.",
      "environment": "Python task environment with astrodynamics dependencies and NAIF kernels.",
      "reference_result": {"system": "Historical sequence", "score": 1.066682},
      "reported_result": {"system": "SimpleTES", "score": 0.820129},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Comparison is meaningful only under the same mission windows, simulator, constraints, and cost normalization."
    },
    {
      "id": "circle-packing-n26",
      "name": "Circle packing in a unit square (n=26)",
      "domain": "mathematics-discovery",
      "registry_suite": "simpletes",
      "task_origin": "Circle-packing problem / EinsteinArena evaluator",
      "task_url": "https://einsteinarena.com/problems/circle-packing",
      "question": "Place 26 non-overlapping circles inside a unit square to maximize the sum of their radii.",
      "input": "A Python seed construction, geometric constraints, and an executable feasibility checker.",
      "output": "Circle centers and radii",
      "metric": {"name": "sum of radii", "direction": "maximize", "unit": "unit-square length"},
      "evaluation": "Recompute boundary and non-overlap constraints, then sum all feasible radii.",
      "environment": "Python; no external task setup listed.",
      "reference_result": {"system": "AlphaEvolve V2", "score": 2.635983},
      "reported_result": {"system": "SimpleTES", "score": 2.635983},
      "outcome": "matched-reference",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Rounded values match; full-precision artifacts are required to establish equality or a strict improvement."
    },
    {
      "id": "circle-packing-n32",
      "name": "Circle packing in a unit square (n=32)",
      "domain": "mathematics-discovery",
      "registry_suite": "simpletes",
      "task_origin": "Circle-packing problem / AlphaEvolve–EFT contract",
      "task_url": "https://arxiv.org/html/2606.29082",
      "question": "Place 32 non-overlapping circles inside a unit square to maximize the sum of their radii.",
      "input": "A Python seed construction, geometric constraints, and an executable feasibility checker.",
      "output": "Circle centers and radii",
      "metric": {"name": "sum of radii", "direction": "maximize", "unit": "unit-square length"},
      "evaluation": "Recompute boundary and non-overlap constraints, then sum all feasible radii.",
      "environment": "Python; no external task setup listed.",
      "reference_result": {"system": "AlphaEvolve V2", "score": 2.939572},
      "reported_result": {"system": "SimpleTES", "score": 2.939572},
      "outcome": "matched-reference",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Rounded values match; full-precision artifacts are required to establish equality or a strict improvement."
    },
    {
      "id": "erdos-minimum-overlap",
      "name": "Erdős minimum overlap",
      "domain": "mathematics-discovery",
      "registry_suite": "simpletes",
      "task_origin": "Erdős minimum-overlap problem / EinsteinArena evaluator",
      "task_url": "https://einsteinarena.com/problems/erdos-min-overlap",
      "question": "Construct a feasible step function with unit mass that minimizes its worst translated one-sided overlap.",
      "input": "A discretized Python construction, integral and range constraints, and a fixed overlap evaluator.",
      "output": "Step-function construction",
      "metric": {"name": "overlap", "direction": "minimize", "unit": "dimensionless bound"},
      "evaluation": "Validate range and mass constraints and recompute the supremum of the translated overlap objective.",
      "environment": "Python; no external task setup listed.",
      "reference_result": {"system": "Together AI", "score": 0.380871},
      "reported_result": {"system": "SimpleTES", "score": 0.380868},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md", "https://teorth.github.io/optimizationproblems/constants/1b.html"],
      "integrity_notes": "This row preserves the rounded comparison table supplied for the task. Other SimpleTES versions report 0.380856, and later independent analysis raised normalization/certification issues; cite the exact artifact and recompute at full precision before claiming a record."
    },
    {
      "id": "galileo-trajectory",
      "name": "Galileo gravity-assist trajectory",
      "domain": "astrodynamics",
      "registry_suite": "simpletes",
      "task_origin": "Galileo historical mission / SimpleTES trajectory evaluator",
      "task_url": "https://arxiv.org/html/2604.19341",
      "question": "Find a feasible Galileo gravity-assist trajectory with lower total propulsive cost.",
      "input": "Mission window, body sequence/search variables, orbital constraints, a seed optimizer, and NAIF ephemeris kernels.",
      "output": "Trajectory-design program and feasible mission parameters",
      "metric": {"name": "propulsive cost", "direction": "minimize", "unit": "normalized mission objective"},
      "evaluation": "The mission simulator validates feasibility and computes the fixed propulsive-cost objective.",
      "environment": "Python task environment with astrodynamics dependencies and NAIF kernels.",
      "reference_result": {"system": "Historical sequence", "score": 0.823681},
      "reported_result": {"system": "SimpleTES", "score": 0.795108},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Comparison is meaningful only under the same mission windows, simulator, constraints, and cost normalization."
    },
    {
      "id": "hadamard-determinant-n29",
      "name": "Hadamard maximum determinant (n=29)",
      "domain": "mathematics-discovery",
      "registry_suite": "simpletes",
      "task_origin": "Hadamard maximal-determinant problem (order 29)",
      "task_url": "https://maths-people.anu.edu.au/~brent/maxdet/order29/",
      "question": "Construct a plus-or-minus-one matrix of order 29 with maximum normalized determinant.",
      "input": "A Python seed construction, discrete matrix constraints, and a determinant evaluator.",
      "output": "29 by 29 sign matrix",
      "metric": {"name": "normalized determinant", "direction": "maximize", "unit": "dimensionless"},
      "evaluation": "Validate every entry and recompute the determinant under the task's normalization.",
      "environment": "Python; no external task setup listed.",
      "reference_result": {"system": "Orrick", "score": 0.935673},
      "reported_result": {"system": "SimpleTES", "score": 0.935673},
      "outcome": "matched-reference",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Rounded values match; use exact determinant arithmetic and the released matrix to verify equality."
    },
    {
      "id": "lasso-regularization-path",
      "name": "LASSO regularization path",
      "domain": "scientific-algorithms",
      "registry_suite": "simpletes",
      "task_origin": "glmnet/sklearn lasso-path evaluation / SimpleTES contract",
      "task_url": "https://arxiv.org/html/2604.19341",
      "question": "Implement a faster solver for the complete LASSO regularization path while retaining solution accuracy.",
      "input": "A seed solver, fixed datasets and regularization path, accuracy checks, and a timing harness.",
      "output": "C++ solver program",
      "metric": {"name": "runtime", "direction": "minimize", "unit": "ms"},
      "evaluation": "Accuracy-gated execution followed by aggregate runtime measurement over the fixed workload.",
      "environment": "C++ invoked through Python with g++ and Eigen.",
      "reference_result": {"system": "glmnet", "score": 4139.4},
      "reported_result": {"system": "SimpleTES", "score": 2502.3},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Runtime comparison requires identical CPU, compiler flags, datasets, tolerance, warmup, and thread policy."
    },
    {
      "id": "mariner-10-trajectory",
      "name": "Mariner 10 gravity-assist trajectory",
      "domain": "astrodynamics",
      "registry_suite": "simpletes",
      "task_origin": "Mariner 10 historical mission / SimpleTES trajectory evaluator",
      "task_url": "https://arxiv.org/html/2604.19341",
      "question": "Find a feasible Mariner 10 gravity-assist trajectory with lower total propulsive cost.",
      "input": "Mission window, body sequence/search variables, orbital constraints, a seed optimizer, and NAIF ephemeris kernels.",
      "output": "Trajectory-design program and feasible mission parameters",
      "metric": {"name": "propulsive cost", "direction": "minimize", "unit": "normalized mission objective"},
      "evaluation": "The mission simulator validates feasibility and computes the fixed propulsive-cost objective.",
      "environment": "Python task environment with astrodynamics dependencies and NAIF kernels.",
      "reference_result": {"system": "Historical sequence", "score": 0.424147},
      "reported_result": {"system": "SimpleTES", "score": 0.326993},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Comparison is meaningful only under the same mission windows, simulator, constraints, and cost normalization."
    },
    {
      "id": "rosetta-trajectory",
      "name": "Rosetta gravity-assist trajectory",
      "domain": "astrodynamics",
      "registry_suite": "simpletes",
      "task_origin": "Rosetta historical mission / SimpleTES trajectory evaluator",
      "task_url": "https://arxiv.org/html/2604.19341",
      "question": "Find a feasible Rosetta gravity-assist trajectory with lower total propulsive cost.",
      "input": "Mission window, body sequence/search variables, orbital constraints, a seed optimizer, and NAIF ephemeris kernels.",
      "output": "Trajectory-design program and feasible mission parameters",
      "metric": {"name": "propulsive cost", "direction": "minimize", "unit": "normalized mission objective"},
      "evaluation": "The mission simulator validates feasibility and computes the fixed propulsive-cost objective.",
      "environment": "Python task environment with astrodynamics dependencies and NAIF kernels.",
      "reference_result": {"system": "Historical sequence", "score": 1.736837},
      "reported_result": {"system": "SimpleTES", "score": 1.552968},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Comparison is meaningful only under the same mission windows, simulator, constraints, and cost normalization."
    },
    {
      "id": "scaling-law-domain-mix",
      "name": "Scaling-law discovery: domain mix",
      "domain": "ai-foundations",
      "registry_suite": "simpletes",
      "task_origin": "SLDBench: domain mixture",
      "task_url": "https://github.com/linhaowei1/SLD",
      "question": "Discover a scaling-law program that extrapolates performance across training-domain mixtures.",
      "input": "Observed training runs, a seed symbolic/numerical predictor, a held-out extrapolation split, and fixed fitting code.",
      "output": "Scaling-law prediction program",
      "metric": {"name": "extrapolation R2", "direction": "maximize", "unit": "coefficient of determination"},
      "evaluation": "Fit on visible observations and compute R2 on the fixed held-out extrapolation split.",
      "environment": "Python with the task dataset and Hugging Face cache.",
      "reference_result": {"system": "SLD Agent (Gemini-2.5-Flash)", "score": 0.991},
      "reported_result": {"system": "SimpleTES", "score": 0.991},
      "outcome": "matched-reference",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Rounded values match; the exact split, preprocessing, and full-precision predictions are required for comparison."
    },
    {
      "id": "scaling-law-lr-bsz",
      "name": "Scaling-law discovery: learning rate and batch size",
      "domain": "ai-foundations",
      "registry_suite": "simpletes",
      "task_origin": "SLDBench: learning rate & batch size",
      "task_url": "https://github.com/linhaowei1/SLD",
      "question": "Discover a scaling-law program that extrapolates across learning-rate and batch-size choices.",
      "input": "Observed training runs, a seed symbolic/numerical predictor, a held-out extrapolation split, and fixed fitting code.",
      "output": "Scaling-law prediction program",
      "metric": {"name": "extrapolation R2", "direction": "maximize", "unit": "coefficient of determination"},
      "evaluation": "Fit on visible observations and compute R2 on the fixed held-out extrapolation split.",
      "environment": "Python with the task dataset and Hugging Face cache.",
      "reference_result": {"system": "SLD Agent (o4-mini)", "score": 0.611},
      "reported_result": {"system": "SimpleTES", "score": 0.712},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "The exact split, preprocessing, and full-precision predictions are required for comparison."
    },
    {
      "id": "scaling-law-parallel",
      "name": "Scaling-law discovery: parallel",
      "domain": "ai-foundations",
      "registry_suite": "simpletes",
      "task_origin": "SLDBench: parallel scaling",
      "task_url": "https://github.com/linhaowei1/SLD",
      "question": "Discover a scaling-law program that extrapolates performance on the parallel-compute split.",
      "input": "Observed training runs, a seed symbolic/numerical predictor, a held-out extrapolation split, and fixed fitting code.",
      "output": "Scaling-law prediction program",
      "metric": {"name": "extrapolation R2", "direction": "maximize", "unit": "coefficient of determination"},
      "evaluation": "Fit on visible observations and compute R2 on the fixed held-out extrapolation split.",
      "environment": "Python with the task dataset and Hugging Face cache.",
      "reference_result": {"system": "SLD Agent (GPT-5)", "score": 1.000},
      "reported_result": {"system": "SimpleTES", "score": 1.000},
      "outcome": "matched-reference",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Rounded values match; the exact split, preprocessing, and full-precision predictions are required for comparison."
    },
    {
      "id": "scaling-law-u-shape",
      "name": "Scaling-law discovery: U-shape",
      "domain": "ai-foundations",
      "registry_suite": "simpletes",
      "task_origin": "SLDBench: U-shaped scaling",
      "task_url": "https://github.com/linhaowei1/SLD",
      "question": "Discover a scaling-law program that extrapolates the U-shaped performance regime.",
      "input": "Observed training runs, a seed symbolic/numerical predictor, a held-out extrapolation split, and fixed fitting code.",
      "output": "Scaling-law prediction program",
      "metric": {"name": "extrapolation R2", "direction": "maximize", "unit": "coefficient of determination"},
      "evaluation": "Fit on visible observations and compute R2 on the fixed held-out extrapolation split.",
      "environment": "Python with the task dataset and Hugging Face cache.",
      "reference_result": {"system": "Goose + GPT-5", "score": -0.232},
      "reported_result": {"system": "SimpleTES", "score": -0.008},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "The exact split, preprocessing, and full-precision predictions are required for comparison."
    },
    {
      "id": "second-autocorrelation-inequality",
      "name": "Second autocorrelation inequality",
      "domain": "mathematics-discovery",
      "registry_suite": "simpletes",
      "task_origin": "Second autocorrelation inequality / EinsteinArena evaluator",
      "task_url": "https://einsteinarena.com/problems/second-autocorrelation-inequality",
      "question": "Construct a feasible function or sequence that improves the second autocorrelation-inequality bound.",
      "input": "A discretized Python construction, inequality constraints, and a fixed bound evaluator.",
      "output": "Mathematical construction",
      "metric": {"name": "bound", "direction": "maximize", "unit": "dimensionless"},
      "evaluation": "Validate construction constraints and recompute the bound from the submitted artifact.",
      "environment": "Python; no external task setup listed.",
      "reference_result": {"system": "Together AI", "score": 0.961206},
      "reported_result": {"system": "SimpleTES", "score": 0.962694},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Source-reported rounded score; full-precision construction and independent feasibility checks are required for a record claim."
    },
    {
      "id": "single-cell-rna-denoising",
      "name": "Single-cell RNA-seq denoising",
      "domain": "scientific-algorithms",
      "registry_suite": "simpletes",
      "task_origin": "OpenProblems denoising / TTT-Discover adaptation",
      "task_url": "https://github.com/openproblems-bio",
      "question": "Discover a denoising policy that reconstructs held-out single-cell gene-expression measurements more accurately.",
      "input": "OpenProblems/Tabula Muris data, a seed Python denoising program, visible training data, and held-out evaluation data.",
      "output": "Denoising program and reconstructed expression matrix",
      "metric": {"name": "denoising score", "direction": "maximize", "unit": "OpenProblems composite score"},
      "evaluation": "Run the submitted policy on the fixed data split and compute the OpenProblems denoising score.",
      "environment": "Python in a bundled task environment with the biological dataset.",
      "reference_result": {"system": "TTT-Discover", "score": 0.73},
      "reported_result": {"system": "SimpleTES", "score": 0.74},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "The data split, preprocessing, allowed retraining, and metric implementation must be frozen for comparison."
    },
    {
      "id": "sum-difference-problem",
      "name": "Sum-Difference problem",
      "domain": "mathematics-discovery",
      "registry_suite": "simpletes",
      "task_origin": "Sum–Difference construction / AlphaEvolve-lineage evaluator",
      "task_url": "https://arxiv.org/abs/2511.02864",
      "question": "Construct a set with a larger sumset-to-difference-set objective ratio under the fixed task definition.",
      "input": "A Python seed construction, discrete constraints, and an exact set-operation evaluator.",
      "output": "Combinatorial set construction",
      "metric": {"name": "ratio", "direction": "maximize", "unit": "dimensionless"},
      "evaluation": "Validate the construction and recompute the objective from its sumset and difference set.",
      "environment": "Python; no external task setup listed.",
      "reference_result": {"system": "AlphaEvolve V2", "score": 1.121936},
      "reported_result": {"system": "SimpleTES + trajectory-level post-trained gpt-oss-120b", "score": 1.144887},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "The current paper reports 1.143975 for the training-free run and 1.144887 after trajectory-level post-training. Verify the released set and exact objective independently."
    },
    {
      "id": "superconducting-qubit-routing",
      "name": "Superconducting qubit routing",
      "domain": "quantum-compilation",
      "registry_suite": "simpletes",
      "task_origin": "QASMBench + MQT Bench / SimpleTES routing contract",
      "task_url": "https://arxiv.org/html/2604.19341",
      "question": "Route two-qubit gates onto a superconducting-device coupling graph while adding as few SWAP gates as possible.",
      "input": "Quantum circuits, target coupling graphs, a Rust seed routing policy, and a fixed compiler/evaluator.",
      "output": "Rust qubit-routing policy",
      "metric": {"name": "added SWAPs", "direction": "minimize", "unit": "aggregate gate count"},
      "evaluation": "Compile the fixed circuit corpus, validate legal routing, and aggregate added SWAP counts.",
      "environment": "Rust toolchain with the task's quantum-circuit corpus and compiler harness.",
      "reference_result": {"system": "LightSABRE", "score": 20063},
      "reported_result": {"system": "SimpleTES", "score": 15147},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Circuit corpus, device topology, compiler version, seed policy, and aggregation must match."
    },
    {
      "id": "third-autocorrelation-inequality",
      "name": "Third autocorrelation inequality",
      "domain": "mathematics-discovery",
      "registry_suite": "simpletes",
      "task_origin": "Third autocorrelation inequality / EinsteinArena evaluator",
      "task_url": "https://einsteinarena.com/problems/third-autocorrelation-inequality",
      "question": "Construct a feasible function or sequence that improves the third autocorrelation-inequality bound.",
      "input": "A discretized Python construction, inequality constraints, and a fixed bound evaluator.",
      "output": "Mathematical construction",
      "metric": {"name": "bound", "direction": "minimize", "unit": "dimensionless"},
      "evaluation": "Validate construction constraints and recompute the bound from the submitted artifact.",
      "environment": "Python; no external task setup listed.",
      "reference_result": {"system": "Together AI", "score": 1.454555},
      "reported_result": {"system": "SimpleTES", "score": 1.453675},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Source-reported rounded score; full-precision construction and independent feasibility checks are required for a record claim."
    },
    {
      "id": "trimul-kernel",
      "name": "TriMul GPU kernel",
      "domain": "ai-foundations",
      "registry_suite": "simpletes",
      "task_origin": "GPUMode TriMul",
      "task_url": "https://github.com/gpu-mode/reference-kernels",
      "question": "Implement a faster triangular matrix-multiplication GPU kernel without changing numerical semantics.",
      "input": "A seed Triton kernel, fixed tensor shapes and dtypes, correctness tests, and a profiling harness.",
      "output": "Triton kernel source",
      "metric": {"name": "runtime", "direction": "minimize", "unit": "ms"},
      "evaluation": "Correctness-gated execution followed by runtime profiling on the fixed benchmark workload.",
      "environment": "Triton with a GPU profiling server; headline result uses H100.",
      "reference_result": {"system": "Human expert", "score": 1.140},
      "reported_result": {"system": "SimpleTES", "score": 1.122},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Runtime depends on hardware and profiler configuration; scores are source-reported and require matched-device replay."
    },
    {
      "id": "voyager-2-trajectory",
      "name": "Voyager 2 gravity-assist trajectory",
      "domain": "astrodynamics",
      "registry_suite": "simpletes",
      "task_origin": "Voyager 2 historical mission / SimpleTES trajectory evaluator",
      "task_url": "https://arxiv.org/html/2604.19341",
      "question": "Find a feasible Voyager 2 gravity-assist trajectory with lower total propulsive cost.",
      "input": "Mission window, body sequence/search variables, orbital constraints, a seed optimizer, and NAIF ephemeris kernels.",
      "output": "Trajectory-design program and feasible mission parameters",
      "metric": {"name": "propulsive cost", "direction": "minimize", "unit": "normalized mission objective"},
      "evaluation": "The mission simulator validates feasibility and computes the fixed propulsive-cost objective.",
      "environment": "Python task environment with astrodynamics dependencies and NAIF kernels.",
      "reference_result": {"system": "Historical sequence", "score": 3.503798},
      "reported_result": {"system": "SimpleTES", "score": 3.430214},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Comparison is meaningful only under the same mission windows, simulator, constraints, and cost normalization."
    },
    {
      "id": "zapbench-h1",
      "name": "ZAPBench whole-brain forecasting (H=1)",
      "domain": "scientific-algorithms",
      "registry_suite": "simpletes",
      "task_origin": "ZAPBench forecasting (H=1)",
      "task_url": "https://github.com/google-research/zapbench",
      "question": "Forecast whole-brain neural activity one step ahead with lower held-out error.",
      "input": "ZAPBench time-series data, a seed forecasting program, visible context, and a fixed held-out split.",
      "output": "Python forecasting program and predictions",
      "metric": {"name": "test MAE", "direction": "minimize", "unit": "normalized activity"},
      "evaluation": "Run the forecaster on the fixed H=1 test split and compute mean absolute error.",
      "environment": "Python task environment with GPU and ZAPBench dataset.",
      "reference_result": {"system": "ERA", "score": 0.0174},
      "reported_result": {"system": "SimpleTES", "score": 0.0165},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Forecast split, normalization, horizon semantics, and use of future information must be identical."
    },
    {
      "id": "zapbench-h4",
      "name": "ZAPBench whole-brain forecasting (H=4)",
      "domain": "scientific-algorithms",
      "registry_suite": "simpletes",
      "task_origin": "ZAPBench forecasting (H=4)",
      "task_url": "https://github.com/google-research/zapbench",
      "question": "Forecast whole-brain neural activity four steps ahead with lower held-out error.",
      "input": "ZAPBench time-series data, a seed forecasting program, visible context, and a fixed held-out split.",
      "output": "Python forecasting program and predictions",
      "metric": {"name": "test MAE", "direction": "minimize", "unit": "normalized activity"},
      "evaluation": "Run the forecaster on the fixed H=4 test split and compute mean absolute error.",
      "environment": "Python task environment with GPU and ZAPBench dataset.",
      "reference_result": {"system": "ERA", "score": 0.0221},
      "reported_result": {"system": "SimpleTES", "score": 0.0211},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Forecast split, normalization, horizon semantics, and use of future information must be identical."
    },
    {
      "id": "zapbench-h8",
      "name": "ZAPBench whole-brain forecasting (H=8)",
      "domain": "scientific-algorithms",
      "registry_suite": "simpletes",
      "task_origin": "ZAPBench forecasting (H=8)",
      "task_url": "https://github.com/google-research/zapbench",
      "question": "Forecast whole-brain neural activity eight steps ahead with lower held-out error.",
      "input": "ZAPBench time-series data, a seed forecasting program, visible context, and a fixed held-out split.",
      "output": "Python forecasting program and predictions",
      "metric": {"name": "test MAE", "direction": "minimize", "unit": "normalized activity"},
      "evaluation": "Run the forecaster on the fixed H=8 test split and compute mean absolute error.",
      "environment": "Python task environment with GPU and ZAPBench dataset.",
      "reference_result": {"system": "ERA", "score": 0.0244},
      "reported_result": {"system": "SimpleTES", "score": 0.0230},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Forecast split, normalization, horizon semantics, and use of future information must be identical."
    },
    {
      "id": "zapbench-h16",
      "name": "ZAPBench whole-brain forecasting (H=16)",
      "domain": "scientific-algorithms",
      "registry_suite": "simpletes",
      "task_origin": "ZAPBench forecasting (H=16)",
      "task_url": "https://github.com/google-research/zapbench",
      "question": "Forecast whole-brain neural activity sixteen steps ahead with lower held-out error.",
      "input": "ZAPBench time-series data, a seed forecasting program, visible context, and a fixed held-out split.",
      "output": "Python forecasting program and predictions",
      "metric": {"name": "test MAE", "direction": "minimize", "unit": "normalized activity"},
      "evaluation": "Run the forecaster on the fixed H=16 test split and compute mean absolute error.",
      "environment": "Python task environment with GPU and ZAPBench dataset.",
      "reference_result": {"system": "ERA", "score": 0.0267},
      "reported_result": {"system": "SimpleTES", "score": 0.0251},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Forecast split, normalization, horizon semantics, and use of future information must be identical."
    },
    {
      "id": "zapbench-h32",
      "name": "ZAPBench whole-brain forecasting (H=32)",
      "domain": "scientific-algorithms",
      "registry_suite": "simpletes",
      "task_origin": "ZAPBench forecasting (H=32)",
      "task_url": "https://github.com/google-research/zapbench",
      "question": "Forecast whole-brain neural activity thirty-two steps ahead with lower held-out error.",
      "input": "ZAPBench time-series data, a seed forecasting program, visible context, and a fixed held-out split.",
      "output": "Python forecasting program and predictions",
      "metric": {"name": "test MAE", "direction": "minimize", "unit": "normalized activity"},
      "evaluation": "Run the forecaster on the fixed H=32 test split and compute mean absolute error.",
      "environment": "Python task environment with GPU and ZAPBench dataset.",
      "reference_result": {"system": "ERA", "score": 0.0283},
      "reported_result": {"system": "SimpleTES", "score": 0.0259},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Forecast split, normalization, horizon semantics, and use of future information must be identical."
    },
    {
      "id": "zoned-neutral-atom-compilation",
      "name": "Zoned neutral-atom compilation",
      "domain": "quantum-compilation",
      "registry_suite": "simpletes",
      "task_origin": "QASMBench + MQT Bench / SimpleTES ZAC-style contract",
      "task_url": "https://arxiv.org/html/2604.19341",
      "question": "Compile circuits for a zoned neutral-atom architecture while minimizing aggregate execution time.",
      "input": "Quantum circuits, architecture constraints, a Python seed compilation policy, and a fixed simulator.",
      "output": "Python compilation policy",
      "metric": {"name": "execution time", "direction": "minimize", "unit": "geometric-mean simulator time"},
      "evaluation": "Validate legal movement and gate scheduling, simulate every circuit, and aggregate execution time.",
      "environment": "Python in the task-specific zoned-neutral-atom environment.",
      "reference_result": {"system": "ZAC", "score": 29187.7},
      "reported_result": {"system": "SimpleTES", "score": 19507.5},
      "outcome": "improved",
      "evidence_urls": ["https://arxiv.org/abs/2604.19341", "https://github.com/wq-will/SimpleTES/blob/main/best_results/README.md"],
      "integrity_notes": "Circuit corpus, architecture parameters, simulator version, and aggregation must match."
    },
    {
      "id": "arc-agi-3-ls20",
      "name": "ARC-AGI-3 ls20 (Agent reasoning)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "arc-agi-3",
      "task_origin": "ARC-AGI-3 public game ls20",
      "task_url": "https://arcprize.org/media/ARC_AGI_3_Technical_Report.pdf",
      "question": "Without natural-language instructions, discover ls20's hidden mechanics and win condition through interaction, then complete its levels efficiently.",
      "input": "A turn-based grid frame of at most 64x64 cells using 16 colors, the currently available actions, and the accumulated frame/action history; the objective and mechanics are withheld.",
      "output": "An action trace that reaches WIN across the game's levels",
      "metric": {
        "name": "Relative Human Action Efficiency (RHAE)",
        "direction": "maximize",
        "unit": "percent"
      },
      "evaluation": "For each completed level, square human-baseline actions divided by agent actions, cap at 1.15, take a level-index-weighted game score, then average games for an aggregate benchmark score.",
      "environment": "Official ARC-AGI Toolkit/API; deterministic turn-based environment; RESET, ACTION1-ACTION7, with ACTION6 accepting coordinates and ACTION7 serving as undo when available.",
      "reference_result": {
        "system": "Human first-attempt upper-median baseline",
        "score": 100
      },
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://arcprize.org/arc-agi/3",
        "https://docs.arcprize.org/available-games",
        "https://docs.arcprize.org/methodology",
        "https://github.com/arcprize/ARC-AGI"
      ],
      "integrity_notes": "This is a public demonstration environment, not a valid stand-alone measure of ARC-AGI-3 progress. Record the exact game version, scorecard, replay, prompt, action budget, and toolkit/API version; do not expose private environments."
    },
    {
      "id": "arc-agi-3-ft09",
      "name": "ARC-AGI-3 ft09 (Elementary Logic)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "arc-agi-3",
      "task_origin": "ARC-AGI-3 public game ft09",
      "task_url": "https://arcprize.org/media/ARC_AGI_3_Technical_Report.pdf",
      "question": "Without natural-language instructions, discover ft09's hidden mechanics and win condition through interaction, then complete its levels efficiently.",
      "input": "A turn-based grid frame of at most 64x64 cells using 16 colors, the currently available actions, and the accumulated frame/action history; the objective and mechanics are withheld.",
      "output": "An action trace that reaches WIN across the game's levels",
      "metric": {
        "name": "Relative Human Action Efficiency (RHAE)",
        "direction": "maximize",
        "unit": "percent"
      },
      "evaluation": "For each completed level, square human-baseline actions divided by agent actions, cap at 1.15, take a level-index-weighted game score, then average games for an aggregate benchmark score.",
      "environment": "Official ARC-AGI Toolkit/API; deterministic turn-based environment; RESET, ACTION1-ACTION7, with ACTION6 accepting coordinates and ACTION7 serving as undo when available.",
      "reference_result": {
        "system": "Human first-attempt upper-median baseline",
        "score": 100
      },
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://arcprize.org/arc-agi/3",
        "https://docs.arcprize.org/available-games",
        "https://docs.arcprize.org/methodology",
        "https://github.com/arcprize/ARC-AGI"
      ],
      "integrity_notes": "This is a public demonstration environment, not a valid stand-alone measure of ARC-AGI-3 progress. Record the exact game version, scorecard, replay, prompt, action budget, and toolkit/API version; do not expose private environments."
    },
    {
      "id": "arc-agi-3-vc33",
      "name": "ARC-AGI-3 vc33 (Orchestration)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "arc-agi-3",
      "task_origin": "ARC-AGI-3 public game vc33",
      "task_url": "https://arcprize.org/media/ARC_AGI_3_Technical_Report.pdf",
      "question": "Without natural-language instructions, discover vc33's hidden mechanics and win condition through interaction, then complete its levels efficiently.",
      "input": "A turn-based grid frame of at most 64x64 cells using 16 colors, the currently available actions, and the accumulated frame/action history; the objective and mechanics are withheld.",
      "output": "An action trace that reaches WIN across the game's levels",
      "metric": {
        "name": "Relative Human Action Efficiency (RHAE)",
        "direction": "maximize",
        "unit": "percent"
      },
      "evaluation": "For each completed level, square human-baseline actions divided by agent actions, cap at 1.15, take a level-index-weighted game score, then average games for an aggregate benchmark score.",
      "environment": "Official ARC-AGI Toolkit/API; deterministic turn-based environment; RESET, ACTION1-ACTION7, with ACTION6 accepting coordinates and ACTION7 serving as undo when available.",
      "reference_result": {
        "system": "Human first-attempt upper-median baseline",
        "score": 100
      },
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://arcprize.org/arc-agi/3",
        "https://docs.arcprize.org/available-games",
        "https://docs.arcprize.org/methodology",
        "https://github.com/arcprize/ARC-AGI"
      ],
      "integrity_notes": "This is a public demonstration environment, not a valid stand-alone measure of ARC-AGI-3 progress. Record the exact game version, scorecard, replay, prompt, action budget, and toolkit/API version; do not expose private environments."
    },
    {
      "id": "dig-bench-p-1",
      "name": "DiG-bench P-1 (tier 1)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-1",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-1's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-2",
      "name": "DiG-bench P-2 (tier 1)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-2",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-2's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-3",
      "name": "DiG-bench P-3 (tier 1)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-3",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-3's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-4",
      "name": "DiG-bench P-4 (tier 2)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-4",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-4's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-5",
      "name": "DiG-bench P-5 (tier 2)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-5",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-5's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-6",
      "name": "DiG-bench P-6 (tier 2)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-6",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-6's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-7",
      "name": "DiG-bench P-7 (tier 3)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-7",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-7's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-8",
      "name": "DiG-bench P-8 (tier 3)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-8",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-8's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-9",
      "name": "DiG-bench P-9 (tier 3)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-9",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-9's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-10",
      "name": "DiG-bench P-10 (tier 4)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-10",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-10's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-11",
      "name": "DiG-bench P-11 (tier 4)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-11",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-11's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-12",
      "name": "DiG-bench P-12 (tier 4)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-12",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-12's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-13",
      "name": "DiG-bench P-13 (tier 5)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-13",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-13's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-14",
      "name": "DiG-bench P-14 (tier 5)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-14",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-14's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-15",
      "name": "DiG-bench P-15 (tier 5)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-15",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-15's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-16",
      "name": "DiG-bench P-16 (tier 6)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-16",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-16's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-17",
      "name": "DiG-bench P-17 (tier 6)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-17",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-17's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-18",
      "name": "DiG-bench P-18 (tier 6)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-18",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-18's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-19",
      "name": "DiG-bench P-19 (tier 7)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-19",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-19's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-20",
      "name": "DiG-bench P-20 (tier 7)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-20",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-20's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    },
    {
      "id": "dig-bench-p-21",
      "name": "DiG-bench P-21 (tier 7)",
      "domain": "interactive-world-discovery",
      "task_type": "interactive-rule-discovery",
      "registry_suite": "dig-bench",
      "task_origin": "DiG-bench public game P-21",
      "task_url": "https://digbench.ai/",
      "question": "Discover P-21's unknown transformation rules and win conditions through experimentation, then apply the learned rules to beat its challenges within the step budget.",
      "input": "A JSON game state containing the observation, level, lives, steps remaining, status, legal actions, and interaction history; no semantic description of the hidden rules.",
      "output": "A legal action sequence that wins the game",
      "metric": {
        "name": "win rate",
        "direction": "maximize",
        "unit": "fraction of runs"
      },
      "evaluation": "Score each run by whether the game is beaten within its fixed step budget; average wins over repeated runs for the game, then average games within the difficulty tier.",
      "environment": "Official text-game server exposed through the DiG-bench API or MCP and a declared model harness; humans and models receive identical states, actions, and step budgets. Some games offer a declared creative mode for unmetered experiments.",
      "reference_result": null,
      "reported_result": null,
      "outcome": "not-yet-scored",
      "evidence_urls": [
        "https://digbench.ai/",
        "https://arxiv.org/abs/2608.12593",
        "https://github.com/discos-research/dig-bench"
      ],
      "integrity_notes": "P-series task is public and may be used for development, so report public-versus-private evaluation separately. Pin the game, server, harness, model, prompt, context policy, run count, and step budget."
    }
  ]
}