Migrate starter graders to hidden verifiers

tasks/starter/2048-advanced-snake-params-001/reference-report.md

@@ -30,13 +30,24 @@
 - Workspace history policy: `base_only`
 - Workspace base ref: `f07a425c106ce205170bc3eb23e722ce1fa0233d`
 
-## Code-Based Graders
+## Public Graders
 
 1. Assertion `python3 -c "import numpy"`: passed (0)
 2. Assertion `python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"`: passed (0)
 3. Assertion `git apply reference.patch`: passed (0)
-4. Assertion `python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"`: passed (0)
-5. Assertion `python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"`: passed (0)
+
+## Hidden Verifier
+
+- Patch: `verifier.patch`
+
+1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0)
+2. Assertion `python3 .agentlab_hidden/check_behavior.py`: passed (0)
+
+```text
+$ python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"
+$ python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"
+```
+
 
 ## Changed Files
 

tasks/starter/2048-advanced-snake-params-001/reference-result.json

@@ -31,28 +31,12 @@
       "returncode": 0,
       "stderr": "",
       "stdout": ""
-    },
-    {
-      "command": "python3 -c \"from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result\"",
-      "passed": true,
-      "returncode": 0,
-      "stderr": "",
-      "stdout": ""
-    },
-    {
-      "command": "python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\"",
-      "passed": true,
-      "returncode": 0,
-      "stderr": "",
-      "stdout": ""
     }
   ],
   "commands_run": [
     "python3 -c \"import numpy\"",
     "python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\"",
-    "git apply reference.patch",
-    "python3 -c \"from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result\"",
-    "python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\""
+    "git apply reference.patch"
   ],
   "cost_usd": null,
   "diff_path": "reference.diff",
@@ -81,20 +65,29 @@
       "passed": true,
       "returncode": 0,
       "type": "code"
-    },
-    {
-      "assertion": "python3 -c \"from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result\"",
-      "passed": true,
-      "returncode": 0,
-      "type": "code"
-    },
-    {
-      "assertion": "python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\"",
-      "passed": true,
-      "returncode": 0,
-      "type": "code"
     }
   ],
+  "hidden_verifier": {
+    "checks": [
+      {
+        "command": "git apply hidden verifier patch: verifier.patch",
+        "passed": true,
+        "returncode": 0,
+        "stderr": "",
+        "stdout": ""
+      },
+      {
+        "command": "python3 .agentlab_hidden/check_behavior.py",
+        "passed": true,
+        "returncode": 0,
+        "stderr": "",
+        "stdout": "$ python3 -c \"from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result\"\n$ python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\"\n"
+      }
+    ],
+    "configured": true,
+    "patch": "verifier.patch",
+    "restore_notes": []
+  },
   "input_tokens": null,
   "lines_added": 2,
   "lines_deleted": 2,

tasks/starter/2048-advanced-snake-params-001/task-card.md

@@ -27,6 +27,11 @@ In simulation.py, persist params when args.heuristic is advanced-snake instead o
 - PYTEST_ADDOPTS=-p no:cacheprovider
 - PYTHONDONTWRITEBYTECODE=1
 
+## Hidden Verifier
+
+- Patch: `verifier.patch`
+- Commands: `1 command configured`
+
 ## Graders
 
 ### Setup
@@ -39,8 +44,7 @@ In simulation.py, persist params when args.heuristic is advanced-snake instead o
 
 ### Target
 
-- `python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"`
-- `python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"`
+None configured.
 
 ## Success Criteria
 

tasks/starter/2048-advanced-snake-params-001/task.yaml

@@ -17,17 +17,17 @@ reference_solution: >
 reference_artifact:
   type: patch
   path: reference.patch
+hidden_verifier:
+  patch: verifier.patch
+  commands:
+    - python3 .agentlab_hidden/check_behavior.py
 setup:
   - python3 -c "import numpy"
 environment:
   PYTHONDONTWRITEBYTECODE: "1"
   PYTEST_ADDOPTS: "-p no:cacheprovider"
 baseline:
   - python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"
-test:
-  - >-
-    python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"
-  - python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"
 success:
   tests_must_pass: true
   max_files_changed: 3

tasks/starter/2048-advanced-snake-params-001/verifier.patch

@@ -0,0 +1,37 @@
+diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py
+new file mode 100644
+index 0000000..0000000
+--- /dev/null
++++ b/.agentlab_hidden/check_behavior.py
+@@ -0,0 +1,31 @@
++from __future__ import annotations
++
++import subprocess
++import sys
++
++COMMANDS = ['python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {\'empty\': 1.0, \'smooth\': 2.0, \'mono\': 3.0, \'snake\': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai=\'expectimax\', heuristic=\'advanced-snake\', depth=1, tag=\'eval-smoke\'); result = run_single_simulation((args, weights, heuristic, {\'name\': heuristic.name, \'version\': heuristic.version, \'description\': heuristic.description})); assert result[\'params\'] == weights, result; assert result[\'tag\'] == \'eval-smoke\', result"', 'python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"']
++
++
++def main() -> int:
++    for command in COMMANDS:
++        print(f"$ {command}")
++        completed = subprocess.run(
++            command,
++            shell=True,
++            text=True,
++            stdout=subprocess.PIPE,
++            stderr=subprocess.STDOUT,
++        )
++        if completed.stdout:
++            print(completed.stdout, end="")
++        if completed.returncode != 0:
++            print(
++                f"hidden verifier command failed with exit code {completed.returncode}",
++                file=sys.stderr,
++            )
++            return completed.returncode
++    return 0
++
++
++if __name__ == "__main__":
++    raise SystemExit(main())

tasks/starter/click-default-map-nargs-001/reference-report.md

@@ -32,7 +32,7 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis
 - Workspace history policy: `base_only`
 - Workspace base ref: `9de2b9c9cba0c82374206783bb4b3853373e575c`
 
-## Code-Based Graders
+## Public Graders
 
 1. Assertion `PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv`: passed (0)
 2. Assertion `python -m pip install -e . "pytest<9"`: passed (0)
@@ -44,9 +44,20 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis
 
 3. Assertion `python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code != 0, result.output; assert "Value must be an iterable" in result.output, result.output'`: passed (0)
 4. Assertion `git apply reference.patch`: passed (0)
-5. Assertion `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'`: passed (0)
-6. Assertion `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'`: passed (0)
-7. Assertion `python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'`: passed (0)
+
+## Hidden Verifier
+
+- Patch: `verifier.patch`
+
+1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0)
+2. Assertion `python .agentlab_hidden/check_behavior.py`: passed (0)
+
+```text
+$ python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'
+$ python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'
+$ python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'
+```
+
 
 ## Changed Files