diff --git a/tasks/starter/2048-advanced-snake-params-001/reference-report.md b/tasks/starter/2048-advanced-snake-params-001/reference-report.md index 51ebb81..2e731dc 100644 --- a/tasks/starter/2048-advanced-snake-params-001/reference-report.md +++ b/tasks/starter/2048-advanced-snake-params-001/reference-report.md @@ -30,13 +30,24 @@ - Workspace history policy: `base_only` - Workspace base ref: `f07a425c106ce205170bc3eb23e722ce1fa0233d` -## Code-Based Graders +## Public Graders 1. Assertion `python3 -c "import numpy"`: passed (0) 2. Assertion `python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"`: passed (0) 3. Assertion `git apply reference.patch`: passed (0) -4. Assertion `python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"`: passed (0) -5. Assertion `python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python3 .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result" +$ python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()" +``` + ## Changed Files diff --git a/tasks/starter/2048-advanced-snake-params-001/reference-result.json b/tasks/starter/2048-advanced-snake-params-001/reference-result.json index 48c7c7d..d679e20 100644 --- a/tasks/starter/2048-advanced-snake-params-001/reference-result.json +++ b/tasks/starter/2048-advanced-snake-params-001/reference-result.json @@ -31,28 +31,12 @@ "returncode": 0, "stderr": "", "stdout": "" - }, - { - "command": "python3 -c \"from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result\"", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\"", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" } ], "commands_run": [ "python3 -c \"import numpy\"", "python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\"", - "git apply reference.patch", - "python3 -c \"from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result\"", - "python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\"" + "git apply reference.patch" ], "cost_usd": null, "diff_path": "reference.diff", @@ -81,20 +65,29 @@ "passed": true, "returncode": 0, "type": "code" - }, - { - "assertion": "python3 -c \"from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result\"", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\"", - "passed": true, - "returncode": 0, - "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python3 .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ python3 -c \"from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result\"\n$ python3 -c \"from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()\"\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 2, "lines_deleted": 2, diff --git a/tasks/starter/2048-advanced-snake-params-001/task-card.md b/tasks/starter/2048-advanced-snake-params-001/task-card.md index 169cf3b..b0d43c1 100644 --- a/tasks/starter/2048-advanced-snake-params-001/task-card.md +++ b/tasks/starter/2048-advanced-snake-params-001/task-card.md @@ -27,6 +27,11 @@ In simulation.py, persist params when args.heuristic is advanced-snake instead o - PYTEST_ADDOPTS=-p no:cacheprovider - PYTHONDONTWRITEBYTECODE=1 +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -39,8 +44,7 @@ In simulation.py, persist params when args.heuristic is advanced-snake instead o ### Target -- `python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"` -- `python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"` +None configured. ## Success Criteria diff --git a/tasks/starter/2048-advanced-snake-params-001/task.yaml b/tasks/starter/2048-advanced-snake-params-001/task.yaml index 2f7e42e..7dd4e17 100644 --- a/tasks/starter/2048-advanced-snake-params-001/task.yaml +++ b/tasks/starter/2048-advanced-snake-params-001/task.yaml @@ -17,6 +17,10 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python3 .agentlab_hidden/check_behavior.py setup: - python3 -c "import numpy" environment: @@ -24,10 +28,6 @@ environment: PYTEST_ADDOPTS: "-p no:cacheprovider" baseline: - python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()" -test: - - >- - python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result" - - python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()" success: tests_must_pass: true max_files_changed: 3 diff --git a/tasks/starter/2048-advanced-snake-params-001/verifier.patch b/tasks/starter/2048-advanced-snake-params-001/verifier.patch new file mode 100644 index 0000000..ab644fe --- /dev/null +++ b/tasks/starter/2048-advanced-snake-params-001/verifier.patch @@ -0,0 +1,37 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,31 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = ['python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {\'empty\': 1.0, \'smooth\': 2.0, \'mono\': 3.0, \'snake\': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai=\'expectimax\', heuristic=\'advanced-snake\', depth=1, tag=\'eval-smoke\'); result = run_single_simulation((args, weights, heuristic, {\'name\': heuristic.name, \'version\': heuristic.version, \'description\': heuristic.description})); assert result[\'params\'] == weights, result; assert result[\'tag\'] == \'eval-smoke\', result"', 'python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"'] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print(f"$ {command}") ++ completed = subprocess.run( ++ command, ++ shell=True, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/click-default-map-nargs-001/reference-report.md b/tasks/starter/click-default-map-nargs-001/reference-report.md index 4e4b640..22a15dd 100644 --- a/tasks/starter/click-default-map-nargs-001/reference-report.md +++ b/tasks/starter/click-default-map-nargs-001/reference-report.md @@ -32,7 +32,7 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis - Workspace history policy: `base_only` - Workspace base ref: `9de2b9c9cba0c82374206783bb4b3853373e575c` -## Code-Based Graders +## Public Graders 1. Assertion `PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv`: passed (0) 2. Assertion `python -m pip install -e . "pytest<9"`: passed (0) @@ -44,9 +44,20 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis 3. Assertion `python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code != 0, result.output; assert "Value must be an iterable" in result.output, result.output'`: passed (0) 4. Assertion `git apply reference.patch`: passed (0) -5. Assertion `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'`: passed (0) -6. Assertion `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'`: passed (0) -7. Assertion `python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output' +$ python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output' +$ python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output' +``` + ## Changed Files diff --git a/tasks/starter/click-default-map-nargs-001/reference-result.json b/tasks/starter/click-default-map-nargs-001/reference-result.json index 57d6326..3a4ed8e 100644 --- a/tasks/starter/click-default-map-nargs-001/reference-result.json +++ b/tasks/starter/click-default-map-nargs-001/reference-result.json @@ -23,7 +23,7 @@ "passed": true, "returncode": 0, "stderr": "\n[notice] A new release of pip is available: 25.1.1 -> 26.1.2\n[notice] To update, run: pip install --upgrade pip\n", - "stdout": "Obtaining file:///private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/agentlab-reference-4ic7u1_p/click-default-map-nargs-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nBuilding wheels for collected packages: click\n Building editable for click (pyproject.toml): started\n Building editable for click (pyproject.toml): finished with status 'done'\n Created wheel for click: filename=click-8.3.3-py3-none-any.whl size=3184 sha256=9188065879e19764b83a5ed36d330f2c5e9c61c89140791d2837a92d244eb670\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-7rmj5m9e/wheels/2b/e9/41/402dc5b1c7ce94bc63b03064fb6ca2f7a984974373562222b4\nSuccessfully built click\nInstalling collected packages: pygments, pluggy, packaging, iniconfig, click, pytest\n\nSuccessfully installed click-8.3.3 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2\n" + "stdout": "Obtaining file:///private/tmp/ael-ref-starter-123/click-default-map-nargs-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nBuilding wheels for collected packages: click\n Building editable for click (pyproject.toml): started\n Building editable for click (pyproject.toml): finished with status 'done'\n Created wheel for click: filename=click-8.3.3-py3-none-any.whl size=3143 sha256=61063a33608d3a9b2a270c3d571753524e3ecba9ea91ca7b340f234efb7b8b88\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-owiiyr2f/wheels/9c/cf/2b/bb7145d73ba8b416e4becb81cd52ab295d6d1d936437f00cd8\nSuccessfully built click\nInstalling collected packages: pygments, pluggy, packaging, iniconfig, click, pytest\n\nSuccessfully installed click-8.3.3 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2\n" }, { "command": "python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option(\"--point\", nargs=2, type=int)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={\"point\":\"3 4\"}); assert result.exit_code != 0, result.output; assert \"Value must be an iterable\" in result.output, result.output'", @@ -38,37 +38,13 @@ "returncode": 0, "stderr": "", "stdout": "" - }, - { - "command": "python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option(\"--point\", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={\"point\":\"3 4\"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, [\"--point\", \"10\", \"20\"], default_map={\"point\":\"3 4\"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option(\"--word-pair\", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={\"word_pair\":\"hello world\"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (\"hello\", \"world\"), result.output'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option(\"--name\")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={\"name\":\"hello world\"}); assert result.exit_code == 0, result.output; assert result.output.strip() == \"hello world\", result.output'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" } ], "commands_run": [ "PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv", "python -m pip install -e . \"pytest<9\"", "python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option(\"--point\", nargs=2, type=int)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={\"point\":\"3 4\"}); assert result.exit_code != 0, result.output; assert \"Value must be an iterable\" in result.output, result.output'", - "git apply reference.patch", - "python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option(\"--point\", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={\"point\":\"3 4\"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, [\"--point\", \"10\", \"20\"], default_map={\"point\":\"3 4\"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'", - "python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option(\"--word-pair\", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={\"word_pair\":\"hello world\"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (\"hello\", \"world\"), result.output'", - "python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option(\"--name\")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={\"name\":\"hello world\"}); assert result.exit_code == 0, result.output; assert result.output.strip() == \"hello world\", result.output'" + "git apply reference.patch" ], "cost_usd": null, "diff_path": "reference.diff", @@ -103,26 +79,29 @@ "passed": true, "returncode": 0, "type": "code" - }, - { - "assertion": "python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option(\"--point\", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={\"point\":\"3 4\"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, [\"--point\", \"10\", \"20\"], default_map={\"point\":\"3 4\"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option(\"--word-pair\", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={\"word_pair\":\"hello world\"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (\"hello\", \"world\"), result.output'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option(\"--name\")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={\"name\":\"hello world\"}); assert result.exit_code == 0, result.output; assert result.output.strip() == \"hello world\", result.output'", - "passed": true, - "returncode": 0, - "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option(\"--point\", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={\"point\":\"3 4\"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, [\"--point\", \"10\", \"20\"], default_map={\"point\":\"3 4\"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'\n$ python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option(\"--word-pair\", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={\"word_pair\":\"hello world\"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (\"hello\", \"world\"), result.output'\n$ python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option(\"--name\")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={\"name\":\"hello world\"}); assert result.exit_code == 0, result.output; assert result.output.strip() == \"hello world\", result.output'\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 5, "lines_deleted": 0, diff --git a/tasks/starter/click-default-map-nargs-001/task-card.md b/tasks/starter/click-default-map-nargs-001/task-card.md index 292a87d..37b24bc 100644 --- a/tasks/starter/click-default-map-nargs-001/task-card.md +++ b/tasks/starter/click-default-map-nargs-001/task-card.md @@ -30,6 +30,11 @@ In Option.consume_value, when a value comes from default_map and is a string for - PYTHONPATH={workspace}/src - VIRTUAL_ENV={workspace}/.agentlab/venv +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -43,9 +48,7 @@ In Option.consume_value, when a value comes from default_map and is a string for ### Target -- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'` -- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'` -- `python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'` +None configured. ## Success Criteria diff --git a/tasks/starter/click-default-map-nargs-001/task.yaml b/tasks/starter/click-default-map-nargs-001/task.yaml index 76a0691..a00abc4 100644 --- a/tasks/starter/click-default-map-nargs-001/task.yaml +++ b/tasks/starter/click-default-map-nargs-001/task.yaml @@ -20,6 +20,10 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python .agentlab_hidden/check_behavior.py setup: - PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv - python -m pip install -e . "pytest<9" @@ -33,13 +37,6 @@ environment: baseline: - >- python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code != 0, result.output; assert "Value must be an iterable" in result.output, result.output' -test: - - >- - python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output' - - >- - python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output' - - >- - python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output' success: tests_must_pass: true max_files_changed: 2 diff --git a/tasks/starter/click-default-map-nargs-001/verifier.patch b/tasks/starter/click-default-map-nargs-001/verifier.patch new file mode 100644 index 0000000..521f027 --- /dev/null +++ b/tasks/starter/click-default-map-nargs-001/verifier.patch @@ -0,0 +1,37 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,31 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = ['python -c \'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output\'', 'python -c \'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output\'', 'python -c \'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output\''] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print(f"$ {command}") ++ completed = subprocess.run( ++ command, ++ shell=True, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/click-help-option-refactor-001/reference-report.md b/tasks/starter/click-help-option-refactor-001/reference-report.md index 2ed7208..ae432cb 100644 --- a/tasks/starter/click-help-option-refactor-001/reference-report.md +++ b/tasks/starter/click-help-option-refactor-001/reference-report.md @@ -32,7 +32,7 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis - Workspace history policy: `base_only` - Workspace base ref: `bfb2ef2dc56d8bc66a833a5c91a122ccb2eb63ba` -## Code-Based Graders +## Public Graders 1. Assertion `PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv`: passed (0) 2. Assertion `python -m pip install -e . "pytest<9"`: passed (0) @@ -46,9 +46,20 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis 4. Assertion `python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output'`: passed (0) 5. Assertion `python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output'`: passed (0) 6. Assertion `git apply reference.patch`: passed (0) -7. Assertion `python -c 'import click; assert hasattr(click, "HelpOption"); cmd = click.Command("demo"); ctx = click.Context(cmd); option = cmd.get_help_option(ctx); assert isinstance(option, click.HelpOption), type(option); assert option.is_flag is True; assert option.is_eager is True; assert option.expose_value is False'`: passed (0) -8. Assertion `python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output'`: passed (0) -9. Assertion `python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output'`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output' +$ python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output' +``` + +3. Assertion `python .agentlab_hidden/check_structure.py`: passed (0) ## Changed Files diff --git a/tasks/starter/click-help-option-refactor-001/reference-result.json b/tasks/starter/click-help-option-refactor-001/reference-result.json index 740df5f..8b0b23b 100644 --- a/tasks/starter/click-help-option-refactor-001/reference-result.json +++ b/tasks/starter/click-help-option-refactor-001/reference-result.json @@ -23,7 +23,7 @@ "passed": true, "returncode": 0, "stderr": "\n[notice] A new release of pip is available: 25.1.1 -> 26.1.2\n[notice] To update, run: pip install --upgrade pip\n", - "stdout": "Obtaining file:///private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/agentlab-reference-4ic7u1_p/click-help-option-refactor-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nBuilding wheels for collected packages: click\n Building editable for click (pyproject.toml): started\n Building editable for click (pyproject.toml): finished with status 'done'\n Created wheel for click: filename=click-8.1.7-py3-none-any.whl size=3027 sha256=e07e21c8bfb3378b02413b3d98cbeefbd1d3c0d4674804303e09f813c752f270\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-eilv3u5s/wheels/75/04/ae/5feab7b60487f8b54b1605c75f430c5aa64f6001b29a469b1d\nSuccessfully built click\nInstalling collected packages: pygments, pluggy, packaging, iniconfig, click, pytest\n\nSuccessfully installed click-8.1.7 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2\n" + "stdout": "Obtaining file:///private/tmp/ael-ref-starter-123/click-help-option-refactor-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nBuilding wheels for collected packages: click\n Building editable for click (pyproject.toml): started\n Building editable for click (pyproject.toml): finished with status 'done'\n Created wheel for click: filename=click-8.1.7-py3-none-any.whl size=2984 sha256=d890f7b0d8ed876a4c1d74a4784ab378e48537a568f9107b58f36dd1afb50781\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-e4bxb3b7/wheels/96/93/54/e80d1af63bd52d8de1d4fb2ae168ad13a8374dd201435ea0db\nSuccessfully built click\nInstalling collected packages: pygments, pluggy, packaging, iniconfig, click, pytest\n\nSuccessfully installed click-8.1.7 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2\n" }, { "command": "python -c 'import click; assert not hasattr(click, \"HelpOption\")'", @@ -52,27 +52,6 @@ "returncode": 0, "stderr": "", "stdout": "" - }, - { - "command": "python -c 'import click; assert hasattr(click, \"HelpOption\"); cmd = click.Command(\"demo\"); ctx = click.Context(cmd); option = cmd.get_help_option(ctx); assert isinstance(option, click.HelpOption), type(option); assert option.is_flag is True; assert option.is_eager is True; assert option.expose_value is False'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python -c 'import click; from click.testing import CliRunner; cmd = click.Command(\"demo\", params=[click.Option([\"--name\"], help=\"Name\")], callback=lambda name: click.echo(\"hello \" + name if name else \"hello\")); result = CliRunner().invoke(cmd, [\"--help\"]); assert result.exit_code == 0, result.output; assert \"Usage: demo [OPTIONS]\" in result.output, result.output; assert \"--help\" in result.output, result.output; assert \"Show this message and exit.\" in result.output, result.output; disabled = click.Command(\"plain\", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, [\"--help\"]); assert disabled_result.exit_code == 2, disabled_result.output; assert \"No such option: --help\" in disabled_result.output, disabled_result.output'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python -c 'import click; from click.testing import CliRunner; cli = click.help_option(\"-h\", \"--halp\")(click.command()(lambda: click.echo(\"ran\"))); runner = CliRunner(); short_help = runner.invoke(cli, [\"-h\"]); long_help = runner.invoke(cli, [\"--halp\"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert \"Usage:\" in short_help.output, short_help.output; assert \"Usage:\" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == \"ran\", normal_run.output'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" } ], "commands_run": [ @@ -81,10 +60,7 @@ "python -c 'import click; assert not hasattr(click, \"HelpOption\")'", "python -c 'import click; from click.testing import CliRunner; cmd = click.Command(\"demo\", params=[click.Option([\"--name\"], help=\"Name\")], callback=lambda name: click.echo(\"hello \" + name if name else \"hello\")); result = CliRunner().invoke(cmd, [\"--help\"]); assert result.exit_code == 0, result.output; assert \"Usage: demo [OPTIONS]\" in result.output, result.output; assert \"--help\" in result.output, result.output; assert \"Show this message and exit.\" in result.output, result.output; disabled = click.Command(\"plain\", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, [\"--help\"]); assert disabled_result.exit_code == 2, disabled_result.output; assert \"No such option: --help\" in disabled_result.output, disabled_result.output'", "python -c 'import click; from click.testing import CliRunner; cli = click.help_option(\"-h\", \"--halp\")(click.command()(lambda: click.echo(\"ran\"))); runner = CliRunner(); short_help = runner.invoke(cli, [\"-h\"]); long_help = runner.invoke(cli, [\"--halp\"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert \"Usage:\" in short_help.output, short_help.output; assert \"Usage:\" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == \"ran\", normal_run.output'", - "git apply reference.patch", - "python -c 'import click; assert hasattr(click, \"HelpOption\"); cmd = click.Command(\"demo\"); ctx = click.Context(cmd); option = cmd.get_help_option(ctx); assert isinstance(option, click.HelpOption), type(option); assert option.is_flag is True; assert option.is_eager is True; assert option.expose_value is False'", - "python -c 'import click; from click.testing import CliRunner; cmd = click.Command(\"demo\", params=[click.Option([\"--name\"], help=\"Name\")], callback=lambda name: click.echo(\"hello \" + name if name else \"hello\")); result = CliRunner().invoke(cmd, [\"--help\"]); assert result.exit_code == 0, result.output; assert \"Usage: demo [OPTIONS]\" in result.output, result.output; assert \"--help\" in result.output, result.output; assert \"Show this message and exit.\" in result.output, result.output; disabled = click.Command(\"plain\", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, [\"--help\"]); assert disabled_result.exit_code == 2, disabled_result.output; assert \"No such option: --help\" in disabled_result.output, disabled_result.output'", - "python -c 'import click; from click.testing import CliRunner; cli = click.help_option(\"-h\", \"--halp\")(click.command()(lambda: click.echo(\"ran\"))); runner = CliRunner(); short_help = runner.invoke(cli, [\"-h\"]); long_help = runner.invoke(cli, [\"--halp\"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert \"Usage:\" in short_help.output, short_help.output; assert \"Usage:\" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == \"ran\", normal_run.output'" + "git apply reference.patch" ], "cost_usd": null, "diff_path": "reference.diff", @@ -133,26 +109,36 @@ "passed": true, "returncode": 0, "type": "code" - }, - { - "assertion": "python -c 'import click; assert hasattr(click, \"HelpOption\"); cmd = click.Command(\"demo\"); ctx = click.Context(cmd); option = cmd.get_help_option(ctx); assert isinstance(option, click.HelpOption), type(option); assert option.is_flag is True; assert option.is_eager is True; assert option.expose_value is False'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python -c 'import click; from click.testing import CliRunner; cmd = click.Command(\"demo\", params=[click.Option([\"--name\"], help=\"Name\")], callback=lambda name: click.echo(\"hello \" + name if name else \"hello\")); result = CliRunner().invoke(cmd, [\"--help\"]); assert result.exit_code == 0, result.output; assert \"Usage: demo [OPTIONS]\" in result.output, result.output; assert \"--help\" in result.output, result.output; assert \"Show this message and exit.\" in result.output, result.output; disabled = click.Command(\"plain\", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, [\"--help\"]); assert disabled_result.exit_code == 2, disabled_result.output; assert \"No such option: --help\" in disabled_result.output, disabled_result.output'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python -c 'import click; from click.testing import CliRunner; cli = click.help_option(\"-h\", \"--halp\")(click.command()(lambda: click.echo(\"ran\"))); runner = CliRunner(); short_help = runner.invoke(cli, [\"-h\"]); long_help = runner.invoke(cli, [\"--halp\"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert \"Usage:\" in short_help.output, short_help.output; assert \"Usage:\" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == \"ran\", normal_run.output'", - "passed": true, - "returncode": 0, - "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ python -c 'import click; from click.testing import CliRunner; cmd = click.Command(\"demo\", params=[click.Option([\"--name\"], help=\"Name\")], callback=lambda name: click.echo(\"hello \" + name if name else \"hello\")); result = CliRunner().invoke(cmd, [\"--help\"]); assert result.exit_code == 0, result.output; assert \"Usage: demo [OPTIONS]\" in result.output, result.output; assert \"--help\" in result.output, result.output; assert \"Show this message and exit.\" in result.output, result.output; disabled = click.Command(\"plain\", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, [\"--help\"]); assert disabled_result.exit_code == 2, disabled_result.output; assert \"No such option: --help\" in disabled_result.output, disabled_result.output'\n$ python -c 'import click; from click.testing import CliRunner; cli = click.help_option(\"-h\", \"--halp\")(click.command()(lambda: click.echo(\"ran\"))); runner = CliRunner(); short_help = runner.invoke(cli, [\"-h\"]); long_help = runner.invoke(cli, [\"--halp\"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert \"Usage:\" in short_help.output, short_help.output; assert \"Usage:\" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == \"ran\", normal_run.output'\n" + }, + { + "command": "python .agentlab_hidden/check_structure.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 39, "lines_deleted": 35, diff --git a/tasks/starter/click-help-option-refactor-001/task-card.md b/tasks/starter/click-help-option-refactor-001/task-card.md index 708ef64..3d00fc0 100644 --- a/tasks/starter/click-help-option-refactor-001/task-card.md +++ b/tasks/starter/click-help-option-refactor-001/task-card.md @@ -30,6 +30,11 @@ Extract a HelpOption subclass in click.decorators, make help_option use it as th - PYTHONPATH={workspace}/src - VIRTUAL_ENV={workspace}/.agentlab/venv +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `2 commands configured` + ## Graders ### Setup @@ -45,9 +50,7 @@ Extract a HelpOption subclass in click.decorators, make help_option use it as th ### Target -- `python -c 'from pathlib import Path; core = Path("src/click/core.py").read_text(); decorators = Path("src/click/decorators.py").read_text(); assert "def show_help(ctx: Context" not in core, "automatic help should no longer define its own nested callback"; assert "def callback(ctx: Context" not in decorators.split("def help_option", 1)[1], "help_option should reuse shared help implementation instead of defining a duplicate callback"; assert ("HelpOption" in core and "HelpOption" in decorators) or ("_make_help_option" in core and "_make_help_option" in decorators), "automatic help and help_option should share a reusable implementation"'` -- `python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output'` -- `python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output'` +None configured. ## Success Criteria diff --git a/tasks/starter/click-help-option-refactor-001/task.yaml b/tasks/starter/click-help-option-refactor-001/task.yaml index 0556fa6..f4455c0 100644 --- a/tasks/starter/click-help-option-refactor-001/task.yaml +++ b/tasks/starter/click-help-option-refactor-001/task.yaml @@ -19,6 +19,11 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python .agentlab_hidden/check_behavior.py + - python .agentlab_hidden/check_structure.py setup: - PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv - python -m pip install -e . "pytest<9" @@ -35,13 +40,6 @@ baseline: python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output' - >- python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output' -test: - - >- - python -c 'from pathlib import Path; core = Path("src/click/core.py").read_text(); decorators = Path("src/click/decorators.py").read_text(); assert "def show_help(ctx: Context" not in core, "automatic help should no longer define its own nested callback"; assert "def callback(ctx: Context" not in decorators.split("def help_option", 1)[1], "help_option should reuse shared help implementation instead of defining a duplicate callback"; assert ("HelpOption" in core and "HelpOption" in decorators) or ("_make_help_option" in core and "_make_help_option" in decorators), "automatic help and help_option should share a reusable implementation"' - - >- - python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output' - - >- - python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output' success: tests_must_pass: true max_files_changed: 3 diff --git a/tasks/starter/click-help-option-refactor-001/verifier.patch b/tasks/starter/click-help-option-refactor-001/verifier.patch new file mode 100644 index 0000000..75550f3 --- /dev/null +++ b/tasks/starter/click-help-option-refactor-001/verifier.patch @@ -0,0 +1,124 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,31 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = ['python -c \'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output\'', 'python -c \'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output\''] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print(f"$ {command}") ++ completed = subprocess.run( ++ command, ++ shell=True, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) +diff --git a/.agentlab_hidden/check_structure.py b/.agentlab_hidden/check_structure.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_structure.py +@@ -0,0 +1,81 @@ ++from __future__ import annotations ++ ++import ast ++from pathlib import Path ++ ++CORE = Path("src/click/core.py") ++DECORATORS = Path("src/click/decorators.py") ++ ++ ++def parse(path: Path) -> ast.Module: ++ return ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) ++ ++ ++def find_class(module: ast.Module, name: str) -> ast.ClassDef: ++ for node in module.body: ++ if isinstance(node, ast.ClassDef) and node.name == name: ++ return node ++ raise AssertionError(f"missing class {name}") ++ ++ ++def find_function(nodes: list[ast.stmt], name: str) -> ast.FunctionDef | ast.AsyncFunctionDef: ++ for node in nodes: ++ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == name: ++ return node ++ raise AssertionError(f"missing function {name}") ++ ++ ++def nested_functions(node: ast.AST) -> list[str]: ++ names: list[str] = [] ++ for child in ast.walk(node): ++ if child is node: ++ continue ++ if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): ++ names.append(child.name) ++ return names ++ ++ ++def referenced_names(node: ast.AST) -> set[str]: ++ names: set[str] = set() ++ for child in ast.walk(node): ++ if isinstance(child, ast.Name): ++ names.add(child.id) ++ elif isinstance(child, ast.Attribute): ++ names.add(child.attr) ++ return names ++ ++ ++def main() -> int: ++ core = parse(CORE) ++ decorators = parse(DECORATORS) ++ command = find_class(core, "Command") ++ get_help_option = find_function(command.body, "get_help_option") ++ help_option = find_function(decorators.body, "help_option") ++ ++ core_nested = nested_functions(get_help_option) ++ decorator_nested = nested_functions(help_option) ++ assert not core_nested, ( ++ "Command.get_help_option should not keep a private nested help callback: " ++ f"{core_nested}" ++ ) ++ assert not decorator_nested, ( ++ "click.help_option should not keep a duplicate nested help callback: " ++ f"{decorator_nested}" ++ ) ++ ++ shared_refs = referenced_names(get_help_option) & referenced_names(help_option) ++ reusable = { ++ name ++ for name in shared_refs ++ if ("help" in name.lower() and "option" in name.lower()) ++ or name in {"HelpOption", "_make_help_option"} ++ } ++ assert reusable, ( ++ "automatic help and click.help_option should reference the same reusable " ++ "help-option implementation" ++ ) ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/click-help-shadowed-option-001/reference-report.md b/tasks/starter/click-help-shadowed-option-001/reference-report.md index 4b9ba9f..7628be0 100644 --- a/tasks/starter/click-help-shadowed-option-001/reference-report.md +++ b/tasks/starter/click-help-shadowed-option-001/reference-report.md @@ -32,7 +32,7 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis - Workspace history policy: `base_only` - Workspace base ref: `7c4571df4ebadf09af6bd6e5d0b579273625c283` -## Code-Based Graders +## Public Graders 1. Assertion `PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv`: passed (0) 2. Assertion `python -m pip install -e . "pytest<9"`: passed (0) @@ -44,8 +44,19 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis 3. Assertion `python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output'`: passed (0) 4. Assertion `git apply reference.patch`: passed (0) -5. Assertion `python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo --help" + chr(39) + " for help."; bad = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, ["foo", "--help"]); assert help_result.exit_code == 0, help_result.output; assert "--help" in help_result.output, help_result.output'`: passed (0) -6. Assertion `python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.option("--help-file","--help")(click.argument("required_arg")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); assert result.exit_code == 2, result.output; assert "Try " not in result.output, result.output'`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo --help" + chr(39) + " for help."; bad = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, ["foo", "--help"]); assert help_result.exit_code == 0, help_result.output; assert "--help" in help_result.output, help_result.output' +$ python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.option("--help-file","--help")(click.argument("required_arg")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); assert result.exit_code == 2, result.output; assert "Try " not in result.output, result.output' +``` + ## Changed Files diff --git a/tasks/starter/click-help-shadowed-option-001/reference-result.json b/tasks/starter/click-help-shadowed-option-001/reference-result.json index 0d5bf76..ce3f2ed 100644 --- a/tasks/starter/click-help-shadowed-option-001/reference-result.json +++ b/tasks/starter/click-help-shadowed-option-001/reference-result.json @@ -23,7 +23,7 @@ "passed": true, "returncode": 0, "stderr": "\n[notice] A new release of pip is available: 25.1.1 -> 26.1.2\n[notice] To update, run: pip install --upgrade pip\n", - "stdout": "Obtaining file:///private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/agentlab-reference-4ic7u1_p/click-help-shadowed-option-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nBuilding wheels for collected packages: click\n Building editable for click (pyproject.toml): started\n Building editable for click (pyproject.toml): finished with status 'done'\n Created wheel for click: filename=click-8.3.2-py3-none-any.whl size=3186 sha256=772562614b318eb7c39c797c59f5c66fd962bdf5cc220f356cb10eba50d0424a\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-i9rwibbj/wheels/fd/c8/ac/7e84f649080d6c72bd7fd3ab580fe1aa2aaa00ca2900b237dd\nSuccessfully built click\nInstalling collected packages: pygments, pluggy, packaging, iniconfig, click, pytest\n\nSuccessfully installed click-8.3.2 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2\n" + "stdout": "Obtaining file:///private/tmp/ael-ref-starter-123/click-help-shadowed-option-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nBuilding wheels for collected packages: click\n Building editable for click (pyproject.toml): started\n Building editable for click (pyproject.toml): finished with status 'done'\n Created wheel for click: filename=click-8.3.2-py3-none-any.whl size=3148 sha256=32e41dc984bfba3b4fc9048adc613c90e2beddb1702a057f7dc89ee87cd6aede\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-wcms6817/wheels/1c/59/60/ad3a269806eeac86612ce5fc68d1b3e8a49705298e0835dfe8\nSuccessfully built click\nInstalling collected packages: pygments, pluggy, packaging, iniconfig, click, pytest\n\nSuccessfully installed click-8.3.2 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2\n" }, { "command": "python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.argument(\"required_arg\")(lambda required_arg, host: None))); cli.add_command(foo); result = CliRunner().invoke(cli, [\"foo\"]); expected = \"Try \" + chr(39) + \"cli foo -h\" + chr(39) + \" for help.\"; assert result.exit_code == 2, result.output; assert expected in result.output, result.output'", @@ -38,29 +38,13 @@ "returncode": 0, "stderr": "", "stdout": "" - }, - { - "command": "python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.argument(\"required_arg\")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, [\"foo\"]); expected = \"Try \" + chr(39) + \"cli foo --help\" + chr(39) + \" for help.\"; bad = \"Try \" + chr(39) + \"cli foo -h\" + chr(39) + \" for help.\"; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, [\"foo\", \"--help\"]); assert help_result.exit_code == 0, help_result.output; assert \"--help\" in help_result.output, help_result.output'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.option(\"--help-file\",\"--help\")(click.argument(\"required_arg\")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, [\"foo\"]); assert result.exit_code == 2, result.output; assert \"Try \" not in result.output, result.output'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" } ], "commands_run": [ "PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv", "python -m pip install -e . \"pytest<9\"", "python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.argument(\"required_arg\")(lambda required_arg, host: None))); cli.add_command(foo); result = CliRunner().invoke(cli, [\"foo\"]); expected = \"Try \" + chr(39) + \"cli foo -h\" + chr(39) + \" for help.\"; assert result.exit_code == 2, result.output; assert expected in result.output, result.output'", - "git apply reference.patch", - "python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.argument(\"required_arg\")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, [\"foo\"]); expected = \"Try \" + chr(39) + \"cli foo --help\" + chr(39) + \" for help.\"; bad = \"Try \" + chr(39) + \"cli foo -h\" + chr(39) + \" for help.\"; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, [\"foo\", \"--help\"]); assert help_result.exit_code == 0, help_result.output; assert \"--help\" in help_result.output, help_result.output'", - "python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.option(\"--help-file\",\"--help\")(click.argument(\"required_arg\")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, [\"foo\"]); assert result.exit_code == 2, result.output; assert \"Try \" not in result.output, result.output'" + "git apply reference.patch" ], "cost_usd": null, "diff_path": "reference.diff", @@ -95,20 +79,29 @@ "passed": true, "returncode": 0, "type": "code" - }, - { - "assertion": "python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.argument(\"required_arg\")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, [\"foo\"]); expected = \"Try \" + chr(39) + \"cli foo --help\" + chr(39) + \" for help.\"; bad = \"Try \" + chr(39) + \"cli foo -h\" + chr(39) + \" for help.\"; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, [\"foo\", \"--help\"]); assert help_result.exit_code == 0, help_result.output; assert \"--help\" in help_result.output, help_result.output'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.option(\"--help-file\",\"--help\")(click.argument(\"required_arg\")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, [\"foo\"]); assert result.exit_code == 2, result.output; assert \"Try \" not in result.output, result.output'", - "passed": true, - "returncode": 0, - "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.argument(\"required_arg\")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, [\"foo\"]); expected = \"Try \" + chr(39) + \"cli foo --help\" + chr(39) + \" for help.\"; bad = \"Try \" + chr(39) + \"cli foo -h\" + chr(39) + \" for help.\"; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, [\"foo\", \"--help\"]); assert help_result.exit_code == 0, help_result.output; assert \"--help\" in help_result.output, help_result.output'\n$ python -c 'import click; from click.testing import CliRunner; cli = click.group(\"cli\", context_settings={\"help_option_names\":[\"-h\",\"--help\"]})(lambda: None); foo = click.command(\"foo\")(click.option(\"--host\",\"-h\")(click.option(\"--help-file\",\"--help\")(click.argument(\"required_arg\")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, [\"foo\"]); assert result.exit_code == 2, result.output; assert \"Try \" not in result.output, result.output'\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 5, "lines_deleted": 1, diff --git a/tasks/starter/click-help-shadowed-option-001/task-card.md b/tasks/starter/click-help-shadowed-option-001/task-card.md index cf7f177..c29dbec 100644 --- a/tasks/starter/click-help-shadowed-option-001/task-card.md +++ b/tasks/starter/click-help-shadowed-option-001/task-card.md @@ -30,6 +30,11 @@ In UsageError.show, derive the available help option names from the current comm - PYTHONPATH={workspace}/src - VIRTUAL_ENV={workspace}/.agentlab/venv +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -43,8 +48,7 @@ In UsageError.show, derive the available help option names from the current comm ### Target -- `python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo --help" + chr(39) + " for help."; bad = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, ["foo", "--help"]); assert help_result.exit_code == 0, help_result.output; assert "--help" in help_result.output, help_result.output'` -- `python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.option("--help-file","--help")(click.argument("required_arg")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); assert result.exit_code == 2, result.output; assert "Try " not in result.output, result.output'` +None configured. ## Success Criteria diff --git a/tasks/starter/click-help-shadowed-option-001/task.yaml b/tasks/starter/click-help-shadowed-option-001/task.yaml index 4b61b4b..57ddd41 100644 --- a/tasks/starter/click-help-shadowed-option-001/task.yaml +++ b/tasks/starter/click-help-shadowed-option-001/task.yaml @@ -20,6 +20,10 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python .agentlab_hidden/check_behavior.py setup: - PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv - python -m pip install -e . "pytest<9" @@ -33,11 +37,6 @@ environment: baseline: - >- python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output' -test: - - >- - python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo --help" + chr(39) + " for help."; bad = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, ["foo", "--help"]); assert help_result.exit_code == 0, help_result.output; assert "--help" in help_result.output, help_result.output' - - >- - python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.option("--help-file","--help")(click.argument("required_arg")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); assert result.exit_code == 2, result.output; assert "Try " not in result.output, result.output' success: tests_must_pass: true max_files_changed: 3 diff --git a/tasks/starter/click-help-shadowed-option-001/verifier.patch b/tasks/starter/click-help-shadowed-option-001/verifier.patch new file mode 100644 index 0000000..53d2f89 --- /dev/null +++ b/tasks/starter/click-help-shadowed-option-001/verifier.patch @@ -0,0 +1,37 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,31 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = ['python -c \'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo --help" + chr(39) + " for help."; bad = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, ["foo", "--help"]); assert help_result.exit_code == 0, help_result.output; assert "--help" in help_result.output, help_result.output\'', 'python -c \'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.option("--help-file","--help")(click.argument("required_arg")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); assert result.exit_code == 2, result.output; assert "Try " not in result.output, result.output\''] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print(f"$ {command}") ++ completed = subprocess.run( ++ command, ++ shell=True, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/click-should-strip-ansi-tests-001/reference-report.md b/tasks/starter/click-should-strip-ansi-tests-001/reference-report.md index 48d93ce..2ca6b0c 100644 --- a/tasks/starter/click-should-strip-ansi-tests-001/reference-report.md +++ b/tasks/starter/click-should-strip-ansi-tests-001/reference-report.md @@ -32,7 +32,7 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis - Workspace history policy: `base_only` - Workspace base ref: `9de2b9c9cba0c82374206783bb4b3853373e575c` -## Code-Based Graders +## Public Graders 1. Assertion `PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv`: passed (0) 2. Assertion `python -m pip install -e . "pytest<9"`: passed (0) @@ -49,10 +49,64 @@ Setup-created untracked coverage caveat: 1802 setup-created untracked paths exis ```text ..................................... [100%] -37 passed in 0.01s +37 passed in 0.02s +``` + + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python .agentlab_hidden/check_test_mutations.py`: passed (0) + +```text +"color", "expected_override"), + [ + (True, False), + (False, True), + (None, None), + ], + ) + @pytest.mark.parametrize( + ("isatty", "is_jupyter", "expected"), + [ + (True, False, False), + (False, True, False), + (False, False, True), + ], + ) + def test_should_strip_ansi( + monkeypatch, + stream, + color: bool | None, + expected_override: bool | None, + isatty: bool, + is_jupyter: bool, + expected: bool, + ) -> None: + monkeypatch.setattr(click._compat, "isatty", lambda x: isatty) + monkeypatch.setattr( + click._compat, "_is_jupyter_kernel_output", lambda x: is_jupyter + ) + + if expected_override is not None: + expected = expected_override +> assert click._compat.should_strip_ansi(stream=stream, color=color) == expected +E assert True == False +E + where True = (stream=, color=None) +E + where = .should_strip_ansi +E + where = click._compat + +tests/test_compat.py:55: AssertionError +=========================== short test summary info ============================ +FAILED tests/test_compat.py::test_should_strip_ansi[True-False-False-None-None-None] +FAILED tests/test_compat.py::test_should_strip_ansi[True-False-False-None-None-stream1] +FAILED tests/test_compat.py::test_should_strip_ansi[True-False-False-None-None-stream2] +FAILED tests/test_compat.py::test_should_strip_ansi[True-False-False-None-None-stream3] +4 failed, 33 passed in 0.02s ``` -7. Assertion `python -c 'import ast, pathlib, subprocess; status = subprocess.check_output(["git", "status", "--short"], text=True).splitlines(); paths = [line[3:] for line in status]; assert paths == ["tests/test_compat.py"], status; source = pathlib.Path("tests/test_compat.py").read_text(); tree = ast.parse(source); funcs = {node.name: node for node in tree.body if isinstance(node, ast.FunctionDef)}; fn = funcs.get("test_should_strip_ansi"); assert fn is not None; parametrize = [dec for dec in fn.decorator_list if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute) and dec.func.attr == "parametrize"]; assert len(parametrize) >= 3, len(parametrize); args = {arg.arg for arg in fn.args.args}; assert {"monkeypatch", "stream", "color", "expected_override", "isatty", "is_jupyter", "expected"} <= args, args; calls = [node for node in ast.walk(fn) if isinstance(node, ast.Call)]; assert any(isinstance(call.func, ast.Attribute) and call.func.attr == "should_strip_ansi" and {"stream", "color"} <= {kw.arg for kw in call.keywords} for call in calls); assert any(isinstance(call.func, ast.Attribute) and call.func.attr == "setattr" and any(isinstance(arg, ast.Constant) and arg.value == "isatty" for arg in call.args) for call in calls); assert any(isinstance(call.func, ast.Attribute) and call.func.attr == "setattr" and any(isinstance(arg, ast.Constant) and arg.value == "_is_jupyter_kernel_output" for arg in call.args) for call in calls); module = ast.unparse(tree); required = ["sys.stdin", "sys.stdout", "sys.stderr", "_is_jupyter_kernel_output"]; missing = [item for item in required if item not in module]; assert not missing, missing'`: passed (0) ## Changed Files diff --git a/tasks/starter/click-should-strip-ansi-tests-001/reference-result.json b/tasks/starter/click-should-strip-ansi-tests-001/reference-result.json index 44896e8..1ff1ad6 100644 --- a/tasks/starter/click-should-strip-ansi-tests-001/reference-result.json +++ b/tasks/starter/click-should-strip-ansi-tests-001/reference-result.json @@ -23,7 +23,7 @@ "passed": true, "returncode": 0, "stderr": "\n[notice] A new release of pip is available: 25.1.1 -> 26.1.2\n[notice] To update, run: pip install --upgrade pip\n", - "stdout": "Obtaining file:///private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/agentlab-reference-4ic7u1_p/click-should-strip-ansi-tests-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nBuilding wheels for collected packages: click\n Building editable for click (pyproject.toml): started\n Building editable for click (pyproject.toml): finished with status 'done'\n Created wheel for click: filename=click-8.3.3-py3-none-any.whl size=3191 sha256=21ab09387619455a88db638bd10db98bb36be44bdc067b1ecc4456f204be73ba\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-4d_407jw/wheels/c8/0f/af/b112ebd25a5021d6d2432305d966a9b4f05756246fd6f9f3ca\nSuccessfully built click\nInstalling collected packages: pygments, pluggy, packaging, iniconfig, click, pytest\n\nSuccessfully installed click-8.3.3 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2\n" + "stdout": "Obtaining file:///private/tmp/ael-ref-starter-123/click-should-strip-ansi-tests-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nBuilding wheels for collected packages: click\n Building editable for click (pyproject.toml): started\n Building editable for click (pyproject.toml): finished with status 'done'\n Created wheel for click: filename=click-8.3.3-py3-none-any.whl size=3148 sha256=5932774231b4dad1ed5ed2d8d3b56bfdccb5913b0b3ed317b1b1da2654b6870f\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-87xit_5z/wheels/65/b7/1b/862575d55ef938fe4cc8dd8229678b5ec87f9bbf9a9c649588\nSuccessfully built click\nInstalling collected packages: pygments, pluggy, packaging, iniconfig, click, pytest\n\nSuccessfully installed click-8.3.3 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2\n" }, { "command": "python -c 'import click, sys; compat = click._compat; compat.isatty = lambda stream: True; compat._is_jupyter_kernel_output = lambda stream: False; assert compat.should_strip_ansi(stream=sys.stdout, color=None) is False; assert compat.should_strip_ansi(stream=sys.stdout, color=True) is False; assert compat.should_strip_ansi(stream=sys.stdout, color=False) is True; compat.isatty = lambda stream: False; compat._is_jupyter_kernel_output = lambda stream: True; assert compat.should_strip_ansi(stream=None, color=None) is False; compat._is_jupyter_kernel_output = lambda stream: False; assert compat.should_strip_ansi(stream=None, color=None) is True'", @@ -51,14 +51,7 @@ "passed": true, "returncode": 0, "stderr": "", - "stdout": "..................................... [100%]\n37 passed in 0.01s\n" - }, - { - "command": "python -c 'import ast, pathlib, subprocess; status = subprocess.check_output([\"git\", \"status\", \"--short\"], text=True).splitlines(); paths = [line[3:] for line in status]; assert paths == [\"tests/test_compat.py\"], status; source = pathlib.Path(\"tests/test_compat.py\").read_text(); tree = ast.parse(source); funcs = {node.name: node for node in tree.body if isinstance(node, ast.FunctionDef)}; fn = funcs.get(\"test_should_strip_ansi\"); assert fn is not None; parametrize = [dec for dec in fn.decorator_list if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute) and dec.func.attr == \"parametrize\"]; assert len(parametrize) >= 3, len(parametrize); args = {arg.arg for arg in fn.args.args}; assert {\"monkeypatch\", \"stream\", \"color\", \"expected_override\", \"isatty\", \"is_jupyter\", \"expected\"} <= args, args; calls = [node for node in ast.walk(fn) if isinstance(node, ast.Call)]; assert any(isinstance(call.func, ast.Attribute) and call.func.attr == \"should_strip_ansi\" and {\"stream\", \"color\"} <= {kw.arg for kw in call.keywords} for call in calls); assert any(isinstance(call.func, ast.Attribute) and call.func.attr == \"setattr\" and any(isinstance(arg, ast.Constant) and arg.value == \"isatty\" for arg in call.args) for call in calls); assert any(isinstance(call.func, ast.Attribute) and call.func.attr == \"setattr\" and any(isinstance(arg, ast.Constant) and arg.value == \"_is_jupyter_kernel_output\" for arg in call.args) for call in calls); module = ast.unparse(tree); required = [\"sys.stdin\", \"sys.stdout\", \"sys.stderr\", \"_is_jupyter_kernel_output\"]; missing = [item for item in required if item not in module]; assert not missing, missing'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" + "stdout": "..................................... [100%]\n37 passed in 0.02s\n" } ], "commands_run": [ @@ -67,8 +60,7 @@ "python -c 'import click, sys; compat = click._compat; compat.isatty = lambda stream: True; compat._is_jupyter_kernel_output = lambda stream: False; assert compat.should_strip_ansi(stream=sys.stdout, color=None) is False; assert compat.should_strip_ansi(stream=sys.stdout, color=True) is False; assert compat.should_strip_ansi(stream=sys.stdout, color=False) is True; compat.isatty = lambda stream: False; compat._is_jupyter_kernel_output = lambda stream: True; assert compat.should_strip_ansi(stream=None, color=None) is False; compat._is_jupyter_kernel_output = lambda stream: False; assert compat.should_strip_ansi(stream=None, color=None) is True'", "python -c 'from pathlib import Path; text = Path(\"tests/test_compat.py\").read_text(); assert \"def test_should_strip_ansi\" not in text'", "git apply reference.patch", - "python -m pytest tests/test_compat.py -q", - "python -c 'import ast, pathlib, subprocess; status = subprocess.check_output([\"git\", \"status\", \"--short\"], text=True).splitlines(); paths = [line[3:] for line in status]; assert paths == [\"tests/test_compat.py\"], status; source = pathlib.Path(\"tests/test_compat.py\").read_text(); tree = ast.parse(source); funcs = {node.name: node for node in tree.body if isinstance(node, ast.FunctionDef)}; fn = funcs.get(\"test_should_strip_ansi\"); assert fn is not None; parametrize = [dec for dec in fn.decorator_list if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute) and dec.func.attr == \"parametrize\"]; assert len(parametrize) >= 3, len(parametrize); args = {arg.arg for arg in fn.args.args}; assert {\"monkeypatch\", \"stream\", \"color\", \"expected_override\", \"isatty\", \"is_jupyter\", \"expected\"} <= args, args; calls = [node for node in ast.walk(fn) if isinstance(node, ast.Call)]; assert any(isinstance(call.func, ast.Attribute) and call.func.attr == \"should_strip_ansi\" and {\"stream\", \"color\"} <= {kw.arg for kw in call.keywords} for call in calls); assert any(isinstance(call.func, ast.Attribute) and call.func.attr == \"setattr\" and any(isinstance(arg, ast.Constant) and arg.value == \"isatty\" for arg in call.args) for call in calls); assert any(isinstance(call.func, ast.Attribute) and call.func.attr == \"setattr\" and any(isinstance(arg, ast.Constant) and arg.value == \"_is_jupyter_kernel_output\" for arg in call.args) for call in calls); module = ast.unparse(tree); required = [\"sys.stdin\", \"sys.stdout\", \"sys.stderr\", \"_is_jupyter_kernel_output\"]; missing = [item for item in required if item not in module]; assert not missing, missing'" + "python -m pytest tests/test_compat.py -q" ], "cost_usd": null, "diff_path": "reference.diff", @@ -115,14 +107,29 @@ "passed": true, "returncode": 0, "type": "code" - }, - { - "assertion": "python -c 'import ast, pathlib, subprocess; status = subprocess.check_output([\"git\", \"status\", \"--short\"], text=True).splitlines(); paths = [line[3:] for line in status]; assert paths == [\"tests/test_compat.py\"], status; source = pathlib.Path(\"tests/test_compat.py\").read_text(); tree = ast.parse(source); funcs = {node.name: node for node in tree.body if isinstance(node, ast.FunctionDef)}; fn = funcs.get(\"test_should_strip_ansi\"); assert fn is not None; parametrize = [dec for dec in fn.decorator_list if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute) and dec.func.attr == \"parametrize\"]; assert len(parametrize) >= 3, len(parametrize); args = {arg.arg for arg in fn.args.args}; assert {\"monkeypatch\", \"stream\", \"color\", \"expected_override\", \"isatty\", \"is_jupyter\", \"expected\"} <= args, args; calls = [node for node in ast.walk(fn) if isinstance(node, ast.Call)]; assert any(isinstance(call.func, ast.Attribute) and call.func.attr == \"should_strip_ansi\" and {\"stream\", \"color\"} <= {kw.arg for kw in call.keywords} for call in calls); assert any(isinstance(call.func, ast.Attribute) and call.func.attr == \"setattr\" and any(isinstance(arg, ast.Constant) and arg.value == \"isatty\" for arg in call.args) for call in calls); assert any(isinstance(call.func, ast.Attribute) and call.func.attr == \"setattr\" and any(isinstance(arg, ast.Constant) and arg.value == \"_is_jupyter_kernel_output\" for arg in call.args) for call in calls); module = ast.unparse(tree); required = [\"sys.stdin\", \"sys.stdout\", \"sys.stderr\", \"_is_jupyter_kernel_output\"]; missing = [item for item in required if item not in module]; assert not missing, missing'", - "passed": true, - "returncode": 0, - "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python .agentlab_hidden/check_test_mutations.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": " [\n (True, False),\n (False, True),\n (None, None),\n ],\n )\n @pytest.mark.parametrize(\n (\"isatty\", \"is_jupyter\", \"expected\"),\n [\n (True, False, False),\n (False, True, False),\n (False, False, True),\n ],\n )\n def test_should_strip_ansi(\n monkeypatch,\n stream,\n color: bool | None,\n expected_override: bool | None,\n isatty: bool,\n is_jupyter: bool,\n expected: bool,\n ) -> None:\n monkeypatch.setattr(click._compat, \"isatty\", lambda x: isatty)\n monkeypatch.setattr(\n click._compat, \"_is_jupyter_kernel_output\", lambda x: is_jupyter\n )\n \n if expected_override is not None:\n expected = expected_override\n> assert click._compat.should_strip_ansi(stream=stream, color=color) == expected\nE assert True == False\nE + where True = (stream=\" mode='r+' encoding='utf-8'>, color=None)\nE + where = .should_strip_ansi\nE + where = click._compat\n\ntests/test_compat.py:55: AssertionError\n__________ test_should_strip_ansi[True-False-False-None-None-stream3] __________\n\nmonkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x10abbfe10>\nstream = \" mode='r+' encoding='utf-8'>\ncolor = None, expected_override = None, isatty = True, is_jupyter = False\nexpected = False\n\n @pytest.mark.parametrize(\n \"stream\",\n [None, sys.stdin, sys.stderr, sys.stdout],\n )\n @pytest.mark.parametrize(\n (\"color\", \"expected_override\"),\n [\n (True, False),\n (False, True),\n (None, None),\n ],\n )\n @pytest.mark.parametrize(\n (\"isatty\", \"is_jupyter\", \"expected\"),\n [\n (True, False, False),\n (False, True, False),\n (False, False, True),\n ],\n )\n def test_should_strip_ansi(\n monkeypatch,\n stream,\n color: bool | None,\n expected_override: bool | None,\n isatty: bool,\n is_jupyter: bool,\n expected: bool,\n ) -> None:\n monkeypatch.setattr(click._compat, \"isatty\", lambda x: isatty)\n monkeypatch.setattr(\n click._compat, \"_is_jupyter_kernel_output\", lambda x: is_jupyter\n )\n \n if expected_override is not None:\n expected = expected_override\n> assert click._compat.should_strip_ansi(stream=stream, color=color) == expected\nE assert True == False\nE + where True = (stream=\" mode='r+' encoding='utf-8'>, color=None)\nE + where = .should_strip_ansi\nE + where = click._compat\n\ntests/test_compat.py:55: AssertionError\n=========================== short test summary info ============================\nFAILED tests/test_compat.py::test_should_strip_ansi[True-False-False-None-None-None]\nFAILED tests/test_compat.py::test_should_strip_ansi[True-False-False-None-None-stream1]\nFAILED tests/test_compat.py::test_should_strip_ansi[True-False-False-None-None-stream2]\nFAILED tests/test_compat.py::test_should_strip_ansi[True-False-False-None-None-stream3]\n4 failed, 33 passed in 0.02s\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 47, "lines_deleted": 2, diff --git a/tasks/starter/click-should-strip-ansi-tests-001/task-card.md b/tasks/starter/click-should-strip-ansi-tests-001/task-card.md index d052732..e65f0e2 100644 --- a/tasks/starter/click-should-strip-ansi-tests-001/task-card.md +++ b/tasks/starter/click-should-strip-ansi-tests-001/task-card.md @@ -30,6 +30,11 @@ Update `tests/test_compat.py` to import Click, pytest, and sys; test `_is_jupyte - PYTHONPATH={workspace}/src - VIRTUAL_ENV={workspace}/.agentlab/venv +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -45,7 +50,6 @@ Update `tests/test_compat.py` to import Click, pytest, and sys; test `_is_jupyte ### Target - `python -m pytest tests/test_compat.py -q` -- `python -c 'import pathlib, subprocess; status = subprocess.check_output(["git", "status", "--short"], text=True).splitlines(); paths = [line[3:] for line in status]; assert paths == ["tests/test_compat.py"], status; source = pathlib.Path("tests/test_compat.py").read_text(); required = ["should_strip_ansi", "color=True", "color=False", "color=None", "sys.stdin", "sys.stdout", "sys.stderr"]; missing = [item for item in required if item not in source]; assert not missing, missing; assert "_is_jupyter_kernel_output" in source or "ipykernel" in source, "tests should cover Jupyter kernel output detection"; assert source.count("should_strip_ansi") >= 4, "tests should exercise should_strip_ansi across override and automatic cases"'` ## Success Criteria diff --git a/tasks/starter/click-should-strip-ansi-tests-001/task.yaml b/tasks/starter/click-should-strip-ansi-tests-001/task.yaml index db2f8fc..156e1b5 100644 --- a/tasks/starter/click-should-strip-ansi-tests-001/task.yaml +++ b/tasks/starter/click-should-strip-ansi-tests-001/task.yaml @@ -23,6 +23,10 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python .agentlab_hidden/check_test_mutations.py setup: - PYENV_VERSION=3.13.5 python3.13 -m venv .agentlab/venv - python -m pip install -e . "pytest<9" @@ -39,8 +43,6 @@ baseline: - python -c 'from pathlib import Path; text = Path("tests/test_compat.py").read_text(); assert "def test_should_strip_ansi" not in text' test: - python -m pytest tests/test_compat.py -q - - >- - python -c 'import pathlib, subprocess; status = subprocess.check_output(["git", "status", "--short"], text=True).splitlines(); paths = [line[3:] for line in status]; assert paths == ["tests/test_compat.py"], status; source = pathlib.Path("tests/test_compat.py").read_text(); required = ["should_strip_ansi", "color=True", "color=False", "color=None", "sys.stdin", "sys.stdout", "sys.stderr"]; missing = [item for item in required if item not in source]; assert not missing, missing; assert "_is_jupyter_kernel_output" in source or "ipykernel" in source, "tests should cover Jupyter kernel output detection"; assert source.count("should_strip_ansi") >= 4, "tests should exercise should_strip_ansi across override and automatic cases"' success: tests_must_pass: true max_files_changed: 1 diff --git a/tasks/starter/click-should-strip-ansi-tests-001/verifier.patch b/tasks/starter/click-should-strip-ansi-tests-001/verifier.patch new file mode 100644 index 0000000..f5d3685 --- /dev/null +++ b/tasks/starter/click-should-strip-ansi-tests-001/verifier.patch @@ -0,0 +1,85 @@ +diff --git a/.agentlab_hidden/check_test_mutations.py b/.agentlab_hidden/check_test_mutations.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_test_mutations.py +@@ -0,0 +1,79 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++from pathlib import Path ++ ++TARGET = Path("src/click/_compat.py") ++TEST_COMMAND = [sys.executable, "-m", "pytest", "tests/test_compat.py", "-q"] ++ ++MUTATIONS = [ ++ ( ++ "explicit color overrides", ++ """ if color is None: ++ if stream is None: ++ stream = sys.stdin ++ return not isatty(stream) and not _is_jupyter_kernel_output(stream) ++ return not color ++""", ++ """ if color is None: ++ if stream is None: ++ stream = sys.stdin ++ return not isatty(stream) and not _is_jupyter_kernel_output(stream) ++ return False ++""", ++ ), ++ ( ++ "Jupyter output detection", ++ """ return not isatty(stream) and not _is_jupyter_kernel_output(stream) ++""", ++ """ return not isatty(stream) ++""", ++ ), ++ ( ++ "TTY detection", ++ """ return not isatty(stream) and not _is_jupyter_kernel_output(stream) ++""", ++ """ return not _is_jupyter_kernel_output(stream) ++""", ++ ), ++] ++ ++ ++def run_pytest(description: str) -> bool: ++ completed = subprocess.run( ++ TEST_COMMAND, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ print(f"$ {' '.join(TEST_COMMAND)} # mutation: {description}") ++ if completed.stdout: ++ print(completed.stdout, end="") ++ return completed.returncode != 0 ++ ++ ++def main() -> int: ++ original = TARGET.read_text(encoding="utf-8") ++ failures: list[str] = [] ++ for description, needle, replacement in MUTATIONS: ++ if needle not in original: ++ failures.append(f"could not locate production behavior for {description}") ++ continue ++ TARGET.write_text(original.replace(needle, replacement, 1), encoding="utf-8") ++ try: ++ if not run_pytest(description): ++ failures.append( ++ f"submitted tests did not catch broken {description} behavior" ++ ) ++ finally: ++ TARGET.write_text(original, encoding="utf-8") ++ if failures: ++ for failure in failures: ++ print(failure, file=sys.stderr) ++ return 1 ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/datawrapper-mcp-docker-requirements-001/reference-report.md b/tasks/starter/datawrapper-mcp-docker-requirements-001/reference-report.md index ebd9867..b6a673a 100644 --- a/tasks/starter/datawrapper-mcp-docker-requirements-001/reference-report.md +++ b/tasks/starter/datawrapper-mcp-docker-requirements-001/reference-report.md @@ -30,13 +30,24 @@ - Workspace history policy: `base_only` - Workspace base ref: `c9a9f3db7f6e2cc943ba81de798bb604d4ca487f` -## Code-Based Graders +## Public Graders 1. Assertion `python3 -c "from pathlib import Path; assert Path('pyproject.toml').is_file(); assert Path('Dockerfile').is_file(); assert Path('deployment/requirements.txt').is_file()"`: passed (0) 2. Assertion `python3 -c "from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); pyproject = Path('pyproject.toml').read_text(); dockerfile = Path('Dockerfile').read_text(); assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile; assert 'prefab-ui==0.8.0' in pyproject and 'fastmcp[apps]==3.1.1' in pyproject and 'datawrapper>=2.0.14' in pyproject, pyproject; assert 'prefab-ui==0.8.0' not in req and 'fastmcp>=3.0.0' in req and 'datawrapper>=2.0.7' in req, req"`: passed (0) 3. Assertion `git apply reference.patch`: passed (0) -4. Assertion `python3 -c "from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); dockerfile = Path('Dockerfile').read_text(); expected = ['fastmcp[apps]==3.1.1', 'datawrapper>=2.0.14', 'prefab-ui==0.8.0', 'pandas>=2.0.0', 'starlette>=0.27.0', 'uvicorn>=0.23.0']; missing = [item for item in expected if item not in req]; assert not missing, missing; assert 'fastmcp>=3.0.0' not in req and 'datawrapper>=2.0.7' not in req, req; assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile"`: passed (0) -5. Assertion `python3 -c "from pathlib import Path; server = Path('datawrapper_mcp/server.py').read_text(); req = Path('deployment/requirements.txt').read_text(); assert 'from fastmcp import FastMCP' in server and 'fastmcp[apps]==3.1.1' in req, req; assert 'from prefab_ui.app import PrefabApp' in server and 'prefab-ui==0.8.0' in req, req"`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python3 .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ python3 -c "from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); dockerfile = Path('Dockerfile').read_text(); expected = ['fastmcp[apps]==3.1.1', 'datawrapper>=2.0.14', 'prefab-ui==0.8.0', 'pandas>=2.0.0', 'starlette>=0.27.0', 'uvicorn>=0.23.0']; missing = [item for item in expected if item not in req]; assert not missing, missing; assert 'fastmcp>=3.0.0' not in req and 'datawrapper>=2.0.7' not in req, req; assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile" +$ python3 -c "from pathlib import Path; server = Path('datawrapper_mcp/server.py').read_text(); req = Path('deployment/requirements.txt').read_text(); assert 'from fastmcp import FastMCP' in server and 'fastmcp[apps]==3.1.1' in req, req; assert 'from prefab_ui.app import PrefabApp' in server and 'prefab-ui==0.8.0' in req, req" +``` + ## Changed Files diff --git a/tasks/starter/datawrapper-mcp-docker-requirements-001/reference-result.json b/tasks/starter/datawrapper-mcp-docker-requirements-001/reference-result.json index c6b821f..11c4b46 100644 --- a/tasks/starter/datawrapper-mcp-docker-requirements-001/reference-result.json +++ b/tasks/starter/datawrapper-mcp-docker-requirements-001/reference-result.json @@ -31,28 +31,12 @@ "returncode": 0, "stderr": "", "stdout": "" - }, - { - "command": "python3 -c \"from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); dockerfile = Path('Dockerfile').read_text(); expected = ['fastmcp[apps]==3.1.1', 'datawrapper>=2.0.14', 'prefab-ui==0.8.0', 'pandas>=2.0.0', 'starlette>=0.27.0', 'uvicorn>=0.23.0']; missing = [item for item in expected if item not in req]; assert not missing, missing; assert 'fastmcp>=3.0.0' not in req and 'datawrapper>=2.0.7' not in req, req; assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile\"", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python3 -c \"from pathlib import Path; server = Path('datawrapper_mcp/server.py').read_text(); req = Path('deployment/requirements.txt').read_text(); assert 'from fastmcp import FastMCP' in server and 'fastmcp[apps]==3.1.1' in req, req; assert 'from prefab_ui.app import PrefabApp' in server and 'prefab-ui==0.8.0' in req, req\"", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" } ], "commands_run": [ "python3 -c \"from pathlib import Path; assert Path('pyproject.toml').is_file(); assert Path('Dockerfile').is_file(); assert Path('deployment/requirements.txt').is_file()\"", "python3 -c \"from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); pyproject = Path('pyproject.toml').read_text(); dockerfile = Path('Dockerfile').read_text(); assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile; assert 'prefab-ui==0.8.0' in pyproject and 'fastmcp[apps]==3.1.1' in pyproject and 'datawrapper>=2.0.14' in pyproject, pyproject; assert 'prefab-ui==0.8.0' not in req and 'fastmcp>=3.0.0' in req and 'datawrapper>=2.0.7' in req, req\"", - "git apply reference.patch", - "python3 -c \"from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); dockerfile = Path('Dockerfile').read_text(); expected = ['fastmcp[apps]==3.1.1', 'datawrapper>=2.0.14', 'prefab-ui==0.8.0', 'pandas>=2.0.0', 'starlette>=0.27.0', 'uvicorn>=0.23.0']; missing = [item for item in expected if item not in req]; assert not missing, missing; assert 'fastmcp>=3.0.0' not in req and 'datawrapper>=2.0.7' not in req, req; assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile\"", - "python3 -c \"from pathlib import Path; server = Path('datawrapper_mcp/server.py').read_text(); req = Path('deployment/requirements.txt').read_text(); assert 'from fastmcp import FastMCP' in server and 'fastmcp[apps]==3.1.1' in req, req; assert 'from prefab_ui.app import PrefabApp' in server and 'prefab-ui==0.8.0' in req, req\"" + "git apply reference.patch" ], "cost_usd": null, "diff_path": "reference.diff", @@ -81,20 +65,29 @@ "passed": true, "returncode": 0, "type": "code" - }, - { - "assertion": "python3 -c \"from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); dockerfile = Path('Dockerfile').read_text(); expected = ['fastmcp[apps]==3.1.1', 'datawrapper>=2.0.14', 'prefab-ui==0.8.0', 'pandas>=2.0.0', 'starlette>=0.27.0', 'uvicorn>=0.23.0']; missing = [item for item in expected if item not in req]; assert not missing, missing; assert 'fastmcp>=3.0.0' not in req and 'datawrapper>=2.0.7' not in req, req; assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile\"", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python3 -c \"from pathlib import Path; server = Path('datawrapper_mcp/server.py').read_text(); req = Path('deployment/requirements.txt').read_text(); assert 'from fastmcp import FastMCP' in server and 'fastmcp[apps]==3.1.1' in req, req; assert 'from prefab_ui.app import PrefabApp' in server and 'prefab-ui==0.8.0' in req, req\"", - "passed": true, - "returncode": 0, - "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python3 .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ python3 -c \"from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); dockerfile = Path('Dockerfile').read_text(); expected = ['fastmcp[apps]==3.1.1', 'datawrapper>=2.0.14', 'prefab-ui==0.8.0', 'pandas>=2.0.0', 'starlette>=0.27.0', 'uvicorn>=0.23.0']; missing = [item for item in expected if item not in req]; assert not missing, missing; assert 'fastmcp>=3.0.0' not in req and 'datawrapper>=2.0.7' not in req, req; assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile\"\n$ python3 -c \"from pathlib import Path; server = Path('datawrapper_mcp/server.py').read_text(); req = Path('deployment/requirements.txt').read_text(); assert 'from fastmcp import FastMCP' in server and 'fastmcp[apps]==3.1.1' in req, req; assert 'from prefab_ui.app import PrefabApp' in server and 'prefab-ui==0.8.0' in req, req\"\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 5, "lines_deleted": 2, diff --git a/tasks/starter/datawrapper-mcp-docker-requirements-001/task-card.md b/tasks/starter/datawrapper-mcp-docker-requirements-001/task-card.md index f981e52..03545f6 100644 --- a/tasks/starter/datawrapper-mcp-docker-requirements-001/task-card.md +++ b/tasks/starter/datawrapper-mcp-docker-requirements-001/task-card.md @@ -27,6 +27,11 @@ Sync deployment/requirements.txt with pyproject.toml by using fastmcp[apps]==3.1 - PYTEST_ADDOPTS=-p no:cacheprovider - PYTHONDONTWRITEBYTECODE=1 +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -39,8 +44,7 @@ Sync deployment/requirements.txt with pyproject.toml by using fastmcp[apps]==3.1 ### Target -- `python3 -c "from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); dockerfile = Path('Dockerfile').read_text(); expected = ['fastmcp[apps]==3.1.1', 'datawrapper>=2.0.14', 'prefab-ui==0.8.0', 'pandas>=2.0.0', 'starlette>=0.27.0', 'uvicorn>=0.23.0']; missing = [item for item in expected if item not in req]; assert not missing, missing; assert 'fastmcp>=3.0.0' not in req and 'datawrapper>=2.0.7' not in req, req; assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile"` -- `python3 -c "from pathlib import Path; server = Path('datawrapper_mcp/server.py').read_text(); req = Path('deployment/requirements.txt').read_text(); assert 'from fastmcp import FastMCP' in server and 'fastmcp[apps]==3.1.1' in req, req; assert 'from prefab_ui.app import PrefabApp' in server and 'prefab-ui==0.8.0' in req, req"` +None configured. ## Success Criteria diff --git a/tasks/starter/datawrapper-mcp-docker-requirements-001/task.yaml b/tasks/starter/datawrapper-mcp-docker-requirements-001/task.yaml index 88be1ef..4bcf4bd 100644 --- a/tasks/starter/datawrapper-mcp-docker-requirements-001/task.yaml +++ b/tasks/starter/datawrapper-mcp-docker-requirements-001/task.yaml @@ -20,6 +20,10 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python3 .agentlab_hidden/check_behavior.py setup: - python3 -c "from pathlib import Path; assert Path('pyproject.toml').is_file(); assert Path('Dockerfile').is_file(); assert Path('deployment/requirements.txt').is_file()" environment: @@ -27,9 +31,6 @@ environment: PYTEST_ADDOPTS: "-p no:cacheprovider" baseline: - python3 -c "from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); pyproject = Path('pyproject.toml').read_text(); dockerfile = Path('Dockerfile').read_text(); assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile; assert 'prefab-ui==0.8.0' in pyproject and 'fastmcp[apps]==3.1.1' in pyproject and 'datawrapper>=2.0.14' in pyproject, pyproject; assert 'prefab-ui==0.8.0' not in req and 'fastmcp>=3.0.0' in req and 'datawrapper>=2.0.7' in req, req" -test: - - python3 -c "from pathlib import Path; req = Path('deployment/requirements.txt').read_text(); dockerfile = Path('Dockerfile').read_text(); expected = ['fastmcp[apps]==3.1.1', 'datawrapper>=2.0.14', 'prefab-ui==0.8.0', 'pandas>=2.0.0', 'starlette>=0.27.0', 'uvicorn>=0.23.0']; missing = [item for item in expected if item not in req]; assert not missing, missing; assert 'fastmcp>=3.0.0' not in req and 'datawrapper>=2.0.7' not in req, req; assert 'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt' in dockerfile, dockerfile" - - python3 -c "from pathlib import Path; server = Path('datawrapper_mcp/server.py').read_text(); req = Path('deployment/requirements.txt').read_text(); assert 'from fastmcp import FastMCP' in server and 'fastmcp[apps]==3.1.1' in req, req; assert 'from prefab_ui.app import PrefabApp' in server and 'prefab-ui==0.8.0' in req, req" success: tests_must_pass: true max_files_changed: 2 diff --git a/tasks/starter/datawrapper-mcp-docker-requirements-001/verifier.patch b/tasks/starter/datawrapper-mcp-docker-requirements-001/verifier.patch new file mode 100644 index 0000000..fd2d645 --- /dev/null +++ b/tasks/starter/datawrapper-mcp-docker-requirements-001/verifier.patch @@ -0,0 +1,37 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,31 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = ['python3 -c "from pathlib import Path; req = Path(\'deployment/requirements.txt\').read_text(); dockerfile = Path(\'Dockerfile\').read_text(); expected = [\'fastmcp[apps]==3.1.1\', \'datawrapper>=2.0.14\', \'prefab-ui==0.8.0\', \'pandas>=2.0.0\', \'starlette>=0.27.0\', \'uvicorn>=0.23.0\']; missing = [item for item in expected if item not in req]; assert not missing, missing; assert \'fastmcp>=3.0.0\' not in req and \'datawrapper>=2.0.7\' not in req, req; assert \'RUN pip install --no-cache-dir -r /app/deployment/requirements.txt\' in dockerfile, dockerfile"', 'python3 -c "from pathlib import Path; server = Path(\'datawrapper_mcp/server.py\').read_text(); req = Path(\'deployment/requirements.txt\').read_text(); assert \'from fastmcp import FastMCP\' in server and \'fastmcp[apps]==3.1.1\' in req, req; assert \'from prefab_ui.app import PrefabApp\' in server and \'prefab-ui==0.8.0\' in req, req"'] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print(f"$ {command}") ++ completed = subprocess.run( ++ command, ++ shell=True, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/httpx-verify-false-client-cert-001/reference-report.md b/tasks/starter/httpx-verify-false-client-cert-001/reference-report.md index b10a3c9..f1a977e 100644 --- a/tasks/starter/httpx-verify-false-client-cert-001/reference-report.md +++ b/tasks/starter/httpx-verify-false-client-cert-001/reference-report.md @@ -13,7 +13,7 @@ - Lines added: `3` - Lines deleted: `4` -Setup-created untracked coverage caveat: 2044 setup-created untracked paths existed outside exact boundary-pattern matching. Changed-file counts/lists and boundary metrics include detected changes, but detection remains best-effort for worktree-only content-preserving edits to those paths. +Setup-created untracked coverage caveat: 1299 setup-created untracked paths existed outside exact boundary-pattern matching. Changed-file counts/lists and boundary metrics include detected changes, but detection remains best-effort for worktree-only content-preserving edits to those paths. ## Run Surface @@ -32,68 +32,74 @@ Setup-created untracked coverage caveat: 2044 setup-created untracked paths exis - Workspace history policy: `base_only` - Workspace base ref: `5043f1479886c4c5f6e3f524465bbe168c6bb37f` -## Code-Based Graders +## Public Graders 1. Assertion `python3 -m venv .agentlab/venv`: passed (0) 2. Assertion `python -m pip install --upgrade pip`: passed (0) ```text -Requirement already satisfied: pip in ./.agentlab/venv/lib/python3.13/site-packages (25.1.1) +Requirement already satisfied: pip in ./.agentlab/venv/lib/python3.9/site-packages (21.2.4) Collecting pip - Downloading pip-26.1.2-py3-none-any.whl.metadata (4.6 kB) -Downloading pip-26.1.2-py3-none-any.whl (1.8 MB) - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.8/1.8 MB 34.3 MB/s eta 0:00:00 + Using cached pip-26.0.1-py3-none-any.whl (1.8 MB) Installing collected packages: pip Attempting uninstall: pip - Found existing installation: pip 25.1.1 - Uninstalling pip-25.1.1: - Successfully uninstalled pip-25.1.1 -Successfully installed pip-26.1.2 + Found existing installation: pip 21.2.4 + Uninstalling pip-21.2.4: + Successfully uninstalled pip-21.2.4 +Successfully installed pip-26.0.1 ``` 3. Assertion `python -m pip install -e . "pytest<9"`: passed (0) ```text -21 kB) -Collecting idna (from httpx==0.28.0) - Using cached idna-3.18-py3-none-any.whl.metadata (6.1 kB) -Collecting iniconfig>=1 (from pytest<9) - Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB) -Collecting packaging>=20 (from pytest<9) - Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB) -Collecting pluggy<2,>=1.5 (from pytest<9) - Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB) -Collecting pygments>=2.7.2 (from pytest<9) - Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB) +data (2.5 kB) +Collecting tomli>=1 (from pytest<9) + Using cached tomli-2.4.1-py3-none-any.whl.metadata (10 kB) Collecting h11>=0.16 (from httpcore==1.*->httpx==0.28.0) Using cached h11-0.16.0-py3-none-any.whl.metadata (8.3 kB) +Collecting typing-extensions>=4.6.0 (from exceptiongroup>=1->pytest<9) + Using cached typing_extensions-4.15.0-py3-none-any.whl.metadata (3.3 kB) Using cached pytest-8.4.2-py3-none-any.whl (365 kB) Using cached httpcore-1.0.9-py3-none-any.whl (78 kB) Using cached pluggy-1.6.0-py3-none-any.whl (20 kB) +Using cached exceptiongroup-1.3.1-py3-none-any.whl (16 kB) Using cached h11-0.16.0-py3-none-any.whl (37 kB) -Using cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB) +Using cached iniconfig-2.1.0-py3-none-any.whl (6.0 kB) Using cached packaging-26.2-py3-none-any.whl (100 kB) Using cached pygments-2.20.0-py3-none-any.whl (1.2 MB) -Downloading anyio-4.13.0-py3-none-any.whl (114 kB) +Using cached tomli-2.4.1-py3-none-any.whl (14 kB) +Using cached typing_extensions-4.15.0-py3-none-any.whl (44 kB) +Using cached anyio-4.12.1-py3-none-any.whl (113 kB) Using cached idna-3.18-py3-none-any.whl (65 kB) -Using cached certifi-2026.5.20-py3-none-any.whl (134 kB) +Using cached certifi-2026.6.17-py3-none-any.whl (133 kB) Building wheels for collected packages: httpx Building editable for httpx (pyproject.toml): started Building editable for httpx (pyproject.toml): finished with status 'done' - Created wheel for httpx: filename=httpx-0.28.0-py3-none-any.whl size=4921 sha256=badff137afab7bef72bbf4e3e0af07e5e778c12aaefefee844082c50fde662e9 - Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-9ml56d2w/wheels/3f/e3/6a/162557ff5d76689efe1dc1cf98ab6eebb739b2cbcda4c53ee2 + Created wheel for httpx: filename=httpx-0.28.0-py3-none-any.whl size=4878 sha256=35161bedc4cde7225abf08a86ca87ad77e324aad97ffc9fc30d3f7b725474ae0 + Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-o1vu442_/wheels/b4/1f/31/93c94facf0f7e16355c86f731264009a167ff283a25bcb8898 Successfully built httpx -Installing collected packages: pygments, pluggy, packaging, iniconfig, idna, h11, certifi, pytest, httpcore, anyio, httpx +Installing collected packages: typing-extensions, tomli, pygments, pluggy, packaging, iniconfig, idna, h11, certifi, httpcore, exceptiongroup, pytest, anyio, httpx -Successfully installed anyio-4.13.0 certifi-2026.5.20 h11-0.16.0 httpcore-1.0.9 httpx-0.28.0 idna-3.18 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2 +Successfully installed anyio-4.12.1 certifi-2026.6.17 exceptiongroup-1.3.1 h11-0.16.0 httpcore-1.0.9 httpx-0.28.0 idna-3.18 iniconfig-2.1.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2 tomli-2.4.1 typing-extensions-4.15.0 ``` 4. Assertion `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'`: passed (0) 5. Assertion `git apply reference.patch`: passed (0) -6. Assertion `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls'`: passed (0) -7. Assertion `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert="client.pem"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem",)], calls'`: passed (0) -8. Assertion `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'`: passed (0) -9. Assertion `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls'`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls' +$ python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert="client.pem"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem",)], calls' +$ python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls' +$ python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls' +``` + ## Changed Files diff --git a/tasks/starter/httpx-verify-false-client-cert-001/reference-result.json b/tasks/starter/httpx-verify-false-client-cert-001/reference-result.json index fb8e443..a85bd65 100644 --- a/tasks/starter/httpx-verify-false-client-cert-001/reference-result.json +++ b/tasks/starter/httpx-verify-false-client-cert-001/reference-result.json @@ -23,14 +23,14 @@ "passed": true, "returncode": 0, "stderr": "", - "stdout": "Requirement already satisfied: pip in ./.agentlab/venv/lib/python3.13/site-packages (25.1.1)\nCollecting pip\n Downloading pip-26.1.2-py3-none-any.whl.metadata (4.6 kB)\nDownloading pip-26.1.2-py3-none-any.whl (1.8 MB)\n \u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501 1.8/1.8 MB 34.3 MB/s eta 0:00:00\nInstalling collected packages: pip\n Attempting uninstall: pip\n Found existing installation: pip 25.1.1\n Uninstalling pip-25.1.1:\n Successfully uninstalled pip-25.1.1\nSuccessfully installed pip-26.1.2\n" + "stdout": "Requirement already satisfied: pip in ./.agentlab/venv/lib/python3.9/site-packages (21.2.4)\nCollecting pip\n Using cached pip-26.0.1-py3-none-any.whl (1.8 MB)\nInstalling collected packages: pip\n Attempting uninstall: pip\n Found existing installation: pip 21.2.4\n Uninstalling pip-21.2.4:\n Successfully uninstalled pip-21.2.4\nSuccessfully installed pip-26.0.1\n" }, { "command": "python -m pip install -e . \"pytest<9\"", "passed": true, "returncode": 0, "stderr": "", - "stdout": "Obtaining file:///private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/agentlab-reference-4ic7u1_p/httpx-verify-false-client-cert-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Installing backend dependencies: started\n Installing backend dependencies: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting anyio (from httpx==0.28.0)\n Downloading anyio-4.13.0-py3-none-any.whl.metadata (4.5 kB)\nCollecting certifi (from httpx==0.28.0)\n Using cached certifi-2026.5.20-py3-none-any.whl.metadata (2.5 kB)\nCollecting httpcore==1.* (from httpx==0.28.0)\n Using cached httpcore-1.0.9-py3-none-any.whl.metadata (21 kB)\nCollecting idna (from httpx==0.28.0)\n Using cached idna-3.18-py3-none-any.whl.metadata (6.1 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.3.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting h11>=0.16 (from httpcore==1.*->httpx==0.28.0)\n Using cached h11-0.16.0-py3-none-any.whl.metadata (8.3 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached httpcore-1.0.9-py3-none-any.whl (78 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached h11-0.16.0-py3-none-any.whl (37 kB)\nUsing cached iniconfig-2.3.0-py3-none-any.whl (7.5 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nDownloading anyio-4.13.0-py3-none-any.whl (114 kB)\nUsing cached idna-3.18-py3-none-any.whl (65 kB)\nUsing cached certifi-2026.5.20-py3-none-any.whl (134 kB)\nBuilding wheels for collected packages: httpx\n Building editable for httpx (pyproject.toml): started\n Building editable for httpx (pyproject.toml): finished with status 'done'\n Created wheel for httpx: filename=httpx-0.28.0-py3-none-any.whl size=4921 sha256=badff137afab7bef72bbf4e3e0af07e5e778c12aaefefee844082c50fde662e9\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-9ml56d2w/wheels/3f/e3/6a/162557ff5d76689efe1dc1cf98ab6eebb739b2cbcda4c53ee2\nSuccessfully built httpx\nInstalling collected packages: pygments, pluggy, packaging, iniconfig, idna, h11, certifi, pytest, httpcore, anyio, httpx\n\nSuccessfully installed anyio-4.13.0 certifi-2026.5.20 h11-0.16.0 httpcore-1.0.9 httpx-0.28.0 idna-3.18 iniconfig-2.3.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2\n" + "stdout": "Obtaining file:///private/tmp/ael-ref-starter-123/httpx-verify-false-client-cert-001\n Installing build dependencies: started\n Installing build dependencies: finished with status 'done'\n Checking if build backend supports build_editable: started\n Checking if build backend supports build_editable: finished with status 'done'\n Getting requirements to build editable: started\n Getting requirements to build editable: finished with status 'done'\n Installing backend dependencies: started\n Installing backend dependencies: finished with status 'done'\n Preparing editable metadata (pyproject.toml): started\n Preparing editable metadata (pyproject.toml): finished with status 'done'\nCollecting pytest<9\n Using cached pytest-8.4.2-py3-none-any.whl.metadata (7.7 kB)\nCollecting anyio (from httpx==0.28.0)\n Using cached anyio-4.12.1-py3-none-any.whl.metadata (4.3 kB)\nCollecting certifi (from httpx==0.28.0)\n Using cached certifi-2026.6.17-py3-none-any.whl.metadata (2.5 kB)\nCollecting httpcore==1.* (from httpx==0.28.0)\n Using cached httpcore-1.0.9-py3-none-any.whl.metadata (21 kB)\nCollecting idna (from httpx==0.28.0)\n Using cached idna-3.18-py3-none-any.whl.metadata (6.1 kB)\nCollecting exceptiongroup>=1 (from pytest<9)\n Using cached exceptiongroup-1.3.1-py3-none-any.whl.metadata (6.7 kB)\nCollecting iniconfig>=1 (from pytest<9)\n Using cached iniconfig-2.1.0-py3-none-any.whl.metadata (2.7 kB)\nCollecting packaging>=20 (from pytest<9)\n Using cached packaging-26.2-py3-none-any.whl.metadata (3.5 kB)\nCollecting pluggy<2,>=1.5 (from pytest<9)\n Using cached pluggy-1.6.0-py3-none-any.whl.metadata (4.8 kB)\nCollecting pygments>=2.7.2 (from pytest<9)\n Using cached pygments-2.20.0-py3-none-any.whl.metadata (2.5 kB)\nCollecting tomli>=1 (from pytest<9)\n Using cached tomli-2.4.1-py3-none-any.whl.metadata (10 kB)\nCollecting h11>=0.16 (from httpcore==1.*->httpx==0.28.0)\n Using cached h11-0.16.0-py3-none-any.whl.metadata (8.3 kB)\nCollecting typing-extensions>=4.6.0 (from exceptiongroup>=1->pytest<9)\n Using cached typing_extensions-4.15.0-py3-none-any.whl.metadata (3.3 kB)\nUsing cached pytest-8.4.2-py3-none-any.whl (365 kB)\nUsing cached httpcore-1.0.9-py3-none-any.whl (78 kB)\nUsing cached pluggy-1.6.0-py3-none-any.whl (20 kB)\nUsing cached exceptiongroup-1.3.1-py3-none-any.whl (16 kB)\nUsing cached h11-0.16.0-py3-none-any.whl (37 kB)\nUsing cached iniconfig-2.1.0-py3-none-any.whl (6.0 kB)\nUsing cached packaging-26.2-py3-none-any.whl (100 kB)\nUsing cached pygments-2.20.0-py3-none-any.whl (1.2 MB)\nUsing cached tomli-2.4.1-py3-none-any.whl (14 kB)\nUsing cached typing_extensions-4.15.0-py3-none-any.whl (44 kB)\nUsing cached anyio-4.12.1-py3-none-any.whl (113 kB)\nUsing cached idna-3.18-py3-none-any.whl (65 kB)\nUsing cached certifi-2026.6.17-py3-none-any.whl (133 kB)\nBuilding wheels for collected packages: httpx\n Building editable for httpx (pyproject.toml): started\n Building editable for httpx (pyproject.toml): finished with status 'done'\n Created wheel for httpx: filename=httpx-0.28.0-py3-none-any.whl size=4878 sha256=35161bedc4cde7225abf08a86ca87ad77e324aad97ffc9fc30d3f7b725474ae0\n Stored in directory: /private/var/folders/3m/s27dbmbs1mn8yp8dmfxmtl9h0000gn/T/pip-ephem-wheel-cache-o1vu442_/wheels/b4/1f/31/93c94facf0f7e16355c86f731264009a167ff283a25bcb8898\nSuccessfully built httpx\nInstalling collected packages: typing-extensions, tomli, pygments, pluggy, packaging, iniconfig, idna, h11, certifi, httpcore, exceptiongroup, pytest, anyio, httpx\n\nSuccessfully installed anyio-4.12.1 certifi-2026.6.17 exceptiongroup-1.3.1 h11-0.16.0 httpcore-1.0.9 httpx-0.28.0 idna-3.18 iniconfig-2.1.0 packaging-26.2 pluggy-1.6.0 pygments-2.20.0 pytest-8.4.2 tomli-2.4.1 typing-extensions-4.15.0\n" }, { "command": "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'", @@ -45,34 +45,6 @@ "returncode": 0, "stderr": "", "stdout": "" - }, - { - "command": "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [(\"client.pem\", \"client.key\")], calls'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=\"client.pem\"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [(\"client.pem\",)], calls'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [(\"client.pem\", \"client.key\")], calls'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" } ], "commands_run": [ @@ -80,11 +52,7 @@ "python -m pip install --upgrade pip", "python -m pip install -e . \"pytest<9\"", "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'", - "git apply reference.patch", - "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [(\"client.pem\", \"client.key\")], calls'", - "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=\"client.pem\"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [(\"client.pem\",)], calls'", - "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'", - "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [(\"client.pem\", \"client.key\")], calls'" + "git apply reference.patch" ], "cost_usd": null, "diff_path": "reference.diff", @@ -125,32 +93,29 @@ "passed": true, "returncode": 0, "type": "code" - }, - { - "assertion": "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [(\"client.pem\", \"client.key\")], calls'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=\"client.pem\"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [(\"client.pem\",)], calls'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [(\"client.pem\", \"client.key\")], calls'", - "passed": true, - "returncode": 0, - "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [(\"client.pem\", \"client.key\")], calls'\n$ python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=\"client.pem\"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [(\"client.pem\",)], calls'\n$ python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'\n$ python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=(\"client.pem\", \"client.key\")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [(\"client.pem\", \"client.key\")], calls'\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 3, "lines_deleted": 4, @@ -163,7 +128,7 @@ "lines_added": 3, "lines_deleted": 4, "n_files_changed": 1, - "setup_created_untracked_coverage_caveat_count": 2044, + "setup_created_untracked_coverage_caveat_count": 1299, "status": "passed" }, "output_tokens": null, @@ -201,7 +166,7 @@ "workspace_history_policy": "base_only" }, "score_notes": [], - "setup_created_untracked_coverage_caveat_count": 2044, + "setup_created_untracked_coverage_caveat_count": 1299, "status": "passed", "success": true, "task_commit": "0cb7e5a2e736628e2f506d259fcf0d48cd2bde82", diff --git a/tasks/starter/httpx-verify-false-client-cert-001/task-card.md b/tasks/starter/httpx-verify-false-client-cert-001/task-card.md index 64040b2..14a2b4e 100644 --- a/tasks/starter/httpx-verify-false-client-cert-001/task-card.md +++ b/tasks/starter/httpx-verify-false-client-cert-001/task-card.md @@ -29,6 +29,11 @@ In create_ssl_context, make the verify=False branch assign the unverified SSL co - PYTHONDONTWRITEBYTECODE=1 - VIRTUAL_ENV={workspace}/.agentlab/venv +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -43,10 +48,7 @@ In create_ssl_context, make the verify=False branch assign the unverified SSL co ### Target -- `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls'` -- `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert="client.pem"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem",)], calls'` -- `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'` -- `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls'` +None configured. ## Success Criteria diff --git a/tasks/starter/httpx-verify-false-client-cert-001/task.yaml b/tasks/starter/httpx-verify-false-client-cert-001/task.yaml index df78b5e..3e5e82f 100644 --- a/tasks/starter/httpx-verify-false-client-cert-001/task.yaml +++ b/tasks/starter/httpx-verify-false-client-cert-001/task.yaml @@ -25,6 +25,10 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python .agentlab_hidden/check_behavior.py setup: - python3 -m venv .agentlab/venv - python -m pip install --upgrade pip @@ -38,15 +42,6 @@ environment: baseline: - >- python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls' -test: - - >- - python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls' - - >- - python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert="client.pem"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem",)], calls' - - >- - python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls' - - >- - python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls' success: tests_must_pass: true max_files_changed: 2 diff --git a/tasks/starter/httpx-verify-false-client-cert-001/verifier.patch b/tasks/starter/httpx-verify-false-client-cert-001/verifier.patch new file mode 100644 index 0000000..ba7487b --- /dev/null +++ b/tasks/starter/httpx-verify-false-client-cert-001/verifier.patch @@ -0,0 +1,37 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,31 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = ['python -c \'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls\'', 'python -c \'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert="client.pem"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem",)], calls\'', "python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'", 'python -c \'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls\''] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print(f"$ {command}") ++ completed = subprocess.run( ++ command, ++ shell=True, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/prettier-duplicate-dangling-comments-001/reference-report.md b/tasks/starter/prettier-duplicate-dangling-comments-001/reference-report.md index 37173fe..e8fd14b 100644 --- a/tasks/starter/prettier-duplicate-dangling-comments-001/reference-report.md +++ b/tasks/starter/prettier-duplicate-dangling-comments-001/reference-report.md @@ -4,6 +4,8 @@ - Agent harness: `reference` - Evaluation suite: `starter-coding` - Evaluation type: `regression` +- Task repository: `https://github.com/prettier/prettier.git` +- Task commit: `80a2fdb5e88ce220b88d9122cd74303d71e4d8c0` - Reference artifact: patch `reference.patch` - Status: `passed` - Outcome: `passed` @@ -11,7 +13,26 @@ - Lines added: `0` - Lines deleted: `18` -## Code-Based Graders +Setup-created untracked coverage caveat: 31106 setup-created untracked paths existed outside exact boundary-pattern matching. Changed-file counts/lists and boundary metrics include detected changes, but detection remains best-effort for worktree-only content-preserving edits to those paths. + +## Run Surface + +- Execution surface: `unknown` +- Runtime version: `unknown` +- Model identity source: `unknown` +- Sandbox mode: `unknown` +- Approval policy: `unknown` +- Tool policy: `unknown` +- Memory scope: `unknown` +- Network policy: `unknown` +- Timeout seconds: `unknown` +- Turn or step budget: `unknown` +- Stop reason: `success` +- Human intervention events: `none` +- Workspace history policy: `base_only` +- Workspace base ref: `d2c78eaaf1ba3b0cd4a12438feb22f5c9ae2ef98` + +## Public Graders 1. Assertion `corepack yarn install --immutable --mode=skip-build`: passed (0) @@ -27,14 +48,25 @@ ➤ YN0000: ┌ Fetch step ➤ YN0000: └ Completed ➤ YN0000: ┌ Link step -➤ YN0000: └ Completed in 2s 405ms -➤ YN0000: · Done with warnings in 2s 658ms +➤ YN0000: └ Completed in 2s 317ms +➤ YN0000: · Done with warnings in 2s 544ms ``` 2. Assertion `node -e 'const prettier = require("./src/index.cjs"); (async () => { const input = `condition ? ifTrue\n: [\n // Hello, world!\n ]\n`; const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 2) { console.error(output); throw new Error(`expected start commit to duplicate the comment twice, saw ${count}`); } })().catch((error) => { console.error(error); process.exit(1); })'`: passed (0) 3. Assertion `git apply reference.patch`: passed (0) -4. Assertion `node -e 'const prettier = require("./src/index.cjs"); (async () => { const cases = [`condition ? ifTrue\n: [\n // Hello, world!\n ]\n`, `condition ? [\n // Hello, world!\n ]\n: ifFalse\n`, `condition ? ifTrue\n: {\n // Hello, world!\n }\n`, `condition ? {\n // Hello, world!\n }\n: ifFalse\n`, `condition ? ifTrue\n: [\n // Hello, world!\n 1,\n ]\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })'`: passed (0) -5. Assertion `git diff --check`: passed (0) +4. Assertion `git diff --check`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python3 .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ node -e 'const prettier = require("./src/index.cjs"); (async () => { const cases = [`condition ? ifTrue\n: [\n // Hello, world!\n ]\n`, `condition ? [\n // Hello, world!\n ]\n: ifFalse\n`, `condition ? ifTrue\n: {\n // Hello, world!\n }\n`, `condition ? {\n // Hello, world!\n }\n: ifFalse\n`, `condition ? ifTrue\n: [\n // Hello, world!\n 1,\n ]\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })' +``` + ## Changed Files diff --git a/tasks/starter/prettier-duplicate-dangling-comments-001/reference-result.json b/tasks/starter/prettier-duplicate-dangling-comments-001/reference-result.json index 9206902..16ac9eb 100644 --- a/tasks/starter/prettier-duplicate-dangling-comments-001/reference-result.json +++ b/tasks/starter/prettier-duplicate-dangling-comments-001/reference-result.json @@ -1,4 +1,13 @@ { + "agent_harness_config": { + "agent_harness": "reference", + "model_name": null, + "runtime_accountability": { + "account": null, + "billing_context": null, + "cost_usd": null + } + }, "agent_name": "reference", "cached_input_tokens": null, "checks": [ @@ -7,7 +16,7 @@ "passed": true, "returncode": 0, "stderr": "", - "stdout": "\u27a4 YN0000: \u00b7 Yarn 4.12.0\n\u27a4 YN0000: \u250c Resolution step\n\u27a4 YN0000: \u2514 Completed\n\u27a4 YN0000: \u250c Post-resolution validation\n\u27a4 YN0002: \u2502 prettier@workspace:. doesn't provide rollup (pd38a67), requested by rollup-plugin-license.\n\u27a4 YN0086: \u2502 Some peer dependencies are incorrectly met by your project; run yarn explain peer-requirements for details, where is the six-letter p-prefixed code.\n\u27a4 YN0086: \u2502 Some peer dependencies are incorrectly met by dependencies; run yarn explain peer-requirements for details.\n\u27a4 YN0000: \u2514 Completed\n\u27a4 YN0000: \u250c Fetch step\n\u27a4 YN0000: \u2514 Completed\n\u27a4 YN0000: \u250c Link step\n\u27a4 YN0000: \u2514 Completed in 2s 405ms\n\u27a4 YN0000: \u00b7 Done with warnings in 2s 658ms\n" + "stdout": "\u27a4 YN0000: \u00b7 Yarn 4.12.0\n\u27a4 YN0000: \u250c Resolution step\n\u27a4 YN0000: \u2514 Completed\n\u27a4 YN0000: \u250c Post-resolution validation\n\u27a4 YN0002: \u2502 prettier@workspace:. doesn't provide rollup (pd38a67), requested by rollup-plugin-license.\n\u27a4 YN0086: \u2502 Some peer dependencies are incorrectly met by your project; run yarn explain peer-requirements for details, where is the six-letter p-prefixed code.\n\u27a4 YN0086: \u2502 Some peer dependencies are incorrectly met by dependencies; run yarn explain peer-requirements for details.\n\u27a4 YN0000: \u2514 Completed\n\u27a4 YN0000: \u250c Fetch step\n\u27a4 YN0000: \u2514 Completed\n\u27a4 YN0000: \u250c Link step\n\u27a4 YN0000: \u2514 Completed in 2s 317ms\n\u27a4 YN0000: \u00b7 Done with warnings in 2s 544ms\n" }, { "command": "node -e 'const prettier = require(\"./src/index.cjs\"); (async () => { const input = `condition ? ifTrue\\n: [\\n // Hello, world!\\n ]\\n`; const output = await prettier.format(input, { parser: \"babel\", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 2) { console.error(output); throw new Error(`expected start commit to duplicate the comment twice, saw ${count}`); } })().catch((error) => { console.error(error); process.exit(1); })'", @@ -23,13 +32,6 @@ "stderr": "", "stdout": "" }, - { - "command": "node -e 'const prettier = require(\"./src/index.cjs\"); (async () => { const cases = [`condition ? ifTrue\\n: [\\n // Hello, world!\\n ]\\n`, `condition ? [\\n // Hello, world!\\n ]\\n: ifFalse\\n`, `condition ? ifTrue\\n: {\\n // Hello, world!\\n }\\n`, `condition ? {\\n // Hello, world!\\n }\\n: ifFalse\\n`, `condition ? ifTrue\\n: [\\n // Hello, world!\\n 1,\\n ]\\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: \"babel\", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, { "command": "git diff --check", "passed": true, @@ -42,7 +44,6 @@ "corepack yarn install --immutable --mode=skip-build", "node -e 'const prettier = require(\"./src/index.cjs\"); (async () => { const input = `condition ? ifTrue\\n: [\\n // Hello, world!\\n ]\\n`; const output = await prettier.format(input, { parser: \"babel\", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 2) { console.error(output); throw new Error(`expected start commit to duplicate the comment twice, saw ${count}`); } })().catch((error) => { console.error(error); process.exit(1); })'", "git apply reference.patch", - "node -e 'const prettier = require(\"./src/index.cjs\"); (async () => { const cases = [`condition ? ifTrue\\n: [\\n // Hello, world!\\n ]\\n`, `condition ? [\\n // Hello, world!\\n ]\\n: ifFalse\\n`, `condition ? ifTrue\\n: {\\n // Hello, world!\\n }\\n`, `condition ? {\\n // Hello, world!\\n }\\n: ifFalse\\n`, `condition ? ifTrue\\n: [\\n // Hello, world!\\n 1,\\n ]\\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: \"babel\", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })'", "git diff --check" ], "cost_usd": null, @@ -73,12 +74,6 @@ "returncode": 0, "type": "code" }, - { - "assertion": "node -e 'const prettier = require(\"./src/index.cjs\"); (async () => { const cases = [`condition ? ifTrue\\n: [\\n // Hello, world!\\n ]\\n`, `condition ? [\\n // Hello, world!\\n ]\\n: ifFalse\\n`, `condition ? ifTrue\\n: {\\n // Hello, world!\\n }\\n`, `condition ? {\\n // Hello, world!\\n }\\n: ifFalse\\n`, `condition ? ifTrue\\n: [\\n // Hello, world!\\n 1,\\n ]\\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: \"babel\", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })'", - "passed": true, - "returncode": 0, - "type": "code" - }, { "assertion": "git diff --check", "passed": true, @@ -86,6 +81,27 @@ "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python3 .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ node -e 'const prettier = require(\"./src/index.cjs\"); (async () => { const cases = [`condition ? ifTrue\\n: [\\n // Hello, world!\\n ]\\n`, `condition ? [\\n // Hello, world!\\n ]\\n: ifFalse\\n`, `condition ? ifTrue\\n: {\\n // Hello, world!\\n }\\n`, `condition ? {\\n // Hello, world!\\n }\\n: ifFalse\\n`, `condition ? ifTrue\\n: [\\n // Hello, world!\\n 1,\\n ]\\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: \"babel\", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })'\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 0, "lines_deleted": 18, @@ -98,6 +114,7 @@ "lines_added": 0, "lines_deleted": 18, "n_files_changed": 1, + "setup_created_untracked_coverage_caveat_count": 31106, "status": "passed" }, "output_tokens": null, @@ -118,10 +135,29 @@ }, "run_dir": ".", "run_id": "prettier-duplicate-dangling-comments-001-reference", + "run_surface": { + "approval_policy": "unknown", + "execution_surface": "unknown", + "human_intervention_events": [], + "memory_scope": "unknown", + "model_identity_source": "unknown", + "network_policy": "unknown", + "runtime_version": "unknown", + "sandbox_mode": "unknown", + "stop_reason": "success", + "timeout_seconds": "unknown", + "tool_policy": "unknown", + "turn_or_step_budget": "unknown", + "workspace_base_ref": "d2c78eaaf1ba3b0cd4a12438feb22f5c9ae2ef98", + "workspace_history_policy": "base_only" + }, "score_notes": [], + "setup_created_untracked_coverage_caveat_count": 31106, "status": "passed", "success": true, + "task_commit": "80a2fdb5e88ce220b88d9122cd74303d71e4d8c0", "task_id": "prettier-duplicate-dangling-comments-001", + "task_repo": "https://github.com/prettier/prettier.git", "task_title": "Prettier should not duplicate dangling comments in experimental ternaries", "transcript_path": null, "trial_id": "prettier-duplicate-dangling-comments-001-reference", diff --git a/tasks/starter/prettier-duplicate-dangling-comments-001/task-card.md b/tasks/starter/prettier-duplicate-dangling-comments-001/task-card.md index be8ff2b..8161fe4 100644 --- a/tasks/starter/prettier-duplicate-dangling-comments-001/task-card.md +++ b/tasks/starter/prettier-duplicate-dangling-comments-001/task-card.md @@ -26,6 +26,11 @@ In the JavaScript ternary printer, stop separately printing dangling comments fr No task-local environment configured. +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -38,7 +43,6 @@ No task-local environment configured. ### Target -- `node -e 'const prettier = require("./src/index.cjs"); (async () => { const cases = [`condition ? ifTrue\n: [\n // Hello, world!\n ]\n`, `condition ? [\n // Hello, world!\n ]\n: ifFalse\n`, `condition ? ifTrue\n: {\n // Hello, world!\n }\n`, `condition ? {\n // Hello, world!\n }\n: ifFalse\n`, `condition ? ifTrue\n: [\n // Hello, world!\n 1,\n ]\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })'` - `git diff --check` ## Success Criteria diff --git a/tasks/starter/prettier-duplicate-dangling-comments-001/task.yaml b/tasks/starter/prettier-duplicate-dangling-comments-001/task.yaml index b579581..933a8ce 100644 --- a/tasks/starter/prettier-duplicate-dangling-comments-001/task.yaml +++ b/tasks/starter/prettier-duplicate-dangling-comments-001/task.yaml @@ -23,14 +23,16 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python3 .agentlab_hidden/check_behavior.py setup: - corepack yarn install --immutable --mode=skip-build baseline: - >- node -e 'const prettier = require("./src/index.cjs"); (async () => { const input = `condition ? ifTrue\n: [\n // Hello, world!\n ]\n`; const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 2) { console.error(output); throw new Error(`expected start commit to duplicate the comment twice, saw ${count}`); } })().catch((error) => { console.error(error); process.exit(1); })' test: - - >- - node -e 'const prettier = require("./src/index.cjs"); (async () => { const cases = [`condition ? ifTrue\n: [\n // Hello, world!\n ]\n`, `condition ? [\n // Hello, world!\n ]\n: ifFalse\n`, `condition ? ifTrue\n: {\n // Hello, world!\n }\n`, `condition ? {\n // Hello, world!\n }\n: ifFalse\n`, `condition ? ifTrue\n: [\n // Hello, world!\n 1,\n ]\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })' - git diff --check success: tests_must_pass: true diff --git a/tasks/starter/prettier-duplicate-dangling-comments-001/verifier.patch b/tasks/starter/prettier-duplicate-dangling-comments-001/verifier.patch new file mode 100644 index 0000000..7c17a76 --- /dev/null +++ b/tasks/starter/prettier-duplicate-dangling-comments-001/verifier.patch @@ -0,0 +1,37 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,31 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = ['node -e \'const prettier = require("./src/index.cjs"); (async () => { const cases = [`condition ? ifTrue\\n: [\\n // Hello, world!\\n ]\\n`, `condition ? [\\n // Hello, world!\\n ]\\n: ifFalse\\n`, `condition ? ifTrue\\n: {\\n // Hello, world!\\n }\\n`, `condition ? {\\n // Hello, world!\\n }\\n: ifFalse\\n`, `condition ? ifTrue\\n: [\\n // Hello, world!\\n 1,\\n ]\\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })\''] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print(f"$ {command}") ++ completed = subprocess.run( ++ command, ++ shell=True, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/react-tabs-selected-focus-overlay-001/reference-report.md b/tasks/starter/react-tabs-selected-focus-overlay-001/reference-report.md index 3b43eed..50e1ff3 100644 --- a/tasks/starter/react-tabs-selected-focus-overlay-001/reference-report.md +++ b/tasks/starter/react-tabs-selected-focus-overlay-001/reference-report.md @@ -4,6 +4,8 @@ - Agent harness: `reference` - Evaluation suite: `starter-coding` - Evaluation type: `regression` +- Task repository: `https://github.com/reactjs/react-tabs.git` +- Task commit: `186631aca0458e0b991c94180bc3a4a785151c04` - Reference artifact: patch `reference.patch` - Status: `passed` - Outcome: `passed` @@ -11,12 +13,40 @@ - Lines added: `0` - Lines deleted: `30` -## Code-Based Graders +## Run Surface + +- Execution surface: `unknown` +- Runtime version: `unknown` +- Model identity source: `unknown` +- Sandbox mode: `unknown` +- Approval policy: `unknown` +- Tool policy: `unknown` +- Memory scope: `unknown` +- Network policy: `unknown` +- Timeout seconds: `unknown` +- Turn or step budget: `unknown` +- Stop reason: `success` +- Human intervention events: `none` +- Workspace history policy: `base_only` +- Workspace base ref: `6988e9a8967cac275d083850c99ea400719f0054` + +## Public Graders 1. Assertion `python3 -c 'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" in css, css; assert "background: #fff" in css and "bottom: -5px" in css, css; assert "&:after" in scss and "&:after" in less'`: passed (0) 2. Assertion `git apply reference.patch`: passed (0) -3. Assertion `python3 -c 'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" not in css, css; assert "&:after" not in scss, scss; assert "&:after" not in less, less; assert ".react-tabs__tab:focus" in css and "&:focus" in scss and "&:focus" in less; assert ".react-tabs__tab--selected" in css and "&--selected" in scss and "&--selected" in less; assert "border-color: #aaa" in css and "border-color: #aaa" in scss and "border-color: #aaa" in less'`: passed (0) -4. Assertion `git diff --check`: passed (0) +3. Assertion `git diff --check`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python3 .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ python3 -c 'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" not in css, css; assert "&:after" not in scss, scss; assert "&:after" not in less, less; assert ".react-tabs__tab:focus" in css and "&:focus" in scss and "&:focus" in less; assert ".react-tabs__tab--selected" in css and "&--selected" in scss and "&--selected" in less; assert "border-color: #aaa" in css and "border-color: #aaa" in scss and "border-color: #aaa" in less' +``` + ## Changed Files diff --git a/tasks/starter/react-tabs-selected-focus-overlay-001/reference-result.json b/tasks/starter/react-tabs-selected-focus-overlay-001/reference-result.json index 2d1aca4..88d6297 100644 --- a/tasks/starter/react-tabs-selected-focus-overlay-001/reference-result.json +++ b/tasks/starter/react-tabs-selected-focus-overlay-001/reference-result.json @@ -1,4 +1,13 @@ { + "agent_harness_config": { + "agent_harness": "reference", + "model_name": null, + "runtime_accountability": { + "account": null, + "billing_context": null, + "cost_usd": null + } + }, "agent_name": "reference", "cached_input_tokens": null, "checks": [ @@ -16,13 +25,6 @@ "stderr": "", "stdout": "" }, - { - "command": "python3 -c 'from pathlib import Path; css = Path(\"style/react-tabs.css\").read_text(); scss = Path(\"style/react-tabs.scss\").read_text(); less = Path(\"style/react-tabs.less\").read_text(); assert \".react-tabs__tab:focus:after\" not in css, css; assert \"&:after\" not in scss, scss; assert \"&:after\" not in less, less; assert \".react-tabs__tab:focus\" in css and \"&:focus\" in scss and \"&:focus\" in less; assert \".react-tabs__tab--selected\" in css and \"&--selected\" in scss and \"&--selected\" in less; assert \"border-color: #aaa\" in css and \"border-color: #aaa\" in scss and \"border-color: #aaa\" in less'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, { "command": "git diff --check", "passed": true, @@ -34,7 +36,6 @@ "commands_run": [ "python3 -c 'from pathlib import Path; css = Path(\"style/react-tabs.css\").read_text(); scss = Path(\"style/react-tabs.scss\").read_text(); less = Path(\"style/react-tabs.less\").read_text(); assert \".react-tabs__tab:focus:after\" in css, css; assert \"background: #fff\" in css and \"bottom: -5px\" in css, css; assert \"&:after\" in scss and \"&:after\" in less'", "git apply reference.patch", - "python3 -c 'from pathlib import Path; css = Path(\"style/react-tabs.css\").read_text(); scss = Path(\"style/react-tabs.scss\").read_text(); less = Path(\"style/react-tabs.less\").read_text(); assert \".react-tabs__tab:focus:after\" not in css, css; assert \"&:after\" not in scss, scss; assert \"&:after\" not in less, less; assert \".react-tabs__tab:focus\" in css and \"&:focus\" in scss and \"&:focus\" in less; assert \".react-tabs__tab--selected\" in css and \"&--selected\" in scss and \"&--selected\" in less; assert \"border-color: #aaa\" in css and \"border-color: #aaa\" in scss and \"border-color: #aaa\" in less'", "git diff --check" ], "cost_usd": null, @@ -61,12 +62,6 @@ "returncode": 0, "type": "code" }, - { - "assertion": "python3 -c 'from pathlib import Path; css = Path(\"style/react-tabs.css\").read_text(); scss = Path(\"style/react-tabs.scss\").read_text(); less = Path(\"style/react-tabs.less\").read_text(); assert \".react-tabs__tab:focus:after\" not in css, css; assert \"&:after\" not in scss, scss; assert \"&:after\" not in less, less; assert \".react-tabs__tab:focus\" in css and \"&:focus\" in scss and \"&:focus\" in less; assert \".react-tabs__tab--selected\" in css and \"&--selected\" in scss and \"&--selected\" in less; assert \"border-color: #aaa\" in css and \"border-color: #aaa\" in scss and \"border-color: #aaa\" in less'", - "passed": true, - "returncode": 0, - "type": "code" - }, { "assertion": "git diff --check", "passed": true, @@ -74,6 +69,27 @@ "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python3 .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ python3 -c 'from pathlib import Path; css = Path(\"style/react-tabs.css\").read_text(); scss = Path(\"style/react-tabs.scss\").read_text(); less = Path(\"style/react-tabs.less\").read_text(); assert \".react-tabs__tab:focus:after\" not in css, css; assert \"&:after\" not in scss, scss; assert \"&:after\" not in less, less; assert \".react-tabs__tab:focus\" in css and \"&:focus\" in scss and \"&:focus\" in less; assert \".react-tabs__tab--selected\" in css and \"&--selected\" in scss and \"&--selected\" in less; assert \"border-color: #aaa\" in css and \"border-color: #aaa\" in scss and \"border-color: #aaa\" in less'\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 0, "lines_deleted": 30, @@ -108,10 +124,28 @@ }, "run_dir": ".", "run_id": "react-tabs-selected-focus-overlay-001-reference", + "run_surface": { + "approval_policy": "unknown", + "execution_surface": "unknown", + "human_intervention_events": [], + "memory_scope": "unknown", + "model_identity_source": "unknown", + "network_policy": "unknown", + "runtime_version": "unknown", + "sandbox_mode": "unknown", + "stop_reason": "success", + "timeout_seconds": "unknown", + "tool_policy": "unknown", + "turn_or_step_budget": "unknown", + "workspace_base_ref": "6988e9a8967cac275d083850c99ea400719f0054", + "workspace_history_policy": "base_only" + }, "score_notes": [], "status": "passed", "success": true, + "task_commit": "186631aca0458e0b991c94180bc3a4a785151c04", "task_id": "react-tabs-selected-focus-overlay-001", + "task_repo": "https://github.com/reactjs/react-tabs.git", "task_title": "React Tabs selected-tab focus overlay should not mask custom borders", "transcript_path": null, "trial_id": "react-tabs-selected-focus-overlay-001-reference", diff --git a/tasks/starter/react-tabs-selected-focus-overlay-001/task-card.md b/tasks/starter/react-tabs-selected-focus-overlay-001/task-card.md index b21d5db..8743533 100644 --- a/tasks/starter/react-tabs-selected-focus-overlay-001/task-card.md +++ b/tasks/starter/react-tabs-selected-focus-overlay-001/task-card.md @@ -26,6 +26,11 @@ Remove the obsolete focus `:after` pseudo-element from `style/react-tabs.css` an No task-local environment configured. +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -38,7 +43,6 @@ None configured. ### Target -- `python3 -c 'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" not in css, css; assert "&:after" not in scss, scss; assert "&:after" not in less, less; assert ".react-tabs__tab:focus" in css and "&:focus" in scss and "&:focus" in less; assert ".react-tabs__tab--selected" in css and "&--selected" in scss and "&--selected" in less; assert "border-color: #aaa" in css and "border-color: #aaa" in scss and "border-color: #aaa" in less'` - `git diff --check` ## Success Criteria diff --git a/tasks/starter/react-tabs-selected-focus-overlay-001/task.yaml b/tasks/starter/react-tabs-selected-focus-overlay-001/task.yaml index e1ba151..129fc59 100644 --- a/tasks/starter/react-tabs-selected-focus-overlay-001/task.yaml +++ b/tasks/starter/react-tabs-selected-focus-overlay-001/task.yaml @@ -23,12 +23,14 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python3 .agentlab_hidden/check_behavior.py baseline: - >- python3 -c 'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" in css, css; assert "background: #fff" in css and "bottom: -5px" in css, css; assert "&:after" in scss and "&:after" in less' test: - - >- - python3 -c 'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" not in css, css; assert "&:after" not in scss, scss; assert "&:after" not in less, less; assert ".react-tabs__tab:focus" in css and "&:focus" in scss and "&:focus" in less; assert ".react-tabs__tab--selected" in css and "&--selected" in scss and "&--selected" in less; assert "border-color: #aaa" in css and "border-color: #aaa" in scss and "border-color: #aaa" in less' - git diff --check success: tests_must_pass: true diff --git a/tasks/starter/react-tabs-selected-focus-overlay-001/verifier.patch b/tasks/starter/react-tabs-selected-focus-overlay-001/verifier.patch new file mode 100644 index 0000000..9695dcd --- /dev/null +++ b/tasks/starter/react-tabs-selected-focus-overlay-001/verifier.patch @@ -0,0 +1,37 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,31 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = ['python3 -c \'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" not in css, css; assert "&:after" not in scss, scss; assert "&:after" not in less, less; assert ".react-tabs__tab:focus" in css and "&:focus" in scss and "&:focus" in less; assert ".react-tabs__tab--selected" in css and "&--selected" in scss and "&--selected" in less; assert "border-color: #aaa" in css and "border-color: #aaa" in scss and "border-color: #aaa" in less\''] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print(f"$ {command}") ++ completed = subprocess.run( ++ command, ++ shell=True, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/remotion-audio-context-autoplay-muted-001/reference-report.md b/tasks/starter/remotion-audio-context-autoplay-muted-001/reference-report.md index 4589ce6..9a81b81 100644 --- a/tasks/starter/remotion-audio-context-autoplay-muted-001/reference-report.md +++ b/tasks/starter/remotion-audio-context-autoplay-muted-001/reference-report.md @@ -4,6 +4,8 @@ - Agent harness: `reference` - Evaluation suite: `starter-coding` - Evaluation type: `regression` +- Task repository: `https://github.com/remotion-dev/remotion.git` +- Task commit: `631ce3d2ab757e44cfb542a108a21f1e9a9c6c0f` - Reference artifact: patch `reference.patch` - Status: `passed` - Outcome: `passed` @@ -11,14 +13,46 @@ - Lines added: `30` - Lines deleted: `15` -## Code-Based Graders +## Run Surface + +- Execution surface: `unknown` +- Runtime version: `unknown` +- Model identity source: `unknown` +- Sandbox mode: `unknown` +- Approval policy: `unknown` +- Tool policy: `unknown` +- Memory scope: `unknown` +- Network policy: `unknown` +- Timeout seconds: `unknown` +- Turn or step budget: `unknown` +- Stop reason: `success` +- Human intervention events: `none` +- Workspace history policy: `focused_substitute` +- Workspace base ref: `focused-two-file-validation` + +## Public Graders 1. Assertion `python3 -c 'from pathlib import Path; playback = Path("packages/player/src/use-playback.ts").read_text(); shared = Path("packages/core/src/audio/shared-audio-tags.tsx").read_text(); assert "sharedAudioContext?.resume?.();" in playback; assert "if (!muted)" not in playback.split("sharedAudioContext?.resume?.();", 1)[0][-80:]; assert "if (getIsResumingAudioContext !== null && !muted)" not in playback; assert "const resumePromise = ctxAndGain.audioContext.resume();" not in shared; assert "return ctxAndGain.audioContext.resume().then(() => {" in shared'`: passed (0) 2. Assertion `git apply reference.patch`: passed (0) -3. Assertion `python3 -c 'from pathlib import Path; import re; playback = Path("packages/player/src/use-playback.ts").read_text(); guarded_resume = re.search(r"if\s*\(\s*!muted\s*\)\s*{\s*sharedAudioContext\?\.resume\?\.\(\);\s*}", playback); assert guarded_resume, "sharedAudioContext.resume() must be guarded by !muted"; unguarded_resume = playback.replace(guarded_resume.group(0), ""); assert "sharedAudioContext?.resume?.();" not in unguarded_resume, "no unguarded sharedAudioContext.resume() calls should remain"; assert re.search(r"if\s*\(\s*getIsResumingAudioContext\s*!==\s*null\s*&&\s*!muted\s*\)", playback), "muted playback must not wait on getIsResumingAudioContext()"; assert "getIsResumingAudioContext.then(" in playback, "non-muted playback should still wait for real audio-context resume work"'`: passed (0) -4. Assertion `python3 -c 'from pathlib import Path; import re; shared = Path("packages/core/src/audio/shared-audio-tags.tsx").read_text(); assert "const resumePromise = ctxAndGain.audioContext.resume();" in shared, "resume() should call AudioContext.resume() once and keep its promise"; assert re.search(r"isResuming\.current\s*=\s*new Promise\(\(resolve\)\s*=>\s*{[\s\S]*?resumePromise\.catch\(\(err\)\s*=>\s*{[\s\S]*?Log\.warn\([\s\S]*?resolve\(\);[\s\S]*?}\);[\s\S]*?}\)\.finally", shared), "resume rejection must resolve the shared resume waiter instead of hanging playback"; assert re.search(r"return\s+resumePromise\s*[\s\S]*?\.then\(\(\)\s*=>\s*{[\s\S]*?nodesToResume\.current\.clear\(\);[\s\S]*?}\)\s*[\s\S]*?\.catch\(\(\)\s*=>\s*{", shared), "resume() must swallow AudioContext.resume() rejection after logging"; assert "return ctxAndGain.audioContext.resume().then" not in shared, "returning the raw resume().then() chain can propagate autoplay-policy rejection"'`: passed (0) -5. Assertion `python3 -c 'import subprocess; changed = subprocess.check_output(["git", "diff", "--name-only"], text=True).splitlines(); assert changed == ["packages/core/src/audio/shared-audio-tags.tsx", "packages/player/src/use-playback.ts"], changed'`: passed (0) -6. Assertion `git diff --check`: passed (0) +3. Assertion `git diff --check`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python3 .agentlab_hidden/check_behavior.py`: passed (0) + +```text +check: assert_playback +check: assert_shared_audio +check: assert_changed_files +``` + + +## Grader Notes + +- Remotion full verify-reference was attempted twice and interrupted during large-repo synthetic-base materialization; this reference artifact was regenerated from the focused two-file validation workspace /private/tmp/ael-remotion-mini-123-fix2 using the same reference patch, hidden verifier patch, and public git diff --check command. ## Changed Files diff --git a/tasks/starter/remotion-audio-context-autoplay-muted-001/reference-result.json b/tasks/starter/remotion-audio-context-autoplay-muted-001/reference-result.json index 20fad8c..7ab690e 100644 --- a/tasks/starter/remotion-audio-context-autoplay-muted-001/reference-result.json +++ b/tasks/starter/remotion-audio-context-autoplay-muted-001/reference-result.json @@ -1,4 +1,13 @@ { + "agent_harness_config": { + "agent_harness": "reference", + "model_name": null, + "runtime_accountability": { + "account": null, + "billing_context": null, + "cost_usd": null + } + }, "agent_name": "reference", "cached_input_tokens": null, "checks": [ @@ -16,27 +25,6 @@ "stderr": "", "stdout": "" }, - { - "command": "python3 -c 'from pathlib import Path; import re; playback = Path(\"packages/player/src/use-playback.ts\").read_text(); guarded_resume = re.search(r\"if\\s*\\(\\s*!muted\\s*\\)\\s*{\\s*sharedAudioContext\\?\\.resume\\?\\.\\(\\);\\s*}\", playback); assert guarded_resume, \"sharedAudioContext.resume() must be guarded by !muted\"; unguarded_resume = playback.replace(guarded_resume.group(0), \"\"); assert \"sharedAudioContext?.resume?.();\" not in unguarded_resume, \"no unguarded sharedAudioContext.resume() calls should remain\"; assert re.search(r\"if\\s*\\(\\s*getIsResumingAudioContext\\s*!==\\s*null\\s*&&\\s*!muted\\s*\\)\", playback), \"muted playback must not wait on getIsResumingAudioContext()\"; assert \"getIsResumingAudioContext.then(\" in playback, \"non-muted playback should still wait for real audio-context resume work\"'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python3 -c 'from pathlib import Path; import re; shared = Path(\"packages/core/src/audio/shared-audio-tags.tsx\").read_text(); assert \"const resumePromise = ctxAndGain.audioContext.resume();\" in shared, \"resume() should call AudioContext.resume() once and keep its promise\"; assert re.search(r\"isResuming\\.current\\s*=\\s*new Promise\\(\\(resolve\\)\\s*=>\\s*{[\\s\\S]*?resumePromise\\.catch\\(\\(err\\)\\s*=>\\s*{[\\s\\S]*?Log\\.warn\\([\\s\\S]*?resolve\\(\\);[\\s\\S]*?}\\);[\\s\\S]*?}\\)\\.finally\", shared), \"resume rejection must resolve the shared resume waiter instead of hanging playback\"; assert re.search(r\"return\\s+resumePromise\\s*[\\s\\S]*?\\.then\\(\\(\\)\\s*=>\\s*{[\\s\\S]*?nodesToResume\\.current\\.clear\\(\\);[\\s\\S]*?}\\)\\s*[\\s\\S]*?\\.catch\\(\\(\\)\\s*=>\\s*{\", shared), \"resume() must swallow AudioContext.resume() rejection after logging\"; assert \"return ctxAndGain.audioContext.resume().then\" not in shared, \"returning the raw resume().then() chain can propagate autoplay-policy rejection\"'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "python3 -c 'import subprocess; changed = subprocess.check_output([\"git\", \"diff\", \"--name-only\"], text=True).splitlines(); assert changed == [\"packages/core/src/audio/shared-audio-tags.tsx\", \"packages/player/src/use-playback.ts\"], changed'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, { "command": "git diff --check", "passed": true, @@ -48,9 +36,6 @@ "commands_run": [ "python3 -c 'from pathlib import Path; playback = Path(\"packages/player/src/use-playback.ts\").read_text(); shared = Path(\"packages/core/src/audio/shared-audio-tags.tsx\").read_text(); assert \"sharedAudioContext?.resume?.();\" in playback; assert \"if (!muted)\" not in playback.split(\"sharedAudioContext?.resume?.();\", 1)[0][-80:]; assert \"if (getIsResumingAudioContext !== null && !muted)\" not in playback; assert \"const resumePromise = ctxAndGain.audioContext.resume();\" not in shared; assert \"return ctxAndGain.audioContext.resume().then(() => {\" in shared'", "git apply reference.patch", - "python3 -c 'from pathlib import Path; import re; playback = Path(\"packages/player/src/use-playback.ts\").read_text(); guarded_resume = re.search(r\"if\\s*\\(\\s*!muted\\s*\\)\\s*{\\s*sharedAudioContext\\?\\.resume\\?\\.\\(\\);\\s*}\", playback); assert guarded_resume, \"sharedAudioContext.resume() must be guarded by !muted\"; unguarded_resume = playback.replace(guarded_resume.group(0), \"\"); assert \"sharedAudioContext?.resume?.();\" not in unguarded_resume, \"no unguarded sharedAudioContext.resume() calls should remain\"; assert re.search(r\"if\\s*\\(\\s*getIsResumingAudioContext\\s*!==\\s*null\\s*&&\\s*!muted\\s*\\)\", playback), \"muted playback must not wait on getIsResumingAudioContext()\"; assert \"getIsResumingAudioContext.then(\" in playback, \"non-muted playback should still wait for real audio-context resume work\"'", - "python3 -c 'from pathlib import Path; import re; shared = Path(\"packages/core/src/audio/shared-audio-tags.tsx\").read_text(); assert \"const resumePromise = ctxAndGain.audioContext.resume();\" in shared, \"resume() should call AudioContext.resume() once and keep its promise\"; assert re.search(r\"isResuming\\.current\\s*=\\s*new Promise\\(\\(resolve\\)\\s*=>\\s*{[\\s\\S]*?resumePromise\\.catch\\(\\(err\\)\\s*=>\\s*{[\\s\\S]*?Log\\.warn\\([\\s\\S]*?resolve\\(\\);[\\s\\S]*?}\\);[\\s\\S]*?}\\)\\.finally\", shared), \"resume rejection must resolve the shared resume waiter instead of hanging playback\"; assert re.search(r\"return\\s+resumePromise\\s*[\\s\\S]*?\\.then\\(\\(\\)\\s*=>\\s*{[\\s\\S]*?nodesToResume\\.current\\.clear\\(\\);[\\s\\S]*?}\\)\\s*[\\s\\S]*?\\.catch\\(\\(\\)\\s*=>\\s*{\", shared), \"resume() must swallow AudioContext.resume() rejection after logging\"; assert \"return ctxAndGain.audioContext.resume().then\" not in shared, \"returning the raw resume().then() chain can propagate autoplay-policy rejection\"'", - "python3 -c 'import subprocess; changed = subprocess.check_output([\"git\", \"diff\", \"--name-only\"], text=True).splitlines(); assert changed == [\"packages/core/src/audio/shared-audio-tags.tsx\", \"packages/player/src/use-playback.ts\"], changed'", "git diff --check" ], "cost_usd": null, @@ -76,24 +61,6 @@ "returncode": 0, "type": "code" }, - { - "assertion": "python3 -c 'from pathlib import Path; import re; playback = Path(\"packages/player/src/use-playback.ts\").read_text(); guarded_resume = re.search(r\"if\\s*\\(\\s*!muted\\s*\\)\\s*{\\s*sharedAudioContext\\?\\.resume\\?\\.\\(\\);\\s*}\", playback); assert guarded_resume, \"sharedAudioContext.resume() must be guarded by !muted\"; unguarded_resume = playback.replace(guarded_resume.group(0), \"\"); assert \"sharedAudioContext?.resume?.();\" not in unguarded_resume, \"no unguarded sharedAudioContext.resume() calls should remain\"; assert re.search(r\"if\\s*\\(\\s*getIsResumingAudioContext\\s*!==\\s*null\\s*&&\\s*!muted\\s*\\)\", playback), \"muted playback must not wait on getIsResumingAudioContext()\"; assert \"getIsResumingAudioContext.then(\" in playback, \"non-muted playback should still wait for real audio-context resume work\"'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python3 -c 'from pathlib import Path; import re; shared = Path(\"packages/core/src/audio/shared-audio-tags.tsx\").read_text(); assert \"const resumePromise = ctxAndGain.audioContext.resume();\" in shared, \"resume() should call AudioContext.resume() once and keep its promise\"; assert re.search(r\"isResuming\\.current\\s*=\\s*new Promise\\(\\(resolve\\)\\s*=>\\s*{[\\s\\S]*?resumePromise\\.catch\\(\\(err\\)\\s*=>\\s*{[\\s\\S]*?Log\\.warn\\([\\s\\S]*?resolve\\(\\);[\\s\\S]*?}\\);[\\s\\S]*?}\\)\\.finally\", shared), \"resume rejection must resolve the shared resume waiter instead of hanging playback\"; assert re.search(r\"return\\s+resumePromise\\s*[\\s\\S]*?\\.then\\(\\(\\)\\s*=>\\s*{[\\s\\S]*?nodesToResume\\.current\\.clear\\(\\);[\\s\\S]*?}\\)\\s*[\\s\\S]*?\\.catch\\(\\(\\)\\s*=>\\s*{\", shared), \"resume() must swallow AudioContext.resume() rejection after logging\"; assert \"return ctxAndGain.audioContext.resume().then\" not in shared, \"returning the raw resume().then() chain can propagate autoplay-policy rejection\"'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "python3 -c 'import subprocess; changed = subprocess.check_output([\"git\", \"diff\", \"--name-only\"], text=True).splitlines(); assert changed == [\"packages/core/src/audio/shared-audio-tags.tsx\", \"packages/player/src/use-playback.ts\"], changed'", - "passed": true, - "returncode": 0, - "type": "code" - }, { "assertion": "git diff --check", "passed": true, @@ -101,6 +68,27 @@ "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python3 .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "check: assert_playback\ncheck: assert_shared_audio\ncheck: assert_changed_files\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 30, "lines_deleted": 15, @@ -134,10 +122,30 @@ }, "run_dir": ".", "run_id": "remotion-audio-context-autoplay-muted-001-reference", - "score_notes": [], + "run_surface": { + "approval_policy": "unknown", + "execution_surface": "unknown", + "human_intervention_events": [], + "memory_scope": "unknown", + "model_identity_source": "unknown", + "network_policy": "unknown", + "runtime_version": "unknown", + "sandbox_mode": "unknown", + "stop_reason": "success", + "timeout_seconds": "unknown", + "tool_policy": "unknown", + "turn_or_step_budget": "unknown", + "workspace_base_ref": "focused-two-file-validation", + "workspace_history_policy": "focused_substitute" + }, + "score_notes": [ + "Remotion full verify-reference was attempted twice and interrupted during large-repo synthetic-base materialization; this reference artifact was regenerated from the focused two-file validation workspace /private/tmp/ael-remotion-mini-123-fix2 using the same reference patch, hidden verifier patch, and public git diff --check command." + ], "status": "passed", "success": true, + "task_commit": "631ce3d2ab757e44cfb542a108a21f1e9a9c6c0f", "task_id": "remotion-audio-context-autoplay-muted-001", + "task_repo": "https://github.com/remotion-dev/remotion.git", "task_title": "Remotion Player muted playback should not wait on AudioContext resume", "transcript_path": null, "trial_id": "remotion-audio-context-autoplay-muted-001-reference", diff --git a/tasks/starter/remotion-audio-context-autoplay-muted-001/task-card.md b/tasks/starter/remotion-audio-context-autoplay-muted-001/task-card.md index 33a603f..435a608 100644 --- a/tasks/starter/remotion-audio-context-autoplay-muted-001/task-card.md +++ b/tasks/starter/remotion-audio-context-autoplay-muted-001/task-card.md @@ -26,6 +26,11 @@ In `shared-audio-tags.tsx`, store the AudioContext resume promise, resolve the s No task-local environment configured. +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -38,9 +43,6 @@ None configured. ### Target -- `python3 -c 'from pathlib import Path; import re; playback = Path("packages/player/src/use-playback.ts").read_text(); guarded_resume = re.search(r"if\s*\(\s*!muted\s*\)\s*{\s*sharedAudioContext\?\.resume\?\.\(\);\s*}", playback); assert guarded_resume, "sharedAudioContext.resume() must be guarded by !muted"; unguarded_resume = playback.replace(guarded_resume.group(0), ""); assert "sharedAudioContext?.resume?.();" not in unguarded_resume, "no unguarded sharedAudioContext.resume() calls should remain"; waits_only_when_unmuted = re.search(r"if\s*\(\s*getIsResumingAudioContext\s*!==\s*null\s*&&\s*!muted\s*\)", playback) or re.search(r"const\s+getIsResumingAudioContext\s*=\s*muted\s*\?\s*null\s*:", playback); assert waits_only_when_unmuted, "muted playback must not wait on getIsResumingAudioContext()"; assert "getIsResumingAudioContext.then(" in playback, "non-muted playback should still wait for real audio-context resume work"'` -- `python3 -c 'from pathlib import Path; import re; shared = Path("packages/core/src/audio/shared-audio-tags.tsx").read_text(); assert re.search(r"ctxAndGain\.audioContext\s*\.?\s*resume\(\)", shared), "resume() should still call AudioContext.resume()"; assert re.search(r"ctxAndGain\.audioContext\s*\.?[\s\S]*?resume\(\)[\s\S]*?\.catch\(\(?err", shared) or re.search(r"resumePromise\.catch\(\(?err", shared), "AudioContext.resume() rejection must be caught"; assert "Log.warn" in shared or "Log.verbose" in shared, "resume rejection should be logged"; assert re.search(r"isResuming\.current[\s\S]*?finally\(\(\)\s*=>\s*{[\s\S]*?isResuming\.current\s*=\s*null", shared), "resume waiter should be cleared"; assert "return ctxAndGain.audioContext.resume().then" not in shared, "returning the raw resume().then() chain can propagate autoplay-policy rejection"'` -- `python3 -c 'import subprocess; changed = subprocess.check_output(["git", "diff", "--name-only"], text=True).splitlines(); assert changed == ["packages/core/src/audio/shared-audio-tags.tsx", "packages/player/src/use-playback.ts"], changed'` - `git diff --check` ## Success Criteria diff --git a/tasks/starter/remotion-audio-context-autoplay-muted-001/task.yaml b/tasks/starter/remotion-audio-context-autoplay-muted-001/task.yaml index 9a9a5fd..d3fe21a 100644 --- a/tasks/starter/remotion-audio-context-autoplay-muted-001/task.yaml +++ b/tasks/starter/remotion-audio-context-autoplay-muted-001/task.yaml @@ -25,12 +25,13 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python3 .agentlab_hidden/check_behavior.py baseline: - python3 -c 'from pathlib import Path; playback = Path("packages/player/src/use-playback.ts").read_text(); shared = Path("packages/core/src/audio/shared-audio-tags.tsx").read_text(); assert "sharedAudioContext?.resume?.();" in playback; assert "if (!muted)" not in playback.split("sharedAudioContext?.resume?.();", 1)[0][-80:]; assert "if (getIsResumingAudioContext !== null && !muted)" not in playback; assert "const resumePromise = ctxAndGain.audioContext.resume();" not in shared; assert "return ctxAndGain.audioContext.resume().then(() => {" in shared' test: - - python3 -c 'from pathlib import Path; import re; playback = Path("packages/player/src/use-playback.ts").read_text(); guarded_resume = re.search(r"if\s*\(\s*!muted\s*\)\s*{\s*sharedAudioContext\?\.resume\?\.\(\);\s*}", playback); assert guarded_resume, "sharedAudioContext.resume() must be guarded by !muted"; unguarded_resume = playback.replace(guarded_resume.group(0), ""); assert "sharedAudioContext?.resume?.();" not in unguarded_resume, "no unguarded sharedAudioContext.resume() calls should remain"; waits_only_when_unmuted = re.search(r"if\s*\(\s*getIsResumingAudioContext\s*!==\s*null\s*&&\s*!muted\s*\)", playback) or re.search(r"const\s+getIsResumingAudioContext\s*=\s*muted\s*\?\s*null\s*:", playback); assert waits_only_when_unmuted, "muted playback must not wait on getIsResumingAudioContext()"; assert "getIsResumingAudioContext.then(" in playback, "non-muted playback should still wait for real audio-context resume work"' - - python3 -c 'from pathlib import Path; import re; shared = Path("packages/core/src/audio/shared-audio-tags.tsx").read_text(); assert re.search(r"ctxAndGain\.audioContext\s*\.?\s*resume\(\)", shared), "resume() should still call AudioContext.resume()"; assert re.search(r"ctxAndGain\.audioContext\s*\.?[\s\S]*?resume\(\)[\s\S]*?\.catch\(\(?err", shared) or re.search(r"resumePromise\.catch\(\(?err", shared), "AudioContext.resume() rejection must be caught"; assert "Log.warn" in shared or "Log.verbose" in shared, "resume rejection should be logged"; assert re.search(r"isResuming\.current[\s\S]*?finally\(\(\)\s*=>\s*{[\s\S]*?isResuming\.current\s*=\s*null", shared), "resume waiter should be cleared"; assert "return ctxAndGain.audioContext.resume().then" not in shared, "returning the raw resume().then() chain can propagate autoplay-policy rejection"' - - python3 -c 'import subprocess; changed = subprocess.check_output(["git", "diff", "--name-only"], text=True).splitlines(); assert changed == ["packages/core/src/audio/shared-audio-tags.tsx", "packages/player/src/use-playback.ts"], changed' - git diff --check success: tests_must_pass: true diff --git a/tasks/starter/remotion-audio-context-autoplay-muted-001/verifier.patch b/tasks/starter/remotion-audio-context-autoplay-muted-001/verifier.patch new file mode 100644 index 0000000..1517b34 --- /dev/null +++ b/tasks/starter/remotion-audio-context-autoplay-muted-001/verifier.patch @@ -0,0 +1,100 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,84 @@ ++from __future__ import annotations ++ ++import re ++import subprocess ++import sys ++from pathlib import Path ++ ++ ++def assert_playback() -> None: ++ playback = Path("packages/player/src/use-playback.ts").read_text() ++ guarded_resume = re.search( ++ r"if\s*\(\s*!muted\s*\)\s*{\s*sharedAudioContext\?\.resume\?\.\(\);\s*}", ++ playback, ++ ) ++ assert guarded_resume, "sharedAudioContext.resume() must be guarded by !muted" ++ unguarded_resume = playback.replace(guarded_resume.group(0), "") ++ assert "sharedAudioContext?.resume?.();" not in unguarded_resume, ( ++ "no unguarded sharedAudioContext.resume() calls should remain" ++ ) ++ waits_only_when_unmuted = re.search( ++ r"if\s*\(\s*getIsResumingAudioContext\s*!==\s*null\s*&&\s*!muted\s*\)", ++ playback, ++ ) or re.search( ++ r"const\s+getIsResumingAudioContext\s*=\s*muted\s*\?\s*null\s*:", ++ playback, ++ ) ++ assert waits_only_when_unmuted, ( ++ "muted playback must not wait on getIsResumingAudioContext()" ++ ) ++ assert "getIsResumingAudioContext.then(" in playback, ( ++ "non-muted playback should still wait for real audio-context resume work" ++ ) ++ ++ ++def assert_shared_audio() -> None: ++ shared = Path("packages/core/src/audio/shared-audio-tags.tsx").read_text() ++ assert re.search(r"ctxAndGain\.audioContext\s*\.?\s*resume\(\)", shared), ( ++ "resume() should still call AudioContext.resume()" ++ ) ++ waiter_body = re.search( ++ r"isResuming\.current\s*=\s*new\s+Promise(?:<[^>]+>)?\s*\(\s*\(\s*resolve\s*\)\s*=>\s*{(?P[\s\S]*?)}\s*\)\.finally", ++ shared, ++ ) ++ assert waiter_body, "isResuming.current should be assigned a shared resume waiter" ++ body = waiter_body.group("body") ++ assert re.search( ++ r"waitUntilActuallyResumed\([\s\S]*?\)\.then\(\s*resolve\s*\)", ++ body, ++ ), "the shared waiter should resolve when AudioContext actually resumes" ++ catch_body = re.search( ++ r"resumePromise\.catch\(\s*\(?\s*\w*\s*\)?\s*=>\s*{(?P[\s\S]*?)\n\s*}\s*\)", ++ body, ++ ) ++ catch_resolve = re.search(r"resumePromise\.catch\(\s*resolve\s*\)", body) ++ assert catch_resolve or (catch_body and "resolve()" in catch_body.group("body")), ( ++ "AudioContext.resume() rejection must resolve the same shared resume waiter" ++ ) ++ assert "Log.warn" in shared or "Log.verbose" in shared, ( ++ "resume rejection should be logged" ++ ) ++ assert re.search( ++ r"isResuming\.current[\s\S]*?finally\(\(\)\s*=>\s*{[\s\S]*?isResuming\.current\s*=\s*null", ++ shared, ++ ), "resume waiter should be cleared" ++ assert "return ctxAndGain.audioContext.resume().then" not in shared, ( ++ "returning the raw resume().then() chain can propagate autoplay-policy rejection" ++ ) ++ ++ ++def assert_changed_files() -> None: ++ changed = subprocess.check_output( ++ ["git", "diff", "--name-only"], ++ text=True, ++ ).splitlines() ++ assert changed == [ ++ "packages/core/src/audio/shared-audio-tags.tsx", ++ "packages/player/src/use-playback.ts", ++ ], changed ++ ++ ++def main() -> int: ++ checks = [assert_playback, assert_shared_audio, assert_changed_files] ++ for check in checks: ++ print(f"check: {check.__name__}") ++ try: ++ check() ++ except Exception as exc: ++ print(f"{check.__name__} failed: {exc}", file=sys.stderr) ++ return 1 ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/todomvc-toggle-all-checkbox-001/reference-report.md b/tasks/starter/todomvc-toggle-all-checkbox-001/reference-report.md index a6da889..8ed6df2 100644 --- a/tasks/starter/todomvc-toggle-all-checkbox-001/reference-report.md +++ b/tasks/starter/todomvc-toggle-all-checkbox-001/reference-report.md @@ -4,6 +4,8 @@ - Agent harness: `reference` - Evaluation suite: `starter-coding` - Evaluation type: `regression` +- Task repository: `https://github.com/tastejs/todomvc.git` +- Task commit: `ff43b02e59dfa604386bb382034b2cd07c2bcd8a` - Reference artifact: patch `reference.patch` - Status: `passed` - Outcome: `passed` @@ -11,7 +13,24 @@ - Lines added: `6` - Lines deleted: `8` -## Code-Based Graders +## Run Surface + +- Execution surface: `unknown` +- Runtime version: `unknown` +- Model identity source: `unknown` +- Sandbox mode: `unknown` +- Approval policy: `unknown` +- Tool policy: `unknown` +- Memory scope: `unknown` +- Network policy: `unknown` +- Timeout seconds: `unknown` +- Turn or step budget: `unknown` +- Stop reason: `success` +- Human intervention events: `none` +- Workspace history policy: `base_only` +- Workspace base ref: `f3b616ee76c94bea59cb1f04d68cd1bd43e183d7` + +## Public Graders 1. Assertion `node --version`: passed (0) @@ -22,8 +41,19 @@ v24.5.0 2. Assertion `node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.checked=!this.checked;this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,false,path); assert.strictEqual(elements[".toggle-all-label"].checked,true,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all-label"],path); assert.strictEqual(events[0].type,"click",path); events[0].handler.call(elements[".toggle-all-label"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,1,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");'`: passed (0) 3. Assertion `node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(!/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}'`: passed (0) 4. Assertion `git apply reference.patch`: passed (0) -5. Assertion `node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,true,path); assert.strictEqual(elements[".toggle-all-label"].checked,undefined,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all"],path); assert.strictEqual(events[0].type,"change",path); elements[".toggle-all"].checked=false; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,false,path); elements[".toggle-all"].checked=true; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,undefined,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");'`: passed (0) -6. Assertion `node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}'`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python3 .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,true,path); assert.strictEqual(elements[".toggle-all-label"].checked,undefined,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all"],path); assert.strictEqual(events[0].type,"change",path); elements[".toggle-all"].checked=false; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,false,path); elements[".toggle-all"].checked=true; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,undefined,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");' +$ node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}' +``` + ## Changed Files diff --git a/tasks/starter/todomvc-toggle-all-checkbox-001/reference-result.json b/tasks/starter/todomvc-toggle-all-checkbox-001/reference-result.json index de41236..4d640b8 100644 --- a/tasks/starter/todomvc-toggle-all-checkbox-001/reference-result.json +++ b/tasks/starter/todomvc-toggle-all-checkbox-001/reference-result.json @@ -1,4 +1,13 @@ { + "agent_harness_config": { + "agent_harness": "reference", + "model_name": null, + "runtime_accountability": { + "account": null, + "billing_context": null, + "cost_usd": null + } + }, "agent_name": "reference", "cached_input_tokens": null, "checks": [ @@ -29,29 +38,13 @@ "returncode": 0, "stderr": "", "stdout": "" - }, - { - "command": "node -e 'const fs=require(\"fs\"),vm=require(\"vm\"),assert=require(\"assert\"); function exercise(path){const elements={\".todo-list\":{},\".todo-count\":{},\".clear-completed\":{},\".main\":{style:{}},\".footer\":{style:{}},\".toggle-all\":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},\".toggle-all-label\":{},\".new-todo\":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:\"1\"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,\"utf8\"),context); const view=new context.app.View({}); view.render(\"toggleAll\",{checked:true}); assert.strictEqual(elements[\".toggle-all\"].checked,true,path); assert.strictEqual(elements[\".toggle-all-label\"].checked,undefined,path); let payload; view.bind(\"toggleAll\",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[\".toggle-all\"],path); assert.strictEqual(events[0].type,\"change\",path); elements[\".toggle-all\"].checked=false; events[0].handler.call(elements[\".toggle-all\"]); assert.strictEqual(payload.completed,false,path); elements[\".toggle-all\"].checked=true; events[0].handler.call(elements[\".toggle-all\"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[\".toggle-all\"].clicked,undefined,path);} exercise(\"examples/javascript-es5/src/view.js\"); exercise(\"examples/javascript-es5/dist/view.js\");'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "node -e 'const fs=require(\"fs\"),assert=require(\"assert\"); for (const path of [\"examples/javascript-es5/index.html\",\"examples/javascript-es5/dist/index.html\"]){const html=fs.readFileSync(path,\"utf8\"); const input=html.match(/]*class=\"toggle-all\"[^>]*>/); assert(input,path); assert(/\\bid=\"toggle-all\"/.test(input[0]),path); assert(/]*class=\"toggle-all-label\"[^>]*for=\"toggle-all\"/.test(html),path);}'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" } ], "commands_run": [ "node --version", "node -e 'const fs=require(\"fs\"),vm=require(\"vm\"),assert=require(\"assert\"); function exercise(path){const elements={\".todo-list\":{},\".todo-count\":{},\".clear-completed\":{},\".main\":{style:{}},\".footer\":{style:{}},\".toggle-all\":{checked:false,click(){this.checked=!this.checked;this.clicked=(this.clicked||0)+1;}},\".toggle-all-label\":{},\".new-todo\":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:\"1\"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,\"utf8\"),context); const view=new context.app.View({}); view.render(\"toggleAll\",{checked:true}); assert.strictEqual(elements[\".toggle-all\"].checked,false,path); assert.strictEqual(elements[\".toggle-all-label\"].checked,true,path); let payload; view.bind(\"toggleAll\",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[\".toggle-all-label\"],path); assert.strictEqual(events[0].type,\"click\",path); events[0].handler.call(elements[\".toggle-all-label\"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[\".toggle-all\"].clicked,1,path);} exercise(\"examples/javascript-es5/src/view.js\"); exercise(\"examples/javascript-es5/dist/view.js\");'", "node -e 'const fs=require(\"fs\"),assert=require(\"assert\"); for (const path of [\"examples/javascript-es5/index.html\",\"examples/javascript-es5/dist/index.html\"]){const html=fs.readFileSync(path,\"utf8\"); const input=html.match(/]*class=\"toggle-all\"[^>]*>/); assert(input,path); assert(!/\\bid=\"toggle-all\"/.test(input[0]),path); assert(/]*class=\"toggle-all-label\"[^>]*for=\"toggle-all\"/.test(html),path);}'", - "git apply reference.patch", - "node -e 'const fs=require(\"fs\"),vm=require(\"vm\"),assert=require(\"assert\"); function exercise(path){const elements={\".todo-list\":{},\".todo-count\":{},\".clear-completed\":{},\".main\":{style:{}},\".footer\":{style:{}},\".toggle-all\":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},\".toggle-all-label\":{},\".new-todo\":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:\"1\"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,\"utf8\"),context); const view=new context.app.View({}); view.render(\"toggleAll\",{checked:true}); assert.strictEqual(elements[\".toggle-all\"].checked,true,path); assert.strictEqual(elements[\".toggle-all-label\"].checked,undefined,path); let payload; view.bind(\"toggleAll\",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[\".toggle-all\"],path); assert.strictEqual(events[0].type,\"change\",path); elements[\".toggle-all\"].checked=false; events[0].handler.call(elements[\".toggle-all\"]); assert.strictEqual(payload.completed,false,path); elements[\".toggle-all\"].checked=true; events[0].handler.call(elements[\".toggle-all\"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[\".toggle-all\"].clicked,undefined,path);} exercise(\"examples/javascript-es5/src/view.js\"); exercise(\"examples/javascript-es5/dist/view.js\");'", - "node -e 'const fs=require(\"fs\"),assert=require(\"assert\"); for (const path of [\"examples/javascript-es5/index.html\",\"examples/javascript-es5/dist/index.html\"]){const html=fs.readFileSync(path,\"utf8\"); const input=html.match(/]*class=\"toggle-all\"[^>]*>/); assert(input,path); assert(/\\bid=\"toggle-all\"/.test(input[0]),path); assert(/]*class=\"toggle-all-label\"[^>]*for=\"toggle-all\"/.test(html),path);}'" + "git apply reference.patch" ], "cost_usd": null, "diff_path": "reference.diff", @@ -89,20 +82,29 @@ "passed": true, "returncode": 0, "type": "code" - }, - { - "assertion": "node -e 'const fs=require(\"fs\"),vm=require(\"vm\"),assert=require(\"assert\"); function exercise(path){const elements={\".todo-list\":{},\".todo-count\":{},\".clear-completed\":{},\".main\":{style:{}},\".footer\":{style:{}},\".toggle-all\":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},\".toggle-all-label\":{},\".new-todo\":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:\"1\"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,\"utf8\"),context); const view=new context.app.View({}); view.render(\"toggleAll\",{checked:true}); assert.strictEqual(elements[\".toggle-all\"].checked,true,path); assert.strictEqual(elements[\".toggle-all-label\"].checked,undefined,path); let payload; view.bind(\"toggleAll\",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[\".toggle-all\"],path); assert.strictEqual(events[0].type,\"change\",path); elements[\".toggle-all\"].checked=false; events[0].handler.call(elements[\".toggle-all\"]); assert.strictEqual(payload.completed,false,path); elements[\".toggle-all\"].checked=true; events[0].handler.call(elements[\".toggle-all\"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[\".toggle-all\"].clicked,undefined,path);} exercise(\"examples/javascript-es5/src/view.js\"); exercise(\"examples/javascript-es5/dist/view.js\");'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "node -e 'const fs=require(\"fs\"),assert=require(\"assert\"); for (const path of [\"examples/javascript-es5/index.html\",\"examples/javascript-es5/dist/index.html\"]){const html=fs.readFileSync(path,\"utf8\"); const input=html.match(/]*class=\"toggle-all\"[^>]*>/); assert(input,path); assert(/\\bid=\"toggle-all\"/.test(input[0]),path); assert(/]*class=\"toggle-all-label\"[^>]*for=\"toggle-all\"/.test(html),path);}'", - "passed": true, - "returncode": 0, - "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python3 .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ node -e 'const fs=require(\"fs\"),vm=require(\"vm\"),assert=require(\"assert\"); function exercise(path){const elements={\".todo-list\":{},\".todo-count\":{},\".clear-completed\":{},\".main\":{style:{}},\".footer\":{style:{}},\".toggle-all\":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},\".toggle-all-label\":{},\".new-todo\":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:\"1\"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,\"utf8\"),context); const view=new context.app.View({}); view.render(\"toggleAll\",{checked:true}); assert.strictEqual(elements[\".toggle-all\"].checked,true,path); assert.strictEqual(elements[\".toggle-all-label\"].checked,undefined,path); let payload; view.bind(\"toggleAll\",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[\".toggle-all\"],path); assert.strictEqual(events[0].type,\"change\",path); elements[\".toggle-all\"].checked=false; events[0].handler.call(elements[\".toggle-all\"]); assert.strictEqual(payload.completed,false,path); elements[\".toggle-all\"].checked=true; events[0].handler.call(elements[\".toggle-all\"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[\".toggle-all\"].clicked,undefined,path);} exercise(\"examples/javascript-es5/src/view.js\"); exercise(\"examples/javascript-es5/dist/view.js\");'\n$ node -e 'const fs=require(\"fs\"),assert=require(\"assert\"); for (const path of [\"examples/javascript-es5/index.html\",\"examples/javascript-es5/dist/index.html\"]){const html=fs.readFileSync(path,\"utf8\"); const input=html.match(/]*class=\"toggle-all\"[^>]*>/); assert(input,path); assert(/\\bid=\"toggle-all\"/.test(input[0]),path); assert(/]*class=\"toggle-all-label\"[^>]*for=\"toggle-all\"/.test(html),path);}'\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 6, "lines_deleted": 8, @@ -138,10 +140,28 @@ }, "run_dir": ".", "run_id": "todomvc-toggle-all-checkbox-001-reference", + "run_surface": { + "approval_policy": "unknown", + "execution_surface": "unknown", + "human_intervention_events": [], + "memory_scope": "unknown", + "model_identity_source": "unknown", + "network_policy": "unknown", + "runtime_version": "unknown", + "sandbox_mode": "unknown", + "stop_reason": "success", + "timeout_seconds": "unknown", + "tool_policy": "unknown", + "turn_or_step_budget": "unknown", + "workspace_base_ref": "f3b616ee76c94bea59cb1f04d68cd1bd43e183d7", + "workspace_history_policy": "base_only" + }, "score_notes": [], "status": "passed", "success": true, + "task_commit": "ff43b02e59dfa604386bb382034b2cd07c2bcd8a", "task_id": "todomvc-toggle-all-checkbox-001", + "task_repo": "https://github.com/tastejs/todomvc.git", "task_title": "TodoMVC toggle-all checkbox should work directly", "transcript_path": null, "trial_id": "todomvc-toggle-all-checkbox-001-reference", diff --git a/tasks/starter/todomvc-toggle-all-checkbox-001/task-card.md b/tasks/starter/todomvc-toggle-all-checkbox-001/task-card.md index e1ff3de..5014b79 100644 --- a/tasks/starter/todomvc-toggle-all-checkbox-001/task-card.md +++ b/tasks/starter/todomvc-toggle-all-checkbox-001/task-card.md @@ -26,6 +26,11 @@ Add id="toggle-all" to the JavaScript ES5 toggle-all checkbox in both index.html No task-local environment configured. +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -39,8 +44,7 @@ No task-local environment configured. ### Target -- `node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,true,path); assert.strictEqual(elements[".toggle-all-label"].checked,undefined,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all"],path); assert.strictEqual(events[0].type,"change",path); elements[".toggle-all"].checked=false; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,false,path); elements[".toggle-all"].checked=true; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,undefined,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");'` -- `node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}'` +None configured. ## Success Criteria diff --git a/tasks/starter/todomvc-toggle-all-checkbox-001/task.yaml b/tasks/starter/todomvc-toggle-all-checkbox-001/task.yaml index 366a2e3..2f2582d 100644 --- a/tasks/starter/todomvc-toggle-all-checkbox-001/task.yaml +++ b/tasks/starter/todomvc-toggle-all-checkbox-001/task.yaml @@ -21,14 +21,15 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python3 .agentlab_hidden/check_behavior.py setup: - node --version baseline: - node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.checked=!this.checked;this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,false,path); assert.strictEqual(elements[".toggle-all-label"].checked,true,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all-label"],path); assert.strictEqual(events[0].type,"click",path); events[0].handler.call(elements[".toggle-all-label"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,1,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");' - node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(!/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}' -test: - - node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,true,path); assert.strictEqual(elements[".toggle-all-label"].checked,undefined,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all"],path); assert.strictEqual(events[0].type,"change",path); elements[".toggle-all"].checked=false; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,false,path); elements[".toggle-all"].checked=true; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,undefined,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");' - - node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}' success: tests_must_pass: true max_files_changed: 4 diff --git a/tasks/starter/todomvc-toggle-all-checkbox-001/verifier.patch b/tasks/starter/todomvc-toggle-all-checkbox-001/verifier.patch new file mode 100644 index 0000000..fb8a123 --- /dev/null +++ b/tasks/starter/todomvc-toggle-all-checkbox-001/verifier.patch @@ -0,0 +1,37 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,31 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = ['node -e \'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,true,path); assert.strictEqual(elements[".toggle-all-label"].checked,undefined,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all"],path); assert.strictEqual(events[0].type,"change",path); elements[".toggle-all"].checked=false; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,false,path); elements[".toggle-all"].checked=true; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,undefined,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");\'', 'node -e \'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(/\\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}\''] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print(f"$ {command}") ++ completed = subprocess.run( ++ command, ++ shell=True, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) diff --git a/tasks/starter/vite-deno-workspace-root-001/reference-report.md b/tasks/starter/vite-deno-workspace-root-001/reference-report.md index d331965..d191c0d 100644 --- a/tasks/starter/vite-deno-workspace-root-001/reference-report.md +++ b/tasks/starter/vite-deno-workspace-root-001/reference-report.md @@ -4,6 +4,8 @@ - Agent harness: `reference` - Evaluation suite: `starter-coding` - Evaluation type: `regression` +- Task repository: `https://github.com/vitejs/vite.git` +- Task commit: `dfc8aa5057dd8ec2b1223980d1e2eeb946ac3384` - Reference artifact: patch `reference.patch` - Status: `passed` - Outcome: `passed` @@ -11,7 +13,24 @@ - Lines added: `39` - Lines deleted: `0` -## Code-Based Graders +## Run Surface + +- Execution surface: `unknown` +- Runtime version: `unknown` +- Model identity source: `unknown` +- Sandbox mode: `unknown` +- Approval policy: `unknown` +- Tool policy: `unknown` +- Memory scope: `unknown` +- Network policy: `unknown` +- Timeout seconds: `unknown` +- Turn or step budget: `unknown` +- Stop reason: `success` +- Human intervention events: `none` +- Workspace history policy: `base_only` +- Workspace base ref: `3bf060a95d87c3a78d9506bdfb443ad8a848fcb4` + +## Public Graders 1. Assertion `node --version`: passed (0) @@ -21,10 +40,21 @@ v24.5.0 2. Assertion `node -e 'const fs=require("node:fs"),os=require("node:os"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:["nested"]}));write(path.join(root,"nested/package.json"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),"vite-deno-root-baseline-")),denoJsonRoot=fixture(tmp,"deno-json","deno.json");assert.notEqual(searchForWorkspaceRoot(path.join(denoJsonRoot,"nested")),denoJsonRoot);const denoJsoncRoot=fixture(tmp,"deno-jsonc","deno.jsonc");assert.notEqual(searchForWorkspaceRoot(path.join(denoJsoncRoot,"nested")),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,"package-workspace");write(path.join(packageWorkspaceRoot,"package.json"),JSON.stringify({workspaces:["nested"]}));write(path.join(packageWorkspaceRoot,"nested/package.json"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,"nested")),packageWorkspaceRoot)'`: passed (0) 3. Assertion `git apply reference.patch`: passed (0) -4. Assertion `node -e 'const fs=require("node:fs"),os=require("node:os"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:["nested"]}));write(path.join(root,"nested/package.json"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),"vite-deno-root-test-")),denoJsonRoot=fixture(tmp,"deno-json","deno.json");assert.equal(searchForWorkspaceRoot(path.join(denoJsonRoot,"nested")),denoJsonRoot);assert.equal(searchForWorkspaceRoot(denoJsonRoot),denoJsonRoot);const denoJsoncRoot=fixture(tmp,"deno-jsonc","deno.jsonc");assert.equal(searchForWorkspaceRoot(path.join(denoJsoncRoot,"nested")),denoJsoncRoot);assert.equal(searchForWorkspaceRoot(denoJsoncRoot),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,"package-workspace");write(path.join(packageWorkspaceRoot,"package.json"),JSON.stringify({workspaces:["nested"]}));write(path.join(packageWorkspaceRoot,"nested/package.json"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,"nested")),packageWorkspaceRoot)'`: passed (0) -5. Assertion `node -e 'const fs=require("node:fs"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");const specPath="packages/vite/src/node/server/__tests__/search-root.spec.ts",dirname=path.dirname(specPath),spec=fs.readFileSync(specPath,"utf8");function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),q=String.fromCharCode(39),pattern=new RegExp("searchForWorkspaceRoot\\(\\s*resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)[\\s\\S]*?expect\\(resolved\\)\\.toBe\\(resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)\\)","g"),matches=[...spec.matchAll(pattern)];assert(matches.length>=5,"expected focused search-root.spec.ts cases");for(const match of matches){const input=path.resolve(dirname,match[1]),expected=path.resolve(dirname,match[2]);assert(fs.existsSync(input),`missing fixture path referenced by search-root.spec.ts: ${match[1]}`);assert(fs.existsSync(expected),`missing expected fixture path referenced by search-root.spec.ts: ${match[2]}`);assert.equal(searchForWorkspaceRoot(input),expected,`search-root.spec.ts expectation failed for ${match[1]}`)}'`: passed (0) -6. Assertion `git add -N -- packages/vite/src/node/server/__tests__/fixtures`: passed (0) -7. Assertion `git diff --check`: passed (0) +4. Assertion `git add -N -- packages/vite/src/node/server/__tests__/fixtures`: passed (0) +5. Assertion `git diff --check`: passed (0) + +## Hidden Verifier + +- Patch: `verifier.patch` + +1. Assertion `git apply hidden verifier patch: verifier.patch`: passed (0) +2. Assertion `python3 .agentlab_hidden/check_behavior.py`: passed (0) + +```text +$ node .agentlab_hidden/check_deno_workspace_root.cjs +$ node .agentlab_hidden/check_search_root_spec.cjs +``` + ## Changed Files diff --git a/tasks/starter/vite-deno-workspace-root-001/reference-result.json b/tasks/starter/vite-deno-workspace-root-001/reference-result.json index 70397d2..fbb9a5b 100644 --- a/tasks/starter/vite-deno-workspace-root-001/reference-result.json +++ b/tasks/starter/vite-deno-workspace-root-001/reference-result.json @@ -1,4 +1,13 @@ { + "agent_harness_config": { + "agent_harness": "reference", + "model_name": null, + "runtime_accountability": { + "account": null, + "billing_context": null, + "cost_usd": null + } + }, "agent_name": "reference", "cached_input_tokens": null, "checks": [ @@ -23,20 +32,6 @@ "stderr": "", "stdout": "" }, - { - "command": "node -e 'const fs=require(\"node:fs\"),os=require(\"node:os\"),path=require(\"node:path\"),vm=require(\"node:vm\"),assert=require(\"node:assert/strict\");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:[\"nested\"]}));write(path.join(root,\"nested/package.json\"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync(\"packages/vite/src/node/server/searchRoot.ts\",\"utf8\").split(\"\\n\").filter((line)=>!line.startsWith(\"import \")).join(\"\\n\");source=source.replace(/export function /g,\"function \").replace(/\\): [A-Za-z][A-Za-z0-9_<>, ]+/g,\")\").replace(/(\\w+): string/g,\"$1\");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+\"\\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot\",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),\"vite-deno-root-test-\")),denoJsonRoot=fixture(tmp,\"deno-json\",\"deno.json\");assert.equal(searchForWorkspaceRoot(path.join(denoJsonRoot,\"nested\")),denoJsonRoot);assert.equal(searchForWorkspaceRoot(denoJsonRoot),denoJsonRoot);const denoJsoncRoot=fixture(tmp,\"deno-jsonc\",\"deno.jsonc\");assert.equal(searchForWorkspaceRoot(path.join(denoJsoncRoot,\"nested\")),denoJsoncRoot);assert.equal(searchForWorkspaceRoot(denoJsoncRoot),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,\"package-workspace\");write(path.join(packageWorkspaceRoot,\"package.json\"),JSON.stringify({workspaces:[\"nested\"]}));write(path.join(packageWorkspaceRoot,\"nested/package.json\"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,\"nested\")),packageWorkspaceRoot)'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, - { - "command": "node -e 'const fs=require(\"node:fs\"),path=require(\"node:path\"),vm=require(\"node:vm\"),assert=require(\"node:assert/strict\");const specPath=\"packages/vite/src/node/server/__tests__/search-root.spec.ts\",dirname=path.dirname(specPath),spec=fs.readFileSync(specPath,\"utf8\");function load(){let source=fs.readFileSync(\"packages/vite/src/node/server/searchRoot.ts\",\"utf8\").split(\"\\n\").filter((line)=>!line.startsWith(\"import \")).join(\"\\n\");source=source.replace(/export function /g,\"function \").replace(/\\): [A-Za-z][A-Za-z0-9_<>, ]+/g,\")\").replace(/(\\w+): string/g,\"$1\");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+\"\\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot\",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),q=String.fromCharCode(39),pattern=new RegExp(\"searchForWorkspaceRoot\\\\(\\\\s*resolve\\\\(dirname, \"+q+\"([^\"+q+\"]+)\"+q+\"\\\\)[\\\\s\\\\S]*?expect\\\\(resolved\\\\)\\\\.toBe\\\\(resolve\\\\(dirname, \"+q+\"([^\"+q+\"]+)\"+q+\"\\\\)\\\\)\",\"g\"),matches=[...spec.matchAll(pattern)];assert(matches.length>=5,\"expected focused search-root.spec.ts cases\");for(const match of matches){const input=path.resolve(dirname,match[1]),expected=path.resolve(dirname,match[2]);assert(fs.existsSync(input),`missing fixture path referenced by search-root.spec.ts: ${match[1]}`);assert(fs.existsSync(expected),`missing expected fixture path referenced by search-root.spec.ts: ${match[2]}`);assert.equal(searchForWorkspaceRoot(input),expected,`search-root.spec.ts expectation failed for ${match[1]}`)}'", - "passed": true, - "returncode": 0, - "stderr": "", - "stdout": "" - }, { "command": "git add -N -- packages/vite/src/node/server/__tests__/fixtures", "passed": true, @@ -56,8 +51,6 @@ "node --version", "node -e 'const fs=require(\"node:fs\"),os=require(\"node:os\"),path=require(\"node:path\"),vm=require(\"node:vm\"),assert=require(\"node:assert/strict\");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:[\"nested\"]}));write(path.join(root,\"nested/package.json\"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync(\"packages/vite/src/node/server/searchRoot.ts\",\"utf8\").split(\"\\n\").filter((line)=>!line.startsWith(\"import \")).join(\"\\n\");source=source.replace(/export function /g,\"function \").replace(/\\): [A-Za-z][A-Za-z0-9_<>, ]+/g,\")\").replace(/(\\w+): string/g,\"$1\");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+\"\\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot\",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),\"vite-deno-root-baseline-\")),denoJsonRoot=fixture(tmp,\"deno-json\",\"deno.json\");assert.notEqual(searchForWorkspaceRoot(path.join(denoJsonRoot,\"nested\")),denoJsonRoot);const denoJsoncRoot=fixture(tmp,\"deno-jsonc\",\"deno.jsonc\");assert.notEqual(searchForWorkspaceRoot(path.join(denoJsoncRoot,\"nested\")),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,\"package-workspace\");write(path.join(packageWorkspaceRoot,\"package.json\"),JSON.stringify({workspaces:[\"nested\"]}));write(path.join(packageWorkspaceRoot,\"nested/package.json\"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,\"nested\")),packageWorkspaceRoot)'", "git apply reference.patch", - "node -e 'const fs=require(\"node:fs\"),os=require(\"node:os\"),path=require(\"node:path\"),vm=require(\"node:vm\"),assert=require(\"node:assert/strict\");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:[\"nested\"]}));write(path.join(root,\"nested/package.json\"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync(\"packages/vite/src/node/server/searchRoot.ts\",\"utf8\").split(\"\\n\").filter((line)=>!line.startsWith(\"import \")).join(\"\\n\");source=source.replace(/export function /g,\"function \").replace(/\\): [A-Za-z][A-Za-z0-9_<>, ]+/g,\")\").replace(/(\\w+): string/g,\"$1\");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+\"\\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot\",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),\"vite-deno-root-test-\")),denoJsonRoot=fixture(tmp,\"deno-json\",\"deno.json\");assert.equal(searchForWorkspaceRoot(path.join(denoJsonRoot,\"nested\")),denoJsonRoot);assert.equal(searchForWorkspaceRoot(denoJsonRoot),denoJsonRoot);const denoJsoncRoot=fixture(tmp,\"deno-jsonc\",\"deno.jsonc\");assert.equal(searchForWorkspaceRoot(path.join(denoJsoncRoot,\"nested\")),denoJsoncRoot);assert.equal(searchForWorkspaceRoot(denoJsoncRoot),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,\"package-workspace\");write(path.join(packageWorkspaceRoot,\"package.json\"),JSON.stringify({workspaces:[\"nested\"]}));write(path.join(packageWorkspaceRoot,\"nested/package.json\"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,\"nested\")),packageWorkspaceRoot)'", - "node -e 'const fs=require(\"node:fs\"),path=require(\"node:path\"),vm=require(\"node:vm\"),assert=require(\"node:assert/strict\");const specPath=\"packages/vite/src/node/server/__tests__/search-root.spec.ts\",dirname=path.dirname(specPath),spec=fs.readFileSync(specPath,\"utf8\");function load(){let source=fs.readFileSync(\"packages/vite/src/node/server/searchRoot.ts\",\"utf8\").split(\"\\n\").filter((line)=>!line.startsWith(\"import \")).join(\"\\n\");source=source.replace(/export function /g,\"function \").replace(/\\): [A-Za-z][A-Za-z0-9_<>, ]+/g,\")\").replace(/(\\w+): string/g,\"$1\");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+\"\\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot\",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),q=String.fromCharCode(39),pattern=new RegExp(\"searchForWorkspaceRoot\\\\(\\\\s*resolve\\\\(dirname, \"+q+\"([^\"+q+\"]+)\"+q+\"\\\\)[\\\\s\\\\S]*?expect\\\\(resolved\\\\)\\\\.toBe\\\\(resolve\\\\(dirname, \"+q+\"([^\"+q+\"]+)\"+q+\"\\\\)\\\\)\",\"g\"),matches=[...spec.matchAll(pattern)];assert(matches.length>=5,\"expected focused search-root.spec.ts cases\");for(const match of matches){const input=path.resolve(dirname,match[1]),expected=path.resolve(dirname,match[2]);assert(fs.existsSync(input),`missing fixture path referenced by search-root.spec.ts: ${match[1]}`);assert(fs.existsSync(expected),`missing expected fixture path referenced by search-root.spec.ts: ${match[2]}`);assert.equal(searchForWorkspaceRoot(input),expected,`search-root.spec.ts expectation failed for ${match[1]}`)}'", "git add -N -- packages/vite/src/node/server/__tests__/fixtures", "git diff --check" ], @@ -93,18 +86,6 @@ "returncode": 0, "type": "code" }, - { - "assertion": "node -e 'const fs=require(\"node:fs\"),os=require(\"node:os\"),path=require(\"node:path\"),vm=require(\"node:vm\"),assert=require(\"node:assert/strict\");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:[\"nested\"]}));write(path.join(root,\"nested/package.json\"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync(\"packages/vite/src/node/server/searchRoot.ts\",\"utf8\").split(\"\\n\").filter((line)=>!line.startsWith(\"import \")).join(\"\\n\");source=source.replace(/export function /g,\"function \").replace(/\\): [A-Za-z][A-Za-z0-9_<>, ]+/g,\")\").replace(/(\\w+): string/g,\"$1\");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+\"\\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot\",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),\"vite-deno-root-test-\")),denoJsonRoot=fixture(tmp,\"deno-json\",\"deno.json\");assert.equal(searchForWorkspaceRoot(path.join(denoJsonRoot,\"nested\")),denoJsonRoot);assert.equal(searchForWorkspaceRoot(denoJsonRoot),denoJsonRoot);const denoJsoncRoot=fixture(tmp,\"deno-jsonc\",\"deno.jsonc\");assert.equal(searchForWorkspaceRoot(path.join(denoJsoncRoot,\"nested\")),denoJsoncRoot);assert.equal(searchForWorkspaceRoot(denoJsoncRoot),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,\"package-workspace\");write(path.join(packageWorkspaceRoot,\"package.json\"),JSON.stringify({workspaces:[\"nested\"]}));write(path.join(packageWorkspaceRoot,\"nested/package.json\"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,\"nested\")),packageWorkspaceRoot)'", - "passed": true, - "returncode": 0, - "type": "code" - }, - { - "assertion": "node -e 'const fs=require(\"node:fs\"),path=require(\"node:path\"),vm=require(\"node:vm\"),assert=require(\"node:assert/strict\");const specPath=\"packages/vite/src/node/server/__tests__/search-root.spec.ts\",dirname=path.dirname(specPath),spec=fs.readFileSync(specPath,\"utf8\");function load(){let source=fs.readFileSync(\"packages/vite/src/node/server/searchRoot.ts\",\"utf8\").split(\"\\n\").filter((line)=>!line.startsWith(\"import \")).join(\"\\n\");source=source.replace(/export function /g,\"function \").replace(/\\): [A-Za-z][A-Za-z0-9_<>, ]+/g,\")\").replace(/(\\w+): string/g,\"$1\");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+\"\\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot\",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),q=String.fromCharCode(39),pattern=new RegExp(\"searchForWorkspaceRoot\\\\(\\\\s*resolve\\\\(dirname, \"+q+\"([^\"+q+\"]+)\"+q+\"\\\\)[\\\\s\\\\S]*?expect\\\\(resolved\\\\)\\\\.toBe\\\\(resolve\\\\(dirname, \"+q+\"([^\"+q+\"]+)\"+q+\"\\\\)\\\\)\",\"g\"),matches=[...spec.matchAll(pattern)];assert(matches.length>=5,\"expected focused search-root.spec.ts cases\");for(const match of matches){const input=path.resolve(dirname,match[1]),expected=path.resolve(dirname,match[2]);assert(fs.existsSync(input),`missing fixture path referenced by search-root.spec.ts: ${match[1]}`);assert(fs.existsSync(expected),`missing expected fixture path referenced by search-root.spec.ts: ${match[2]}`);assert.equal(searchForWorkspaceRoot(input),expected,`search-root.spec.ts expectation failed for ${match[1]}`)}'", - "passed": true, - "returncode": 0, - "type": "code" - }, { "assertion": "git add -N -- packages/vite/src/node/server/__tests__/fixtures", "passed": true, @@ -118,6 +99,27 @@ "type": "code" } ], + "hidden_verifier": { + "checks": [ + { + "command": "git apply hidden verifier patch: verifier.patch", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "" + }, + { + "command": "python3 .agentlab_hidden/check_behavior.py", + "passed": true, + "returncode": 0, + "stderr": "", + "stdout": "$ node .agentlab_hidden/check_deno_workspace_root.cjs\n$ node .agentlab_hidden/check_search_root_spec.cjs\n" + } + ], + "configured": true, + "patch": "verifier.patch", + "restore_notes": [] + }, "input_tokens": null, "lines_added": 39, "lines_deleted": 0, @@ -154,10 +156,28 @@ }, "run_dir": ".", "run_id": "vite-deno-workspace-root-001-reference", + "run_surface": { + "approval_policy": "unknown", + "execution_surface": "unknown", + "human_intervention_events": [], + "memory_scope": "unknown", + "model_identity_source": "unknown", + "network_policy": "unknown", + "runtime_version": "unknown", + "sandbox_mode": "unknown", + "stop_reason": "success", + "timeout_seconds": "unknown", + "tool_policy": "unknown", + "turn_or_step_budget": "unknown", + "workspace_base_ref": "3bf060a95d87c3a78d9506bdfb443ad8a848fcb4", + "workspace_history_policy": "base_only" + }, "score_notes": [], "status": "passed", "success": true, + "task_commit": "dfc8aa5057dd8ec2b1223980d1e2eeb946ac3384", "task_id": "vite-deno-workspace-root-001", + "task_repo": "https://github.com/vitejs/vite.git", "task_title": "Vite should detect Deno workspace roots", "transcript_path": null, "trial_id": "vite-deno-workspace-root-001-reference", diff --git a/tasks/starter/vite-deno-workspace-root-001/task-card.md b/tasks/starter/vite-deno-workspace-root-001/task-card.md index 69b28ac..3a36b4e 100644 --- a/tasks/starter/vite-deno-workspace-root-001/task-card.md +++ b/tasks/starter/vite-deno-workspace-root-001/task-card.md @@ -26,6 +26,11 @@ Add a Deno workspace-root detector in `packages/vite/src/node/server/searchRoot. No task-local environment configured. +## Hidden Verifier + +- Patch: `verifier.patch` +- Commands: `1 command configured` + ## Graders ### Setup @@ -38,8 +43,6 @@ No task-local environment configured. ### Target -- `node -e 'const fs=require("node:fs"),os=require("node:os"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:["nested"]}));write(path.join(root,"nested/package.json"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),"vite-deno-root-test-")),denoJsonRoot=fixture(tmp,"deno-json","deno.json");assert.equal(searchForWorkspaceRoot(path.join(denoJsonRoot,"nested")),denoJsonRoot);assert.equal(searchForWorkspaceRoot(denoJsonRoot),denoJsonRoot);const denoJsoncRoot=fixture(tmp,"deno-jsonc","deno.jsonc");assert.equal(searchForWorkspaceRoot(path.join(denoJsoncRoot,"nested")),denoJsoncRoot);assert.equal(searchForWorkspaceRoot(denoJsoncRoot),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,"package-workspace");write(path.join(packageWorkspaceRoot,"package.json"),JSON.stringify({workspaces:["nested"]}));write(path.join(packageWorkspaceRoot,"nested/package.json"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,"nested")),packageWorkspaceRoot)'` -- `node -e 'const fs=require("node:fs"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");const specPath="packages/vite/src/node/server/__tests__/search-root.spec.ts",dirname=path.dirname(specPath),spec=fs.readFileSync(specPath,"utf8");function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),q=String.fromCharCode(39),pattern=new RegExp("searchForWorkspaceRoot\\(\\s*resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)[\\s\\S]*?expect\\(resolved\\)\\.toBe\\(resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)\\)","g"),matches=[...spec.matchAll(pattern)];assert(matches.length>=5,"expected focused search-root.spec.ts cases");for(const match of matches){const input=path.resolve(dirname,match[1]),expected=path.resolve(dirname,match[2]);assert(fs.existsSync(input),`missing fixture path referenced by search-root.spec.ts: ${match[1]}`);assert(fs.existsSync(expected),`missing expected fixture path referenced by search-root.spec.ts: ${match[2]}`);assert.equal(searchForWorkspaceRoot(input),expected,`search-root.spec.ts expectation failed for ${match[1]}`)}'` - `git add -N -- packages/vite/src/node/server/__tests__/fixtures` - `git diff --check` diff --git a/tasks/starter/vite-deno-workspace-root-001/task.yaml b/tasks/starter/vite-deno-workspace-root-001/task.yaml index 901edf6..915165f 100644 --- a/tasks/starter/vite-deno-workspace-root-001/task.yaml +++ b/tasks/starter/vite-deno-workspace-root-001/task.yaml @@ -25,16 +25,16 @@ reference_solution: > reference_artifact: type: patch path: reference.patch +hidden_verifier: + patch: verifier.patch + commands: + - python3 .agentlab_hidden/check_behavior.py setup: - node --version baseline: - >- node -e 'const fs=require("node:fs"),os=require("node:os"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:["nested"]}));write(path.join(root,"nested/package.json"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),"vite-deno-root-baseline-")),denoJsonRoot=fixture(tmp,"deno-json","deno.json");assert.notEqual(searchForWorkspaceRoot(path.join(denoJsonRoot,"nested")),denoJsonRoot);const denoJsoncRoot=fixture(tmp,"deno-jsonc","deno.jsonc");assert.notEqual(searchForWorkspaceRoot(path.join(denoJsoncRoot,"nested")),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,"package-workspace");write(path.join(packageWorkspaceRoot,"package.json"),JSON.stringify({workspaces:["nested"]}));write(path.join(packageWorkspaceRoot,"nested/package.json"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,"nested")),packageWorkspaceRoot)' test: - - >- - node -e 'const fs=require("node:fs"),os=require("node:os"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:["nested"]}));write(path.join(root,"nested/package.json"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),"vite-deno-root-test-")),denoJsonRoot=fixture(tmp,"deno-json","deno.json");assert.equal(searchForWorkspaceRoot(path.join(denoJsonRoot,"nested")),denoJsonRoot);assert.equal(searchForWorkspaceRoot(denoJsonRoot),denoJsonRoot);const denoJsoncRoot=fixture(tmp,"deno-jsonc","deno.jsonc");assert.equal(searchForWorkspaceRoot(path.join(denoJsoncRoot,"nested")),denoJsoncRoot);assert.equal(searchForWorkspaceRoot(denoJsoncRoot),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,"package-workspace");write(path.join(packageWorkspaceRoot,"package.json"),JSON.stringify({workspaces:["nested"]}));write(path.join(packageWorkspaceRoot,"nested/package.json"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,"nested")),packageWorkspaceRoot)' - - >- - node -e 'const fs=require("node:fs"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");const specPath="packages/vite/src/node/server/__tests__/search-root.spec.ts",dirname=path.dirname(specPath),spec=fs.readFileSync(specPath,"utf8");function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),q=String.fromCharCode(39),pattern=new RegExp("searchForWorkspaceRoot\\(\\s*resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)[\\s\\S]*?expect\\(resolved\\)\\.toBe\\(resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)\\)","g"),matches=[...spec.matchAll(pattern)];assert(matches.length>=5,"expected focused search-root.spec.ts cases");for(const match of matches){const input=path.resolve(dirname,match[1]),expected=path.resolve(dirname,match[2]);assert(fs.existsSync(input),`missing fixture path referenced by search-root.spec.ts: ${match[1]}`);assert(fs.existsSync(expected),`missing expected fixture path referenced by search-root.spec.ts: ${match[2]}`);assert.equal(searchForWorkspaceRoot(input),expected,`search-root.spec.ts expectation failed for ${match[1]}`)}' - git add -N -- packages/vite/src/node/server/__tests__/fixtures - git diff --check success: diff --git a/tasks/starter/vite-deno-workspace-root-001/verifier.patch b/tasks/starter/vite-deno-workspace-root-001/verifier.patch new file mode 100644 index 0000000..f617e77 --- /dev/null +++ b/tasks/starter/vite-deno-workspace-root-001/verifier.patch @@ -0,0 +1,170 @@ +diff --git a/.agentlab_hidden/check_behavior.py b/.agentlab_hidden/check_behavior.py +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_behavior.py +@@ -0,0 +1,33 @@ ++from __future__ import annotations ++ ++import subprocess ++import sys ++ ++COMMANDS = [ ++ ["node", ".agentlab_hidden/check_deno_workspace_root.cjs"], ++ ["node", ".agentlab_hidden/check_search_root_spec.cjs"], ++] ++ ++ ++def main() -> int: ++ for command in COMMANDS: ++ print("$ " + " ".join(command)) ++ completed = subprocess.run( ++ command, ++ text=True, ++ stdout=subprocess.PIPE, ++ stderr=subprocess.STDOUT, ++ ) ++ if completed.stdout: ++ print(completed.stdout, end="") ++ if completed.returncode != 0: ++ print( ++ f"hidden verifier command failed with exit code {completed.returncode}", ++ file=sys.stderr, ++ ) ++ return completed.returncode ++ return 0 ++ ++ ++if __name__ == "__main__": ++ raise SystemExit(main()) +diff --git a/.agentlab_hidden/check_deno_workspace_root.cjs b/.agentlab_hidden/check_deno_workspace_root.cjs +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_deno_workspace_root.cjs +@@ -0,0 +1,64 @@ ++const fs = require("node:fs"); ++const os = require("node:os"); ++const path = require("node:path"); ++const vm = require("node:vm"); ++const assert = require("node:assert/strict"); ++ ++function write(file, data) { ++ fs.mkdirSync(path.dirname(file), {recursive: true}); ++ fs.writeFileSync(file, data); ++} ++ ++function fixture(tmp, name, configName) { ++ const root = path.join(tmp, name); ++ write(path.join(root, configName), JSON.stringify({workspace: ["nested"]})); ++ write(path.join(root, "nested/package.json"), JSON.stringify({private: true})); ++ return root; ++} ++ ++function loadSearchForWorkspaceRoot() { ++ let source = fs ++ .readFileSync("packages/vite/src/node/server/searchRoot.ts", "utf8") ++ .split("\n") ++ .filter((line) => !line.startsWith("import ")) ++ .join("\n"); ++ source = source ++ .replace(/export function /g, "function ") ++ .replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g, ")") ++ .replace(/(\w+): string/g, "$1"); ++ const sandbox = { ++ fs, ++ dirname: path.dirname, ++ join: path.join, ++ isFileReadable(file) { ++ try { ++ fs.accessSync(file, fs.constants.R_OK); ++ return true; ++ } catch { ++ return false; ++ } ++ }, ++ exports: {}, ++ }; ++ vm.runInNewContext( ++ source + "\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot", ++ sandbox, ++ ); ++ return sandbox.exports.searchForWorkspaceRoot; ++} ++ ++const searchForWorkspaceRoot = loadSearchForWorkspaceRoot(); ++const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "vite-deno-root-test-")); ++const denoJsonRoot = fixture(tmp, "deno-json", "deno.json"); ++assert.equal(searchForWorkspaceRoot(path.join(denoJsonRoot, "nested")), denoJsonRoot); ++assert.equal(searchForWorkspaceRoot(denoJsonRoot), denoJsonRoot); ++const denoJsoncRoot = fixture(tmp, "deno-jsonc", "deno.jsonc"); ++assert.equal(searchForWorkspaceRoot(path.join(denoJsoncRoot, "nested")), denoJsoncRoot); ++assert.equal(searchForWorkspaceRoot(denoJsoncRoot), denoJsoncRoot); ++const packageWorkspaceRoot = path.join(tmp, "package-workspace"); ++write(path.join(packageWorkspaceRoot, "package.json"), JSON.stringify({workspaces: ["nested"]})); ++write(path.join(packageWorkspaceRoot, "nested/package.json"), JSON.stringify({private: true})); ++assert.equal( ++ searchForWorkspaceRoot(path.join(packageWorkspaceRoot, "nested")), ++ packageWorkspaceRoot, ++); +diff --git a/.agentlab_hidden/check_search_root_spec.cjs b/.agentlab_hidden/check_search_root_spec.cjs +new file mode 100644 +index 0000000..0000000 +--- /dev/null ++++ b/.agentlab_hidden/check_search_root_spec.cjs +@@ -0,0 +1,55 @@ ++const fs = require("node:fs"); ++const path = require("node:path"); ++const vm = require("node:vm"); ++const assert = require("node:assert/strict"); ++ ++const specPath = "packages/vite/src/node/server/__tests__/search-root.spec.ts"; ++const dirname = path.dirname(specPath); ++const spec = fs.readFileSync(specPath, "utf8"); ++ ++function loadSearchForWorkspaceRoot() { ++ let source = fs ++ .readFileSync("packages/vite/src/node/server/searchRoot.ts", "utf8") ++ .split("\n") ++ .filter((line) => !line.startsWith("import ")) ++ .join("\n"); ++ source = source ++ .replace(/export function /g, "function ") ++ .replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g, ")") ++ .replace(/(\w+): string/g, "$1"); ++ const sandbox = { ++ fs, ++ dirname: path.dirname, ++ join: path.join, ++ isFileReadable(file) { ++ try { ++ fs.accessSync(file, fs.constants.R_OK); ++ return true; ++ } catch { ++ return false; ++ } ++ }, ++ exports: {}, ++ }; ++ vm.runInNewContext( ++ source + "\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot", ++ sandbox, ++ ); ++ return sandbox.exports.searchForWorkspaceRoot; ++} ++ ++assert(spec.includes("test('deno'"), "missing deno nested spec case"); ++assert(spec.includes("test('deno at root'"), "missing deno root spec case"); ++assert( ++ spec.includes("fixtures/deno/nested") && spec.includes("fixtures/deno"), ++ "search-root.spec.ts should cover the deno workspace fixture", ++); ++ ++const fixtureRoot = path.resolve(dirname, "fixtures/deno"); ++const nested = path.resolve(dirname, "fixtures/deno/nested"); ++assert(fs.existsSync(path.join(fixtureRoot, "deno.json")), "missing deno.json fixture"); ++assert(fs.existsSync(path.join(nested, "package.json")), "missing nested package fixture"); ++ ++const searchForWorkspaceRoot = loadSearchForWorkspaceRoot(); ++assert.equal(searchForWorkspaceRoot(nested), fixtureRoot); ++assert.equal(searchForWorkspaceRoot(fixtureRoot), fixtureRoot);