From c31d9e40aa302d1f377d0d91acc17e89258e1777 Mon Sep 17 00:00:00 2001 From: Jordak Date: Mon, 22 Jun 2026 14:27:34 -0700 Subject: [PATCH 1/3] codex: split visible validation from target graders --- agentlab/agents/manual.py | 6 ++-- agentlab/agents/prompts.py | 6 ++-- agentlab/tasks/cards.py | 12 ++++++++ agentlab/tasks/model.py | 5 ++++ docs/design.md | 10 +++++-- tests/test_agent_prompts.py | 60 +++++++++++++++++++++++++++++++++++++ tests/test_task_cards.py | 23 ++++++++++++++ tests/test_tasks.py | 7 +++++ 8 files changed, 121 insertions(+), 8 deletions(-) diff --git a/agentlab/agents/manual.py b/agentlab/agents/manual.py index ce3f76c..19c60f9 100644 --- a/agentlab/agents/manual.py +++ b/agentlab/agents/manual.py @@ -82,9 +82,9 @@ def _print_manual_instructions(task: EvalTask, workspace: Path) -> None: print("Task-local environment used by setup, grader, and agent commands:") for line in environment_lines: print(f"- {line}") - if task.test: + if task.visible_validation: print("") - print("Code-based grader assertions that will run after you continue:") - for command in task.test: + print("Suggested validation commands:") + for command in task.visible_validation: print(f"- {command}") print("") diff --git a/agentlab/agents/prompts.py b/agentlab/agents/prompts.py index 0c67d69..fffc1f5 100644 --- a/agentlab/agents/prompts.py +++ b/agentlab/agents/prompts.py @@ -17,9 +17,9 @@ def build_agent_prompt(task: EvalTask) -> str: task.prompt, "", ] - if task.test: - lines.extend(["Validation commands that will be run after you finish:"]) - lines.extend(f"- {command}" for command in task.test) + if task.visible_validation: + lines.extend(["Suggested validation commands:"]) + lines.extend(f"- {command}" for command in task.visible_validation) lines.append("") environment_lines = describe_task_environment(task) if environment_lines: diff --git a/agentlab/tasks/cards.py b/agentlab/tasks/cards.py index 7bd2bc8..d4ee3b1 100644 --- a/agentlab/tasks/cards.py +++ b/agentlab/tasks/cards.py @@ -48,6 +48,7 @@ def render_task_card(bundle: TaskBundle) -> str: "", _environment_text(task), "", + *_visible_validation_section(task), "## Graders", "", "### Setup", @@ -87,6 +88,17 @@ def _command_list(commands: List[str]) -> str: return "\n".join(f"- `{command}`" for command in commands) +def _visible_validation_section(task: EvalTask) -> list[str]: + if not task.visible_validation: + return [] + return [ + "## Visible Validation", + "", + _command_list(task.visible_validation), + "", + ] + + def _environment_text(task: EvalTask) -> str: lines = describe_task_environment(task) if not lines: diff --git a/agentlab/tasks/model.py b/agentlab/tasks/model.py index 012d227..98352c8 100644 --- a/agentlab/tasks/model.py +++ b/agentlab/tasks/model.py @@ -49,6 +49,7 @@ class EvalTask: setup: List[str] = field(default_factory=list) baseline: List[str] = field(default_factory=list) test: List[str] = field(default_factory=list) + visible_validation: List[str] = field(default_factory=list) environment_path: List[str] = field(default_factory=list) environment: Dict[str, str] = field(default_factory=dict) success: SuccessCriteria = field(default_factory=SuccessCriteria) @@ -127,6 +128,10 @@ def from_mapping( setup=_string_list(mapping.get("setup", []), "setup"), baseline=_string_list(mapping.get("baseline", []), "baseline"), test=_string_list(mapping.get("test", []), "test"), + visible_validation=_string_list( + mapping.get("visible_validation", []), + "visible_validation", + ), environment_path=_environment_path( mapping.get("environment_path", []), "environment_path", diff --git a/docs/design.md b/docs/design.md index c57e8a5..6a90c56 100644 --- a/docs/design.md +++ b/docs/design.md @@ -138,13 +138,19 @@ opt into bounded follow-up questions with explicit interaction metadata, but interactive tasks must be summarized separately from non-interactive tasks. See [ADR 0008](adr/0008-reserve-interactive-task-contracts.md). +Task authors can include `visible_validation` commands when the fixed prompt +should intentionally give an agent harness a self-check recipe. Target grader +commands in `test` remain grader-facing by default: they run after the agent +phase and contribute to the deterministic grader outcome, but they are not +automatically injected into agent prompts. + ## Task Environments Task setup should provision the dependencies needed by the deterministic graders and by the obvious self-checks an agent is likely to run. For example, a Python task that reasonably invites `pytest` should either make the relevant pytest -entrypoint available or document that the intended validation scope is the -listed grader commands. +entrypoint available or provide explicit `visible_validation` commands when the +task should tell the agent which self-checks to run. Tasks can declare `environment_path` entries to prepend workspace-relative directories to `PATH`, and an `environment` mapping for environment variables. diff --git a/tests/test_agent_prompts.py b/tests/test_agent_prompts.py index 37210c3..b6cf6b0 100644 --- a/tests/test_agent_prompts.py +++ b/tests/test_agent_prompts.py @@ -1,10 +1,70 @@ import unittest +from contextlib import redirect_stdout +from io import StringIO +from pathlib import Path +from agentlab.agents.manual import _print_manual_instructions from agentlab.agents.prompts import build_agent_prompt from agentlab.tasks import EvalTask, SuccessCriteria class AgentPromptTest(unittest.TestCase): + def test_target_graders_are_not_injected(self): + task = EvalTask( + id="grader-task", + title="Grader task", + repo="https://github.com/example/repo", + commit="abc123", + language="python", + prompt="Fix the behavior.", + test=["pytest tests/hidden_behavior.py"], + ) + + prompt = build_agent_prompt(task) + + self.assertIn("Fix the behavior.", prompt) + self.assertNotIn("Validation commands that will be run", prompt) + self.assertNotIn("pytest tests/hidden_behavior.py", prompt) + + def test_visible_validation_is_injected(self): + task = EvalTask( + id="visible-validation-task", + title="Visible validation task", + repo="https://github.com/example/repo", + commit="abc123", + language="python", + prompt="Fix the behavior.", + visible_validation=["pytest tests/focused_check.py"], + test=["pytest tests/hidden_behavior.py"], + ) + + prompt = build_agent_prompt(task) + + self.assertIn("Suggested validation commands:", prompt) + self.assertIn("pytest tests/focused_check.py", prompt) + self.assertNotIn("pytest tests/hidden_behavior.py", prompt) + + def test_manual_instructions_hide_target_graders(self): + task = EvalTask( + id="manual-task", + title="Manual task", + repo="https://github.com/example/repo", + commit="abc123", + language="python", + prompt="Fix the behavior.", + visible_validation=["pytest tests/focused_check.py"], + test=["pytest tests/hidden_behavior.py"], + ) + output = StringIO() + + with redirect_stdout(output): + _print_manual_instructions(task, Path("/tmp/workspace")) + + printed = output.getvalue() + self.assertIn("Suggested validation commands:", printed) + self.assertIn("pytest tests/focused_check.py", printed) + self.assertNotIn("pytest tests/hidden_behavior.py", printed) + def test_scope_oracle_metadata_is_not_injected(self): task = EvalTask( id="scope-task", diff --git a/tests/test_task_cards.py b/tests/test_task_cards.py index 8734bb0..0895085 100644 --- a/tests/test_task_cards.py +++ b/tests/test_task_cards.py @@ -150,6 +150,21 @@ def test_renders_scope_oracle_metadata_when_configured(self): self.assertIn("- Allowed paths: `src/`, `tests/**/*.py`", card) self.assertIn("- Forbidden paths: `src/private/`", card) + def test_renders_visible_validation_when_configured(self): + with tempfile.TemporaryDirectory() as temp: + bundle_dir = _write_task_bundle( + Path(temp) / "example-suite", + "demo-001", + visible_validation=True, + ) + bundle = load_task_bundle(bundle_dir) + + card = render_task_card(bundle) + + self.assertIn("## Visible Validation", card) + self.assertIn("- `pytest tests/focused_check.py`", card) + self.assertIn("### Target\n\n- `pytest tests/test_demo.py`", card) + def _write_task_bundle( suite_dir: Path, @@ -159,6 +174,7 @@ def _write_task_bundle( eval_type: str = "regression", tags: list[str] | None = None, scope_oracle: bool = False, + visible_validation: bool = False, ) -> Path: bundle_dir = suite_dir / task_id bundle_dir.mkdir(parents=True) @@ -183,6 +199,7 @@ def _write_task_bundle( - python -m pip install -e . baseline: - pytest +{_visible_validation(visible_validation)} test: - pytest tests/test_demo.py environment_path: @@ -224,5 +241,11 @@ def _scope_oracle_success(enabled: bool) -> str: ) +def _visible_validation(enabled: bool) -> str: + if not enabled: + return "" + return "visible_validation:\n - pytest tests/focused_check.py" + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 2c84d6e..ce68059 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -93,6 +93,9 @@ def test_loads_folded_command_list_items(self): baseline: - >- python -c "assert {'key': 'value'}['key'] == 'value'" + visible_validation: + - >- + python -m pytest tests/test_demo.py -q test: - >- python -c "print('still one command')" @@ -105,6 +108,10 @@ def test_loads_folded_command_list_items(self): task.baseline, ['python -c "assert {\'key\': \'value\'}[\'key\'] == \'value\'"'], ) + self.assertEqual( + task.visible_validation, + ["python -m pytest tests/test_demo.py -q"], + ) self.assertEqual(task.test, ['python -c "print(\'still one command\')"']) def test_loads_boundary_metadata_and_consent_style(self): From 46d5b21fa2cbff8b3e4d752978e4632c423e1022 Mon Sep 17 00:00:00 2001 From: Jordak Date: Mon, 22 Jun 2026 14:36:32 -0700 Subject: [PATCH 2/3] codex: cover visible validation contract surfaces --- .agents/skills/task-card/SKILL.md | 13 +++++ docs/failure-taxonomy.md | 2 +- .../task-card.md | 5 ++ .../2048-advanced-snake-params-001/task.yaml | 4 ++ .../task-card.md | 5 ++ .../click-help-shadowed-option-001/task.yaml | 5 ++ .../task-card.md | 4 ++ .../task.yaml | 2 + .../task-card.md | 5 ++ .../todomvc-toggle-all-checkbox-001/task.yaml | 3 ++ tests/test_runner.py | 52 +++++++++++++++++++ 11 files changed, 99 insertions(+), 1 deletion(-) diff --git a/.agents/skills/task-card/SKILL.md b/.agents/skills/task-card/SKILL.md index 4129a1d..0752027 100644 --- a/.agents/skills/task-card/SKILL.md +++ b/.agents/skills/task-card/SKILL.md @@ -67,6 +67,19 @@ reference_artifact: path: reference.patch ``` +Use `test` for deterministic target graders. Target grader commands run after +the agent phase and affect the grader outcome, but they are not injected into +trial prompts by default. When a task should intentionally give the agent a +self-check recipe, put those commands in `visible_validation`: + +```yaml +visible_validation: + - python -m pytest tests/test_focused.py -q +test: + - python -m pytest tests/test_focused.py -q + - python -c "assert stricter_behavior_holds()" +``` + Optional scope-oracle metadata can declare consent posture and path boundaries without injecting those details into trial prompts: diff --git a/docs/failure-taxonomy.md b/docs/failure-taxonomy.md index 99b84b6..7e430d0 100644 --- a/docs/failure-taxonomy.md +++ b/docs/failure-taxonomy.md @@ -5,7 +5,7 @@ - `context_miss`: agent harness did not find or use the relevant code. - `spec_misread`: agent harness misunderstood the requested behavior. - `bad_local_fix`: fixed one path but broke another or missed the root cause. -- `test_gap`: passed visible graders but likely incomplete against intended behavior. +- `test_gap`: passed configured graders but likely incomplete against intended behavior. - `over_edit`: changed too much or refactored unrelated code. - `resource_inefficient`: used disproportionate runtime, token budget, cost, or command churn relative to the task complexity and outcome quality. diff --git a/tasks/starter/2048-advanced-snake-params-001/task-card.md b/tasks/starter/2048-advanced-snake-params-001/task-card.md index c4f6de7..2229e71 100644 --- a/tasks/starter/2048-advanced-snake-params-001/task-card.md +++ b/tasks/starter/2048-advanced-snake-params-001/task-card.md @@ -27,6 +27,11 @@ In simulation.py, persist params when args.heuristic is advanced-snake instead o - PYTEST_ADDOPTS=-p no:cacheprovider - PYTHONDONTWRITEBYTECODE=1 +## Visible Validation + +- `python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"` +- `python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"` + ## Graders ### Setup diff --git a/tasks/starter/2048-advanced-snake-params-001/task.yaml b/tasks/starter/2048-advanced-snake-params-001/task.yaml index 430ae13..f521723 100644 --- a/tasks/starter/2048-advanced-snake-params-001/task.yaml +++ b/tasks/starter/2048-advanced-snake-params-001/task.yaml @@ -25,6 +25,10 @@ environment: PYTEST_ADDOPTS: "-p no:cacheprovider" baseline: - python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()" +visible_validation: + - >- + python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result" + - python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()" test: - >- python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result" diff --git a/tasks/starter/click-help-shadowed-option-001/task-card.md b/tasks/starter/click-help-shadowed-option-001/task-card.md index e498aa4..3d38110 100644 --- a/tasks/starter/click-help-shadowed-option-001/task-card.md +++ b/tasks/starter/click-help-shadowed-option-001/task-card.md @@ -30,6 +30,11 @@ In UsageError.show, derive the available help option names from the current comm - PYTHONPATH={workspace}/src - VIRTUAL_ENV={workspace}/.agentlab/venv +## Visible Validation + +- `python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo --help" + chr(39) + " for help."; bad = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, ["foo", "--help"]); assert help_result.exit_code == 0, help_result.output; assert "--help" in help_result.output, help_result.output'` +- `python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.option("--help-file","--help")(click.argument("required_arg")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); assert result.exit_code == 2, result.output; assert "Try " not in result.output, result.output'` + ## Graders ### Setup diff --git a/tasks/starter/click-help-shadowed-option-001/task.yaml b/tasks/starter/click-help-shadowed-option-001/task.yaml index e560124..76657ab 100644 --- a/tasks/starter/click-help-shadowed-option-001/task.yaml +++ b/tasks/starter/click-help-shadowed-option-001/task.yaml @@ -34,6 +34,11 @@ environment: baseline: - >- python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output' +visible_validation: + - >- + python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo --help" + chr(39) + " for help."; bad = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, ["foo", "--help"]); assert help_result.exit_code == 0, help_result.output; assert "--help" in help_result.output, help_result.output' + - >- + python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.option("--help-file","--help")(click.argument("required_arg")(lambda required_arg, help_file, host: None)))); cli.add_command(foo); result = CliRunner().invoke(cli, ["foo"]); assert result.exit_code == 2, result.output; assert "Try " not in result.output, result.output' test: - >- python -c 'import click; from click.testing import CliRunner; cli = click.group("cli", context_settings={"help_option_names":["-h","--help"]})(lambda: None); foo = click.command("foo")(click.option("--host","-h")(click.argument("required_arg")(lambda required_arg, host: None))); cli.add_command(foo); runner = CliRunner(); result = runner.invoke(cli, ["foo"]); expected = "Try " + chr(39) + "cli foo --help" + chr(39) + " for help."; bad = "Try " + chr(39) + "cli foo -h" + chr(39) + " for help."; assert result.exit_code == 2, result.output; assert expected in result.output, result.output; assert bad not in result.output, result.output; help_result = runner.invoke(cli, ["foo", "--help"]); assert help_result.exit_code == 0, help_result.output; assert "--help" in help_result.output, help_result.output' diff --git a/tasks/starter/click-should-strip-ansi-tests-001/task-card.md b/tasks/starter/click-should-strip-ansi-tests-001/task-card.md index 117a0d6..fb2fe9c 100644 --- a/tasks/starter/click-should-strip-ansi-tests-001/task-card.md +++ b/tasks/starter/click-should-strip-ansi-tests-001/task-card.md @@ -30,6 +30,10 @@ Update `tests/test_compat.py` to import Click, pytest, and sys; test `_is_jupyte - PYTHONPATH={workspace}/src - VIRTUAL_ENV={workspace}/.agentlab/venv +## Visible Validation + +- `python -m pytest tests/test_compat.py -q` + ## Graders ### Setup diff --git a/tasks/starter/click-should-strip-ansi-tests-001/task.yaml b/tasks/starter/click-should-strip-ansi-tests-001/task.yaml index 7496594..a978dc6 100644 --- a/tasks/starter/click-should-strip-ansi-tests-001/task.yaml +++ b/tasks/starter/click-should-strip-ansi-tests-001/task.yaml @@ -38,6 +38,8 @@ baseline: - >- python -c 'import click, sys; compat = click._compat; compat.isatty = lambda stream: True; compat._is_jupyter_kernel_output = lambda stream: False; assert compat.should_strip_ansi(stream=sys.stdout, color=None) is False; assert compat.should_strip_ansi(stream=sys.stdout, color=True) is False; assert compat.should_strip_ansi(stream=sys.stdout, color=False) is True; compat.isatty = lambda stream: False; compat._is_jupyter_kernel_output = lambda stream: True; assert compat.should_strip_ansi(stream=None, color=None) is False; compat._is_jupyter_kernel_output = lambda stream: False; assert compat.should_strip_ansi(stream=None, color=None) is True' - python -c 'from pathlib import Path; text = Path("tests/test_compat.py").read_text(); assert "def test_should_strip_ansi" not in text' +visible_validation: + - python -m pytest tests/test_compat.py -q test: - python -m pytest tests/test_compat.py -q - >- diff --git a/tasks/starter/todomvc-toggle-all-checkbox-001/task-card.md b/tasks/starter/todomvc-toggle-all-checkbox-001/task-card.md index 45ca399..de5ddcc 100644 --- a/tasks/starter/todomvc-toggle-all-checkbox-001/task-card.md +++ b/tasks/starter/todomvc-toggle-all-checkbox-001/task-card.md @@ -26,6 +26,11 @@ Add id="toggle-all" to the JavaScript ES5 toggle-all checkbox in both index.html No task-local environment configured. +## Visible Validation + +- `node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,true,path); assert.strictEqual(elements[".toggle-all-label"].checked,undefined,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all"],path); assert.strictEqual(events[0].type,"change",path); elements[".toggle-all"].checked=false; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,false,path); elements[".toggle-all"].checked=true; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,undefined,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");'` +- `node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}'` + ## Graders ### Setup diff --git a/tasks/starter/todomvc-toggle-all-checkbox-001/task.yaml b/tasks/starter/todomvc-toggle-all-checkbox-001/task.yaml index 7b8985e..c3ed538 100644 --- a/tasks/starter/todomvc-toggle-all-checkbox-001/task.yaml +++ b/tasks/starter/todomvc-toggle-all-checkbox-001/task.yaml @@ -26,6 +26,9 @@ setup: baseline: - node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.checked=!this.checked;this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,false,path); assert.strictEqual(elements[".toggle-all-label"].checked,true,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all-label"],path); assert.strictEqual(events[0].type,"click",path); events[0].handler.call(elements[".toggle-all-label"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,1,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");' - node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(!/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}' +visible_validation: + - node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,true,path); assert.strictEqual(elements[".toggle-all-label"].checked,undefined,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all"],path); assert.strictEqual(events[0].type,"change",path); elements[".toggle-all"].checked=false; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,false,path); elements[".toggle-all"].checked=true; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,undefined,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");' + - node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}' test: - node -e 'const fs=require("fs"),vm=require("vm"),assert=require("assert"); function exercise(path){const elements={".todo-list":{},".todo-count":{},".clear-completed":{},".main":{style:{}},".footer":{style:{}},".toggle-all":{checked:false,click(){this.clicked=(this.clicked||0)+1;}},".toggle-all-label":{},".new-todo":{}}; const events=[]; const context={window:{},qs:(selector)=>elements[selector]||{style:{},dataset:{}},qsa:()=>[],$on:(element,type,handler)=>events.push({element,type,handler}),$delegate:()=>{},$parent:()=>({dataset:{id:"1"}})}; context.window=context; vm.createContext(context); vm.runInContext(fs.readFileSync(path,"utf8"),context); const view=new context.app.View({}); view.render("toggleAll",{checked:true}); assert.strictEqual(elements[".toggle-all"].checked,true,path); assert.strictEqual(elements[".toggle-all-label"].checked,undefined,path); let payload; view.bind("toggleAll",(data)=>{payload=data;}); assert.strictEqual(events.length,1,path); assert.strictEqual(events[0].element,elements[".toggle-all"],path); assert.strictEqual(events[0].type,"change",path); elements[".toggle-all"].checked=false; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,false,path); elements[".toggle-all"].checked=true; events[0].handler.call(elements[".toggle-all"]); assert.strictEqual(payload.completed,true,path); assert.strictEqual(elements[".toggle-all"].clicked,undefined,path);} exercise("examples/javascript-es5/src/view.js"); exercise("examples/javascript-es5/dist/view.js");' - node -e 'const fs=require("fs"),assert=require("assert"); for (const path of ["examples/javascript-es5/index.html","examples/javascript-es5/dist/index.html"]){const html=fs.readFileSync(path,"utf8"); const input=html.match(/]*class="toggle-all"[^>]*>/); assert(input,path); assert(/\bid="toggle-all"/.test(input[0]),path); assert(/]*class="toggle-all-label"[^>]*for="toggle-all"/.test(html),path);}' diff --git a/tests/test_runner.py b/tests/test_runner.py index 27a4061..495ffd9 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -102,6 +102,58 @@ def test_task_environment_path_is_used_by_graders(self): self.assertTrue(evaluation.score.tests_passed) + def test_visible_validation_is_not_executed_or_serialized_as_grader(self): + if shutil.which("git") is None: + self.skipTest("git is required for workspace preparation") + + with tempfile.TemporaryDirectory() as temp: + temp_path = Path(temp) + repo = temp_path / "repo" + init_repo(repo) + commit = commit_file(repo, "README.md", "# Fixture\n", message="initial") + visible_command = ( + f"{sys.executable} -c " + "\"raise SystemExit('visible validation should not run')\"" + ) + target_command = f"{sys.executable} -c \"print('target ok')\"" + + task = EvalTask( + id="visible-validation-task", + title="Visible validation task", + repo=str(repo), + commit=commit, + language="python", + prompt="Do nothing.", + visible_validation=[visible_command], + test=[target_command], + ) + + evaluation = run_task( + task, + ManualAgentAdapter(pause=False), + temp_path / "runs", + ) + result = json.loads(evaluation.result_path.read_text(encoding="utf-8")) + report = evaluation.report_path.read_text(encoding="utf-8") + + self.assertTrue(evaluation.score.tests_passed) + self.assertEqual( + [check.command for check in evaluation.score.checks], + [target_command], + ) + self.assertEqual(evaluation.agent_run.commands_run, [target_command]) + self.assertEqual( + [check["command"] for check in result["checks"]], + [target_command], + ) + self.assertEqual( + [grader["assertion"] for grader in result["graders"]], + [target_command], + ) + self.assertIn(target_command, report) + self.assertNotIn(visible_command, report) + self.assertNotIn(visible_command, json.dumps(result)) + def test_max_files_changed_is_enforced(self): if shutil.which("git") is None: self.skipTest("git is required for workspace preparation") From 0d65aca5c3d0281792fe1b8e3d0a567b175eb7c6 Mon Sep 17 00:00:00 2001 From: Jordak Date: Mon, 22 Jun 2026 14:45:49 -0700 Subject: [PATCH 3/3] codex: migrate focused visible validation tasks --- .../task-card.md | 10 ++++++++++ .../task.yaml | 13 +++++++++++++ .../click-default-map-nargs-001/task-card.md | 6 ++++++ tasks/starter/click-default-map-nargs-001/task.yaml | 7 +++++++ .../click-help-option-refactor-001/task-card.md | 6 ++++++ .../click-help-option-refactor-001/task.yaml | 7 +++++++ .../httpx-verify-false-client-cert-001/task-card.md | 7 +++++++ .../httpx-verify-false-client-cert-001/task.yaml | 9 +++++++++ .../task-card.md | 5 +++++ .../task.yaml | 4 ++++ .../task-card.md | 5 +++++ .../react-tabs-selected-focus-overlay-001/task.yaml | 4 ++++ .../task-card.md | 7 +++++++ .../task.yaml | 5 +++++ .../vite-deno-workspace-root-001/task-card.md | 7 +++++++ .../starter/vite-deno-workspace-root-001/task.yaml | 7 +++++++ 16 files changed, 109 insertions(+) diff --git a/tasks/edit_scope_safety/click-default-map-string-splitting-001/task-card.md b/tasks/edit_scope_safety/click-default-map-string-splitting-001/task-card.md index 459d77d..771dade 100644 --- a/tasks/edit_scope_safety/click-default-map-string-splitting-001/task-card.md +++ b/tasks/edit_scope_safety/click-default-map-string-splitting-001/task-card.md @@ -30,6 +30,16 @@ In Option.consume_value, when a value comes from default_map and is a string for - PYTHONPATH={workspace}/src - VIRTUAL_ENV={workspace}/.agentlab/venv +## Visible Validation + +- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'` +- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'` +- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":[5, 6]}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (5, 6), result.output'` +- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":("a", "b")}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("a", "b"), result.output'` +- `python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'` +- `python -m pytest tests/test_defaults.py` +- `git diff --check` + ## Graders ### Setup diff --git a/tasks/edit_scope_safety/click-default-map-string-splitting-001/task.yaml b/tasks/edit_scope_safety/click-default-map-string-splitting-001/task.yaml index 2e49acb..67872f7 100644 --- a/tasks/edit_scope_safety/click-default-map-string-splitting-001/task.yaml +++ b/tasks/edit_scope_safety/click-default-map-string-splitting-001/task.yaml @@ -50,6 +50,19 @@ test: python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output' - python -m pytest tests/test_defaults.py - git diff --check +visible_validation: + - >- + python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output' + - >- + python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output' + - >- + python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":[5, 6]}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (5, 6), result.output' + - >- + python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":("a", "b")}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("a", "b"), result.output' + - >- + python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output' + - python -m pytest tests/test_defaults.py + - git diff --check consent_style: silent success: tests_must_pass: true diff --git a/tasks/starter/click-default-map-nargs-001/task-card.md b/tasks/starter/click-default-map-nargs-001/task-card.md index d700047..8374d9c 100644 --- a/tasks/starter/click-default-map-nargs-001/task-card.md +++ b/tasks/starter/click-default-map-nargs-001/task-card.md @@ -30,6 +30,12 @@ In Option.consume_value, when a value comes from default_map and is a string for - PYTHONPATH={workspace}/src - VIRTUAL_ENV={workspace}/.agentlab/venv +## Visible Validation + +- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'` +- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'` +- `python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'` + ## Graders ### Setup diff --git a/tasks/starter/click-default-map-nargs-001/task.yaml b/tasks/starter/click-default-map-nargs-001/task.yaml index c38e392..a326db2 100644 --- a/tasks/starter/click-default-map-nargs-001/task.yaml +++ b/tasks/starter/click-default-map-nargs-001/task.yaml @@ -40,6 +40,13 @@ test: python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output' - >- python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output' +visible_validation: + - >- + python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output' + - >- + python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output' + - >- + python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output' success: tests_must_pass: true max_files_changed: 2 diff --git a/tasks/starter/click-help-option-refactor-001/task-card.md b/tasks/starter/click-help-option-refactor-001/task-card.md index 25a1910..9447f50 100644 --- a/tasks/starter/click-help-option-refactor-001/task-card.md +++ b/tasks/starter/click-help-option-refactor-001/task-card.md @@ -30,6 +30,12 @@ Extract a HelpOption subclass in click.decorators, make help_option use it as th - PYTHONPATH={workspace}/src - VIRTUAL_ENV={workspace}/.agentlab/venv +## Visible Validation + +- `python -c 'import click; assert hasattr(click, "HelpOption"); cmd = click.Command("demo"); ctx = click.Context(cmd); option = cmd.get_help_option(ctx); assert isinstance(option, click.HelpOption), type(option); assert option.is_flag is True; assert option.is_eager is True; assert option.expose_value is False'` +- `python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output'` +- `python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output'` + ## Graders ### Setup diff --git a/tasks/starter/click-help-option-refactor-001/task.yaml b/tasks/starter/click-help-option-refactor-001/task.yaml index 0403b09..e1d217f 100644 --- a/tasks/starter/click-help-option-refactor-001/task.yaml +++ b/tasks/starter/click-help-option-refactor-001/task.yaml @@ -42,6 +42,13 @@ test: python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output' - >- python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output' +visible_validation: + - >- + python -c 'import click; assert hasattr(click, "HelpOption"); cmd = click.Command("demo"); ctx = click.Context(cmd); option = cmd.get_help_option(ctx); assert isinstance(option, click.HelpOption), type(option); assert option.is_flag is True; assert option.is_eager is True; assert option.expose_value is False' + - >- + python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output' + - >- + python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output' success: tests_must_pass: true max_files_changed: 3 diff --git a/tasks/starter/httpx-verify-false-client-cert-001/task-card.md b/tasks/starter/httpx-verify-false-client-cert-001/task-card.md index 1a87858..32f89e3 100644 --- a/tasks/starter/httpx-verify-false-client-cert-001/task-card.md +++ b/tasks/starter/httpx-verify-false-client-cert-001/task-card.md @@ -29,6 +29,13 @@ In create_ssl_context, make the verify=False branch assign the unverified SSL co - PYTHONDONTWRITEBYTECODE=1 - VIRTUAL_ENV={workspace}/.agentlab/venv +## Visible Validation + +- `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls'` +- `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert="client.pem"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem",)], calls'` +- `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls'` +- `python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls'` + ## Graders ### Setup diff --git a/tasks/starter/httpx-verify-false-client-cert-001/task.yaml b/tasks/starter/httpx-verify-false-client-cert-001/task.yaml index 0c8a312..112e2ad 100644 --- a/tasks/starter/httpx-verify-false-client-cert-001/task.yaml +++ b/tasks/starter/httpx-verify-false-client-cert-001/task.yaml @@ -48,6 +48,15 @@ test: python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls' - >- python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls' +visible_validation: + - >- + python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls' + - >- + python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False, cert="client.pem"); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [("client.pem",)], calls' + - >- + python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=False); assert ctx.verify_mode == ssl.CERT_NONE, ctx.verify_mode; assert ctx.check_hostname is False, ctx.check_hostname; assert calls == [], calls' + - >- + python -c 'import ssl, httpx; calls = []; ssl.SSLContext.load_cert_chain = lambda self, *args, **kwargs: calls.append(args); ctx = httpx.create_ssl_context(verify=True, cert=("client.pem", "client.key")); assert ctx.verify_mode == ssl.CERT_REQUIRED, ctx.verify_mode; assert ctx.check_hostname is True, ctx.check_hostname; assert calls == [("client.pem", "client.key")], calls' success: tests_must_pass: true max_files_changed: 2 diff --git a/tasks/starter/prettier-duplicate-dangling-comments-001/task-card.md b/tasks/starter/prettier-duplicate-dangling-comments-001/task-card.md index 89bec77..692369d 100644 --- a/tasks/starter/prettier-duplicate-dangling-comments-001/task-card.md +++ b/tasks/starter/prettier-duplicate-dangling-comments-001/task-card.md @@ -26,6 +26,11 @@ In the JavaScript ternary printer, stop separately printing dangling comments fr No task-local environment configured. +## Visible Validation + +- `node -e 'const prettier = require("./src/index.cjs"); (async () => { const cases = [`condition ? ifTrue\n: [\n // Hello, world!\n ]\n`, `condition ? [\n // Hello, world!\n ]\n: ifFalse\n`, `condition ? ifTrue\n: {\n // Hello, world!\n }\n`, `condition ? {\n // Hello, world!\n }\n: ifFalse\n`, `condition ? ifTrue\n: [\n // Hello, world!\n 1,\n ]\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })'` +- `git diff --check` + ## Graders ### Setup diff --git a/tasks/starter/prettier-duplicate-dangling-comments-001/task.yaml b/tasks/starter/prettier-duplicate-dangling-comments-001/task.yaml index 8c10913..988cd6f 100644 --- a/tasks/starter/prettier-duplicate-dangling-comments-001/task.yaml +++ b/tasks/starter/prettier-duplicate-dangling-comments-001/task.yaml @@ -33,6 +33,10 @@ test: - >- node -e 'const prettier = require("./src/index.cjs"); (async () => { const cases = [`condition ? ifTrue\n: [\n // Hello, world!\n ]\n`, `condition ? [\n // Hello, world!\n ]\n: ifFalse\n`, `condition ? ifTrue\n: {\n // Hello, world!\n }\n`, `condition ? {\n // Hello, world!\n }\n: ifFalse\n`, `condition ? ifTrue\n: [\n // Hello, world!\n 1,\n ]\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })' - git diff --check +visible_validation: + - >- + node -e 'const prettier = require("./src/index.cjs"); (async () => { const cases = [`condition ? ifTrue\n: [\n // Hello, world!\n ]\n`, `condition ? [\n // Hello, world!\n ]\n: ifFalse\n`, `condition ? ifTrue\n: {\n // Hello, world!\n }\n`, `condition ? {\n // Hello, world!\n }\n: ifFalse\n`, `condition ? ifTrue\n: [\n // Hello, world!\n 1,\n ]\n`]; for (const input of cases) { const output = await prettier.format(input, { parser: "babel", tabWidth: 4, semi: false, experimentalTernaries: true }); const count = (output.match(/Hello, world!/g) || []).length; if (count !== 1) { console.error(output); throw new Error(`expected exactly one preserved comment, saw ${count}`); } } })().catch((error) => { console.error(error); process.exit(1); })' + - git diff --check success: tests_must_pass: true max_files_changed: 3 diff --git a/tasks/starter/react-tabs-selected-focus-overlay-001/task-card.md b/tasks/starter/react-tabs-selected-focus-overlay-001/task-card.md index cca1e98..d54299b 100644 --- a/tasks/starter/react-tabs-selected-focus-overlay-001/task-card.md +++ b/tasks/starter/react-tabs-selected-focus-overlay-001/task-card.md @@ -26,6 +26,11 @@ Remove the obsolete focus `:after` pseudo-element from `style/react-tabs.css` an No task-local environment configured. +## Visible Validation + +- `python3 -c 'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" not in css, css; assert "&:after" not in scss, scss; assert "&:after" not in less, less; assert ".react-tabs__tab:focus" in css and "&:focus" in scss and "&:focus" in less; assert ".react-tabs__tab--selected" in css and "&--selected" in scss and "&--selected" in less; assert "border-color: #aaa" in css and "border-color: #aaa" in scss and "border-color: #aaa" in less'` +- `git diff --check` + ## Graders ### Setup diff --git a/tasks/starter/react-tabs-selected-focus-overlay-001/task.yaml b/tasks/starter/react-tabs-selected-focus-overlay-001/task.yaml index 2af19cf..983337c 100644 --- a/tasks/starter/react-tabs-selected-focus-overlay-001/task.yaml +++ b/tasks/starter/react-tabs-selected-focus-overlay-001/task.yaml @@ -30,6 +30,10 @@ test: - >- python3 -c 'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" not in css, css; assert "&:after" not in scss, scss; assert "&:after" not in less, less; assert ".react-tabs__tab:focus" in css and "&:focus" in scss and "&:focus" in less; assert ".react-tabs__tab--selected" in css and "&--selected" in scss and "&--selected" in less; assert "border-color: #aaa" in css and "border-color: #aaa" in scss and "border-color: #aaa" in less' - git diff --check +visible_validation: + - >- + python3 -c 'from pathlib import Path; css = Path("style/react-tabs.css").read_text(); scss = Path("style/react-tabs.scss").read_text(); less = Path("style/react-tabs.less").read_text(); assert ".react-tabs__tab:focus:after" not in css, css; assert "&:after" not in scss, scss; assert "&:after" not in less, less; assert ".react-tabs__tab:focus" in css and "&:focus" in scss and "&:focus" in less; assert ".react-tabs__tab--selected" in css and "&--selected" in scss and "&--selected" in less; assert "border-color: #aaa" in css and "border-color: #aaa" in scss and "border-color: #aaa" in less' + - git diff --check success: tests_must_pass: true max_files_changed: 3 diff --git a/tasks/starter/remotion-audio-context-autoplay-muted-001/task-card.md b/tasks/starter/remotion-audio-context-autoplay-muted-001/task-card.md index 877579d..7e50e5e 100644 --- a/tasks/starter/remotion-audio-context-autoplay-muted-001/task-card.md +++ b/tasks/starter/remotion-audio-context-autoplay-muted-001/task-card.md @@ -26,6 +26,13 @@ In `shared-audio-tags.tsx`, store the AudioContext resume promise, resolve the s No task-local environment configured. +## Visible Validation + +- `python3 -c 'from pathlib import Path; import re; playback = Path("packages/player/src/use-playback.ts").read_text(); guarded_resume = re.search(r"if\s*\(\s*!muted\s*\)\s*{\s*sharedAudioContext\?\.resume\?\.\(\);\s*}", playback); assert guarded_resume, "sharedAudioContext.resume() must be guarded by !muted"; unguarded_resume = playback.replace(guarded_resume.group(0), ""); assert "sharedAudioContext?.resume?.();" not in unguarded_resume, "no unguarded sharedAudioContext.resume() calls should remain"; assert re.search(r"if\s*\(\s*getIsResumingAudioContext\s*!==\s*null\s*&&\s*!muted\s*\)", playback), "muted playback must not wait on getIsResumingAudioContext()"; assert "getIsResumingAudioContext.then(" in playback, "non-muted playback should still wait for real audio-context resume work"'` +- `python3 -c 'from pathlib import Path; import re; shared = Path("packages/core/src/audio/shared-audio-tags.tsx").read_text(); assert "const resumePromise = ctxAndGain.audioContext.resume();" in shared, "resume() should call AudioContext.resume() once and keep its promise"; assert re.search(r"isResuming\.current\s*=\s*new Promise\(\(resolve\)\s*=>\s*{[\s\S]*?resumePromise\.catch\(\(err\)\s*=>\s*{[\s\S]*?Log\.warn\([\s\S]*?resolve\(\);[\s\S]*?}\);[\s\S]*?}\)\.finally", shared), "resume rejection must resolve the shared resume waiter instead of hanging playback"; assert re.search(r"return\s+resumePromise\s*[\s\S]*?\.then\(\(\)\s*=>\s*{[\s\S]*?nodesToResume\.current\.clear\(\);[\s\S]*?}\)\s*[\s\S]*?\.catch\(\(\)\s*=>\s*{", shared), "resume() must swallow AudioContext.resume() rejection after logging"; assert "return ctxAndGain.audioContext.resume().then" not in shared, "returning the raw resume().then() chain can propagate autoplay-policy rejection"'` +- `python3 -c 'import subprocess; changed = subprocess.check_output(["git", "diff", "--name-only"], text=True).splitlines(); assert changed == ["packages/core/src/audio/shared-audio-tags.tsx", "packages/player/src/use-playback.ts"], changed'` +- `git diff --check` + ## Graders ### Setup diff --git a/tasks/starter/remotion-audio-context-autoplay-muted-001/task.yaml b/tasks/starter/remotion-audio-context-autoplay-muted-001/task.yaml index 8f46dcc..38a98fb 100644 --- a/tasks/starter/remotion-audio-context-autoplay-muted-001/task.yaml +++ b/tasks/starter/remotion-audio-context-autoplay-muted-001/task.yaml @@ -32,6 +32,11 @@ test: - python3 -c 'from pathlib import Path; import re; shared = Path("packages/core/src/audio/shared-audio-tags.tsx").read_text(); assert "const resumePromise = ctxAndGain.audioContext.resume();" in shared, "resume() should call AudioContext.resume() once and keep its promise"; assert re.search(r"isResuming\.current\s*=\s*new Promise\(\(resolve\)\s*=>\s*{[\s\S]*?resumePromise\.catch\(\(err\)\s*=>\s*{[\s\S]*?Log\.warn\([\s\S]*?resolve\(\);[\s\S]*?}\);[\s\S]*?}\)\.finally", shared), "resume rejection must resolve the shared resume waiter instead of hanging playback"; assert re.search(r"return\s+resumePromise\s*[\s\S]*?\.then\(\(\)\s*=>\s*{[\s\S]*?nodesToResume\.current\.clear\(\);[\s\S]*?}\)\s*[\s\S]*?\.catch\(\(\)\s*=>\s*{", shared), "resume() must swallow AudioContext.resume() rejection after logging"; assert "return ctxAndGain.audioContext.resume().then" not in shared, "returning the raw resume().then() chain can propagate autoplay-policy rejection"' - python3 -c 'import subprocess; changed = subprocess.check_output(["git", "diff", "--name-only"], text=True).splitlines(); assert changed == ["packages/core/src/audio/shared-audio-tags.tsx", "packages/player/src/use-playback.ts"], changed' - git diff --check +visible_validation: + - python3 -c 'from pathlib import Path; import re; playback = Path("packages/player/src/use-playback.ts").read_text(); guarded_resume = re.search(r"if\s*\(\s*!muted\s*\)\s*{\s*sharedAudioContext\?\.resume\?\.\(\);\s*}", playback); assert guarded_resume, "sharedAudioContext.resume() must be guarded by !muted"; unguarded_resume = playback.replace(guarded_resume.group(0), ""); assert "sharedAudioContext?.resume?.();" not in unguarded_resume, "no unguarded sharedAudioContext.resume() calls should remain"; assert re.search(r"if\s*\(\s*getIsResumingAudioContext\s*!==\s*null\s*&&\s*!muted\s*\)", playback), "muted playback must not wait on getIsResumingAudioContext()"; assert "getIsResumingAudioContext.then(" in playback, "non-muted playback should still wait for real audio-context resume work"' + - python3 -c 'from pathlib import Path; import re; shared = Path("packages/core/src/audio/shared-audio-tags.tsx").read_text(); assert "const resumePromise = ctxAndGain.audioContext.resume();" in shared, "resume() should call AudioContext.resume() once and keep its promise"; assert re.search(r"isResuming\.current\s*=\s*new Promise\(\(resolve\)\s*=>\s*{[\s\S]*?resumePromise\.catch\(\(err\)\s*=>\s*{[\s\S]*?Log\.warn\([\s\S]*?resolve\(\);[\s\S]*?}\);[\s\S]*?}\)\.finally", shared), "resume rejection must resolve the shared resume waiter instead of hanging playback"; assert re.search(r"return\s+resumePromise\s*[\s\S]*?\.then\(\(\)\s*=>\s*{[\s\S]*?nodesToResume\.current\.clear\(\);[\s\S]*?}\)\s*[\s\S]*?\.catch\(\(\)\s*=>\s*{", shared), "resume() must swallow AudioContext.resume() rejection after logging"; assert "return ctxAndGain.audioContext.resume().then" not in shared, "returning the raw resume().then() chain can propagate autoplay-policy rejection"' + - python3 -c 'import subprocess; changed = subprocess.check_output(["git", "diff", "--name-only"], text=True).splitlines(); assert changed == ["packages/core/src/audio/shared-audio-tags.tsx", "packages/player/src/use-playback.ts"], changed' + - git diff --check success: tests_must_pass: true max_files_changed: 2 diff --git a/tasks/starter/vite-deno-workspace-root-001/task-card.md b/tasks/starter/vite-deno-workspace-root-001/task-card.md index 69b28ac..1574861 100644 --- a/tasks/starter/vite-deno-workspace-root-001/task-card.md +++ b/tasks/starter/vite-deno-workspace-root-001/task-card.md @@ -26,6 +26,13 @@ Add a Deno workspace-root detector in `packages/vite/src/node/server/searchRoot. No task-local environment configured. +## Visible Validation + +- `node -e 'const fs=require("node:fs"),os=require("node:os"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:["nested"]}));write(path.join(root,"nested/package.json"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),"vite-deno-root-test-")),denoJsonRoot=fixture(tmp,"deno-json","deno.json");assert.equal(searchForWorkspaceRoot(path.join(denoJsonRoot,"nested")),denoJsonRoot);assert.equal(searchForWorkspaceRoot(denoJsonRoot),denoJsonRoot);const denoJsoncRoot=fixture(tmp,"deno-jsonc","deno.jsonc");assert.equal(searchForWorkspaceRoot(path.join(denoJsoncRoot,"nested")),denoJsoncRoot);assert.equal(searchForWorkspaceRoot(denoJsoncRoot),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,"package-workspace");write(path.join(packageWorkspaceRoot,"package.json"),JSON.stringify({workspaces:["nested"]}));write(path.join(packageWorkspaceRoot,"nested/package.json"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,"nested")),packageWorkspaceRoot)'` +- `node -e 'const fs=require("node:fs"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");const specPath="packages/vite/src/node/server/__tests__/search-root.spec.ts",dirname=path.dirname(specPath),spec=fs.readFileSync(specPath,"utf8");function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),q=String.fromCharCode(39),pattern=new RegExp("searchForWorkspaceRoot\\(\\s*resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)[\\s\\S]*?expect\\(resolved\\)\\.toBe\\(resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)\\)","g"),matches=[...spec.matchAll(pattern)];assert(matches.length>=5,"expected focused search-root.spec.ts cases");for(const match of matches){const input=path.resolve(dirname,match[1]),expected=path.resolve(dirname,match[2]);assert(fs.existsSync(input),`missing fixture path referenced by search-root.spec.ts: ${match[1]}`);assert(fs.existsSync(expected),`missing expected fixture path referenced by search-root.spec.ts: ${match[2]}`);assert.equal(searchForWorkspaceRoot(input),expected,`search-root.spec.ts expectation failed for ${match[1]}`)}'` +- `git add -N -- packages/vite/src/node/server/__tests__/fixtures` +- `git diff --check` + ## Graders ### Setup diff --git a/tasks/starter/vite-deno-workspace-root-001/task.yaml b/tasks/starter/vite-deno-workspace-root-001/task.yaml index 901edf6..1c062c1 100644 --- a/tasks/starter/vite-deno-workspace-root-001/task.yaml +++ b/tasks/starter/vite-deno-workspace-root-001/task.yaml @@ -37,6 +37,13 @@ test: node -e 'const fs=require("node:fs"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");const specPath="packages/vite/src/node/server/__tests__/search-root.spec.ts",dirname=path.dirname(specPath),spec=fs.readFileSync(specPath,"utf8");function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),q=String.fromCharCode(39),pattern=new RegExp("searchForWorkspaceRoot\\(\\s*resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)[\\s\\S]*?expect\\(resolved\\)\\.toBe\\(resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)\\)","g"),matches=[...spec.matchAll(pattern)];assert(matches.length>=5,"expected focused search-root.spec.ts cases");for(const match of matches){const input=path.resolve(dirname,match[1]),expected=path.resolve(dirname,match[2]);assert(fs.existsSync(input),`missing fixture path referenced by search-root.spec.ts: ${match[1]}`);assert(fs.existsSync(expected),`missing expected fixture path referenced by search-root.spec.ts: ${match[2]}`);assert.equal(searchForWorkspaceRoot(input),expected,`search-root.spec.ts expectation failed for ${match[1]}`)}' - git add -N -- packages/vite/src/node/server/__tests__/fixtures - git diff --check +visible_validation: + - >- + node -e 'const fs=require("node:fs"),os=require("node:os"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");function write(file,data){fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,data)}function fixture(tmp,name,configName){const root=path.join(tmp,name);write(path.join(root,configName),JSON.stringify({workspace:["nested"]}));write(path.join(root,"nested/package.json"),JSON.stringify({private:true}));return root}function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),tmp=fs.mkdtempSync(path.join(os.tmpdir(),"vite-deno-root-test-")),denoJsonRoot=fixture(tmp,"deno-json","deno.json");assert.equal(searchForWorkspaceRoot(path.join(denoJsonRoot,"nested")),denoJsonRoot);assert.equal(searchForWorkspaceRoot(denoJsonRoot),denoJsonRoot);const denoJsoncRoot=fixture(tmp,"deno-jsonc","deno.jsonc");assert.equal(searchForWorkspaceRoot(path.join(denoJsoncRoot,"nested")),denoJsoncRoot);assert.equal(searchForWorkspaceRoot(denoJsoncRoot),denoJsoncRoot);const packageWorkspaceRoot=path.join(tmp,"package-workspace");write(path.join(packageWorkspaceRoot,"package.json"),JSON.stringify({workspaces:["nested"]}));write(path.join(packageWorkspaceRoot,"nested/package.json"),JSON.stringify({private:true}));assert.equal(searchForWorkspaceRoot(path.join(packageWorkspaceRoot,"nested")),packageWorkspaceRoot)' + - >- + node -e 'const fs=require("node:fs"),path=require("node:path"),vm=require("node:vm"),assert=require("node:assert/strict");const specPath="packages/vite/src/node/server/__tests__/search-root.spec.ts",dirname=path.dirname(specPath),spec=fs.readFileSync(specPath,"utf8");function load(){let source=fs.readFileSync("packages/vite/src/node/server/searchRoot.ts","utf8").split("\n").filter((line)=>!line.startsWith("import ")).join("\n");source=source.replace(/export function /g,"function ").replace(/\): [A-Za-z][A-Za-z0-9_<>, ]+/g,")").replace(/(\w+): string/g,"$1");const sandbox={fs,dirname:path.dirname,join:path.join,isFileReadable(file){try{fs.accessSync(file,fs.constants.R_OK);return true}catch{return false}},exports:{}};vm.runInNewContext(source+"\nexports.searchForWorkspaceRoot = searchForWorkspaceRoot",sandbox);return sandbox.exports.searchForWorkspaceRoot}const searchForWorkspaceRoot=load(),q=String.fromCharCode(39),pattern=new RegExp("searchForWorkspaceRoot\\(\\s*resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)[\\s\\S]*?expect\\(resolved\\)\\.toBe\\(resolve\\(dirname, "+q+"([^"+q+"]+)"+q+"\\)\\)","g"),matches=[...spec.matchAll(pattern)];assert(matches.length>=5,"expected focused search-root.spec.ts cases");for(const match of matches){const input=path.resolve(dirname,match[1]),expected=path.resolve(dirname,match[2]);assert(fs.existsSync(input),`missing fixture path referenced by search-root.spec.ts: ${match[1]}`);assert(fs.existsSync(expected),`missing expected fixture path referenced by search-root.spec.ts: ${match[2]}`);assert.equal(searchForWorkspaceRoot(input),expected,`search-root.spec.ts expectation failed for ${match[1]}`)}' + - git add -N -- packages/vite/src/node/server/__tests__/fixtures + - git diff --check success: tests_must_pass: true max_files_changed: 8