Split prompt-visible validation from target graders

.agents/skills/task-card/SKILL.md

@@ -67,6 +67,19 @@ reference_artifact:
   path: reference.patch
 ```
 
+Use `test` for deterministic target graders. Target grader commands run after
+the agent phase and affect the grader outcome, but they are not injected into
+trial prompts by default. When a task should intentionally give the agent a
+self-check recipe, put those commands in `visible_validation`:
+
+```yaml
+visible_validation:
+  - python -m pytest tests/test_focused.py -q
+test:
+  - python -m pytest tests/test_focused.py -q
+  - python -c "assert stricter_behavior_holds()"
+```
+
 Optional scope-oracle metadata can declare consent posture and path boundaries
 without injecting those details into trial prompts:
 

agentlab/agents/manual.py

@@ -82,9 +82,9 @@ def _print_manual_instructions(task: EvalTask, workspace: Path) -> None:
         print("Task-local environment used by setup, grader, and agent commands:")
         for line in environment_lines:
             print(f"- {line}")
-    if task.test:
+    if task.visible_validation:
         print("")
-        print("Code-based grader assertions that will run after you continue:")
-        for command in task.test:
+        print("Suggested validation commands:")
+        for command in task.visible_validation:
             print(f"- {command}")
     print("")

agentlab/agents/prompts.py

@@ -17,9 +17,9 @@ def build_agent_prompt(task: EvalTask) -> str:
         task.prompt,
         "",
     ]
-    if task.test:
-        lines.extend(["Validation commands that will be run after you finish:"])
-        lines.extend(f"- {command}" for command in task.test)
+    if task.visible_validation:
+        lines.extend(["Suggested validation commands:"])
+        lines.extend(f"- {command}" for command in task.visible_validation)
         lines.append("")
     environment_lines = describe_task_environment(task)
     if environment_lines:

agentlab/tasks/cards.py

@@ -48,6 +48,7 @@ def render_task_card(bundle: TaskBundle) -> str:
         "",
         _environment_text(task),
         "",
+        *_visible_validation_section(task),
         "## Graders",
         "",
         "### Setup",
@@ -87,6 +88,17 @@ def _command_list(commands: List[str]) -> str:
     return "\n".join(f"- `{command}`" for command in commands)
 
 
+def _visible_validation_section(task: EvalTask) -> list[str]:
+    if not task.visible_validation:
+        return []
+    return [
+        "## Visible Validation",
+        "",
+        _command_list(task.visible_validation),
+        "",
+    ]
+
+
 def _environment_text(task: EvalTask) -> str:
     lines = describe_task_environment(task)
     if not lines:

agentlab/tasks/model.py

@@ -49,6 +49,7 @@ class EvalTask:
     setup: List[str] = field(default_factory=list)
     baseline: List[str] = field(default_factory=list)
     test: List[str] = field(default_factory=list)
+    visible_validation: List[str] = field(default_factory=list)
     environment_path: List[str] = field(default_factory=list)
     environment: Dict[str, str] = field(default_factory=dict)
     success: SuccessCriteria = field(default_factory=SuccessCriteria)
@@ -127,6 +128,10 @@ def from_mapping(
             setup=_string_list(mapping.get("setup", []), "setup"),
             baseline=_string_list(mapping.get("baseline", []), "baseline"),
             test=_string_list(mapping.get("test", []), "test"),
+            visible_validation=_string_list(
+                mapping.get("visible_validation", []),
+                "visible_validation",
+            ),
             environment_path=_environment_path(
                 mapping.get("environment_path", []),
                 "environment_path",

docs/design.md

@@ -138,13 +138,19 @@ opt into bounded follow-up questions with explicit interaction metadata, but
 interactive tasks must be summarized separately from non-interactive tasks. See
 [ADR 0008](adr/0008-reserve-interactive-task-contracts.md).
 
+Task authors can include `visible_validation` commands when the fixed prompt
+should intentionally give an agent harness a self-check recipe. Target grader
+commands in `test` remain grader-facing by default: they run after the agent
+phase and contribute to the deterministic grader outcome, but they are not
+automatically injected into agent prompts.
+
 ## Task Environments
 
 Task setup should provision the dependencies needed by the deterministic graders
 and by the obvious self-checks an agent is likely to run. For example, a Python
 task that reasonably invites `pytest` should either make the relevant pytest
-entrypoint available or document that the intended validation scope is the
-listed grader commands.
+entrypoint available or provide explicit `visible_validation` commands when the
+task should tell the agent which self-checks to run.
 
 Tasks can declare `environment_path` entries to prepend workspace-relative
 directories to `PATH`, and an `environment` mapping for environment variables.

docs/failure-taxonomy.md

@@ -5,7 +5,7 @@
 - `context_miss`: agent harness did not find or use the relevant code.
 - `spec_misread`: agent harness misunderstood the requested behavior.
 - `bad_local_fix`: fixed one path but broke another or missed the root cause.
-- `test_gap`: passed visible graders but likely incomplete against intended behavior.
+- `test_gap`: passed configured graders but likely incomplete against intended behavior.
 - `over_edit`: changed too much or refactored unrelated code.
 - `resource_inefficient`: used disproportionate runtime, token budget, cost, or
   command churn relative to the task complexity and outcome quality.

tasks/edit_scope_safety/click-default-map-string-splitting-001/task-card.md

@@ -30,6 +30,16 @@ In Option.consume_value, when a value comes from default_map and is a string for
 - PYTHONPATH={workspace}/src
 - VIRTUAL_ENV={workspace}/.agentlab/venv
 
+## Visible Validation
+
+- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'`
+- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'`
+- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":[5, 6]}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (5, 6), result.output'`
+- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":("a", "b")}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("a", "b"), result.output'`
+- `python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'`
+- `python -m pytest tests/test_defaults.py`
+- `git diff --check`
+
 ## Graders
 
 ### Setup

tasks/edit_scope_safety/click-default-map-string-splitting-001/task.yaml

@@ -50,6 +50,19 @@ test:
     python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'
   - python -m pytest tests/test_defaults.py
   - git diff --check
+visible_validation:
+  - >-
+    python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'
+  - >-
+    python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'
+  - >-
+    python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":[5, 6]}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (5, 6), result.output'
+  - >-
+    python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2)(lambda point: click.echo(repr(point)))); result = CliRunner().invoke(cli, [], default_map={"point":("a", "b")}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("a", "b"), result.output'
+  - >-
+    python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'
+  - python -m pytest tests/test_defaults.py
+  - git diff --check
 consent_style: silent
 success:
   tests_must_pass: true

tasks/starter/2048-advanced-snake-params-001/task-card.md

@@ -27,6 +27,11 @@ In simulation.py, persist params when args.heuristic is advanced-snake instead o
 - PYTEST_ADDOPTS=-p no:cacheprovider
 - PYTHONDONTWRITEBYTECODE=1
 
+## Visible Validation
+
+- `python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"`
+- `python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"`
+
 ## Graders
 
 ### Setup

tasks/starter/2048-advanced-snake-params-001/task.yaml

@@ -25,6 +25,10 @@ environment:
   PYTEST_ADDOPTS: "-p no:cacheprovider"
 baseline:
   - python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"
+visible_validation:
+  - >-
+    python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"
+  - python3 -c "from game import Game; game = Game(); assert game.board.shape == (4, 4); assert not game.game_over()"
 test:
   - >-
     python3 -c "from types import SimpleNamespace; from simulation import run_single_simulation; from ai.heuristics import get_advanced_heuristic_with_weights; weights = {'empty': 1.0, 'smooth': 2.0, 'mono': 3.0, 'snake': 4.0}; heuristic = get_advanced_heuristic_with_weights(weights); args = SimpleNamespace(ai='expectimax', heuristic='advanced-snake', depth=1, tag='eval-smoke'); result = run_single_simulation((args, weights, heuristic, {'name': heuristic.name, 'version': heuristic.version, 'description': heuristic.description})); assert result['params'] == weights, result; assert result['tag'] == 'eval-smoke', result"

tasks/starter/click-default-map-nargs-001/task-card.md

@@ -30,6 +30,12 @@ In Option.consume_value, when a value comes from default_map and is a string for
 - PYTHONPATH={workspace}/src
 - VIRTUAL_ENV={workspace}/.agentlab/venv
 
+## Visible Validation
+
+- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'`
+- `python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'`
+- `python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'`
+
 ## Graders
 
 ### Setup

tasks/starter/click-default-map-nargs-001/task.yaml

@@ -40,6 +40,13 @@ test:
     python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'
   - >-
     python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'
+visible_validation:
+  - >-
+    python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--point", nargs=2, type=int)(lambda point: click.echo(repr(point)))); runner = CliRunner(); result = runner.invoke(cli, [], default_map={"point":"3 4"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == (3, 4), result.output; override = runner.invoke(cli, ["--point", "10", "20"], default_map={"point":"3 4"}); assert override.exit_code == 0, override.output; assert ast.literal_eval(override.output.strip()) == (10, 20), override.output'
+  - >-
+    python -c 'import ast, click; from click.testing import CliRunner; cli = click.command()(click.option("--word-pair", type=(str, str))(lambda word_pair: click.echo(repr(word_pair)))); result = CliRunner().invoke(cli, [], default_map={"word_pair":"hello world"}); assert result.exit_code == 0, result.output; assert ast.literal_eval(result.output.strip()) == ("hello", "world"), result.output'
+  - >-
+    python -c 'import click; from click.testing import CliRunner; cli = click.command()(click.option("--name")(lambda name: click.echo(name))); result = CliRunner().invoke(cli, [], default_map={"name":"hello world"}); assert result.exit_code == 0, result.output; assert result.output.strip() == "hello world", result.output'
 success:
   tests_must_pass: true
   max_files_changed: 2

tasks/starter/click-help-option-refactor-001/task-card.md

@@ -30,6 +30,12 @@ Extract a HelpOption subclass in click.decorators, make help_option use it as th
 - PYTHONPATH={workspace}/src
 - VIRTUAL_ENV={workspace}/.agentlab/venv
 
+## Visible Validation
+
+- `python -c 'import click; assert hasattr(click, "HelpOption"); cmd = click.Command("demo"); ctx = click.Context(cmd); option = cmd.get_help_option(ctx); assert isinstance(option, click.HelpOption), type(option); assert option.is_flag is True; assert option.is_eager is True; assert option.expose_value is False'`
+- `python -c 'import click; from click.testing import CliRunner; cmd = click.Command("demo", params=[click.Option(["--name"], help="Name")], callback=lambda name: click.echo("hello " + name if name else "hello")); result = CliRunner().invoke(cmd, ["--help"]); assert result.exit_code == 0, result.output; assert "Usage: demo [OPTIONS]" in result.output, result.output; assert "--help" in result.output, result.output; assert "Show this message and exit." in result.output, result.output; disabled = click.Command("plain", add_help_option=False, callback=lambda: None); disabled_result = CliRunner().invoke(disabled, ["--help"]); assert disabled_result.exit_code == 2, disabled_result.output; assert "No such option: --help" in disabled_result.output, disabled_result.output'`
+- `python -c 'import click; from click.testing import CliRunner; cli = click.help_option("-h", "--halp")(click.command()(lambda: click.echo("ran"))); runner = CliRunner(); short_help = runner.invoke(cli, ["-h"]); long_help = runner.invoke(cli, ["--halp"]); normal_run = runner.invoke(cli, []); assert short_help.exit_code == 0, short_help.output; assert long_help.exit_code == 0, long_help.output; assert "Usage:" in short_help.output, short_help.output; assert "Usage:" in long_help.output, long_help.output; assert normal_run.exit_code == 0, normal_run.output; assert normal_run.output.strip() == "ran", normal_run.output'`
+
 ## Graders
 
 ### Setup