Add hidden verifier patch support

agentlab/cli/task.py

@@ -337,6 +337,7 @@ def _print_smoke_test_result(evaluation: object) -> None:
 def _print_failed_reference_checks(verification: ReferenceVerification) -> None:
     checks = verification.setup_checks + verification.baseline_checks
     checks += [verification.artifact_check] + verification.target_checks
+    checks += verification.hidden_verifier.checks
     for check in checks:
         if check.passed:
             continue

agentlab/evidence/results.py

@@ -107,6 +107,11 @@ def to_result_dict(run: Any) -> Dict[str, Any]:
         "diff_path": str(run.agent_run.diff_path),
         "run_dir": str(run.run_dir),
     }
+    hidden_verifier = _hidden_verifier_to_dict(
+        getattr(run, "hidden_verifier", None)
+    )
+    if hidden_verifier is not None:
+        result["hidden_verifier"] = hidden_verifier
     if setup_created_untracked_changed_paths:
         result["setup_created_untracked_changed_paths"] = (
             setup_created_untracked_changed_paths
@@ -202,6 +207,11 @@ def reference_verification_to_result_dict(verification: Any) -> Dict[str, Any]:
         "diff_path": _display_path(verification.diff_path, output_dir),
         "run_dir": _display_path(output_dir, output_dir),
     }
+    hidden_verifier = _hidden_verifier_to_dict(
+        getattr(verification, "hidden_verifier", None)
+    )
+    if hidden_verifier is not None:
+        result["hidden_verifier"] = hidden_verifier
     if setup_created_untracked_changed_paths:
         result["setup_created_untracked_changed_paths"] = (
             setup_created_untracked_changed_paths
@@ -243,6 +253,20 @@ def _check_to_dict(check: CheckResult) -> Dict[str, Any]:
     }
 
 
+def _hidden_verifier_to_dict(hidden_verifier: Any) -> Dict[str, Any] | None:
+    if hidden_verifier is None or not getattr(hidden_verifier, "configured", False):
+        return None
+    return {
+        "configured": True,
+        "patch": getattr(hidden_verifier, "patch", None),
+        "checks": [
+            _check_to_dict(check)
+            for check in getattr(hidden_verifier, "checks", [])
+        ],
+        "restore_notes": list(getattr(hidden_verifier, "restore_notes", [])),
+    }
+
+
 def _reference_artifact_to_dict(artifact: Any) -> Dict[str, Any] | None:
     if artifact is None:
         return None

agentlab/evidence/review_proposals.py

@@ -405,6 +405,13 @@ def _default_evidence(context: ReviewProposalContext) -> tuple[str, ...]:
             passed = check.get("passed")
             if command:
                 evidence.append(f"check: passed={passed}; command={command}")
+    for check in _hidden_verifier_checks(result)[:3]:
+        command = str(check.get("command") or "").strip()
+        passed = check.get("passed")
+        if command:
+            evidence.append(
+                f"hidden verifier: passed={passed}; command={command}"
+            )
 
     if context.report_excerpt:
         evidence.append(f"report.md excerpt: {context.report_excerpt}")
@@ -429,6 +436,14 @@ def _looks_like_setup_issue(result: OutcomeEvidence) -> bool:
                     str(check.get("command") or ""),
                 ]
             )
+    for check in _hidden_verifier_checks(result):
+        text_parts.extend(
+            [
+                str(check.get("stdout") or ""),
+                str(check.get("stderr") or ""),
+                str(check.get("command") or ""),
+            ]
+        )
     text = "\n".join(text_parts).lower()
     return any(
         needle in text
@@ -444,6 +459,16 @@ def _looks_like_setup_issue(result: OutcomeEvidence) -> bool:
     )
 
 
+def _hidden_verifier_checks(result: OutcomeEvidence) -> list[Mapping[str, Any]]:
+    hidden_verifier = result.raw.get("hidden_verifier")
+    if not isinstance(hidden_verifier, Mapping):
+        return []
+    checks = hidden_verifier.get("checks")
+    if not isinstance(checks, list):
+        return []
+    return [check for check in checks if isinstance(check, Mapping)]
+
+
 def _read_excerpt(path: Path | None, max_chars: int = 600) -> str:
     if path is None:
         return ""

agentlab/execution/hidden_verifier.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import shutil
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Mapping
+
+from agentlab.execution.commands import isolated_git_env
+from agentlab.execution.commands import run_commands
+from agentlab.execution.commands import run_git
+from agentlab.execution.scoring import CheckResult
+from agentlab.tasks import EvalTask
+
+
+@dataclass(frozen=True)
+class HiddenVerifierResult:
+    configured: bool = False
+    patch: str | None = None
+    checks: list[CheckResult] = field(default_factory=list)
+    restore_notes: list[str] = field(default_factory=list)
+
+
+def run_hidden_verifier(
+    task: EvalTask,
+    workspace: Path,
+    env: Mapping[str, str],
+) -> HiddenVerifierResult:
+    verifier = task.hidden_verifier
+    if verifier is None:
+        return HiddenVerifierResult()
+    if task.source_path is None:
+        return HiddenVerifierResult(
+            configured=True,
+            patch=verifier.patch,
+            checks=[
+                CheckResult(
+                    command=f"hidden verifier patch: {verifier.patch}",
+                    returncode=1,
+                    stderr="hidden verifier requires task.source_path",
+                )
+            ],
+        )
+
+    patch_path = task.source_path.parent / verifier.patch
+    with _workspace_snapshot(workspace) as snapshot:
+        checks: list[CheckResult] = []
+        restore_notes: list[str] = []
+        try:
+            apply_check = _apply_hidden_patch(workspace, patch_path, verifier.patch)
+            checks.append(apply_check)
+            if apply_check.passed:
+                checks.extend(run_commands(verifier.commands, workspace, env=env))
+        finally:
+            restore_notes.extend(snapshot.restore())
+        if restore_notes:
+            checks.append(
+                CheckResult(
+                    command="restore hidden verifier workspace",
+                    returncode=1,
+                    stderr="\n".join(restore_notes),
+                )
+            )
+
+    return HiddenVerifierResult(
+        configured=True,
+        patch=verifier.patch,
+        checks=checks,
+        restore_notes=restore_notes,
+    )
+
+
+def _apply_hidden_patch(
+    workspace: Path,
+    patch_path: Path,
+    display_path: str,
+) -> CheckResult:
+    completed = run_git(
+        ["apply", str(patch_path.resolve())],
+        cwd=workspace,
+        env=isolated_git_env(),
+    )
+    return CheckResult(
+        command=f"git apply hidden verifier patch: {display_path}",
+        returncode=completed.returncode,
+        stdout=completed.stdout,
+        stderr=completed.stderr,
+    )
+
+
+class _WorkspaceSnapshot:
+    def __init__(self, workspace: Path, snapshot: Path):
+        self._workspace = workspace
+        self._snapshot = snapshot
+
+    def restore(self) -> list[str]:
+        notes: list[str] = []
+        try:
+            _replace_worktree_contents(self._snapshot, self._workspace)
+        except OSError as exc:
+            notes.append(f"failed to restore hidden verifier worktree: {exc}")
+        return notes
+
+
+class _workspace_snapshot:
+    def __init__(self, workspace: Path):
+        self._workspace = workspace
+        self._temp: tempfile.TemporaryDirectory[str] | None = None
+        self._snapshot: _WorkspaceSnapshot | None = None
+
+    def __enter__(self) -> _WorkspaceSnapshot:
+        self._temp = tempfile.TemporaryDirectory(prefix="agentlab-hidden-verifier-")
+        root = Path(self._temp.name)
+        snapshot_path = root / "worktree"
+        _copy_worktree_contents(self._workspace, snapshot_path)
+        self._snapshot = _WorkspaceSnapshot(self._workspace, snapshot_path)
+        return self._snapshot
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        if self._temp is not None:
+            self._temp.cleanup()
+
+
+def _copy_worktree_contents(source: Path, destination: Path) -> None:
+    destination.mkdir(parents=True, exist_ok=True)
+    for child in source.iterdir():
+        target = destination / child.name
+        if child.is_symlink():
+            shutil.copy2(child, target, follow_symlinks=False)
+        elif child.is_dir():
+            shutil.copytree(child, target, symlinks=True)
+        else:
+            shutil.copy2(child, target)
+
+
+def _replace_worktree_contents(snapshot: Path, workspace: Path) -> None:
+    for child in workspace.iterdir():
+        if child.is_dir() and not child.is_symlink():
+            shutil.rmtree(child)
+        else:
+            child.unlink()
+    _copy_worktree_contents(snapshot, workspace)

agentlab/execution/phases.py

@@ -7,6 +7,8 @@
 from agentlab.execution.commands import run_commands
 from agentlab.execution.changed_paths import capture_change_baseline
 from agentlab.execution.changed_paths import capture_diff_details_preserving_index
+from agentlab.execution.hidden_verifier import HiddenVerifierResult
+from agentlab.execution.hidden_verifier import run_hidden_verifier
 from agentlab.tasks.environment import build_task_environment
 from agentlab.runtime.patches import count_patch_lines
 from agentlab.execution.scoring import CheckResult
@@ -35,6 +37,9 @@ class TaskExecution:
     baseline_checks: list[CheckResult] = field(default_factory=list)
     action_checks: list[CheckResult] = field(default_factory=list)
     target_checks: list[CheckResult] = field(default_factory=list)
+    hidden_verifier: HiddenVerifierResult = field(
+        default_factory=HiddenVerifierResult
+    )
     files_changed: list[str] = field(default_factory=list)
     lines_added: int = 0
     lines_deleted: int = 0
@@ -84,7 +89,12 @@ def execute_task_phases(
     patch_stats = count_patch_lines(
         resolved_diff_path.read_text(encoding="utf-8")
     )
-    target_checks = run_commands(task.test, prepared.path, env=task_env)
+    hidden_verifier = run_hidden_verifier(task, prepared.path, task_env)
+    target_checks = (
+        []
+        if hidden_verifier.restore_notes
+        else run_commands(task.test, prepared.path, env=task_env)
+    )
     all_checks = (
         setup_checks
         + baseline_checks
@@ -96,6 +106,7 @@ def execute_task_phases(
         all_checks,
         captured_diff.files_changed,
         agent_error=action_result.agent_error,
+        hidden_checks=hidden_verifier.checks,
     )
     return TaskExecution(
         task=task,
@@ -105,6 +116,7 @@ def execute_task_phases(
         baseline_checks=baseline_checks,
         action_checks=action_result.checks,
         target_checks=target_checks,
+        hidden_verifier=hidden_verifier,
         files_changed=captured_diff.files_changed,
         lines_added=patch_stats.lines_added,
         lines_deleted=patch_stats.lines_deleted,

agentlab/execution/runner.py

@@ -2,13 +2,14 @@
 
 import time
 import uuid
-from dataclasses import dataclass, replace
+from dataclasses import dataclass, field, replace
 from pathlib import Path
 
 from agentlab.agents.base import AgentAdapter
 from agentlab.agents.base import AgentRun
 from agentlab.reports.trial_markdown import render_markdown_report
 from agentlab.execution.scoring import Score
+from agentlab.execution.hidden_verifier import HiddenVerifierResult
 from agentlab.execution.phases import TaskActionResult
 from agentlab.execution.phases import execute_task_phases
 from agentlab.tasks import EvalTask
@@ -22,6 +23,9 @@ class EvaluationRun:
     run_dir: Path
     report_path: Path
     result_path: Path
+    hidden_verifier: HiddenVerifierResult = field(
+        default_factory=HiddenVerifierResult
+    )
     workspace_history_policy: str = "unknown"
     workspace_base_ref: str = "unknown"
 
@@ -73,6 +77,7 @@ def run_agent(workspace: Path, _task_env: object) -> TaskActionResult:
         run_dir=run_dir,
         report_path=report_path,
         result_path=result_path,
+        hidden_verifier=execution.hidden_verifier,
         workspace_history_policy=execution.workspace_history_policy,
         workspace_base_ref=execution.workspace_base_ref,
     )

agentlab/execution/scoring.py

@@ -35,17 +35,25 @@ def calculate_grader_outcome(
     checks: Iterable[CheckResult],
     files_changed: Sequence[str],
     agent_error: Optional[str] = None,
+    hidden_checks: Iterable[CheckResult] = (),
 ) -> Score:
     check_results = list(checks)
+    hidden_check_results = list(hidden_checks)
     notes = _outcome_notes(task, files_changed)
-    checks_passed = (
+    public_checks_passed = (
         all(check.passed for check in check_results)
         if task.success.tests_must_pass
         else True
     )
+    hidden_checks_passed = all(check.passed for check in hidden_check_results)
 
     return Score(
-        tests_passed=agent_error is None and checks_passed and not notes,
+        tests_passed=(
+            agent_error is None
+            and public_checks_passed
+            and hidden_checks_passed
+            and not notes
+        ),
         checks=check_results,
         notes=notes,
     )

agentlab/reports/operability_evidence.py

@@ -118,10 +118,31 @@ def _verifier_state_facts(results: list[OutcomeEvidence]) -> list[tuple[str, obj
         ("final_grader_status", _result_values(results, lambda result: result.status)),
         ("checks_array", _coverage(results, lambda result: bool(result.checks))),
         ("graders_array", _coverage(results, lambda result: bool(result.graders))),
+        (
+            "hidden_verifier_configured",
+            _coverage(results, _hidden_verifier_configured),
+        ),
+        (
+            "hidden_verifier_checks",
+            _coverage(results, _hidden_verifier_checks),
+        ),
         ("intermediate_verifier_movement", UNKNOWN),
     ]
 
 
+def _hidden_verifier_configured(result: OutcomeEvidence) -> bool:
+    hidden_verifier = result.raw.get("hidden_verifier")
+    return isinstance(hidden_verifier, Mapping)
+
+
+def _hidden_verifier_checks(result: OutcomeEvidence) -> bool:
+    hidden_verifier = result.raw.get("hidden_verifier")
+    if not isinstance(hidden_verifier, Mapping):
+        return False
+    checks = hidden_verifier.get("checks")
+    return isinstance(checks, list) and bool(checks)
+
+
 def _halt_reasons_facts(results: list[OutcomeEvidence]) -> list[tuple[str, object]]:
     return [
         (