Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion examples/experiment_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,14 @@
EvaluationSpec,
Evaluator,
ExperimentStore,
SandboxError,
SandboxTrialExecutor,
TrialCase,
TrialRunner,
build_default_images,
compare_experiments,
)
from tbcore.testing import docker_available

AGENT = """from pathlib import Path
import json
Expand Down Expand Up @@ -140,7 +142,19 @@ def main() -> None:
"--image", help="already built sandbox image; default builds bundled images"
)
args = parser.parse_args()
print(json.dumps(run(args.store, args.image), indent=2))
if not docker_available():
parser.exit(
1,
"error: Docker is not available. Install Docker, start the daemon (Docker Desktop "
"on Windows/macOS) in Linux-container mode, and retry.\n",
)
try:
result = run(args.store, args.image)
except SandboxError as exc:
parser.exit(1, f"error: {exc}\n")
except subprocess.CalledProcessError as exc:
parser.exit(1, f"error: {' '.join(exc.cmd)} failed:\n{exc.stderr}")
print(json.dumps(result, indent=2))


if __name__ == "__main__":
Expand Down
25 changes: 24 additions & 1 deletion tests/test_experiment_example.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import shutil
from pathlib import Path

import pytest
from examples import experiment_workflow

from tbcore import CapturePolicy, ExperimentStore
from tbcore import CapturePolicy, ExperimentStore, SandboxError


def test_documented_example_and_disposable_backup_recovery(tmp_path: Path, monkeypatch):
Expand Down Expand Up @@ -35,3 +36,25 @@ def __call__(self, case, workspace, attempt_id, context):
encoding="utf-8"
) == "Write the target answer."
recovered.close()


def test_main_exits_cleanly_when_docker_is_unavailable(tmp_path: Path, monkeypatch, capsys):
monkeypatch.setattr(experiment_workflow, "docker_available", lambda: False)
monkeypatch.setattr("sys.argv", ["experiment_workflow.py", "--store", str(tmp_path)])
with pytest.raises(SystemExit) as exited:
experiment_workflow.main()
assert exited.value.code == 1
assert "Docker is not available" in capsys.readouterr().err


def test_main_reports_sandbox_errors_without_traceback(tmp_path: Path, monkeypatch, capsys):
def fail(store_root, image):
raise SandboxError("docker build failed for tbcore/sandbox:latest")

monkeypatch.setattr(experiment_workflow, "docker_available", lambda: True)
monkeypatch.setattr(experiment_workflow, "run", fail)
monkeypatch.setattr("sys.argv", ["experiment_workflow.py", "--store", str(tmp_path)])
with pytest.raises(SystemExit) as exited:
experiment_workflow.main()
assert exited.value.code == 1
assert "error: docker build failed" in capsys.readouterr().err
Loading