Codex/prepbench workspace contract

.env.example

@@ -1,13 +1,24 @@
 # Copy this file to `.env` and fill in your credentials.
+# Recommended native DeepSeek setup for comparable simulator runs.
+PREPBENCH_SIMULATOR_BASE_URL=https://api.deepseek.com
+PREPBENCH_SIMULATOR_MODEL=deepseek-v4-flash
+PREPBENCH_SIMULATOR_THINKING=disabled
+PREPBENCH_SIMULATOR_TEMPERATURE=0
+PREPBENCH_SIMULATOR_API_KEY=your_api_key
+
+# OpenRouter alternative:
+# PREPBENCH_SIMULATOR_BASE_URL=https://openrouter.ai/api/v1
+# PREPBENCH_SIMULATOR_MODEL=deepseek/deepseek-v4-flash
+# PREPBENCH_SIMULATOR_THINKING is not sent by default for non-DeepSeek endpoints.
+# PREPBENCH_SIMULATOR_API_KEY=your_openrouter_api_key
+
 # The simulator checks keys in this order:
 # PREPBENCH_SIMULATOR_API_KEY, OPENROUTER_API_KEY, OPENAI_API_KEY.
-PREPBENCH_SIMULATOR_MODEL=your-model-name
-PREPBENCH_SIMULATOR_API_KEY=your_api_key
 # OPENROUTER_API_KEY=your_openrouter_api_key
 # OPENAI_API_KEY=your_openai_api_key
 
 # Optional simulator runtime settings
-# PREPBENCH_SIMULATOR_BASE_URL=https://openrouter.ai/api/v1
 # PREPBENCH_SIMULATOR_MAX_TOKENS=8192
 # PREPBENCH_SIMULATOR_TIMEOUT=120
+# PREPBENCH_SIMULATOR_REASONING_EFFORT=high
 # PREPBENCH_SOLUTIONS_ROOT=/absolute/path/to/reference_solutions

.github/workflows/ci.yml

@@ -21,6 +21,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          pip install -e .
 
       - name: Run release validation
         run: |

.gitignore

@@ -55,9 +55,10 @@ htmlcov/
 tmp/
 
 @output/
-@output copy/
+@runs/
 
-@output.zip
+# Local implementation plans are not part of the public benchmark surface.
+docs/plans/
 
 # Legacy/local reference-solution override paths.
 data/case_*/solution.py

Makefile

@@ -4,12 +4,13 @@ PYTHON ?= python3
 
 install:
 	$(PYTHON) -m pip install -r requirements.txt
+	$(PYTHON) -m pip install -e .
 
 check:
-	$(PYTHON) -m compileall -q src/prepbench src/evaluate src/py2flow src/simulator/*.py examples tests scripts/validate_dataset.py scripts/verify_reference_outputs.py scripts/execute_workflow.py
+	$(PYTHON) -m compileall -q src/prepbench src/evaluate src/py2flow src/simulator/*.py examples tests scripts/validate_dataset.py scripts/verify_reference_outputs.py scripts/execute_workflow.py scripts/prepare_run.py scripts/evaluate_submission.py
 	PYTHONPATH=src $(PYTHON) -m unittest discover -s tests
 	$(PYTHON) scripts/validate_dataset.py
-	PYTHONPATH=src $(PYTHON) -m evaluate.batch --help >/dev/null
+	$(PYTHON) scripts/evaluate_submission.py --help >/dev/null
 
 check-all: check
 	$(PYTHON) -m compileall -q reference/solutions
@@ -29,4 +30,4 @@ clean: clean-outputs
 	rm -rf build dist src/prepbench.egg-info .pytest_cache
 
 clean-outputs:
-	rm -rf @output @out-info
+	rm -rf @output @runs