Decouple PrepAgent prompts and isolate prepagent output namespace

Author: zzzbitz

README.md

@@ -102,6 +102,10 @@ Integration contract:
 Reference implementation:
 - `examples/prep_agent/run_prepagent.py`
 - `examples/prep_agent/README.md`
+- `examples/prep_agent/prompts/`
+
+PrepAgent output root is isolated from benchmark run modes:
+- `@output/<model_info>/prepagent/case_xxx/solution/...`
 
 ## Reproduce Paper Experiments (Secondary Path)
 

agents/clarify_agent.py

@@ -20,12 +20,23 @@ class InteractAction:
 
 
 class ClarifyAgent:
-    def __init__(self, model_name: str, data_head: Optional[DataHead] = None) -> None:
+    def __init__(
+        self,
+        model_name: str,
+        data_head: Optional[DataHead] = None,
+        *,
+        prompt_dir: Optional[Path] = None,
+        template_dir: Optional[Path] = None,
+        prompt_name: str = "clarify_agent",
+        template_name: str = "clarify_agent.jinja2",
+    ) -> None:
         self.model_name = model_name
         self.data_head = data_head or DataHead()
-        template_dir = Path(__file__).parent / "prompts" / "templates"
-        self.jinja_env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True)
-        self.template = self.jinja_env.get_template("clarify_agent.jinja2")
+        self.prompt_dir = prompt_dir
+        self.prompt_name = prompt_name
+        tmpl_dir = template_dir or (Path(__file__).parent / "prompts" / "templates")
+        self.jinja_env = Environment(loader=FileSystemLoader(tmpl_dir), trim_blocks=True, lstrip_blocks=True)
+        self.template = self.jinja_env.get_template(template_name)
 
     def _collect_context(
         self,
@@ -57,7 +68,11 @@ def _collect_context(
         }
 
     def _build_prompt(self, ctx: Dict[str, Any]) -> str:
-        cfg = load_prompt_yaml("clarify_agent", required_keys=("system", "guidelines"))
+        cfg = load_prompt_yaml(
+            self.prompt_name,
+            required_keys=("system", "guidelines"),
+            prompt_dir=self.prompt_dir,
+        )
         return self.template.render(
             system_prompt_text=cfg["system"],
             guidelines_text=cfg["guidelines"],

agents/code_agent.py

@@ -16,12 +16,24 @@
 
 
 class CodeAgent:
-    def __init__(self, model_name: str, data_head: Optional[DataHead] = None):
+    def __init__(
+        self,
+        model_name: str,
+        data_head: Optional[DataHead] = None,
+        *,
+        prompt_dir: Optional[Path] = None,
+        template_dir: Optional[Path] = None,
+        prompt_name: str = "code_agent",
+        template_name: str = "code_agent.jinja2",
+    ):
         self.model_name = model_name
         self.data_head = data_head or DataHead()
+        self.prompt_dir = prompt_dir
+        self.prompt_name = prompt_name
         # Setup Jinja2 environment
-        template_dir = Path(__file__).parent / "prompts" / "templates"
-        self.jinja_env = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True)
+        tmpl_dir = template_dir or (Path(__file__).parent / "prompts" / "templates")
+        self.jinja_env = Environment(loader=FileSystemLoader(tmpl_dir), trim_blocks=True, lstrip_blocks=True)
+        self.template_name = template_name
 
     def _collect_context(self, session_state: Dict[str, Any]) -> Dict:
         mode = str(session_state.get("run_mode", ""))
@@ -57,9 +69,12 @@ def _collect_context(self, session_state: Dict[str, Any]) -> Dict:
 
     def _build_prompt(self, ctx: Dict, feedback: Optional[Dict[str, Any]] = None) -> str:
         """Builds the prompt using the Jinja2 template."""
-        prompt_name = "code_agent"
-        cfg = load_prompt_yaml(prompt_name, required_keys=("system", "guidelines"))
-        template = self.jinja_env.get_template("code_agent.jinja2")
+        cfg = load_prompt_yaml(
+            self.prompt_name,
+            required_keys=("system", "guidelines"),
+            prompt_dir=self.prompt_dir,
+        )
+        template = self.jinja_env.get_template(self.template_name)
 
         # Render the template with all the context and feedback
         return template.render(

agents/flow_agent.py

@@ -17,16 +17,26 @@
 class FlowAgent:
     """Agent that generates flow.json DAG via LLM."""
 
-    def __init__(self, model_name: Optional[str] = None):
+    def __init__(
+        self,
+        model_name: Optional[str] = None,
+        *,
+        prompt_dir: Optional[Path] = None,
+        template_dir: Optional[Path] = None,
+        prompt_name: str = "flow_agent",
+        template_name: str = "flow_agent.jinja2",
+    ):
         self.model_name = model_name or get_model_name()
+        self.prompt_dir = prompt_dir
+        self.prompt_name = prompt_name
         # Setup Jinja2 environment
-        template_dir = Path(__file__).parent / "prompts" / "templates"
+        tmpl_dir = template_dir or (Path(__file__).parent / "prompts" / "templates")
         self.jinja_env = Environment(
-            loader=FileSystemLoader(template_dir),
+            loader=FileSystemLoader(tmpl_dir),
             trim_blocks=True,
             lstrip_blocks=True,
         )
-        self.template = self.jinja_env.get_template("flow_agent.jinja2")
+        self.template = self.jinja_env.get_template(template_name)
 
     def _collect_context(self, session_state: Dict[str, Any]) -> Dict[str, Any]:
         """Collect context for prompt rendering."""
@@ -44,8 +54,9 @@ def _build_prompt(
     ) -> str:
         """Builds the prompt using the Jinja2 template."""
         cfg = load_prompt_yaml(
-            "flow_agent",
+            self.prompt_name,
             required_keys=("system", "core_guidelines", "operator_definitions"),
+            prompt_dir=self.prompt_dir,
         )
 
         return self.template.render(

agents/profile_agent.py

@@ -20,12 +20,31 @@ class ProfileAgent:
     3. summarize() - If round 2 executed, generate final summary
     """
 
-    def __init__(self, model_name: str) -> None:
+    def __init__(
+        self,
+        model_name: str,
+        *,
+        prompt_dir: Path | None = None,
+        template_dir: Path | None = None,
+        profile_prompt_name: str = "profile_agent",
+        profile_template_name: str = "profile_agent.jinja2",
+        decide_prompt_name: str = "profile_agent_decide",
+        decide_template_name: str = "profile_agent_decide.jinja2",
+        summarize_prompt_name: str = "profile_agent_summarize",
+        summarize_template_name: str = "profile_agent_summarize.jinja2",
+    ) -> None:
         self.model_name = model_name
         self._llm = None
-        template_dir = Path(__file__).parent / "prompts" / "templates"
+        self.prompt_dir = prompt_dir
+        tmpl_dir = template_dir or (Path(__file__).parent / "prompts" / "templates")
+        self.profile_prompt_name = profile_prompt_name
+        self.profile_template_name = profile_template_name
+        self.decide_prompt_name = decide_prompt_name
+        self.decide_template_name = decide_template_name
+        self.summarize_prompt_name = summarize_prompt_name
+        self.summarize_template_name = summarize_template_name
         self.jinja_env = Environment(
-            loader=FileSystemLoader(template_dir),
+            loader=FileSystemLoader(tmpl_dir),
             trim_blocks=True,
             lstrip_blocks=True,
         )
@@ -37,9 +56,13 @@ def _get_llm(self):
                 raise RuntimeError("LLM configuration not found.")
         return self._llm
 
-    def _render_prompt(self, name: str, context: Dict[str, Any]) -> str:
-        prompt_cfg = load_prompt_yaml(name, required_keys=("system", "guidelines"))
-        template = self.jinja_env.get_template(f"{name}.jinja2")
+    def _render_prompt(self, prompt_name: str, template_name: str, context: Dict[str, Any]) -> str:
+        prompt_cfg = load_prompt_yaml(
+            prompt_name,
+            required_keys=("system", "guidelines"),
+            prompt_dir=self.prompt_dir,
+        )
+        template = self.jinja_env.get_template(template_name)
         return template.render(
             system_prompt_text=prompt_cfg["system"],
             guidelines_text=prompt_cfg["guidelines"],
@@ -59,7 +82,8 @@ def generate_profile_code(
             (code, raw_response, messages)
         """
         prompt_content = self._render_prompt(
-            "profile_agent",
+            self.profile_prompt_name,
+            self.profile_template_name,
             {
                 "query_md": session_state.get("query", ""),
                 "task_dir": session_state.get("task_dir", ""),
@@ -102,7 +126,8 @@ def decide_or_summarize(
             stderr_section = f"- Stderr:\n```\n{stderr[:1000]}\n```"
 
         prompt_content = self._render_prompt(
-            "profile_agent_decide",
+            self.decide_prompt_name,
+            self.decide_template_name,
             {
                 "query_md": session_state.get("query", ""),
                 "task_dir": session_state.get("task_dir", ""),
@@ -206,7 +231,8 @@ def summarize(
         r2_stdout = (round2_result.get("stdout") or "")[:2000]
 
         prompt_content = self._render_prompt(
-            "profile_agent_summarize",
+            self.summarize_prompt_name,
+            self.summarize_template_name,
             {
                 "query_md": session_state.get("query", ""),
                 "round1_status": r1_status,

core/orchestration/code_phase.py

@@ -64,12 +64,13 @@ def run_code_phase(
     config: ExperimentConfig,
     session_state: Dict[str, Any],
     output_root: Path,
+    code_agent: Optional[CodeAgent] = None,
 ) -> Dict[str, Any]:
     """Run code generation phase with retry on execution errors."""
     rounds_root = output_root / "rounds"
     rounds_root.mkdir(parents=True, exist_ok=True)
 
-    coder = CodeAgent(config.model_name)
+    coder = code_agent or CodeAgent(config.model_name)
     executor = CodeExecutor()
 
     prev_code: Optional[str] = None

core/orchestration/flow_phase.py

@@ -17,6 +17,7 @@ def run_flow_impl(
     config: ExperimentConfig,
     output_root: Path,
     solution_text: str,
+    flow_agent: Optional[FlowAgent] = None,
 ) -> Dict[str, Any]:
     """Shared flow execution logic for flow mode and e2e."""
     from py2flow.errors import FlowExecutionError, FlowValidationError
@@ -47,7 +48,7 @@ def run_flow_impl(
     }
 
     max_rounds = getattr(cfg, "max_rounds_debug", 3)
-    flow_agent = FlowAgent(model_name=cfg.model_name)
+    flow_agent = flow_agent or FlowAgent(model_name=cfg.model_name)
 
     feedback = None
     stopped_reason = "unknown"

core/orchestration/profile_phase.py

@@ -74,12 +74,13 @@ def run_profile_phase(
     input_dir: Path,
     inputs: list[str],
     inputs_preview: Dict[str, Any],
+    profile_agent: Optional[ProfileAgent] = None,
 ) -> dict[str, Any]:
     """Two-round profile flow with LLM decision."""
     profile_root = output_root / "profile"
     profile_root.mkdir(parents=True, exist_ok=True)
 
-    profile_agent = ProfileAgent(config.model_name)
+    profile_agent = profile_agent or ProfileAgent(config.model_name)
     executor = CodeExecutor()
     input_files = {p.name: p for p in sorted(input_dir.glob("*.csv"))}
 

docs/BYOA_E2E.md

@@ -31,9 +31,19 @@ PrepBench provides a reference `PrepAgent` pipeline for E2E:
 ./scripts/run_prepagent.sh --case 1 --model openai/gpt-5.2
 ```
 
+PrepAgent writes to an isolated root:
+- `@output/<model_info>/prepagent/case_xxx/solution/...`
+
+And can be evaluated with:
+
+```bash
+python -m evaluate.batch --results-root @output/<model_info>/prepagent --candidate-kind auto
+```
+
 Source:
 - `examples/prep_agent/run_prepagent.py`
 - `examples/prep_agent/README.md`
+- `examples/prep_agent/prompts/` (PrepAgent-owned prompts/templates)
 
 ## 1) Public Inputs Per Case
 

examples/prep_agent/README.md

@@ -2,29 +2,42 @@
 
 This is a reference implementation for participants who want to build and evaluate their own agent framework on PrepBench.
 
-Pipeline order is fixed:
+Pipeline order is fixed and self-contained in this folder:
 1. profile
-2. clarify (via local user simulator API)
+2. clarify/interact (via local user simulator API)
 3. code generation + execution
 4. flow generation + execution (only when code outputs exist)
 
+## Folder Structure
+
+- `run_prepagent.py`: entrypoint and pipeline orchestration.
+- `prompts/*.yaml`: PrepAgent-owned prompt configs.
+- `prompts/templates/*.jinja2`: PrepAgent-owned prompt templates.
+  - `prep_profile*`: profile stage
+  - `prep_interact*`: interact stage
+  - `prep_code*`: code stage
+  - `prep_flow*`: flow stage
+
+PrepAgent does not reuse the benchmark cache/reuse policy in `run.py`.
+Each case run starts from a clean output directory to keep behavior deterministic for external users.
+
 ## Run
 
 ```bash
 ./scripts/run_prepagent.sh --case 1 --model openai/gpt-5.2
 ```
 
-Outputs follow the standard evaluation layout under `@output/<model_info>/e2e/case_xxx/solution/`:
+Outputs follow the standard evaluation layout under `@output/<model_info>/prepagent/case_xxx/solution/`:
 - `cand/*.csv` (code track)
 - `flow_cand/*.csv` (flow track)
 
 ## Evaluate
 
 ```bash
-python -m evaluate.batch --results-root @output/<model_info>/e2e --candidate-kind auto
+python -m evaluate.batch --results-root @output/<model_info>/prepagent --candidate-kind auto
 ```
 
 ## Notes
 
-- This reference implementation reuses prompts and agent modules under `agents/`.
+- This reference implementation reuses core executors/parsers, but uses its own prompt assets under `examples/prep_agent/prompts`.
 - User simulator and flow mode require reference solutions in `simulator/assets/solutions/case_xxx.py`.