From 5dc0537c325128d1bb2f1d556fba09e240d5031c Mon Sep 17 00:00:00 2001 From: Jay Rainton Date: Wed, 17 Sep 2025 11:02:24 -0700 Subject: [PATCH 01/11] adding in initial code --- .../app_world_adapter/app_world_adapter.py | 104 ++++++++++++++++++ src/gepa/examples/app-world/train_appworld.py | 66 +++++++++++ 2 files changed, 170 insertions(+) create mode 100644 src/gepa/adapters/app_world_adapter/app_world_adapter.py create mode 100644 src/gepa/examples/app-world/train_appworld.py diff --git a/src/gepa/adapters/app_world_adapter/app_world_adapter.py b/src/gepa/adapters/app_world_adapter/app_world_adapter.py new file mode 100644 index 000000000..45116ebd2 --- /dev/null +++ b/src/gepa/adapters/app_world_adapter/app_world_adapter.py @@ -0,0 +1,104 @@ +import json +import os +import subprocess +from datetime import datetime +from pathlib import Path +from typing import Dict, Any + +from pydantic import BaseModel +from appworld_experiments.code.simplified.agent import Agent +from appworld_experiments.code.simplified import SimplifiedFullCodeReflexionAgent + +from gepa import EvaluationBatch, GEPAAdapter +from appworld import AppWorld + + +class AppWorldTask(BaseModel): + task_id: str + +class AppWorldAdapter(GEPAAdapter): + + def __init__( + self, + agent_config: Dict[str, Any], + experiment_name: str, + ): + self.agent = Agent.from_dict(agent_config) + self.experiment_name = experiment_name + + def evaluate( + self, + batch: list[AppWorldTask], + candidate: dict[str, str], + capture_traces: bool = False, + ) -> EvaluationBatch: + outputs = [] + scores = [] + trajectories = [] + + example_run_id = "_temp_gepa_run" + "_" + datetime.now().strftime("%Y%m%d%H%M%S") + instruct_prompt = candidate["instruction_prompt"] + num_tasks = len(batch) + self.agent.logger.initialize( + experiment_name=self.experiment_name + example_run_id, + num_tasks=num_tasks, + num_processes=1, + process_index=0, + ) + self.agent.code_prompt_template = instruct_prompt + + for example in batch: + task_id = example.task_id + test_tracker = self.agent.solve_task(task_id, self.experiment_name + example_run_id) + try: + success = test_tracker.success + score = int(success) + failed_reason_list = [] + for failure in test_tracker.failures: + failed_reason_list.append(json.dumps(failure, indent=2)) + failed_reason = ','.join(failed_reason_list) + except Exception as e: + #TODO: need to handle case for failed code execution + success = False + score = 0 + failed_reason = "Did not terminate" + outputs.append( + f"App World outputs are omitted. Please see directory for detailed logging." + ) + scores.append(score) + trajectories.append( + { + "messages": self.agent.messages, + "instruction_prompt": instruct_prompt, + "failed_reason": str(failed_reason), + "success": success, + } + ) + return EvaluationBatch( + outputs=outputs, + scores=scores, + trajectories=trajectories, + ) + + def make_reflective_dataset( + self, + candidate: dict[str, str], + eval_batch: EvaluationBatch, + components_to_update: list[str], + ): + reflective_dataset = {"instruction_prompt": []} + for score, trajectory in zip(eval_batch.scores, eval_batch.trajectories, strict=False): + if trajectory["success"]: + feedback = "Successfully solved the task!" + else: + feedback = ( + f"Failed to solve the task. Reason: {trajectory['failed_reason']}" + ) + reflective_dataset["instruction_prompt"].append( + { + "Message History": trajectory["messages"], + "Instruction Prompt": candidate["instruction_prompt"], + "Feedback": feedback, + } + ) + return reflective_dataset diff --git a/src/gepa/examples/app-world/train_appworld.py b/src/gepa/examples/app-world/train_appworld.py new file mode 100644 index 000000000..5ed583805 --- /dev/null +++ b/src/gepa/examples/app-world/train_appworld.py @@ -0,0 +1,66 @@ +import os +import argparse +from appworld.common.path_store import path_store +from gepa import optimize +import litellm +from gepa.adapters.app_world_adapter.app_world_adapter import ( + AppWorldTask, + AppWorldAdapter, +) +from appworld.common.utils import jsonnet_load, read_file +from appworld.task import Task, load_task_ids + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--experiment_name", type=str, default="fdklsjfaskl") + args = parser.parse_args() + experiment_name = args.experiment_name + experiment_file_path = os.path.join(path_store.experiment_configs, experiment_name + ".jsonnet") + experiment_config = jsonnet_load( + experiment_file_path, + APPWORLD_EXPERIMENT_PROMPTS_PATH=path_store.experiment_prompts, + APPWORLD_EXPERIMENT_CONFIGS_PATH=path_store.experiment_configs, + APPWORLD_EXPERIMENT_CODE_PATH=path_store.experiment_code, + ) + runner_config = experiment_config.pop("config") + dataset_name = runner_config.pop("dataset") + agent_config = runner_config.pop("agent") + code_prompt_file_path = agent_config['code_prompt_file_path'] + code_prompt_template = read_file(code_prompt_file_path.replace("/", os.sep)) + task_ids = load_task_ids(dataset_name) + for task_id in task_ids: + Task.load(task_id=task_id) + trainset = [ + AppWorldTask(task_id=task_id) for task_id in task_ids[0:20] + ] + valset = [AppWorldTask(task_id=task_id) for task_id in task_ids[20:22]] + testset = [AppWorldTask(task_id=task_id) for task_id in task_ids[30:40]] + + reflection_lm_name = "openai/gpt-5" + reflection_lm = ( + lambda prompt: litellm.completion( + model=reflection_lm_name, + messages=[{"role": "user", "content": prompt}], + reasoning_effort="high", + ) + .choices[0] + .message.content + ) + + adapter = AppWorldAdapter(agent_config, experiment_name=experiment_name) + testset_results_before_opt = adapter.evaluate(testset, {"instruction_prompt": code_prompt_template}, capture_traces=True) + + optimized_results = optimize( + seed_candidate={"instruction_prompt": code_prompt_template}, + trainset=trainset, + valset=valset, + adapter=adapter, + reflection_lm="openai/gpt-5", + max_metric_calls=5 + ) + + testset_results_after_opt = adapter.evaluate( + testset, + {"instruction_prompt": optimized_results.best_candidate["instruction_prompt"]}, + capture_traces=True, + ) \ No newline at end of file From af4c397fa25f9b7f41e0e55fe857e7f2e64b2739 Mon Sep 17 00:00:00 2001 From: Jay Rainton Date: Thu, 18 Sep 2025 17:25:10 -0700 Subject: [PATCH 02/11] appworld adapter --- .../app_world_adapter/app_world_adapter.py | 7 +- src/gepa/examples/app-world/train_appworld.py | 70 ++++++++++++++----- 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/gepa/adapters/app_world_adapter/app_world_adapter.py b/src/gepa/adapters/app_world_adapter/app_world_adapter.py index 45116ebd2..a0f959afd 100644 --- a/src/gepa/adapters/app_world_adapter/app_world_adapter.py +++ b/src/gepa/adapters/app_world_adapter/app_world_adapter.py @@ -11,7 +11,7 @@ from gepa import EvaluationBatch, GEPAAdapter from appworld import AppWorld - +from appworld.evaluator import TestTracker class AppWorldTask(BaseModel): task_id: str @@ -45,7 +45,7 @@ def evaluate( num_processes=1, process_index=0, ) - self.agent.code_prompt_template = instruct_prompt + self.agent.gepa_prompt_replace = instruct_prompt for example in batch: task_id = example.task_id @@ -53,6 +53,7 @@ def evaluate( try: success = test_tracker.success score = int(success) + #score = len(test_tracker.passes) / test_tracker._num_tests failed_reason_list = [] for failure in test_tracker.failures: failed_reason_list.append(json.dumps(failure, indent=2)) @@ -61,7 +62,7 @@ def evaluate( #TODO: need to handle case for failed code execution success = False score = 0 - failed_reason = "Did not terminate" + failed_reason = "\n\n".join(test_tracker) outputs.append( f"App World outputs are omitted. Please see directory for detailed logging." ) diff --git a/src/gepa/examples/app-world/train_appworld.py b/src/gepa/examples/app-world/train_appworld.py index 5ed583805..545651d96 100644 --- a/src/gepa/examples/app-world/train_appworld.py +++ b/src/gepa/examples/app-world/train_appworld.py @@ -23,44 +23,76 @@ APPWORLD_EXPERIMENT_CODE_PATH=path_store.experiment_code, ) runner_config = experiment_config.pop("config") - dataset_name = runner_config.pop("dataset") agent_config = runner_config.pop("agent") - code_prompt_file_path = agent_config['code_prompt_file_path'] - code_prompt_template = read_file(code_prompt_file_path.replace("/", os.sep)) - task_ids = load_task_ids(dataset_name) - for task_id in task_ids: + + initial_prompt_from_app_world = """ +You are a super intelligent AI Assistant whose job is to complete day-to-day tasks by writing code to interact with apps on behalf of your supervisor. You are working in a REPL (Read-Eval-Print-Loop) environment where you can execute code iteratively, see results, and refine your approach. + +You are also provided with curated cheatsheet of strategies, apis specific information, valid assumptions list, insights, code and a reflection that goes over the diagnosis of all previous mistakes made while answering the question. + +## Instructions: + +- **REPL Environment**: Execute code **one small block at a time**; after each block you see output and then decide the next step. Each block is a separate step. +- **Iterative Development**: Use feedback from each execution to guide your next actions. Build incrementally. +- **Explore and Debug (NO GUESSING)**: + - Use `print()` to inspect intermediate values and data structures. + - When **unsure** about an app, which APIs exist, or what an API takes/returns: + 1) **Discover** the available APIs for the app: + ```python + print(apis.api_docs.show_api_descriptions(app_name='')) + ``` + 2) **Read docs** for chosen endpoint(s): + ```python + print(apis.api_docs.show_api_doc(app_name='', api_name='')) + ``` + 3) **Call** the API **exactly as documented**, using **keyword arguments only**. + - On error: re-check the doc, `print` the offending inputs, retry with the **minimal valid payload**, then expand. +- **Signal Completion**: When the task is actually done, call `apis.supervisor.complete_task()` with appropriate arguments. + +Solving a task can take up to 40 interactions between you and the Python REPL. Each code block you write gets executed immediately and you see the results before writing the next block. When you call `apis.supervisor.complete_task()` or reach 40 iterations, evaluation will begin. +""" + + train_task_ids = load_task_ids('train') + val_task_ids = load_task_ids('dev') + test_task_ids = load_task_ids('test_normal') + for task_id in train_task_ids: + Task.load(task_id=task_id) + for task_id in val_task_ids: + Task.load(task_id=task_id) + for task_id in test_task_ids: Task.load(task_id=task_id) trainset = [ - AppWorldTask(task_id=task_id) for task_id in task_ids[0:20] + AppWorldTask(task_id=task_id) for task_id in train_task_ids[:10] ] - valset = [AppWorldTask(task_id=task_id) for task_id in task_ids[20:22]] - testset = [AppWorldTask(task_id=task_id) for task_id in task_ids[30:40]] + valset = [AppWorldTask(task_id=task_id) for task_id in val_task_ids[:5]] + testset = [AppWorldTask(task_id=task_id) for task_id in test_task_ids[:10]] - reflection_lm_name = "openai/gpt-5" + reflection_lm_name = "together_ai/deepseek-ai/DeepSeek-V3.1" reflection_lm = ( lambda prompt: litellm.completion( model=reflection_lm_name, messages=[{"role": "user", "content": prompt}], - reasoning_effort="high", + chat_template_kwargs={"thinking": True}, ) .choices[0] .message.content ) adapter = AppWorldAdapter(agent_config, experiment_name=experiment_name) - testset_results_before_opt = adapter.evaluate(testset, {"instruction_prompt": code_prompt_template}, capture_traces=True) + #testset_results_before_opt = adapter.evaluate(testset, {"instruction_prompt": initial_prompt_from_app_world}, capture_traces=True) optimized_results = optimize( - seed_candidate={"instruction_prompt": code_prompt_template}, + seed_candidate={"instruction_prompt": initial_prompt_from_app_world}, trainset=trainset, valset=valset, adapter=adapter, - reflection_lm="openai/gpt-5", - max_metric_calls=5 + reflection_lm=reflection_lm, + max_metric_calls=20, + run_dir="gepa_app_world", ) - testset_results_after_opt = adapter.evaluate( - testset, - {"instruction_prompt": optimized_results.best_candidate["instruction_prompt"]}, - capture_traces=True, - ) \ No newline at end of file + # testset_results_after_opt = adapter.evaluate( + # testset, + # {"instruction_prompt": optimized_results.best_candidate["instruction_prompt"]}, + # capture_traces=True + # ) \ No newline at end of file From 6a2811ec23798a13bcde11d56eb0f114e51cfeac Mon Sep 17 00:00:00 2001 From: Jay Rainton Date: Thu, 18 Sep 2025 17:54:27 -0700 Subject: [PATCH 03/11] no more thinking --- src/gepa/examples/app-world/train_appworld.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gepa/examples/app-world/train_appworld.py b/src/gepa/examples/app-world/train_appworld.py index 545651d96..1dbd3da2d 100644 --- a/src/gepa/examples/app-world/train_appworld.py +++ b/src/gepa/examples/app-world/train_appworld.py @@ -67,12 +67,12 @@ valset = [AppWorldTask(task_id=task_id) for task_id in val_task_ids[:5]] testset = [AppWorldTask(task_id=task_id) for task_id in test_task_ids[:10]] + reflection_lm_name = "together_ai/deepseek-ai/DeepSeek-V3.1" reflection_lm = ( lambda prompt: litellm.completion( model=reflection_lm_name, messages=[{"role": "user", "content": prompt}], - chat_template_kwargs={"thinking": True}, ) .choices[0] .message.content From d9bb85375eff3d1d69580ddc2aba33e02ea5b1d7 Mon Sep 17 00:00:00 2001 From: Jay Rainton Date: Fri, 24 Oct 2025 14:47:45 -0700 Subject: [PATCH 04/11] fix baseAgent --- src/gepa/adapters/app_world_adapter/app_world_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gepa/adapters/app_world_adapter/app_world_adapter.py b/src/gepa/adapters/app_world_adapter/app_world_adapter.py index a0f959afd..cb77b9730 100644 --- a/src/gepa/adapters/app_world_adapter/app_world_adapter.py +++ b/src/gepa/adapters/app_world_adapter/app_world_adapter.py @@ -6,7 +6,7 @@ from typing import Dict, Any from pydantic import BaseModel -from appworld_experiments.code.simplified.agent import Agent +from appworld_experiments.code.simplified.base_agent import BaseAgent from appworld_experiments.code.simplified import SimplifiedFullCodeReflexionAgent from gepa import EvaluationBatch, GEPAAdapter @@ -23,7 +23,7 @@ def __init__( agent_config: Dict[str, Any], experiment_name: str, ): - self.agent = Agent.from_dict(agent_config) + self.agent = BaseAgent.from_dict(agent_config) self.experiment_name = experiment_name def evaluate( From d7eb7f10c14d57ccc174cf732428f1836dd70471 Mon Sep 17 00:00:00 2001 From: Jay Rainton Date: Tue, 18 Nov 2025 17:04:49 -0800 Subject: [PATCH 05/11] adding in bird adapters --- .../adapters/bird_adapter/bird_adapter.py | 90 +++++++++++++++++++ src/gepa/examples/bird/train_bird.py | 73 +++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 src/gepa/adapters/bird_adapter/bird_adapter.py create mode 100644 src/gepa/examples/bird/train_bird.py diff --git a/src/gepa/adapters/bird_adapter/bird_adapter.py b/src/gepa/adapters/bird_adapter/bird_adapter.py new file mode 100644 index 000000000..3da015a41 --- /dev/null +++ b/src/gepa/adapters/bird_adapter/bird_adapter.py @@ -0,0 +1,90 @@ +import json +import os +import subprocess +from datetime import datetime +from pathlib import Path +from typing import Dict, Any + +from pydantic import BaseModel + +from gepa import EvaluationBatch, GEPAAdapter +import io +import contextlib +import sys +from llm.src.gpt_request import collect_response_from_gpt +from llm.src.evaluation import execute_model + +class BirdTask(BaseModel): + question: str + db_path: str + knowledge: str + ground_truth: str + +class BirdAdapter(GEPAAdapter): + + def __init__( + self, + model_name: str, + ): + self.model_name = model_name + + def evaluate( + self, + batch: list[BirdTask], + candidate: dict[str, str], + capture_traces: bool = False, + ) -> EvaluationBatch: + outputs = [] + scores = [] + trajectories = [] + + instruct_prompt = candidate["instruction_prompt"] + + for example in batch: + responses, message_list = collect_response_from_gpt(db_path_list=[example.db_path], question_list=[example.question], api_key="", engine=self.model_name, knowledge_list=[example.knowledge], system_message=instruct_prompt) + predict_sql, predict_db_name = responses[0].split('\t----- bird -----\t') + predict_db_path = ... + ground_truth_sql = example.ground_truth.strip() + result = execute_model(predict_sql, ground_truth_sql, predict_db_path, 0, 30.0) + score = result['res'] + if score == 1: + success = True + else: + success = False + outputs.append("No logging here...") + scores.append(score) + trajectories.append( + { + "messages": message_list[0], + "instruction_prompt": instruct_prompt, + "success": success, + } + ) + return EvaluationBatch( + outputs=outputs, + scores=scores, + trajectories=trajectories, + ) + + def make_reflective_dataset( + self, + candidate: dict[str, str], + eval_batch: EvaluationBatch, + components_to_update: list[str], + ): + reflective_dataset = {"instruction_prompt": []} + for score, trajectory in zip(eval_batch.scores, eval_batch.trajectories, strict=False): + if trajectory["success"]: + feedback = "Successfully solved the task!" + else: + feedback = ( + f"Failed to solve the task." + ) + reflective_dataset["instruction_prompt"].append( + { + "Message History": trajectory["messages"], + "Instruction Prompt": candidate["instruction_prompt"], + "Feedback": feedback, + } + ) + return reflective_dataset diff --git a/src/gepa/examples/bird/train_bird.py b/src/gepa/examples/bird/train_bird.py new file mode 100644 index 000000000..6fef3d38d --- /dev/null +++ b/src/gepa/examples/bird/train_bird.py @@ -0,0 +1,73 @@ +import os +import argparse +from gepa import optimize +import litellm +from gepa.adapters.bird_adapter.bird_adapter import ( + BirdTask, + BirdAdapter, +) +from llm.src.gpt_request import decouple_question_schema +import json +import random + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model_name", type=str, default=None) + args = parser.parse_args() + if args.model_name is None: + raise ValueError(f"--model_name is set to None!") + print(f"Running with model: {args.model_name}") + + train_path = ... + train_data = json.load(open(train_path, 'r')) + train_db_root_path = ... + train_question_list, train_db_path_list, train_knowledge_list, train_ground_truth_list = decouple_question_schema(datasets=train_data, db_root_path=train_db_root_path) + + eval_path = ... + eval_data = json.load(open(eval_path, 'r')) + eval_db_root_path = ... + eval_question_list, eval_db_path_list, eval_knowledge_list, eval_ground_truth_list = decouple_question_schema(datasets=eval_data, db_root_path=eval_db_root_path) + + fullset = [] + for train_question, train_db_path, train_knowledge, train_ground_truth in zip(train_question_list, train_db_path_list, train_knowledge_list, train_ground_truth_list): + fullset.append(BirdTask(question=train_question, db_path=train_db_path, knowledge=train_knowledge, ground_truth=train_ground_truth)) + + random.seed(42) + random.shuffle(fullset) + trainset = fullset[:1000] + valset = fullset[1000:1500] + + reflection_lm_name = args.model_name + print(f"Running with reflector model: {reflection_lm_name}") + reflection_lm = ( + lambda prompt: litellm.completion( + model=reflection_lm_name, + messages=[{"role": "user", "content": prompt}], + ) + .choices[0] + .message.content + ) + + adapter = BirdAdapter(model_name=reflection_lm_name) + + model_name = reflection_lm_name.split('/')[-1] + + initial_prompt_from_cleaned = "You are an expert SQL developer. Your goal is to answer the natural question and produce an SQL command that would solve the issue at hand. The user will give some information about what the tables look like and some external knowledge along with the question they want answered. Make sure the SQL query you provide is wrapped around the SQL markdown: ```sql```" + + run_dir = f"gepa_bird_{model_name}_train_1000_val_500_minibatch_20" + optimized_results = optimize( + seed_candidate={"instruction_prompt": initial_prompt_from_cleaned}, + trainset=trainset, + valset=valset, + adapter=adapter, + reflection_lm=reflection_lm, + max_metric_calls=4535, + display_progress_bar=True, + run_dir=run_dir, + reflection_minibatch_size=20 + ) + + optimized_instruction_prompt = optimized_results.best_candidate["instruction_prompt"] + + with open(f"{run_dir}/best_prompt.txt", 'w') as f: + f.write(optimized_instruction_prompt) \ No newline at end of file From 34ae41e1cf5c6e5d8eeaee9663770dc19514273a Mon Sep 17 00:00:00 2001 From: Jay Rainton Date: Mon, 29 Dec 2025 16:07:26 -0800 Subject: [PATCH 06/11] updating train appworld --- src/gepa/examples/app-world/train_appworld.py | 133 ++++++++++++------ 1 file changed, 87 insertions(+), 46 deletions(-) diff --git a/src/gepa/examples/app-world/train_appworld.py b/src/gepa/examples/app-world/train_appworld.py index 1dbd3da2d..780601de3 100644 --- a/src/gepa/examples/app-world/train_appworld.py +++ b/src/gepa/examples/app-world/train_appworld.py @@ -9,12 +9,15 @@ ) from appworld.common.utils import jsonnet_load, read_file from appworld.task import Task, load_task_ids +import json if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--experiment_name", type=str, default="fdklsjfaskl") + parser.add_argument("--experiment_name", type=str, default=None) + parser.add_argument("--save-price-path", type=str, default='appworld_gepa_pricing.jsonl') args = parser.parse_args() experiment_name = args.experiment_name + price_path_file = args.save_price_path experiment_file_path = os.path.join(path_store.experiment_configs, experiment_name + ".jsonnet") experiment_config = jsonnet_load( experiment_file_path, @@ -24,33 +27,63 @@ ) runner_config = experiment_config.pop("config") agent_config = runner_config.pop("agent") + print(f"Running with agent config: {agent_config}") - initial_prompt_from_app_world = """ -You are a super intelligent AI Assistant whose job is to complete day-to-day tasks by writing code to interact with apps on behalf of your supervisor. You are working in a REPL (Read-Eval-Print-Loop) environment where you can execute code iteratively, see results, and refine your approach. - -You are also provided with curated cheatsheet of strategies, apis specific information, valid assumptions list, insights, code and a reflection that goes over the diagnosis of all previous mistakes made while answering the question. - -## Instructions: - -- **REPL Environment**: Execute code **one small block at a time**; after each block you see output and then decide the next step. Each block is a separate step. -- **Iterative Development**: Use feedback from each execution to guide your next actions. Build incrementally. -- **Explore and Debug (NO GUESSING)**: - - Use `print()` to inspect intermediate values and data structures. - - When **unsure** about an app, which APIs exist, or what an API takes/returns: - 1) **Discover** the available APIs for the app: - ```python - print(apis.api_docs.show_api_descriptions(app_name='')) - ``` - 2) **Read docs** for chosen endpoint(s): - ```python - print(apis.api_docs.show_api_doc(app_name='', api_name='')) - ``` - 3) **Call** the API **exactly as documented**, using **keyword arguments only**. - - On error: re-check the doc, `print` the offending inputs, retry with the **minimal valid payload**, then expand. -- **Signal Completion**: When the task is actually done, call `apis.supervisor.complete_task()` with appropriate arguments. - -Solving a task can take up to 40 interactions between you and the Python REPL. Each code block you write gets executed immediately and you see the results before writing the next block. When you call `apis.supervisor.complete_task()` or reach 40 iterations, evaluation will begin. -""" + initial_prompt_from_cleaned = """I am your supervisor and you are a super intelligent AI Assistant whose job is to achieve my day-to-day tasks completely autonomously. + +To do this, you will need to interact with app/s (e.g., spotify, venmo etc) using their associated APIs on my behalf. For this you will undertake a *multi-step conversation* using a python REPL environment. That is, you will write the python code and the environment will execute it and show you the result, based on which, you will write python code for the next step and so on, until you've achieved the goal. This environment will let you interact with app/s using their associated APIs on my behalf. + +Here are three key APIs that you need to know to get more information + +# To get a list of apps that are available to you. + +```python +print(apis.api_docs.show_app_descriptions()) +``` + +# To get the list of apis under any app listed above, e.g. spotify + +```python +print(apis.api_docs.show_api_descriptions(app_name='spotify')) +``` + +# To get the specification of a particular api, e.g. spotify app's login api + +```python +print(apis.api_docs.show_api_doc(app_name='spotify', api_name='login')) +``` + +Each code execution will produce an output that you can use in subsequent calls. Using these APIs, you can now generate code, that I will execute, to solve the task. + +You are also provided with a curated cheatsheet of strategies, API-specific information, common mistakes, and proven solutions to help you solve the task effectively. + +**Cheatsheet**: - Read the **Cheatsheet** first, then execute the task by explicitly leveraging each relevant section: +### CHEATSHEET BEGIN +## STRATEGIES AND HARD RULES +[shr-00001] Make sure to end code blocks with ``` followed by a newline(\\n). +[shr-00005] Always look at API specifications (using apis.api_docs.show_api_doc) before calling an API. +[shr-00006] Write small chunks of code and only one chunk of code in every step. Make sure everything is working correctly before making any irreversible change. + +## APIs TO USE FOR SPECIFIC INFORMATION +[api-00004] You can use the "supervisor" app to get information about my accounts and use the "phone" app to get information about friends and family. + +## USEFUL CODE SNIPPETS AND TEMPLATES + +## COMMON MISTAKES AND CORRECT STRATEGIES + +## PROBLEM-SOLVING HEURISTICS AND WORKFLOWS +[psw-00002] Remember you can use the variables in your code in subsequent code blocks. +[psw-00007] Many APIs return items in "pages". Make sure to run through all the pages by looping over `page_index`. + +## VERIFICATION CHECKLIST + +## TROUBLESHOOTING AND PITFALLS: + +## OTHERS +[misc-00003] Remember that the email addresses, access tokens and variables (e.g. spotify_password) in the example above are not valid anymore. +[misc-00008] Once you have completed the task, make sure to call apis.supervisor.complete_task(). If the task asked for some information, return it as the answer argument, i.e. call apis.supervisor.complete_task(answer=). Many tasks do not require an answer, so in those cases, just call apis.supervisor.complete_task() i.e. do not pass any argument. + +### CHEATSHEET END""" train_task_ids = load_task_ids('train') val_task_ids = load_task_ids('dev') @@ -61,38 +94,46 @@ Task.load(task_id=task_id) for task_id in test_task_ids: Task.load(task_id=task_id) + print(f"Length of original train dataset: {len(train_task_ids)}") + print(f"Length of original val dataset: {len(val_task_ids)}") + print(f"Length of original test dataset: {len(test_task_ids)}") trainset = [ - AppWorldTask(task_id=task_id) for task_id in train_task_ids[:10] + AppWorldTask(task_id=task_id) for task_id in train_task_ids[:] ] - valset = [AppWorldTask(task_id=task_id) for task_id in val_task_ids[:5]] - testset = [AppWorldTask(task_id=task_id) for task_id in test_task_ids[:10]] - + valset = [AppWorldTask(task_id=task_id) for task_id in val_task_ids[:]] - reflection_lm_name = "together_ai/deepseek-ai/DeepSeek-V3.1" - reflection_lm = ( - lambda prompt: litellm.completion( + gepa_prompt_gen_file = price_path_file + with open(gepa_prompt_gen_file, "w"): + pass + reflection_lm_name = "sambanova/DeepSeek-V3.1" + print(f"Running with reflector model: {reflection_lm_name}") + def call_lm(prompt): + response = litellm.completion( model=reflection_lm_name, messages=[{"role": "user", "content": prompt}], ) - .choices[0] - .message.content - ) + input_tokens = response.usage.prompt_tokens + output_tokens = response.usage.completion_tokens + with open(gepa_prompt_gen_file, "a") as f: + f.write(json.dumps({'input_tokens': input_tokens, 'output_tokens': output_tokens}) + "\n") + return response.choices[0].message.content + reflection_lm = call_lm adapter = AppWorldAdapter(agent_config, experiment_name=experiment_name) - #testset_results_before_opt = adapter.evaluate(testset, {"instruction_prompt": initial_prompt_from_app_world}, capture_traces=True) + run_dir = "gepa_app_world_deepseek-v3-1" optimized_results = optimize( - seed_candidate={"instruction_prompt": initial_prompt_from_app_world}, + seed_candidate={"instruction_prompt": initial_prompt_from_cleaned}, trainset=trainset, valset=valset, adapter=adapter, reflection_lm=reflection_lm, - max_metric_calls=20, - run_dir="gepa_app_world", + max_metric_calls=1434, + display_progress_bar=True, + run_dir=run_dir, ) - # testset_results_after_opt = adapter.evaluate( - # testset, - # {"instruction_prompt": optimized_results.best_candidate["instruction_prompt"]}, - # capture_traces=True - # ) \ No newline at end of file + optimized_instruction_prompt = optimized_results.best_candidate["instruction_prompt"] + + with open(f"{run_dir}/best_prompt.txt", 'w') as f: + f.write(optimized_instruction_prompt) \ No newline at end of file From be398a28391f0eb161639737ee5653c4ddf3590e Mon Sep 17 00:00:00 2001 From: Jay Rainton Date: Mon, 29 Dec 2025 16:11:06 -0800 Subject: [PATCH 07/11] updating name for pricing save path --- src/gepa/examples/app-world/train_appworld.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gepa/examples/app-world/train_appworld.py b/src/gepa/examples/app-world/train_appworld.py index 780601de3..f7979c11a 100644 --- a/src/gepa/examples/app-world/train_appworld.py +++ b/src/gepa/examples/app-world/train_appworld.py @@ -14,7 +14,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--experiment_name", type=str, default=None) - parser.add_argument("--save-price-path", type=str, default='appworld_gepa_pricing.jsonl') + parser.add_argument("--save-price-path", type=str, default='appworld_gepa_prompt_modification_pricing.jsonl') args = parser.parse_args() experiment_name = args.experiment_name price_path_file = args.save_price_path From 41ede17c5ba048ba2ee114bea55ba91d17e844f1 Mon Sep 17 00:00:00 2001 From: Jay Rainton Date: Mon, 29 Dec 2025 22:08:35 -0800 Subject: [PATCH 08/11] adding appworld data dir to be ignored --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 72521af03..178deb1a0 100644 --- a/.gitignore +++ b/.gitignore @@ -207,4 +207,5 @@ __marimo__/ # Experiments runs/ -gepa_terminus/ \ No newline at end of file +gepa_terminus/ +data/ \ No newline at end of file From efa866ea224e42bdfb3bdbd543fe0d81115e63ec Mon Sep 17 00:00:00 2001 From: Jay Rainton Date: Mon, 29 Dec 2025 22:09:40 -0800 Subject: [PATCH 09/11] updating app_world_adapter to point to different path --- src/gepa/adapters/app_world_adapter/app_world_adapter.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/gepa/adapters/app_world_adapter/app_world_adapter.py b/src/gepa/adapters/app_world_adapter/app_world_adapter.py index cb77b9730..5627376d2 100644 --- a/src/gepa/adapters/app_world_adapter/app_world_adapter.py +++ b/src/gepa/adapters/app_world_adapter/app_world_adapter.py @@ -6,8 +6,7 @@ from typing import Dict, Any from pydantic import BaseModel -from appworld_experiments.code.simplified.base_agent import BaseAgent -from appworld_experiments.code.simplified import SimplifiedFullCodeReflexionAgent +from appworld_experiments.code.gepa.gepa_agent import GEPAAgent from gepa import EvaluationBatch, GEPAAdapter from appworld import AppWorld @@ -23,7 +22,7 @@ def __init__( agent_config: Dict[str, Any], experiment_name: str, ): - self.agent = BaseAgent.from_dict(agent_config) + self.agent = GEPAAgent.from_dict(agent_config) self.experiment_name = experiment_name def evaluate( From ec72937a9603b9f41b12c5e8bcb63531aae40609 Mon Sep 17 00:00:00 2001 From: snova-jayr Date: Tue, 30 Dec 2025 00:55:41 -0800 Subject: [PATCH 10/11] adding in redirect to not print to console for better readability --- .../app_world_adapter/app_world_adapter.py | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/src/gepa/adapters/app_world_adapter/app_world_adapter.py b/src/gepa/adapters/app_world_adapter/app_world_adapter.py index 5627376d2..e9caa774b 100644 --- a/src/gepa/adapters/app_world_adapter/app_world_adapter.py +++ b/src/gepa/adapters/app_world_adapter/app_world_adapter.py @@ -1,5 +1,7 @@ import json import os +import io +import contextlib import subprocess from datetime import datetime from pathlib import Path @@ -46,34 +48,35 @@ def evaluate( ) self.agent.gepa_prompt_replace = instruct_prompt - for example in batch: - task_id = example.task_id - test_tracker = self.agent.solve_task(task_id, self.experiment_name + example_run_id) - try: - success = test_tracker.success - score = int(success) - #score = len(test_tracker.passes) / test_tracker._num_tests - failed_reason_list = [] - for failure in test_tracker.failures: - failed_reason_list.append(json.dumps(failure, indent=2)) - failed_reason = ','.join(failed_reason_list) - except Exception as e: - #TODO: need to handle case for failed code execution - success = False - score = 0 - failed_reason = "\n\n".join(test_tracker) - outputs.append( - f"App World outputs are omitted. Please see directory for detailed logging." - ) - scores.append(score) - trajectories.append( - { - "messages": self.agent.messages, - "instruction_prompt": instruct_prompt, - "failed_reason": str(failed_reason), - "success": success, - } - ) + with contextlib.redirect_stdout(io.StringIO()): + for example in batch: + task_id = example.task_id + test_tracker = self.agent.solve_task(task_id, self.experiment_name + example_run_id) + try: + success = test_tracker.success + score = int(success) + #score = len(test_tracker.passes) / test_tracker._num_tests + failed_reason_list = [] + for failure in test_tracker.failures: + failed_reason_list.append(json.dumps(failure, indent=2)) + failed_reason = ','.join(failed_reason_list) + except Exception as e: + #TODO: need to handle case for failed code execution + success = False + score = 0 + failed_reason = "\n\n".join(test_tracker) + outputs.append( + f"App World outputs are omitted. Please see directory for detailed logging." + ) + scores.append(score) + trajectories.append( + { + "messages": self.agent.messages, + "instruction_prompt": instruct_prompt, + "failed_reason": str(failed_reason), + "success": success, + } + ) return EvaluationBatch( outputs=outputs, scores=scores, From adb24872e066dbad6b73b6e1cf0cb0e0429b7af2 Mon Sep 17 00:00:00 2001 From: snova-jayr Date: Tue, 30 Dec 2025 16:02:59 -0800 Subject: [PATCH 11/11] adding in test file to run test --- src/gepa/examples/app-world/test_appworld.py | 42 ++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/gepa/examples/app-world/test_appworld.py diff --git a/src/gepa/examples/app-world/test_appworld.py b/src/gepa/examples/app-world/test_appworld.py new file mode 100644 index 000000000..89f5519eb --- /dev/null +++ b/src/gepa/examples/app-world/test_appworld.py @@ -0,0 +1,42 @@ +import os +import argparse +from appworld.common.path_store import path_store +from gepa.core.state import GEPAState +from gepa.core.result import GEPAResult +from appworld.common.utils import jsonnet_load, read_file +from appworld.task import Task, load_task_ids +import litellm +from gepa.adapters.app_world_adapter.app_world_adapter import ( + AppWorldTask, + AppWorldAdapter, +) +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--experiment_name", type=str, required=True) + parser.add_argument("--appworld_bin_dir_path", type=str, required=True) + parser.add_argument("--test_type", type=str, choices=["normal", "challenge"], required=True, help="Choose either normal or challenge mode.") + args = parser.parse_args() + experiment_name = args.experiment_name + gepa_state_path = args.appworld_bin_dir_path + test_type = args.test_type + experiment_file_path = os.path.join(path_store.experiment_configs, experiment_name + ".jsonnet") + experiment_config = jsonnet_load( + experiment_file_path, + APPWORLD_EXPERIMENT_PROMPTS_PATH=path_store.experiment_prompts, + APPWORLD_EXPERIMENT_CONFIGS_PATH=path_store.experiment_configs, + APPWORLD_EXPERIMENT_CODE_PATH=path_store.experiment_code, + ) + runner_config = experiment_config.pop("config") + agent_config = runner_config.pop("agent") + + gepa_state = GEPAState.load(gepa_state_path) + result = GEPAResult.from_state(gepa_state) + + test_task_ids = load_task_ids(f'test_{test_type}') + for task_id in test_task_ids: + Task.load(task_id=task_id) + print(f"Length of original test {test_type} dataset: {len(test_task_ids)}") + testset = [AppWorldTask(task_id=task_id) for task_id in test_task_ids[:]] + + adapter = AppWorldAdapter(agent_config, experiment_name=experiment_name) + testset_results_before_opt = adapter.evaluate(testset, {"instruction_prompt": result.best_candidate["instruction_prompt"]}, capture_traces=True) \ No newline at end of file