From a580dfd5ec86bcf54aceea688c6423e0a624ea11 Mon Sep 17 00:00:00 2001 From: Bipin Date: Thu, 30 Jul 2026 15:11:02 -0700 Subject: [PATCH] fix: use prompt template settings for sdk experiments --- src/galileo/experiment.py | 25 +++++++++++----------- src/galileo/experiments.py | 22 +++++++++++++++++-- tests/test_experiment.py | 44 ++++++++++++++++++++++++++++++++++++++ tests/test_experiments.py | 34 +++++++++++++++++++++++++++-- 4 files changed, 108 insertions(+), 17 deletions(-) diff --git a/src/galileo/experiment.py b/src/galileo/experiment.py index 6c815d541..c1cc7996e 100644 --- a/src/galileo/experiment.py +++ b/src/galileo/experiment.py @@ -11,7 +11,7 @@ from galileo.exceptions import NotFoundError from galileo.experiment_tags import upsert_experiment_tag from galileo.experiments import Experiments as ExperimentsService -from galileo.experiments import _default_prompt_settings +from galileo.experiments import _default_prompt_settings, _prompt_template_settings from galileo.export import ExportClient from galileo.job_progress import get_run_scorer_jobs, job_progress from galileo.prompts import PromptTemplate, get_prompt @@ -98,9 +98,9 @@ class Experiment(StateManagementMixin): passed to run() completely overrides any settings stored in the prompt template itself. The Runners service uses ONLY the settings provided at job creation time. - If you don't provide prompt_settings to run(), default values will be used. - To use the template's settings, retrieve them first using get_prompt_template_settings() - and pass them explicitly. + If you don't provide prompt_settings to run(), the selected prompt template + version's settings will be used when available; otherwise default values + will be used. **Experiment Immutability:** Once an experiment has been run and has traces, it cannot be run again. @@ -442,7 +442,7 @@ def create(self) -> Experiment: self._prompt_template = get_prompt(name=self.prompt_name) # Determine effective prompt settings - # Priority: 1. explicit prompt_settings, 2. model parameter, 3. defaults for prompt-template flow + # Priority: 1. explicit prompt_settings, 2. model parameter, 3. template settings, 4. defaults effective_prompt_settings = self.prompt_settings if self.model_alias: if effective_prompt_settings is None: @@ -456,8 +456,9 @@ def create(self) -> Experiment: settings_dict["model_alias"] = self.model_alias effective_prompt_settings = PromptRunSettings(**settings_dict) elif self._prompt_template is not None and effective_prompt_settings is None: - # Default prompt settings for prompt-template flow (same as ExperimentsService.run()) - effective_prompt_settings = _default_prompt_settings() + effective_prompt_settings = ( + _prompt_template_settings(self._prompt_template) or _default_prompt_settings() + ) # Set up metrics if provided scorer_settings: list[ScorerConfig] | None = None @@ -962,10 +963,8 @@ def get_prompt_template_settings(self) -> PromptRunSettings | None: """ Get the settings from the associated prompt template. - WARNING: These settings are NOT automatically used when running the experiment. - The Runners service ignores template settings and only uses the prompt_settings - passed to the run() method. Use this method to retrieve template settings if - you want to apply them to the job. + Explicit prompt_settings still take precedence, but these template settings + are used by default for prompt-template experiment creation when available. Returns ------- @@ -983,8 +982,8 @@ def get_prompt_template_settings(self) -> PromptRunSettings | None: # Get settings from template template_settings = experiment.get_prompt_template_settings() - # Note: Current run() doesn't accept prompt_settings parameter - # This would require updating the run() signature + # These settings are also used automatically when no explicit + # prompt_settings are passed. """ if self._prompt_template is None: if self.prompt_id: diff --git a/src/galileo/experiments.py b/src/galileo/experiments.py index c2573d2e7..cbdc88520 100644 --- a/src/galileo/experiments.py +++ b/src/galileo/experiments.py @@ -19,6 +19,7 @@ list_experiments_projects_project_id_experiments_get, ) from galileo.resources.models import ExperimentResponse, HTTPValidationError, PromptRunSettings, ScorerConfig, TaskType +from galileo.resources.types import Unset from galileo.schema.datasets import DatasetRecord from galileo.schema.experiment_group import ExperimentGroupResponse from galileo.schema.metrics import GalileoMetrics, LocalMetricConfig, Metric @@ -60,6 +61,22 @@ def _default_prompt_settings(model_alias: str = "GPT-4o") -> PromptRunSettings: ) +def _prompt_template_settings(prompt_template: PromptTemplate | None) -> PromptRunSettings | None: + if prompt_template is None: + return None + + selected_version = getattr(prompt_template, "selected_version", None) + settings = getattr(selected_version, "settings", None) + if settings is None or isinstance(settings, Unset): + return None + if isinstance(settings, PromptRunSettings): + return settings + if isinstance(settings, dict) and settings: + return PromptRunSettings.from_dict(settings) + + return None + + MAX_REQUEST_SIZE_BYTES = 10 * 1024 * 1024 # 10 MB MAX_INGEST_BATCH_SIZE = 128 DATASET_CONTENT_PAGE_SIZE = 1000 @@ -193,9 +210,10 @@ def run( if isinstance(prompt_settings, dict): prompt_settings = PromptRunSettings.from_dict(prompt_settings) - # Only set default prompt_settings for prompt-driven flow (when a template is provided) + # Prompt-driven runs need settings in the create+trigger request. Reuse the + # selected template settings when available; fall back only for legacy templates. if prompt_template is not None and prompt_settings is None: - prompt_settings = _default_prompt_settings() + prompt_settings = _prompt_template_settings(prompt_template) or _default_prompt_settings() # Single API call: create experiment + trigger job via trigger=True # Only forward group kwargs when set so existing callers/tests aren't affected. diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 3fe9ba53a..a40f816eb 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -621,6 +621,50 @@ def test_create_fills_default_prompt_settings_for_prompt_template( assert call_kwargs["prompt_settings"].temperature == 0.8 assert call_kwargs["prompt_settings"].max_tokens == 256 + @patch("galileo.experiment.create_metric_configs") + @patch("galileo.experiment.get_prompt") + @patch("galileo.experiment.load_dataset_and_records") + @patch("galileo.shared.project_resolver.Projects") + @patch("galileo.experiment.ExperimentsService") + def test_create_uses_prompt_template_settings_when_available( + self, + mock_experiments_class: MagicMock, + mock_projects_class: MagicMock, + mock_load_dataset: MagicMock, + mock_get_prompt: MagicMock, + mock_create_metrics: MagicMock, + reset_configuration: None, + mock_experiment_response: MagicMock, + mock_project: MagicMock, + ) -> None: + mock_projects_service = MagicMock() + mock_projects_class.return_value = mock_projects_service + mock_projects_service.get_with_env_fallbacks.return_value = mock_project + + mock_dataset = MagicMock() + mock_load_dataset.return_value = (mock_dataset, []) + + saved_settings = PromptRunSettings(model_alias="GPT 5.4 Mini (custom)", temperature=0.0, max_tokens=-1) + mock_prompt = MagicMock() + mock_prompt.selected_version_id = str(uuid4()) + mock_prompt.selected_version.settings = saved_settings + mock_get_prompt.return_value = mock_prompt + + mock_create_metrics.return_value = (None, []) + + mock_experiments_service = MagicMock() + mock_experiments_class.return_value = mock_experiments_service + mock_experiments_service.get.return_value = None + mock_experiments_service.create.return_value = mock_experiment_response + + Experiment( + name="Test Experiment", dataset_name="test-dataset", prompt_name="test-prompt", project_name="Test Project" + ).create() + + actual_settings = mock_experiments_service.create.call_args.kwargs["prompt_settings"] + assert actual_settings is saved_settings + assert actual_settings.model_alias == "GPT 5.4 Mini (custom)" + @patch("galileo.experiment.create_metric_configs") @patch("galileo.experiment.get_prompt") @patch("galileo.experiment.load_dataset_and_records") diff --git a/tests/test_experiments.py b/tests/test_experiments.py index bb4af895c..fb9c7973f 100644 --- a/tests/test_experiments.py +++ b/tests/test_experiments.py @@ -81,7 +81,10 @@ def experiment_response(): ) -def prompt_template(): +def prompt_template(settings: PromptRunSettings | dict | None = None): + if settings is None: + settings = {} + return PromptTemplate( prompt_template=BasePromptTemplateResponse( all_available_versions=[1, 2, 3], @@ -99,7 +102,7 @@ def prompt_template(): lines_edited=0, lines_removed=0, model_changed=False, - settings={}, + settings=settings, settings_changed=False, template="test", updated_at=datetime.now(), @@ -997,6 +1000,33 @@ def test_run_experiment_w_prompt_template_and_metrics( # ScorerSettings.create NOT called for trigger=True flow (API handles it) mock_scorer_settings_class.return_value.create.assert_not_called() + @travel(datetime(2012, 1, 1), tick=False) + @patch.object(galileo.datasets.Datasets, "get") + @patch.object(galileo.experiments.Experiments, "create", return_value=experiment_response()) + @patch.object(galileo.experiments.Experiments, "get", return_value=experiment_response()) + @patch.object(galileo.experiments.Projects, "get_with_env_fallbacks", return_value=project()) + def test_run_experiment_w_prompt_template_uses_template_settings( + self, + mock_get_project: Mock, + mock_get_experiment: Mock, + mock_create_experiment: Mock, + mock_get_dataset: Mock, + dataset_content: DatasetContent, + ) -> None: + saved_settings = PromptRunSettings(model_alias="GPT 5.4 Mini (custom)", temperature=0.0, max_tokens=-1) + + run_experiment( + "test_experiment", + project="awesome-new-project", + dataset_id=str(UUID(int=0)), + prompt_template=prompt_template(saved_settings), + ) + + mock_create_experiment.assert_called_once() + actual_settings = mock_create_experiment.call_args.kwargs["prompt_settings"] + assert actual_settings is saved_settings + assert actual_settings.model_alias == "GPT 5.4 Mini (custom)" + @travel(datetime(2012, 1, 1), tick=False) @patch.object(galileo.datasets.Datasets, "get") @patch.object(galileo.experiments.Experiments, "create", return_value=experiment_response())