From 9fc52de7898bcfc0e25d3d38008a0d3d924a87a6 Mon Sep 17 00:00:00 2001 From: Vikram Shenoy Date: Thu, 12 Mar 2026 17:10:34 -0400 Subject: [PATCH] 2.0 Lite Support, Inference Adapter extensibility, SM Inference Adapter support --- .gitignore | 5 +- README.md | 279 ++- build.sh | 24 + docs/Optimizers.md | 6 +- .../optimized_prompt/few_shot.json | 16 +- .../optimized_prompt/system_prompt.txt | 32 +- .../optimized_prompt/user_prompt.txt | 7 +- ..._user_and_system_prompt_optimization.ipynb | 1501 ++++++++++++----- .../optimized_prompt/few_shot.json | 16 +- .../optimized_prompt/system_prompt.txt | 20 +- .../optimized_prompt/user_prompt.txt | 5 +- ...le_user_prompt_template_optimization.ipynb | 1355 +++++++++------ src/amzn_nova_prompt_optimizer/__version__.py | 2 +- .../core/inference/__init__.py | 16 +- .../core/inference/adapter.py | 142 +- .../core/inference/bedrock_adapter.py | 181 ++ .../core/inference/dspy_compatible.py | 333 ++++ .../core/inference/sagemaker_adapter.py | 378 +++++ .../custom_adapters/custom_chat_adapter.py | 18 +- .../optimizers/miprov2/miprov2_optimizer.py | 41 +- .../nova_meta_prompter/nova_mp_optimizer.py | 2 +- .../nova_prompt_optimizer.py | 90 +- .../inference/test_dspy_compatible_adapter.py | 461 +++++ tests/core/inference/test_dspy_integration.py | 297 ++++ .../core/inference/test_inference_adapter.py | 4 +- .../core/inference/test_sagemaker_adapter.py | 353 ++++ .../miprov2/test_miprov2_optimizer.py | 7 - .../test_nova_meta_prompter.py | 6 +- .../test_nova_prompt_optimizer.py | 91 +- .../test_separate_adapters_integration.py | 269 +++ 30 files changed, 4817 insertions(+), 1140 deletions(-) create mode 100755 build.sh create mode 100644 src/amzn_nova_prompt_optimizer/core/inference/bedrock_adapter.py create mode 100644 src/amzn_nova_prompt_optimizer/core/inference/dspy_compatible.py create mode 100644 src/amzn_nova_prompt_optimizer/core/inference/sagemaker_adapter.py create mode 100644 tests/core/inference/test_dspy_compatible_adapter.py create mode 100644 tests/core/inference/test_dspy_integration.py create mode 100644 tests/core/inference/test_sagemaker_adapter.py create mode 100644 tests/integration/test_separate_adapters_integration.py diff --git a/.gitignore b/.gitignore index fa90de1..e2f9148 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,7 @@ ENV/ *.swo # Misc -.DS_Store \ No newline at end of file +.DS_Store + +# Design documentation (not for commit) +design_docs/ \ No newline at end of file diff --git a/README.md b/README.md index 9aa5525..3dd636f 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ # Nova Prompt Optimizer -A Python SDK for optimizing prompts for Nova. +A Python SDK for optimizing prompts for Amazon Nova and other models deployed on AWS. ## πŸ“š Table of contents * [Installation](#installation) * [Pre-Requisites](#pre-requisites) +* [What's New](#-whats-new) * [Quick Start: Facility Support Analyzer Dataset](#-quick-start) +* [Quick Start: SageMaker Endpoints](#-quick-start-sagemaker-endpoints) * [Core Concepts](#core-concepts) * [Input Adapters](#input-adapters) * [1. Prompt Adapter](#1-prompt-adapter) @@ -16,6 +18,9 @@ A Python SDK for optimizing prompts for Nova. * [Optimizers](#optimizers) * [NovaPromptOptimizer](#novapromptoptimizer) * [Evaluator](#evaluator) +* [Advanced Features](#advanced-features) + * [Separate Inference Adapters](#separate-inference-adapters) + * [SageMaker Endpoint Support](#sagemaker-endpoint-support) * [Optimization Recommendations](#optimization-recommendations) * [Preview Status](#-preview-status) * [Interaction with AWS Bedrock](#interaction-with-aws-bedrock) @@ -50,6 +55,44 @@ export AWS_SECRET_ACCESS_KEY="..." 5. Wait for approval (instant in most cases) +## πŸŽ‰ What's New + +### SageMaker Endpoint Support +Optimize prompts for models deployed on Amazon SageMaker! The optimizer now supports: +- **SageMaker endpoints** as task models for optimization +- **Automatic Bedrock integration** for meta-prompting with Nova 2.0 Lite +- **OpenAI-compatible** payload format for SageMaker endpoints + +### Separate Inference Adapters +Use different models for different optimization phases: +- **Meta-prompting**: Automatically uses Bedrock with Nova 2.0 Lite (or specify your own) +- **Task optimization**: Use any supported backend (Bedrock, SageMaker, etc.) + +### Example: Optimize for SageMaker +```python +from amzn_nova_prompt_optimizer.core.inference import SageMakerInferenceAdapter +from amzn_nova_prompt_optimizer.core.optimizers import NovaPromptOptimizer + +# Your SageMaker endpoint +sagemaker_adapter = SageMakerInferenceAdapter( + endpoint_name="my-model-endpoint", + region_name="us-west-2" +) + +# Optimizer automatically uses Bedrock Nova 2.0 Lite for meta-prompting +optimizer = NovaPromptOptimizer( + prompt_adapter=prompt_adapter, + inference_adapter=sagemaker_adapter, # Your SageMaker endpoint + dataset_adapter=dataset_adapter, + metric_adapter=metric_adapter +) + +optimized_prompt = optimizer.optimize(mode="lite") +``` + +See the [Quick Start for SageMaker](#-quick-start-sagemaker-endpoints) section below for a complete example. + + ## 🏁 Quick Start ### Facility Support Analyzer Dataset The Facility Support Analyzer dataset consists of emails that are to be classified based on category, urgency and sentiment. @@ -57,6 +100,88 @@ The Facility Support Analyzer dataset consists of emails that are to be classifi Please see the [samples](samples/facility-support-analyzer/) folder for example notebooks of how to optimize a prompt in the scenario where a [user prompt template is to be optimized](samples/facility-support-analyzer/user_prompt_only) and the scenario where a [user and system prompt is to be optimized together](samples/facility-support-analyzer/system_and_user_prompt) +## πŸš€ Quick Start: SageMaker Endpoints + +Optimize prompts for models deployed on Amazon SageMaker in just a few steps: + +### 1. Install and Configure +```bash +pip install nova-prompt-optimizer +export AWS_ACCESS_KEY_ID="..." +export AWS_SECRET_ACCESS_KEY="..." +``` + +### 2. Prepare Your Dataset +```jsonl +{"input": "What is machine learning?", "answer": "Machine learning is..."} +{"input": "Explain neural networks", "answer": "Neural networks are..."} +``` + +### 3. Run Optimization +```python +from amzn_nova_prompt_optimizer.core.inference import SageMakerInferenceAdapter +from amzn_nova_prompt_optimizer.core.input_adapters.dataset_adapter import JSONDatasetAdapter +from amzn_nova_prompt_optimizer.core.input_adapters.prompt_adapter import TextPromptAdapter +from amzn_nova_prompt_optimizer.core.input_adapters.metric_adapter import MetricAdapter +from amzn_nova_prompt_optimizer.core.optimizers import NovaPromptOptimizer + +# Define your metric +class ExactMatchMetric(MetricAdapter): + def apply(self, prediction: str, ground_truth: str) -> float: + return 1.0 if prediction.strip() == ground_truth.strip() else 0.0 + + def batch_apply(self, predictions: list, ground_truths: list) -> float: + matches = sum(self.apply(p, g) for p, g in zip(predictions, ground_truths)) + return matches / len(predictions) if predictions else 0.0 + +# Setup SageMaker adapter for your endpoint +sagemaker_adapter = SageMakerInferenceAdapter( + endpoint_name="YOUR-ENDPOINT-NAME", + region_name="us-west-2" +) + +# Load dataset +dataset_adapter = JSONDatasetAdapter( + input_columns={"input"}, + output_columns={"answer"} +) +dataset_adapter.adapt("your_dataset.jsonl") +train_set, test_set = dataset_adapter.split(0.5) + +# Setup initial prompt +prompt_adapter = TextPromptAdapter() +prompt_adapter.set_system_prompt(content="You are a helpful assistant.") +prompt_adapter.set_user_prompt( + content="Answer this question: {{input}}", + variables={"input"} +) +prompt_adapter.adapt() + +# Setup metric +metric_adapter = ExactMatchMetric() + +# Create optimizer (automatically uses Bedrock Nova 2.0 Lite for meta-prompting) +optimizer = NovaPromptOptimizer( + prompt_adapter=prompt_adapter, + inference_adapter=sagemaker_adapter, # Your SageMaker endpoint + dataset_adapter=dataset_adapter, + metric_adapter=metric_adapter +) + +# Run optimization +optimized_prompt = optimizer.optimize(mode="lite") + +# Save results +optimized_prompt.save("optimized_prompts/") +``` + +**What happens during optimization:** +1. **Meta-Prompting Phase**: Uses Bedrock with Nova 2.0 Lite to generate an improved prompt structure (~30 seconds) +2. **Task Optimization Phase**: Tests multiple prompt variations on your SageMaker endpoint (~5-15 minutes) + +For more details, see the [SageMaker Quick Start Guide](docs/QUICK_START_SAGEMAKER.md). + + ## Core Concepts ### Input Adapters @@ -88,32 +213,57 @@ prompt_adapter.adapt() Learn More about the Prompt Adapter [here](docs/PromptAdapter.md) ### 2. Inference Adapter -**Responsibility:** Ability to call an inference backend for the models e.g. Bedrock, etc. +**Responsibility:** Ability to call an inference backend for the models e.g. Bedrock, SageMaker, etc. -**Sample use of Inference Adapter** +**Sample use of Bedrock Inference Adapter** ```python -from amzn_nova_prompt_optimizer.core.inference.adapter import BedrockInferenceAdapter +from amzn_nova_prompt_optimizer.core.inference import BedrockInferenceAdapter inference_adapter = BedrockInferenceAdapter(region_name="us-east-1") ``` -You can pass `rate_limit` into constructor of InferenceAdapter to limit the max TPS of bedrock call to avoid throttle. Default to 2 if not set. +**Sample use of SageMaker Inference Adapter** + +```python +from amzn_nova_prompt_optimizer.core.inference import SageMakerInferenceAdapter + +inference_adapter = SageMakerInferenceAdapter( + endpoint_name="my-model-endpoint", + region_name="us-west-2" +) +``` + +You can pass `rate_limit` into constructor of InferenceAdapter to limit the max TPS of calls to avoid throttle. Default to 2 if not set. ```python -from amzn_nova_prompt_optimizer.core.inference.adapter import BedrockInferenceAdapter +from amzn_nova_prompt_optimizer.core.inference import BedrockInferenceAdapter inference_adapter = BedrockInferenceAdapter(region_name="us-east-1", rate_limit=10) # Max 10 TPS ``` -**Supported Inference Adapters:** `BedrockInferenceAdapter` +**Supported Inference Adapters:** +- `BedrockInferenceAdapter` - For Amazon Bedrock models +- `SageMakerInferenceAdapter` - For SageMaker endpoints (OpenAI-compatible format) **Core Functions** Call the model using the parameters ```python -# Call the model with the passed parametrs -inference_output = inference_adapter.call_model(model_id: str, system_prompt: str, messages: List[Dict[str, str]], inf_config: Dict[str, Any]) +# Call the model with the passed parameters +inference_output = inference_adapter.call_model( + model_id: str, + system_prompt: str, + messages: List[Dict[str, str]], + inf_config: Dict[str, Any] +) +``` + +Test endpoint connectivity (SageMaker) +```python +# Test if endpoint is accessible +if inference_adapter.test_connection(): + print("βœ“ Endpoint is ready") ``` The Inference Adapter accepts the `system_prompt` as a string. @@ -256,9 +406,23 @@ nova_prompt_optimizer = NovaPromptOptimizer(prompt_adapter=prompt_adapter, infer optimized_prompt_adapter = nova_prompt_optimizer.optimize(mode="lite") ``` -NovaPromptOptimizer uses Premier for Meta Prompting and then uses MIPROv2 with 20 candidates and 50 trials with Premier as Prompting model and task model dependent on the mode it's set at. +NovaPromptOptimizer uses Nova 2.0 Lite for Meta Prompting and then uses MIPROv2 with 20 candidates and 30 trials. The task model depends on the mode it's set at. + +**Automatic Meta-Prompting with Bedrock:** +When you don't provide a `meta_prompt_inference_adapter`, NovaPromptOptimizer automatically creates a BedrockInferenceAdapter with Nova 2.0 Lite for the meta-prompting phase. This means you can use any inference adapter (including SageMaker) for your task model, and the optimizer will handle meta-prompting with Bedrock automatically. + +**Optimization Modes:** + +| Mode | Meta-Prompt Model | Task Model | Use Case | +|------|-------------------|------------|----------| +| `micro` | Nova 2.0 Lite | Nova Micro | Fast, cost-effective | +| `lite` | Nova 2.0 Lite | Nova Lite | Balanced (default) | +| `pro` | Nova 2.0 Lite | Nova Pro | High quality | +| `lite-2` | Nova 2.0 Lite | Nova 2.0 Lite | Maximum quality | + You can specify enable_json_fallback=False to disable the behavior that MIPROv2 will [fallback to use JSONAdapter to parse LM model output](https://github.com/stanfordnlp/dspy/blob/main/dspy/adapters/chat_adapter.py#L44-L51). This will force MIPROv2 use structured output (pydantic model) to parse LM output. +**Custom Mode:** You could also define a custom mode and pass your own parameter values to NovaPromptOptimizer ```python @@ -314,10 +478,103 @@ evaluator.save("eval_results.jsonl") WARNING amzn_nova_prompt_optimizer.core.inference: Warn: Prompt Variables not found in User Prompt, injecting them at the end of the prompt ``` +## Advanced Features + +### Separate Inference Adapters + +Use different inference adapters for meta-prompting and task optimization phases. This is particularly useful when optimizing prompts for SageMaker endpoints while using Bedrock for meta-prompting. + +**Example:** +```python +from amzn_nova_prompt_optimizer.core.inference import ( + BedrockInferenceAdapter, + SageMakerInferenceAdapter +) +from amzn_nova_prompt_optimizer.core.optimizers import NovaPromptOptimizer + +# Bedrock for meta-prompting (generates optimized prompts) +meta_adapter = BedrockInferenceAdapter(region_name="us-east-1") + +# SageMaker for task model (the model being optimized) +task_adapter = SageMakerInferenceAdapter( + endpoint_name="my-endpoint", + region_name="us-west-2" +) + +# Create optimizer with separate adapters +optimizer = NovaPromptOptimizer( + prompt_adapter=prompt_adapter, + inference_adapter=task_adapter, # For task optimization + dataset_adapter=dataset_adapter, + metric_adapter=metric_adapter, + meta_prompt_inference_adapter=meta_adapter # For meta-prompting +) + +optimized_prompt = optimizer.optimize(mode="lite") +``` + +**Benefits:** +- Use the best model for each optimization phase +- Optimize SageMaker endpoints with Bedrock intelligence +- Cross-region support +- Independent rate limiting per adapter + +For more details, see the [Separate Inference Adapters Guide](docs/SeparateInferenceAdapters.md). + +### SageMaker Endpoint Support + +The SDK now supports optimizing prompts for models deployed on Amazon SageMaker. SageMaker endpoints must use an OpenAI-compatible message format: + +**Required Payload Format:** +```json +{ + "messages": [ + {"role": "system", "content": "You are helpful"}, + {"role": "user", "content": "Hello"} + ], + "max_tokens": 1000, + "temperature": 0.7, + "top_p": 0.9, + "top_k": 50 +} +``` + +**Expected Response Format:** +```json +{ + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I help you?" + } + } + ] +} +``` + +**Testing Your Endpoint:** +```python +from amzn_nova_prompt_optimizer.core.inference import SageMakerInferenceAdapter + +adapter = SageMakerInferenceAdapter( + endpoint_name="my-endpoint", + region_name="us-west-2" +) + +# Test connectivity +if adapter.test_connection(): + print("βœ“ Endpoint is ready for optimization") +else: + print("βœ— Endpoint connection failed") +``` + +For a complete guide, see the [SageMaker Quick Start](docs/QUICK_START_SAGEMAKER.md). + ## Optimization Recommendations 1. Provide representative real-world evaluation sets and split them into training and testing sets. Ensure dataset is balanced on output label when splitting train and test sets. 2. For evaluation sets, the ground truth column should be as close to the inference output as possible. e.g. If the inference output is {"answer": "POSITIVE"} ground truth should also be in the same format {"answer": "POSITIVE"} -3. For NovaPromptOptimizer, choose the mode (mode= "premier" | ""pro" | "lite" | "micro") based on your Nova Model of choice. By default, we use "pro". +3. For NovaPromptOptimizer, choose the mode (mode= "lite-2" | ""pro" | "lite" | "micro") based on your Nova Model of choice. By default, we use "pro". 4. The `apply` function of the evaluation metric should return a numerical value between 0 and 1 for NovaPromptOptimizer or MIPROv2. ## ⚠️ Preview Status diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..b22c6c1 --- /dev/null +++ b/build.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e + +echo "🧹 Cleaning previous builds..." +rm -rf dist/ build/ *.egg-info src/*.egg-info + +echo "πŸ”¨ Building package..." +python3 -m build + +echo "" +echo "βœ… Build complete!" +echo "" +echo "πŸ“¦ Generated files:" +ls -lh dist/ + +echo "" +echo "πŸš€ To install locally:" +echo " pip install dist/nova_prompt_optimizer-*-py3-none-any.whl" +echo "" +echo "πŸ“€ To distribute:" +echo " - Share the .whl file directly" +echo " - Upload to internal PyPI: twine upload --repository-url dist/*" +echo " - Upload to S3: aws s3 cp dist/*.whl s3://your-bucket/packages/" +echo " - Attach to GitHub release" diff --git a/docs/Optimizers.md b/docs/Optimizers.md index db8df7d..7aa00ce 100644 --- a/docs/Optimizers.md +++ b/docs/Optimizers.md @@ -34,7 +34,7 @@ nova_mp_optimization_adapter = NovaMPOptimizationAdapter(prompt_adapter=prompt_a nova_mp_optimized_prompt_adapter = nova_mp_optimization_adapter.optimize(max_retries=5) ``` -Nova Meta Prompter uses Premier for Meta Prompting. Max Retries to retry optimization if optimized prompts do not contain prompt variables. +Nova Meta Prompter uses Nova 2.0 Lite for Meta Prompting. Max Retries to retry optimization if optimized prompts do not contain prompt variables. #### MIPROv2 @@ -55,7 +55,7 @@ from amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer import mipro_optimization_adapter = MIPROv2OptimizationAdapter(prompt_adapter=prompt_adapter, dataset_adapter=train_dataset_adapter, metric_adapter=metric_adapter) mipro_prompt_adapter = mipro_optimization_adapter.optimize(task_model_id="us.amazon.nova-lite-v1:0", - prompter_model_id ="us.amazon.nova-premier-v1:0", + prompter_model_id ="us.amazon.nova-2-lite-v1:0", num_candidates=None, num_threads= 2, num_trials=None, @@ -66,7 +66,7 @@ mipro_prompt_adapter = mipro_optimization_adapter.optimize(task_model_id="us.ama enable_json_fallback = False) ``` -MIPROv2 uses Premier for Prompting and the task model provided as `task_model_id`. +MIPROv2 uses Nova 2.0 Lite for Prompting and the task model provided as `task_model_id`. By default, it uses "medium" optimization i.e. Generating 6 instruction candidates and num_trials proportional to it You can specify enable_json_fallback=False to disable the behavior that MIPROv2 will [fallback to use JSONAdapter to parse LM model output](https://github.com/stanfordnlp/dspy/blob/main/dspy/adapters/chat_adapter.py#L44-L51). This will force MIPROv2 use structured output (pydantic model) to parse LM output. diff --git a/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/few_shot.json b/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/few_shot.json index e8964bd..5f2a401 100644 --- a/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/few_shot.json +++ b/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/few_shot.json @@ -3,7 +3,7 @@ "role": "user", "content": [ { - "text": "Extract and return a json with the following keys and values from the input: [Subject: Urgent Request for Training and Support\n\nHi ProCare Support Team,\n\nI hope this message finds you well. My name is [Sender], and I manage a residential complex that has been utilizing ProCare Facility Solutions for our maintenance and cleaning needs for the past year. Your services have been instrumental in keeping our environment safe and well-maintained.\n\nHowever, I am reaching out with an urgent request. We are in immediate need of comprehensive training for our in-house maintenance team. Given the complexity of our facility's systems, particularly the HVAC and electrical components, it's crucial that our team is well-versed in best practices and troubleshooting techniques. \n\nWe have encountered several issues recently that have highlighted gaps in our current knowledge and capabilities. While we have managed to address these problems temporarily, a more sustainable solution is necessary. We need a detailed training program that covers routine maintenance, emergency repairs, and energy efficiency practices.\n\nCould you please provide information on the available training programs and how quickly we can schedule a session? Time is of the essence, as we aim to prevent any further disruptions to our facility's operations.\n\nThank you for your prompt attention to this matter. I look forward to your swift response.\n\nBest regards,\n[Sender]]\n- \"urgency\" as one of `high`, `medium`, `low`\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\n- \"categories\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\n\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces." + "text": "The input provided to you is: Subject: Inquiry About HVAC Maintenance Best Practices\n\nHi ProCare Support Team,\n\nI hope this message finds you well. My name is Alex, and I'm a novice electrician currently learning the ropes of working with HVAC units. I've recently started gaining hands-on experience and have been exploring various aspects of facility maintenance.\n\nI wanted to reach out to you with a few general inquiries regarding best practices for maintaining HVAC systems. Given your expertise in facility management and maintenance, I believe your insights would be incredibly valuable to me as I continue to develop my skills.\n\nSo far, I've been following basic maintenance routines like checking filters and ensuring proper airflow, but I would love to know if there are any specific tips or advanced techniques that I should be aware of. Additionally, are there any common pitfalls or mistakes that I should avoid while working with HVAC units?\n\nI haven't encountered any major issues yet, but I want to make sure I'm on the right track and doing everything correctly from the start. Any guidance or resources you could provide would be greatly appreciated. Also, if there are any training programs or support resources available for someone in my position, I would be very interested in learning more about them.\n\nThank you for your time and assistance. I look forward to hearing from you soon.\n\nBest regards, \nAlex" } ] }, @@ -11,7 +11,7 @@ "role": "assistant", "content": [ { - "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": true, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"neutral\", \"urgency\": \"high\"}" + "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": true, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": true}, \"sentiment\": \"neutral\", \"urgency\": \"low\"}" } ] }, @@ -19,7 +19,7 @@ "role": "user", "content": [ { - "text": "Extract and return a json with the following keys and values from the input: [Subject: Immediate Assistance Required for Facility Management Issue\n\nDear ProCare Support Team,\n\nI hope this message finds you well. My name is [Sender], and I am currently residing in one of the residential complexes managed by ProCare Facility Solutions. I am reaching out to you with an urgent matter that requires immediate attention.\n\nOver the past few days, I have noticed significant issues with the facility management in my building. Specifically, there have been recurring problems with the HVAC system, which has resulted in inconsistent heating and cooling. This has made it quite challenging to maintain a comfortable living environment, especially given the current weather conditions.\n\nI have already attempted to address this issue by contacting the building's maintenance staff directly, but unfortunately, the problem persists. Given the urgency of the situation, I am requesting that a qualified technician be dispatched as soon as possible to resolve this matter.\n\nYour prompt assistance in this regard would be greatly appreciated, as it is crucial for me to have a stable and comfortable living environment to focus on my studies.\n\nThank you for your understanding and swift action.\n\nBest regards,\n[Sender]]\n- \"urgency\" as one of `high`, `medium`, `low`\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\n- \"categories\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\n\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces." + "text": "The input provided to you is: Subject: Request for Training and Support on Latest Facility Management Trends\n\nHi ProCare Support Team,\n\nHope you're all doing great! My name is Alex, and I've been following ProCare Facility Solutions for a while now. I really appreciate the top-notch services you provide, especially your commitment to sustainability and quality.\n\nI'm reaching out because I think there's a fantastic opportunity for us to stay ahead of the curve with some of the latest online trends in facility management. As someone who grew up with the internet, I've noticed a lot of new tools and practices emerging that could really benefit our operations. I believe integrating these trends could enhance our efficiency and overall service quality.\n\nI've done some preliminary research and tried to implement a few of these trends myself, but I think a more structured training program from your end would be incredibly beneficial. Specifically, I'm interested in learning more about the latest best practices in energy efficiency and environmental impact reduction, as well as any new technologies that could streamline our maintenance and cleaning processes.\n\nCould you please provide some guidance or set up a training session to help us get up to speed with these new trends? I think it would be a great addition to our current practices and help us maintain our high standards.\n\nThanks a lot for your time and assistance. Looking forward to hearing from you soon!\n\nBest regards,\nAlex" } ] }, @@ -27,7 +27,7 @@ "role": "assistant", "content": [ { - "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": true, \"general_inquiries\": false}, \"sentiment\": \"negative\", \"urgency\": \"high\"}" + "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": true, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": true, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": true, \"general_inquiries\": false}, \"sentiment\": \"positive\", \"urgency\": \"low\"}" } ] }, @@ -35,7 +35,7 @@ "role": "user", "content": [ { - "text": "Extract and return a json with the following keys and values from the input: [Subject: Scheduling Cleaning Services for My Home\n\nHi ProCare Support Team,\n\nI hope this message finds you well. My name is Alex, and I\u2019m a high school student currently living in a residential complex that uses your services. I\u2019ve heard great things about ProCare Facility Solutions from my neighbors and have seen firsthand the excellent work your team does.\n\nI\u2019m reaching out because I need to schedule a cleaning service for my home. With school and college applications taking up most of my time, I haven\u2019t been able to keep up with the cleaning as much as I\u2019d like. I\u2019m looking for a regular cleaning schedule, preferably weekly, to help maintain a clean and healthy environment.\n\nI haven\u2019t taken any steps yet to resolve this, as I wanted to get in touch with you directly to understand the best options available. Could you please guide me on how to set up a cleaning schedule and what the next steps would be?\n\nThank you for your assistance. I look forward to hearing from you soon.\n\nBest regards,\nAlex]\n- \"urgency\" as one of `high`, `medium`, `low`\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\n- \"categories\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\n\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces." + "text": "The input provided to you is: Subject: Immediate Attention Required for Sustainability Practices\n\nDear ProCare Support Team,\n\nI hope this message finds you well, though I must admit, I am quite frustrated as I write this. My name is [Sender], and I have been utilizing your services for my dairy farm's facility management for the past year. While I initially chose ProCare Facility Solutions for your reputed expertise and commitment to quality, recent experiences have left me questioning that decision.\n\nAs a dairy farmer who deeply values sustainable practices, I was particularly drawn to your promise of implementing best practices for energy efficiency and environmental impact reduction. However, I have noticed a significant lack of follow-through in this area. Despite multiple assurances, there has been no visible effort to incorporate eco-friendly solutions into the maintenance and cleaning services provided to my farm.\n\nI have already reached out to your team on two separate occasions to address this issue, but the responses have been unsatisfactory and vague at best. I was assured that someone would look into it, yet here we are, with no tangible progress or updates.\n\nI am requesting immediate and concrete action to rectify this situation. Specifically, I need a detailed plan outlining how ProCare intends to integrate sustainable practices into the services provided to my farm. This includes the use of eco-friendly cleaning products, energy-efficient maintenance solutions, and any other measures that align with your advertised commitment to sustainability.\n\nI trust that you will treat this matter with the urgency it deserves and provide a satisfactory resolution promptly. Failure to do so will force me to reconsider my association with ProCare Facility Solutions.\n\nLooking forward to your prompt response.\n\nBest regards,\n[Sender]" } ] }, @@ -43,7 +43,7 @@ "role": "assistant", "content": [ { - "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": true, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"neutral\", \"urgency\": \"low\"}" + "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": true, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": true, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": true, \"emergency_repair_services\": false, \"facility_management_issues\": true, \"general_inquiries\": false}, \"sentiment\": \"negative\", \"urgency\": \"high\"}" } ] }, @@ -51,7 +51,7 @@ "role": "user", "content": [ { - "text": "Extract and return a json with the following keys and values from the input: [Subject: Immediate Attention Required: Emergency Repair Needed\n\nHey ProCare Support Team,\n\nThis is [Sender], and I have to say, I'm not impressed. I've been using your services for a while now, and I expected better. Right now, I'm dealing with a major issue that needs your immediate attention.\n\nThe HVAC system in my building has completely failed, and it's causing a lot of problems. The temperature is unbearable, and it's affecting everyone here. This isn't just an inconvenience; it's a serious problem that needs to be fixed right away. I've tried resetting the system and checking the circuit breakers, but nothing has worked.\n\nI need your team to come out and fix this immediately. This kind of failure is unacceptable, and I expect a prompt response. If this isn't resolved quickly, I might have to reconsider using your services in the future.\n\nLooking forward to your swift action.\n\n[Sender]]\n- \"urgency\" as one of `high`, `medium`, `low`\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\n- \"categories\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\n\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces." + "text": "The input provided to you is: Subject: Scheduled Maintenance Request for Bathroom Plumbing\n\nDear ProCare Facility Solutions Support Team,\n\nI hope this message finds you well. My name is [Sender], and I have been utilizing your services for my residential property for some time now. I have always appreciated the quality and reliability of your maintenance services.\n\nI am writing to inform you of a minor issue that has recently come up. There seems to be a small problem with the plumbing in my bathroom. While it is not an urgent matter, I would like to have it addressed at your earliest convenience to prevent any potential complications.\n\nI have not taken any steps to resolve the issue myself, as I trust your team\u2019s expertise in handling such matters. Could you please arrange for a technician to visit my home and take a look at the problem as part of your scheduled maintenance services?\n\nThank you for your attention to this matter. I look forward to your prompt response.\n\nBest regards,\n[Sender]" } ] }, @@ -59,7 +59,7 @@ "role": "assistant", "content": [ { - "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": true, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": true, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"negative\", \"urgency\": \"high\"}" + "text": "{\"categories\": {\"routine_maintenance_requests\": true, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"neutral\", \"urgency\": \"medium\"}" } ] } diff --git a/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/system_prompt.txt b/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/system_prompt.txt index ffa7b3f..ce2510d 100644 --- a/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/system_prompt.txt +++ b/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/system_prompt.txt @@ -1 +1,31 @@ -Analyze the customer service email to classify its content accurately. Return a JSON object with "urgency" (high/medium/low), "sentiment" (negative/neutral/positive), and "categories" (boolean flags for predefined types). Ensure compliance with data privacy standards by omitting personal identifiers. Follow the format strictly: `{"urgency": "...", "sentiment": "...", "categories": {"...": true/false, ...}}`. \ No newline at end of file +Generate a JSON object from the provided customer support email input. The JSON must contain three keys: "urgency", "sentiment", and "categories". + +**Key Requirements:** +1. **"urgency"**: Must be one of `high`, `medium`, or `low`, based on how time-sensitive the request is. +2. **"sentiment"**: Must be one of `negative`, `neutral`, or `positive`, reflecting the emotional tone of the email. +3. **"categories"**: Must be a dictionary where: + - The keys are from this fixed list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. + - Each value is a boolean (`true` or `false`) indicating whether the category best matches the content of the email. Only categories that are clearly relevant should be set to `true`; all others should be `false`. + +**Steps to Follow:** +1. **Read the email carefully** to understand the main request, tone, and urgency. +2. **Determine urgency**: + - Use `high` if the issue requires immediate attention (e.g., an emergency or something causing significant disruption). + - Use `medium` for issues that need prompt attention but are not emergencies. + - Use `low` for non-urgent requests that can be addressed at a convenient time. +3. **Determine sentiment**: + - Use `positive` if the email expresses satisfaction, gratitude, or a constructive tone. + - Use `neutral` if the email is factual, informative, or lacks strong emotional language. + - Use `negative` if the email expresses dissatisfaction, frustration, or complaint. +4. **Identify categories**: + - For each category in the list, decide whether it accurately describes the main focus of the email. + - Set the corresponding value to `true` if the category applies; otherwise, set it to `false`. + - Only one or a few categories should be `true`β€”choose the most relevant ones based on the email content. +5. **Output the JSON**: + - The final output must be a **single-line JSON string** with no additional text, formatting, or encapsulation (e.g., no ```json...```). + - Ensure the JSON is compact with no unnecessary whitespace or newlines. + +**Example Output Format:** +```{"urgency": "medium", "sentiment": "positive", "categories": {"emergency_repair_services": false, "routine_maintenance_requests": true, "quality_and_safety_concerns": false, "specialized_cleaning_services": false, "general_inquiries": false, "sustainability_and_environmental_practices": false, "training_and_support_requests": false, "cleaning_services_scheduling": false, "customer_feedback_and_complaints": false, "facility_management_issues": false}}``` + +Use this format to process the input email and return the JSON object. \ No newline at end of file diff --git a/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/user_prompt.txt b/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/user_prompt.txt index b9eb091..a459644 100644 --- a/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/user_prompt.txt +++ b/samples/facility-support-analyzer/system_and_user_prompt/optimized_prompt/user_prompt.txt @@ -1,6 +1 @@ -Extract and return a json with the following keys and values from the input: [{{input}}] -- "urgency" as one of `high`, `medium`, `low` -- "sentiment" as one of `negative`, `neutral`, `positive` -- "categories" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues` - -Your complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces. \ No newline at end of file +The input provided to you is: {{input}} \ No newline at end of file diff --git a/samples/facility-support-analyzer/system_and_user_prompt/sample_user_and_system_prompt_optimization.ipynb b/samples/facility-support-analyzer/system_and_user_prompt/sample_user_and_system_prompt_optimization.ipynb index 7349c7f..bcae076 100644 --- a/samples/facility-support-analyzer/system_and_user_prompt/sample_user_and_system_prompt_optimization.ipynb +++ b/samples/facility-support-analyzer/system_and_user_prompt/sample_user_and_system_prompt_optimization.ipynb @@ -10,20 +10,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "8954cb65-5eb1-4e76-92bb-331be63f7e73", "metadata": {}, "outputs": [], "source": [ "import os\n", "# Setup your AWS Access Key and Secret Key as environment variables.\n", - "os.environ[\"AWS_ACCESS_KEY_ID\"]\n", - "os.environ[\"AWS_SECRET_ACCESS_KEY\"] " + "#os.environ[\"AWS_ACCESS_KEY_ID\"]\n", + "#os.environ[\"AWS_SECRET_ACCESS_KEY\"] \n", + "\n", + "import sys\n", + "sys.path.insert(0, '../../../src')\n" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "id": "8d981604", "metadata": {}, "outputs": [], @@ -44,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 3, "id": "da047488-0220-4e49-b8d0-361c5afcdfb4", "metadata": {}, "outputs": [], @@ -74,17 +77,17 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 4, "id": "621dc01d-6896-4a37-ad64-63f27c597f75", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 20, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -115,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 5, "id": "f093a481-31a0-4871-99a0-670a60d67b1d", "metadata": {}, "outputs": [], @@ -221,12 +224,12 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 6, "id": "ea654ddd-59d0-495e-8e56-29fe0ed6dd7b", "metadata": {}, "outputs": [], "source": [ - "from amzn_nova_prompt_optimizer.core.inference.adapter import BedrockInferenceAdapter\n", + "from amzn_nova_prompt_optimizer.core.inference import BedrockInferenceAdapter\n", "\n", "inference_adapter = BedrockInferenceAdapter(region_name=\"us-east-1\")" ] @@ -251,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 7, "id": "9f158430-ed74-470c-8657-2b49b57ae79f", "metadata": {}, "outputs": [], @@ -263,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 8, "id": "33974a07-c556-4238-a39c-12601fa01e14", "metadata": {}, "outputs": [ @@ -271,18 +274,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:46:37 INFO amzn_nova_prompt_optimizer.core.evaluation: Cache miss - Running new inference on Dataset\n", - "Running inference: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100/100 [00:48<00:00, 2.05it/s]\n", - "2025/07/02 20:47:25 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Batch Evaluation on Dataset, using `batch_apply` metric\n", - "2025/07/02 20:47:25 INFO amzn_nova_prompt_optimizer.core.evaluation: Using cached inference results\n", - "2025/07/02 20:47:25 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Evaluation on Dataset, using `apply` metric\n" + "2026/01/29 18:19:53 INFO amzn_nova_prompt_optimizer.core.evaluation: Cache miss - Running new inference on Dataset\n", + "Running inference: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100/100 [00:39<00:00, 2.55it/s]\n", + "2026/01/29 18:20:32 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Batch Evaluation on Dataset, using `batch_apply` metric\n", + "2026/01/29 18:20:32 INFO amzn_nova_prompt_optimizer.core.evaluation: Using cached inference results\n", + "2026/01/29 18:20:32 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Evaluation on Dataset, using `apply` metric\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Original Prompt Evaluation Score = {'is_valid_json': 0.31, 'correct_categories': 0.281, 'correct_sentiment': 0.2, 'correct_urgency': 0.26, 'total': 0.247}\n" + "Original Prompt Evaluation Score = {'is_valid_json': 0.24, 'correct_categories': 0.22099999999999997, 'correct_sentiment': 0.12, 'correct_urgency': 0.21, 'total': 0.18366666666666667}\n" ] } ], @@ -339,16 +342,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:18:53 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_meta_prompter.nova_mp_optimizer: Optimizing prompt using Nova Meta Prompter\n", - "2025/07/02 20:18:59 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using us.amazon.nova-pro-v1:0 for Evaluation\n", - "2025/07/02 20:18:59 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using us.amazon.nova-premier-v1:0 for Prompting\n", - "2025/07/02 20:18:59 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.custom_adapters.custom_chat_adapter: Initializing CustomChatAdapter with enable_json_fallback=False\n", - "2025/07/02 20:18:59 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using Nova tips for MIPROv2 optimization\n", - "2025/07/02 20:18:59 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 18:20:34 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer: No meta_prompt_inference_adapter provided. Creating default BedrockInferenceAdapter with Nova 2.0 Lite for meta-prompting.\n", + "2026/01/29 18:20:34 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer: Using separate inference adapters: Meta-prompting with BedrockInferenceAdapter, Task optimization with BedrockInferenceAdapter\n", + "2026/01/29 18:20:34 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_meta_prompter.nova_mp_optimizer: Optimizing prompt using Nova Meta Prompter with Model: us.amazon.nova-2-lite-v1:0\n", + "2026/01/29 18:20:36 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using us.amazon.nova-pro-v1:0 for Evaluation\n", + "2026/01/29 18:20:36 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using us.amazon.nova-2-lite-v1:0 for Prompting\n", + "2026/01/29 18:20:36 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.custom_adapters.custom_chat_adapter: Initializing CustomChatAdapter with enable_json_fallback=False\n", + "2026/01/29 18:20:36 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using Nova tips for MIPROv2 optimization\n", + "2026/01/29 18:20:36 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==\n", - "2025/07/02 20:18:59 INFO dspy.teleprompt.mipro_optimizer_v2: These will be used as few-shot example candidates for our program and for creating instructions.\n", + "2026/01/29 18:20:36 INFO dspy.teleprompt.mipro_optimizer_v2: These will be used as few-shot example candidates for our program and for creating instructions.\n", "\n", - "2025/07/02 20:18:59 INFO dspy.teleprompt.mipro_optimizer_v2: Bootstrapping N=20 sets of demonstrations...\n" + "2026/01/29 18:20:36 INFO dspy.teleprompt.mipro_optimizer_v2: Bootstrapping N=20 sets of demonstrations...\n" ] }, { @@ -364,7 +369,9 @@ "name": "stderr", "output_type": "stream", "text": [ - " 8%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 4/50 [00:06<01:10, 1.54s/it]\n" + " 0%| | 0/50 [00:00\n", - "2025/07/02 20:20:07 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + " 2%|▏ | 1/50 [00:01<00:57, 1.17s/it]\n", + "2026/01/29 18:21:27 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Entering patched_propose_instructions, patching GroundedProposer with NovaGroundedProposer\n", + "2026/01/29 18:21:27 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Patched GroundedProposer, current GroundedProposer class=\n", + "2026/01/29 18:21:27 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "==> STEP 2: PROPOSE INSTRUCTION CANDIDATES <==\n", - "2025/07/02 20:20:07 INFO dspy.teleprompt.mipro_optimizer_v2: We will use the few-shot examples from the previous step, a generated dataset summary, a summary of the program code, and a randomly selected prompting tip to propose instructions.\n", - "2025/07/02 20:20:07 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_grounded_proposer: Initializing NovaGroundedProposer\n" + "2026/01/29 18:21:27 INFO dspy.teleprompt.mipro_optimizer_v2: We will use the few-shot examples from the previous step, a generated dataset summary, a summary of the program code, and a randomly selected prompting tip to propose instructions.\n", + "2026/01/29 18:21:27 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_grounded_proposer: Initializing NovaGroundedProposer\n" ] }, { @@ -639,7 +646,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:20:27 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 18:21:41 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "Proposing N=20 instructions...\n", "\n" ] @@ -674,139 +681,701 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: Proposed Instructions for Predictor 0:\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: Proposed Instructions for Predictor 0:\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 0: Task: Extract and return a json with specified keys and values based on the input.\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 0: Task: Extract and return a JSON object with specific keys and values from the provided input.\n", "\n", "Context:\n", - "- The json must include \"urgency\", \"sentiment\", and \"categories\".\n", - "- \"urgency\" can be `high`, `medium`, or `low`.\n", - "- \"sentiment\" can be `negative`, `neutral`, or `positive`.\n", - "- \"categories\" is a dictionary with boolean values indicating if the category matches the input.\n", + "- The JSON must include the following keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The value for \"urgency\" must be one of `high`, `medium`, or `low`.\n", + "- The value for \"sentiment\" must be one of `negative`, `neutral`, or `positive`.\n", + "- The \"categories\" key must be a dictionary where each key is a category from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. The corresponding value for each category must be a boolean (True/False) indicating whether the category is the best match for the input.\n", "\n", "Instructions:\n", - "- The json MUST be valid and readable directly.\n", - "- DO NOT enclose the json in ```json...```.\n", - "- DO NOT include newlines or unnecessary whitespaces.\n", - "- Only include the keys mentioned: \"urgency\", \"sentiment\", and \"categories\".\n", - "\n", - "Any other section from Original Prompt:\n", - "- The categories to consider are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`.\n", + "- Analyze the input to determine the appropriate values for \"urgency\" and \"sentiment\".\n", + "- Evaluate which category or categories best match the input and set the corresponding boolean values in the \"categories\" dictionary to True; set all others to False.\n", + "- The output must be a valid JSON string with no additional text, formatting, or encapsulation (e.g., no ```json...```).\n", + "- The JSON must be compact with no unnecessary whitespace or newlines.\n", "\n", "Response Format:\n", - "- The response MUST be a single-line valid json string.\n", - "- The json MUST contain \"urgency\", \"sentiment\", and \"categories\" keys.\n", + "- The response MUST be a valid JSON string containing only the keys \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra characters or formatting.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 1: You are an expert in facility management and customer service analytics, specializing in the classification, sentiment analysis, and prioritization of customer communications. Your task is to process an incoming email and generate a structured JSON output that categorizes the email, assesses its sentiment, and determines its urgency level.\n", + "\n", + "**Instructions:**\n", + "\n", + "1. **Analyze the Input Email:**\n", + " - Carefully read the email’s subject and body to understand the content, context, and tone.\n", + " - Identify keywords, phrases, and contextual clues that indicate the nature of the request or issue.\n", + "\n", + "2. **Categorize the Email:**\n", + " - Determine which of the following categories best match the email content. Assign `true` to all categories that apply and `false` to those that do not.\n", + " - `emergency_repair_services`: Requests for immediate repair of critical systems or structures.\n", + " - `routine_maintenance_requests`: Requests for regular or scheduled maintenance activities.\n", + " - `quality_and_safety_concerns`: Issues related to safety standards, health hazards, or quality assurance.\n", + " - `specialized_cleaning_services`: Requests for specialized or non-routine cleaning services (e.g., deep cleaning, window washing).\n", + " - `general_inquiries`: General questions or information requests without a specific action required.\n", + " - `sustainability_and_environmental_practices`: Requests or feedback related to eco-friendly practices or sustainability.\n", + " - `training_and_support_requests`: Requests for training sessions, additional support, or resources.\n", + " - `cleaning_services_scheduling`: Requests to schedule, reschedule, or adjust cleaning services.\n", + " - `customer_feedback_and_complaints`: Feedback, complaints, or expressions of dissatisfaction regarding services.\n", + " - `facility_management_issues`: Broader issues affecting facility operations or management.\n", + "\n", + "3. **Assess Sentiment:**\n", + " - Determine the overall sentiment of the email. The sentiment should be one of the following:\n", + " - `positive`: The tone is appreciative, satisfied, or optimistic.\n", + " - `neutral`: The tone is factual, informative, or indifferent without strong positive or negative emotions.\n", + " - `negative`: The tone expresses dissatisfaction, frustration, anger, or criticism.\n", + "\n", + "4. **Determine Urgency:**\n", + " - Assess the urgency level of the request based on indicators in the text (e.g., words like \"urgent,\" \"immediate,\" \"as soon as possible,\" or phrases indicating a time-sensitive issue). The urgency level should be one of the following:\n", + " - `high`: The issue requires immediate attention and could have significant consequences if not addressed promptly.\n", + " - `medium`: The issue is important but does not require immediate action; it can be addressed within a reasonable timeframe.\n", + " - `low`: The issue is non-urgent and can be addressed at a convenient time.\n", + "\n", + "5. **Generate the JSON Output:**\n", + " - Construct a JSON object with the following structure:\n", + " ```json\n", + " {\n", + " \"urgency\": \"high/medium/low\",\n", + " \"sentiment\": \"positive/neutral/negative\",\n", + " \"categories\": {\n", + " \"emergency_repair_services\": true/false,\n", + " \"routine_maintenance_requests\": true/false,\n", + " \"quality_and_safety_concerns\": true/false,\n", + " \"specialized_cleaning_services\": true/false,\n", + " \"general_inquiries\": true/false,\n", + " \"sustainability_and_environmental_practices\": true/false,\n", + " \"training_and_support_requests\": true/false,\n", + " \"cleaning_services_scheduling\": true/false,\n", + " \"customer_feedback_and_complaints\": true/false,\n", + " \"facility_management_issues\": true/false\n", + " }\n", + " }\n", + " ```\n", + " - Ensure the JSON is valid, compact, and on a single line with no additional text, formatting, or encapsulation (e.g., no ```json...```).\n", + "\n", + "**Example Output:**\n", + "```json\n", + "{\"urgency\": \"high\", \"sentiment\": \"negative\", \"categories\": {\"emergency_repair_services\": true, \"routine_maintenance_requests\": false, \"quality_and_safety_concerns\": false, \"specialized_cleaning_services\": false, \"general_inquiries\": false, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": false, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": true, \"facility_management_issues\": false}}\n", + "```\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 1: As a customer service analyst, analyze the provided email to determine its content and context. Extract and return a JSON object with \"urgency\", \"sentiment\", and \"categories\" keys. Ensure the JSON is valid, with \"urgency\" as `high`, `medium`, or `low`, \"sentiment\" as `negative`, `neutral`, or `positive`, and \"categories\" as a dictionary of booleans for predefined categories. Do not include any additional formatting or whitespace.\n", + "**Important:**\n", + "- Your analysis should be thorough and accurate, reflecting a deep understanding of facility management and customer service dynamics.\n", + "- The output must be a valid JSON string with no extra characters or formatting.\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 2: Given an email input, analyze the content to determine its sentiment, urgency, and relevant categories. Return a JSON object with \"urgency\" (high, medium, low), \"sentiment\" (negative, neutral, positive), and \"categories\" (boolean values for predefined categories). Ensure the JSON is valid, concise, and includes only the specified keys. If the email's intent is ambiguous, prioritize clarity by assigning the most likely category based on context.\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 2: Generate a JSON object from the provided email input that includes the following keys: \"urgency\", \"sentiment\", and \"categories\". Follow these guidelines:\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 3: Given the customer support email, analyze the content to determine the urgency, sentiment, and relevant categories. Return a JSON object with \"urgency\" (high, medium, low), \"sentiment\" (negative, neutral, positive), and \"categories\" (boolean values for predefined types). Ensure the JSON is valid, concise, and includes only the specified keys. If the email's intent is unclear, prioritize categories based on explicit mentions or implied needs.\n", + "1. **Urgency**: Determine the urgency level of the request and assign it as a value (`high`, `medium`, or `low`).\n", + "2. **Sentiment**: Assess the emotional tone of the email and assign it as a value (`negative`, `neutral`, or `positive`).\n", + "3. **Categories**: Create a dictionary where each key is one of the following categories: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Assign `True` to each category that best matches the content of the email and `False` to all others.\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 4: Analyze the customer service inquiry and return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and \"categories\" (boolean values for predefined topics). Use urgency indicators like \"immediate\" or \"critical\" for high urgency, and sentiment cues from language tone. Ensure all categories are evaluated based on content relevance.\n", + "Ensure the output is a valid JSON string with no additional text, formatting, or encapsulation. The JSON should be compact, on a single line, and contain only the specified keys.\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 5: Given a customer support email, analyze the content to determine the urgency, sentiment, and relevant categories. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and a \"categories\" dictionary with boolean values for each category. Ensure the email is parsed for keywords related to maintenance, cleaning, training, and facility management issues. Use the email's tone to assess sentiment and urgency, prioritizing phrases like \"urgent\" or \"immediate\" for high urgency.\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 3: Generate a JSON object from the provided email input that includes three main components: urgency level, sentiment analysis, and category classification. Follow these steps:\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 6: Analyze the input email to classify its content into predefined categories, determine sentiment, and assess urgency. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and \"categories\" (boolean values for each category). Ensure the JSON is valid and concise.\n", + "1. **Urgency Level**: Determine the urgency of the request from the email. The urgency must be one of `high`, `medium`, or `low`. Consider factors such as time sensitivity, potential impact, and any explicit urgency indicators in the email.\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 7: Given the critical nature of customer service, accurately categorize the email, assess sentiment, and determine urgency. Incorrect categorization can lead to delayed responses, while misjudging sentiment or urgency may affect customer satisfaction. Ensure the output JSON is precise, with \"urgency\" (high, medium, low), \"sentiment\" (negative, neutral, positive), and \"categories\" as boolean values. Focus on the email's content to make informed decisions, especially for high-urgency or negative-sentiment cases.\n", + "2. **Sentiment Analysis**: Assess the overall sentiment of the email. The sentiment must be one of `negative`, `neutral`, or `positive`. Look for emotional cues, tone, and language used in the email to determine the sentiment.\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 8: Given the critical nature of customer service in maintaining client satisfaction and operational efficiency, your task is to analyze the provided email and generate a precise JSON output. This output will directly influence how quickly and appropriately we respond to customer needs. Ensure that you accurately determine the urgency, sentiment, and relevant categories for each email. Mistakes in categorization or misjudgment of urgency and sentiment could lead to delayed responses or inappropriate handling of customer issues. Pay close attention to the details and context provided in the email to make informed decisions.\n", + "3. **Category Classification**: Classify the email into relevant categories. The categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, and `facility_management_issues`. For each category, decide whether it best matches the content of the email and set the corresponding value to `true` if it matches, otherwise `false`. Only one category should ideally be `true`, but multiple categories can be `true` if they are equally relevant.\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 9: Analyze the provided email and generate a JSON object with \"urgency\", \"sentiment\", and \"categories\" keys. \"urgency\" must be one of [\"high\", \"medium\", \"low\"], \"sentiment\" one of [\"negative\", \"neutral\", \"positive\"], and \"categories\" a dictionary mapping each of the following to a boolean: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Ensure no extraneous whitespace and maintain valid JSON syntax.\n", + "4. **Output Format**: Return the result as a compact JSON string with the following structure:\n", + "```json\n", + "{\"urgency\": \"urgency_level\", \"sentiment\": \"sentiment_analysis\", \"categories\": {\"emergency_repair_services\": boolean, \"routine_maintenance_requests\": boolean, \"quality_and_safety_concerns\": boolean, \"specialized_cleaning_services\": boolean, \"general_inquiries\": boolean, \"sustainability_and_environmental_practices\": boolean, \"training_and_support_requests\": boolean, \"cleaning_services_scheduling\": boolean, \"customer_feedback_and_complaints\": boolean, \"facility_management_issues\": boolean}}\n", + "```\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 10: Given an email, analyze its content to determine the urgency, sentiment, and relevant categories. Provide the output as a JSON object with \"urgency\" (high, medium, low), \"sentiment\" (negative, neutral, positive), and \"categories\" (boolean values for predefined types). Ensure the JSON is concise and valid, without extraneous formatting.\n", + "Ensure the JSON is on a single line with no extra formatting or additional text.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 4: Generate a JSON object from the provided email input with the following structure and requirements:\n", + "\n", + "### **Output Structure**\n", + "Return a JSON object with **exactly** these three keys:\n", + "1. `\"urgency\"` – Must be one of: `\"high\"`, `\"medium\"`, or `\"low\"`.\n", + "2. `\"sentiment\"` – Must be one of: `\"negative\"`, `\"neutral\"`, or `\"positive\"`.\n", + "3. `\"categories\"` – A dictionary where **each key** is one of the following categories, and the **value** is a boolean (`true` or `false`) indicating whether the category applies to the email:\n", + " - `\"emergency_repair_services\"`\n", + " - `\"routine_maintenance_requests\"`\n", + " - `\"quality_and_safety_concerns\"`\n", + " - `\"specialized_cleaning_services\"`\n", + " - `\"general_inquiries\"`\n", + " - `\"sustainability_and_environmental_practices\"`\n", + " - `\"training_and_support_requests\"`\n", + " - `\"cleaning_services_scheduling\"`\n", + " - `\"customer_feedback_and_complaints\"`\n", + " - `\"facility_management_issues\"`\n", + "\n", + "### **Instructions**\n", + "1. **Analyze the Email Content**:\n", + " - Read the **subject** and **body** of the email to understand the intent, tone, and urgency.\n", + " - Identify keywords, phrases, and context clues that indicate:\n", + " - **Urgency**: Look for words like β€œurgent”, β€œas soon as possible”, β€œemergency”, or time-sensitive language. Also consider category-based urgency (e.g., emergency repairs are inherently high urgency).\n", + " - **Sentiment**: Assess the toneβ€”positive (e.g., praise, gratitude), negative (e.g., complaints, frustration), or neutral (e.g., factual inquiries).\n", + " - **Categories**: Match the email to one or more categories based on intent. For example:\n", + " - Requests for training β†’ `\"training_and_support_requests\": true`.\n", + " - Complaints or feedback β†’ `\"customer_feedback_and_complaints\": true`.\n", + " - Requests for routine repairs β†’ `\"routine_maintenance_requests\": true`.\n", + " - If multiple categories seem relevant, **only set the most accurate one(s) to `true`** and all others to `false`.\n", + "\n", + "2. **Determine Values**:\n", + " - **Urgency**: Use logic:\n", + " - If the email mentions an emergency, immediate action, or has high-impact consequences β†’ `\"high\"`.\n", + " - If the issue is important but not urgent (e.g., routine maintenance, general inquiries) β†’ `\"medium\"`.\n", + " - If the request is informational, low-priority, or can be addressed later β†’ `\"low\"`.\n", + " - **Sentiment**: Use tone analysis:\n", + " - Positive: Gratitude, praise, or neutral-but-polite language.\n", + " - Negative: Complaints, frustration, or negative descriptions.\n", + " - Neutral: Factual, inquisitive, or lacks strong emotional language.\n", + " - **Categories**: For each category in the list, set it to `true` **only if** the email clearly matches that category. All others should be `false`. If no category fits well, default all to `false`.\n", + "\n", + "3. **Output Format**:\n", + " - Return **only** a valid JSON string **without any additional text, formatting, or code blocks** (e.g., no ```json…```).\n", + " - The JSON must be **compact**β€”no extra spaces, line breaks, or unnecessary formatting.\n", + " - Ensure the output is a **single line**.\n", + "\n", + "### **Example**\n", + "For this input:\n", + "```\n", + "Subject: Scheduled Maintenance Request for Minor Plumbing Issue\n", + "Body: [Email body requesting a minor plumbing repair]\n", + "```\n", + "Return:\n", + "```json\n", + "{\"urgency\":\"medium\",\"sentiment\":\"positive\",\"categories\":{\"emergency_repair_services\":false,\"routine_maintenance_requests\":true,\"quality_and_safety_concerns\":false,\"specialized_cleaning_services\":false,\"general_inquiries\":false,\"sustainability_and_environmental_practices\":false,\"training_and_support_requests\":false,\"cleaning_services_scheduling\":false,\"customer_feedback_and_complaints\":false,\"facility_management_issues\":false}}\n", + "```\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 11: Analyze the customer service email to classify its content accurately. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and \"categories\" (boolean flags for predefined types). Ensure compliance with data privacy standards by omitting personal identifiers. Follow the format strictly: `{\"urgency\": \"...\", \"sentiment\": \"...\", \"categories\": {\"...\": true/false, ...}}`.\n", + "### **Important Notes**\n", + "- Do **not** add any explanations, comments, or extra textβ€”only the JSON.\n", + "- If unsure about urgency or sentiment, default to `\"medium\"` and `\"neutral\"` respectively.\n", + "- If no category fits, set all category values to `false`.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 5: Generate a JSON object from the provided customer support email input. The JSON must contain three keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "\n", + "**Key Requirements:**\n", + "1. **\"urgency\"**: Must be one of `high`, `medium`, or `low`, based on how time-sensitive the request is.\n", + "2. **\"sentiment\"**: Must be one of `negative`, `neutral`, or `positive`, reflecting the emotional tone of the email.\n", + "3. **\"categories\"**: Must be a dictionary where:\n", + " - The keys are from this fixed list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`.\n", + " - Each value is a boolean (`true` or `false`) indicating whether the category best matches the content of the email. Only categories that are clearly relevant should be set to `true`; all others should be `false`.\n", + "\n", + "**Steps to Follow:**\n", + "1. **Read the email carefully** to understand the main request, tone, and urgency.\n", + "2. **Determine urgency**:\n", + " - Use `high` if the issue requires immediate attention (e.g., an emergency or something causing significant disruption).\n", + " - Use `medium` for issues that need prompt attention but are not emergencies.\n", + " - Use `low` for non-urgent requests that can be addressed at a convenient time.\n", + "3. **Determine sentiment**:\n", + " - Use `positive` if the email expresses satisfaction, gratitude, or a constructive tone.\n", + " - Use `neutral` if the email is factual, informative, or lacks strong emotional language.\n", + " - Use `negative` if the email expresses dissatisfaction, frustration, or complaint.\n", + "4. **Identify categories**:\n", + " - For each category in the list, decide whether it accurately describes the main focus of the email.\n", + " - Set the corresponding value to `true` if the category applies; otherwise, set it to `false`.\n", + " - Only one or a few categories should be `true`β€”choose the most relevant ones based on the email content.\n", + "5. **Output the JSON**:\n", + " - The final output must be a **single-line JSON string** with no additional text, formatting, or encapsulation (e.g., no ```json...```).\n", + " - Ensure the JSON is compact with no unnecessary whitespace or newlines.\n", + "\n", + "**Example Output Format:**\n", + "```{\"urgency\": \"medium\", \"sentiment\": \"positive\", \"categories\": {\"emergency_repair_services\": false, \"routine_maintenance_requests\": true, \"quality_and_safety_concerns\": false, \"specialized_cleaning_services\": false, \"general_inquiries\": false, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": false, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": false, \"facility_management_issues\": false}}```\n", + "\n", + "Use this format to process the input email and return the JSON object.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 6: Analyze the provided input text and generate a JSON object with the following structure:\n", + "- \"urgency\": one of `high`, `medium`, or `low`, based on the urgency level of the request.\n", + "- \"sentiment\": one of `negative`, `neutral`, or `positive`, based on the emotional tone of the input.\n", + "- \"categories\": a dictionary where each key is one of the following categories (`emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`) and the value is a boolean (`true` or `false`) indicating whether the input matches that category.\n", + "\n", + "The output must be a valid JSON string with no additional text, formatting, or encapsulation. Ensure the JSON is compact and on a single line.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 7: Analyze and classify customer support emails into structured categories, sentiment, and urgency levels.**\n", + "\n", + "**Context:**\n", + "You are an advanced AI system designed to process formal, professional emails addressing facility management services. Your objective is to extract and return a structured JSON object containing three key pieces of information: **urgency**, **sentiment**, and **categories**. The accuracy of this classification is critical for efficient customer service triage, resource allocation, and ensuring timely responsesβ€”especially in high-stakes scenarios such as emergency repairs or urgent facility management issues.\n", + "\n", + "**Instructions:**\n", + "1. **Urgency Level (`urgency`):**\n", + " - Determine the urgency level of the request based on keywords, context, and the nature of the issue described in the email.\n", + " - The value for `urgency` **must be one of** `high`, `medium`, or `low`.\n", + " - **High**: The issue is time-sensitive, poses immediate risk, or requires urgent intervention (e.g., \"urgent,\" \"immediate,\" \"emergency,\" or mentions of critical systems failing).\n", + " - **Medium**: The issue is important but does not pose an immediate risk; it requires prompt but not emergency attention (e.g., delays causing frustration or inconvenience).\n", + " - **Low**: The issue is routine, non-urgent, or can be addressed at a later time without significant impact.\n", + "\n", + "2. **Sentiment Analysis (`sentiment`):**\n", + " - Assess the emotional tone of the email by analyzing word choice, punctuation, and overall context.\n", + " - The value for `sentiment` **must be one of** `negative`, `neutral`, or `positive`.\n", + " - **Negative**: The sender expresses dissatisfaction, frustration, anger, or complaints.\n", + " - **Neutral**: The tone is factual, informative, or neutral without strong positive or negative emotions.\n", + " - **Positive**: The sender expresses satisfaction, gratitude, enthusiasm, or positive feedback.\n", + "\n", + "3. **Category Classification (`categories`):**\n", + " - Evaluate which of the predefined categories best match the content of the email. Categories include:\n", + " - `emergency_repair_services`\n", + " - `routine_maintenance_requests`\n", + " - `quality_and_safety_concerns`\n", + " - `specialized_cleaning_services`\n", + " - `general_inquiries`\n", + " - `sustainability_and_environmental_practices`\n", + " - `training_and_support_requests`\n", + " - `cleaning_services_scheduling`\n", + " - `customer_feedback_and_complaints`\n", + " - `facility_management_issues`\n", + " - For each category, determine whether it is a **best match** for the input. If a category is relevant, set its value to `true`; otherwise, set it to `false`.\n", + " - Only **one or more** categories may be `true` depending on the content of the email. All others must be `false`.\n", + "\n", + "**Output Requirements:**\n", + "- Return a **valid JSON string** containing only the keys `\"urgency\"`, `\"sentiment\"`, and `\"categories\"`.\n", + "- The JSON **must be compact** with no unnecessary whitespace or newlines.\n", + "- The response **must not include** any additional text, formatting, or encapsulation (e.g., no ```json...```).\n", + "- The JSON must be structured as follows:\n", + " ```json\n", + " {\"urgency\": \"high\", \"sentiment\": \"neutral\", \"categories\": {\"emergency_repair_services\": true, \"routine_maintenance_requests\": false, \"quality_and_safety_concerns\": false, \"specialized_cleaning_services\": false, \"general_inquiries\": false, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": false, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": false, \"facility_management_issues\": false}}\n", + " ```\n", + "\n", + "**Example:**\n", + "Given the input:\n", + "> Subject: Urgent Assistance Needed for HVAC System Issue\n", + "> \n", + "> Hi ProCare Support Team,\n", + "> \n", + "> I hope this message finds you well. My name is [Sender], and I’ve been a satisfied client of ProCare Facility Solutions for the past year. Your team has always provided exceptional service, and I truly appreciate the dedication and professionalism you bring to maintaining my property.\n", + "> \n", + "> However, I’m currently facing an urgent issue that requires immediate attention. Over the past few days, I’ve noticed a significant drop in the efficiency of the HVAC system in my home. Despite the routine maintenance checks, the system seems to be struggling to maintain a consistent temperature, which is crucial for my training and recovery as a professional athlete.\n", + "> \n", + "> I’ve already tried adjusting the thermostat and checking the filters, but the problem persists. Given the high stakes of my athletic performance, I need this issue resolved as quickly as possible to ensure my living environment remains optimal for my needs.\n", + "> \n", + "> Could you please arrange for an emergency repair at the earliest convenience? Your prompt assistance in this matter would be greatly appreciated, as it directly impacts my daily routine and overall well-being.\n", + "> \n", + ">\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 8: Your task is to **accurately classify and prioritize incoming customer inquiries** by extracting and returning a **strictly formatted JSON object** from the provided input. This is critical for efficient customer service operations, where misclassification can lead to delayed responses, misrouted requests, or missed urgency levelsβ€”potentially damaging client relationships and operational efficiency.\n", + "\n", + "### Requirements:\n", + "1. **Output Structure**: Return a JSON object **exactly** in the following format:\n", + " ```json\n", + " {\"urgency\": \"high\"|\"medium\"|\"low\", \"sentiment\": \"negative\"|\"neutral\"|\"positive\", \"categories\": {\"emergency_repair_services\": false, \"routine_maintenance_requests\": false, \"quality_and_safety_concerns\": false, \"specialized_cleaning_services\": false, \"general_inquiries\": false, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": false, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": false, \"facility_management_issues\": false}}\n", + " ```\n", + " - The `urgency` value **must** be one of: `high`, `medium`, or `low`.\n", + " - The `sentiment` value **must** be one of: `negative`, `neutral`, or `positive`.\n", + " - The `categories` object **must** contain **all ten category keys**, each mapped to a boolean (`true` or `false`) indicating whether the input aligns with that category. **Only the most relevant category(s) should be `true`; all others must be `false`.**\n", + "\n", + "2. **Analysis Steps**:\n", + " - **Urgency Determination**: Assess the content for time-sensitive language, explicit requests for immediate action, or keywords indicating urgency (e.g., β€œASAP”, β€œimmediate”, β€œemergency”). Assign `high` if the request demands prompt attention, `medium` for standard timely handling, and `low` for non-urgent, informational queries.\n", + " - **Sentiment Analysis**: Determine the emotional tone of the input. Use `negative` for complaints, frustration, or dissatisfaction; `positive` for praise, satisfaction, or gratitude; and `neutral` for factual, inquisitive, or routine requests without strong emotional cues.\n", + " - **Category Classification**: Identify the **primary category** (or categories, if applicable) that best describes the input’s intent. Refer to the list of categories and set the corresponding value to `true`. All other categories must be `false`. Categories include:\n", + " - `emergency_repair_services`: Urgent, unexpected repairs (e.g., broken pipes, HVAC failure).\n", + " - `routine_maintenance_requests`: Scheduled or regular maintenance (e.g., filter changes, system checks).\n", + " - `quality_and_safety_concerns`: Issues related to safety standards, health hazards, or compliance.\n", + " - `specialized_cleaning_services`: Requests for non-routine cleaning (e.g., post-construction, deep cleaning).\n", + " - `general_inquiries`: Broad, non-specific questions or information requests.\n", + " - `sustainability_and_environmental_practices`: Queries or requests related to eco-friendly practices, green initiatives, or sustainability.\n", + " - `training_and_support_requests`: Requests for educational resources, skill development, or professional training.\n", + " - `cleaning_services_scheduling`: Scheduling or coordinating cleaning services.\n", + " - `customer_feedback_and_complaints`: Expressions of dissatisfaction, complaints, or feedback.\n", + " - `facility_management_issues`: General facility-related problems or administrative queries.\n", + "\n", + "3. **Output Constraints**:\n", + " - The JSON **must** be a valid, minified string **without any additional text, formatting, or encapsulation** (e.g., no ```json…```).\n", + " - The entire JSON **must** be on **a single line** with no unnecessary whitespace or newlines.\n", + "\n", + "### Example:\n", + "Given the input:\n", + "> Subject: Urgent Leak in Office Building \n", + "> Body: We have a major water leak in the main office area that’s causing significant damage. This is an emergency and requires immediate attention.\n", + "\n", + "Your output **must** be:\n", + "```json\n", + "{\"urgency\": \"high\", \"sentiment\": \"negative\", \"categories\": {\"emergency_repair_services\": true, \"routine_maintenance_requests\": false, \"quality_and_safety_concerns\": false, \"specialized_cleaning_services\": false, \"general_inquiries\": false, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": false, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": false, \"facility_management_issues\": false}}\n", + "```\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 12: Analyze the provided customer service email and generate a JSON object containing the following:\n", + "### Consequence of Inaccuracy:\n", + "- **Incorrect urgency levels** may result in missed deadlines or escalated client frustration.\n", + "- **Misclassified categories** can lead to requests being routed to the wrong department, causing delays and poor customer experience.\n", + "- **Improper sentiment analysis** may cause the system to overlook negative feedback or fail to recognize opportunities to strengthen client\n", "\n", - "1. **categories**: A dictionary where keys are predefined service categories (e.g., \"specialized_cleaning_services\", \"customer_feedback_and_complaints\") and values are booleans indicating relevance.\n", - "2. **sentiment**: A string (\"positive\", \"neutral\", or \"negative\") reflecting the email's emotional tone.\n", - "3. **urgency**: A string (\"high\", \"medium\", or \"low\") based on language cues.\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 9: Analyze the provided customer message and return a JSON object with the following structure and constraints:\n", "\n", - "**Rules**:\n", - "- Use only the categories: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`.\n", - "- Maintain valid JSON syntax without formatting.\n", - "- Example: `{\"categories\": {\"complaint\": true}, \"sentiment\": \"negative\", \"urgency\": \"high\"}`\n", + "**Output Structure:**\n", + "```json\n", + "{\n", + " \"urgency\": \"high|medium|low\",\n", + " \"sentiment\": \"negative|neutral|positive\",\n", + " \"categories\": {\n", + " \"emergency_repair_services\": true|false,\n", + " \"routine_maintenance_requests\": true|false,\n", + " \"quality_and_safety_concerns\": true|false,\n", + " \"specialized_cleaning_services\": true|false,\n", + " \"general_inquiries\": true|false,\n", + " \"sustainability_and_environmental_practices\": true|false,\n", + " \"training_and_support_requests\": true|false,\n", + " \"cleaning_services_scheduling\": true|false,\n", + " \"customer_feedback_and_complaints\": true|false,\n", + " \"facility_management_issues\": true|false\n", + " }\n", + "}\n", + "```\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 13: Analyze the input text to classify the urgency, sentiment, and relevant categories, then return a JSON object with \"urgency\", \"sentiment\", and \"categories\" keys. Ensure \"urgency\" is \"high\", \"medium\", or \"low\", \"sentiment\" is \"negative\", \"neutral\", or \"positive\", and \"categories\" contains boolean values for predefined categories.\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 14: Given the input email, analyze its content to determine the urgency, sentiment, and relevant categories. Return a JSON object with \"urgency\" (high, medium, low), \"sentiment\" (negative, neutral, positive), and \"categories\" (boolean values for predefined types). Ensure the output is a valid, single-line JSON string without unnecessary whitespace. If the email's content is ambiguous, prioritize categories that best match explicit mentions of issues or requests.\n", + "**Instructions:**\n", + "1. **Urgency**: Determine the urgency level of the request. Use:\n", + " - `high`: Issues indicating immediate impact, emergencies, or critical operational disruptions.\n", + " - `medium`: Requests needing timely attention but not urgent.\n", + " - `low`: General inquiries or non-urgent requests.\n", + "\n", + "2. **Sentiment**: Assess the emotional tone of the message. Use:\n", + " - `negative`: Dissatisfaction, complaints, or concerns.\n", + " - `neutral`: Factual, inquisitive, or content tone without strong emotion.\n", + " - `positive`: Satisfaction, praise, or gratitude.\n", + "\n", + "3. **Categories**: Identify which predefined categories best match the content of the message. Set the corresponding category keys to `true` and all others to `false`. Categories include:\n", + " - `emergency_repair_services`: Requests for urgent repairs or breakdowns.\n", + " - `routine_maintenance_requests`: Scheduled or routine maintenance tasks.\n", + " - `quality_and_safety_concerns`: Issues or questions about service quality or safety standards.\n", + " - `specialized_cleaning_services`: Inquiries or requests for specialized or deep cleaning.\n", + " - `general_inquiries`: General questions without a specific service request.\n", + " - `sustainability_and_environmental_practices`: Questions or requests related to eco-friendly practices.\n", + " - `training_and_support_requests`: Requests for training or additional support.\n", + " - `cleaning_services_scheduling`: Requests to schedule cleaning services.\n", + " - `customer_feedback_and_complaints`: Feedback or complaints about services.\n", + " - `facility_management_issues`: Broader issues related to facility management.\n", + "\n", + "**Response Format:**\n", + "- Return **only** a valid JSON string matching the structure above.\n", + "- The JSON must be **compact** (no unnecessary whitespace or newlines).\n", + "- Do not include any additional text, formatting, or encapsulation (e.g., no ```json...```).\n", + "\n", + "Example Output (for reference, not part of the response):\n", + "```json\n", + "{\"urgency\":\"low\",\"sentiment\":\"neutral\",\"categories\":{\"emergency_repair_services\":false,\"routine_maintenance_requests\":false,\"quality_and_safety_concerns\":true,\"specialized_cleaning_services\":false,\"general_inquiries\":false,\"sustainability_and_environmental_practices\":false,\"training_and_support_requests\":false,\"cleaning_services_scheduling\":false,\"customer_feedback_and_complaints\":false,\"facility_management_issues\":false}}\n", + "```\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 10: Generate a JSON object that classifies the input email or support ticket into predefined categories, assesses its sentiment, and determines its urgency level. Follow these steps:\n", + "\n", + "1. **Categorization**: Identify which of the following categories best match the input. Set each category to `true` if it applies, and `false` otherwise. The categories are:\n", + " - `emergency_repair_services`\n", + " - `routine_maintenance_requests`\n", + " - `quality_and_safety_concerns`\n", + " - `specialized_cleaning_services`\n", + " - `general_inquiries`\n", + " - `sustainability_and_environmental_practices`\n", + " - `training_and_support_requests`\n", + " - `cleaning_services_scheduling`\n", + " - `customer_feedback_and_complaints`\n", + " - `facility_management_issues`\n", + "\n", + "2. **Sentiment Analysis**: Determine the emotional tone of the input. The sentiment must be one of `negative`, `neutral`, or `positive`. Consider the language, tone, and context of the message to assess whether the sender is expressing dissatisfaction, neutrality, or satisfaction.\n", + "\n", + "3. **Urgency Assessment**: Evaluate how urgent the request is. The urgency level must be one of `high`, `medium`, or `low`. Consider factors such as deadlines, immediate needs, and the potential impact of delaying the request.\n", + "\n", + "4. **Output Format**: Return the result as a compact JSON string with the following structure:\n", + " ```json\n", + " {\"urgency\": \"urgency_level\", \"sentiment\": \"sentiment_value\", \"categories\": {\"category1\": boolean, \"category2\": boolean, ...}}\n", + " ```\n", + " - Ensure the JSON is valid and on a single line with no extra formatting or encapsulation (e.g., no ```json...```).\n", + " - The JSON must only include the keys `\"urgency\"`, `\"sentiment\"`, and `\"categories\"`.\n", + "\n", + "**Tip**: Think creatively about how urgency and sentiment might be inferred from subtle cues in the text, such as phrasing, tone, and context. For example, a message that mentions \"as soon as possible\" or \"immediately\" may indicate higher urgency, while expressions of gratitude or praise may suggest a positive sentiment.\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 15: Analyze the input email to classify the request. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (positive/neutral/negative), and \"categories\" (boolean values for predefined types). Ensure no extra keys or formatting.\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 11: Generate a JSON object with the following structure from the provided customer service email (subject and body):\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 16: Given the critical nature of customer service in maintaining client satisfaction and operational efficiency, your task is to analyze the provided email and accurately categorize its content, sentiment, and urgency. Misclassification can lead to delayed responses or inappropriate prioritization, impacting customer trust and service quality. Ensure that the JSON output strictly adheres to the required format, with precise categorization under \"categories\", accurate sentiment analysis (\"negative\", \"neutral\", \"positive\"), and correct urgency assessment (\"high\", \"medium\", \"low\"). The output must be a valid, single-line JSON string without any additional formatting or whitespace.\n", + "```json\n", + "{\n", + " \"urgency\": \"\",\n", + " \"sentiment\": \"\",\n", + " \"categories\": {\n", + " \"emergency_repair_services\": ,\n", + " \"routine_maintenance_requests\": ,\n", + " \"quality_and_safety_concerns\": ,\n", + " \"specialized_cleaning_services\": ,\n", + " \"general_inquiries\": ,\n", + " \"sustainability_and_environmental_practices\": ,\n", + " \"training_and_support_requests\": ,\n", + " \"cleaning_services_scheduling\": ,\n", + " \"customer_feedback_and_complaints\": ,\n", + " \"facility_management_issues\": \n", + " }\n", + "}\n", + "```\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 17: ## Task\n", - "Analyze the given customer service email and extract relevant information to generate a structured JSON response.\n", + "**Rules:**\n", + "1. **\"urgency\"** must be one of: `high`, `medium`, or `low`. Determine urgency based on keywords, context, and the identified categories.\n", + "2. **\"sentiment\"** must be one of: `negative`, `neutral`, or `positive`. Assess the tone of the email to determine sentiment.\n", + "3. **\"categories\"** must be a dictionary where each key is a category from the list above. The corresponding value must be a boolean (`true` or `false`) indicating whether the category best matches the input. Only one or more categories should be `true` based on the content of the email.\n", + "4. The output **must** be a valid JSON string with no additional text, formatting, or encapsulation (e.g., no ```json...```).\n", + "5. The JSON **must** be compact with no unnecessary whitespace or newlines, and on a single line.\n", "\n", - "## Context\n", - "- The input is a customer service email related to facility management.\n", - "- The output must be a JSON object with \"urgency\", \"sentiment\", and \"categories\" keys.\n", + "**Example:**\n", + "For the input email:\n", + "```\n", + "Subject: Inquiry About Scheduling Deep Cleaning Services\n", "\n", - "## Instructions\n", - "1. **Identify Urgency**: Determine if the email requires `high`, `medium`, or `low` urgency based on keywords and context.\n", - "2. **Evaluate Sentiment**: Assess the sentiment as `positive`, `neutral`, or `negative` by interpreting the tone and language used.\n", - "3. **Categorize Request**: Match the email content to one or more predefined categories with boolean values.\n", + "Dear ProCare Facility Solutions Support Team,\n", "\n", - "## Categories\n", - "- emergency_repair_services\n", - "- routine_maintenance_requests\n", - "- quality_and_safety_concerns\n", - "- specialized_cleaning_services\n", - "- general_inquiries\n", - "- sustainability_and_environmental_practices\n", - "- training_and_support_requests\n", - "- cleaning_services_scheduling\n", - "- customer_feedback_and_complaints\n", - "- facility_management_issues\n", + "I hope this message finds you well. My name is [Sender], and I am a researcher who has recently been in touch with Daniel through various academic forums. We have exchanged ideas and supported each other's work, which has been quite enriching.\n", "\n", - "## Response Format\n", - "- Return a single-line JSON string with \"urgency\", \"sentiment\", and \"categories\" keys.\n", + "I am writing to inquire about scheduling cleaning services for an upcoming visit to my residential property. I have heard great things about your comprehensive cleaning services and would like to ensure that my home is in pristine condition for my guests. Specifically, I am interested in a deep cleaning service, including window washing and carpet maintenance.\n", + "\n", + "I have not yet taken any steps to schedule this service, as I wanted to confirm the availability and any specific requirements you might have. Could you please provide me with the available dates and any additional information needed to proceed with the booking?\n", + "\n", + "Thank you for your assistance. I look forward to your response.\n", + "\n", + "Best regards,\n", + "[Sender]\n", + "```\n", "\n", - "Example:\n", + "The output should be:\n", "```json\n", - "{\"urgency\": \"high\", \"sentiment\": \"negative\", \"categories\": {\"emergency_repair_services\": true, \"general_inquiries\": false}}\n", + "{\"urgency\":\"medium\",\"sentiment\":\"positive\",\"categories\":{\"emergency_repair_services\":false,\"routine_maintenance_requests\":false,\"quality_and_safety_concerns\":false,\"specialized_cleaning_services\":true,\"general_inquiries\":false,\"sustainability_and_environmental_practices\":false,\"training_and_support_requests\":false,\"cleaning_services_scheduling\":true,\"customer_feedback_and_complaints\":false,\"facility_management_issues\":false}}\n", "```\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 18: Analyze the input email to classify its content. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and \"categories\" (boolean values for predefined types like maintenance or cleaning). Ensure the JSON is valid and concise.\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 12: You are an expert in analyzing customer service emails for facility management and cleaning services. Your task is to **classify and analyze incoming customer emails** by extracting key information and returning a structured JSON object. Follow these steps:\n", + "\n", + "1. **Input Processing**: Carefully read and understand the email content, including the subject and body text. Pay attention to the context, intent, and any specific details mentioned by the sender.\n", + "\n", + "2. **Sentiment Analysis**: Determine the emotional tone of the email. The sentiment must be one of the following:\n", + " - `positive`: The sender expresses satisfaction, gratitude, or appreciation.\n", + " - `neutral`: The sender is factual, informative, or neutral in tone without strong emotions.\n", + " - `negative`: The sender expresses dissatisfaction, complaints, or frustration.\n", + "\n", + "3. **Urgency Assessment**: Evaluate the urgency level of the request based on the language used and any time-sensitive phrases. The urgency must be one of the following:\n", + " - `high`: The issue requires immediate attention or could lead to significant consequences if not addressed promptly.\n", + " - `medium`: The issue is important but does not require immediate action; it should be addressed soon.\n", + " - `low`: The issue is routine or non-urgent and can be addressed at a convenient time.\n", + "\n", + "4. **Categorization**: Identify which category or categories best match the content of the email. Use the following list of categories and set the corresponding boolean value to `True` if the category applies; otherwise, set it to `False`:\n", + " - `emergency_repair_services`: Requests for urgent repairs or services that cannot wait.\n", + " - `routine_maintenance_requests`: Requests for regular or scheduled maintenance.\n", + " - `quality_and_safety_concerns`: Issues related to safety, quality, or compliance.\n", + " - `specialized_cleaning_services`: Requests for specialized or deep cleaning services.\n", + " - `general_inquiries`: General questions or information requests.\n", + " - `sustainability_and_environmental_practices`: Requests or mentions related to eco-friendly practices or sustainability.\n", + " - `training_and_support_requests`: Requests for training or additional support.\n", + " - `cleaning_services_scheduling`: Requests to schedule cleaning services.\n", + " - `customer_feedback_and_complaints`: Feedback, praise, or complaints about services.\n", + " - `facility_management_issues`: Broader issues related to facility management.\n", + "\n", + "5. **Output Generation**: Construct a JSON object with the following structure:\n", + " ```json\n", + " {\n", + " \"urgency\": \"high/medium/low\",\n", + " \"sentiment\": \"positive/neutral/negative\",\n", + " \"categories\": {\n", + " \"emergency_repair_services\": false,\n", + " \"routine_maintenance_requests\": false,\n", + " \"quality_and_safety_concerns\": false,\n", + " \"specialized_cleaning_services\": false,\n", + " \"general_inquiries\": false,\n", + " \"sustainability_and_environmental_practices\": false,\n", + " \"training_and_support_requests\": false,\n", + " \"cleaning_services_scheduling\": false,\n", + " \"customer_feedback_and_complaints\": false,\n", + " \"facility_management_issues\": false\n", + " }\n", + " }\n", + " ```\n", + "\n", + "6. **Response Format**: Return **only** the JSON object as a valid JSON string. The JSON must be compact, on a single line, with no additional text, formatting, or encapsulation (e.g., no ```json...```).\n", + "\n", + "Ensure your analysis is accurate, concise, and adheres strictly to the required format.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 13: Analyze the provided customer communication (email) and return a JSON object with the following structure:\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: 19: Given a customer service email, analyze its content to determine the urgency, sentiment, and relevant categories. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and a \"categories\" dictionary indicating true/false for each predefined category. Ensure the JSON is valid and concise, without unnecessary formatting. If the email contains ambiguous phrasing, prioritize the most likely interpretation based on context.\n", + "```json\n", + "{\n", + " \"urgency\": \"high|medium|low\",\n", + " \"sentiment\": \"negative|neutral|positive\",\n", + " \"categories\": {\n", + " \"emergency_repair_services\": false,\n", + " \"routine_maintenance_requests\": false,\n", + " \"quality_and_safety_concerns\": false,\n", + " \"specialized_cleaning_services\": false,\n", + " \"general_inquiries\": false,\n", + " \"sustainability_and_environmental_practices\": false,\n", + " \"training_and_support_requests\": false,\n", + " \"cleaning_services_scheduling\": false,\n", + " \"customer_feedback_and_complaints\": false,\n", + " \"facility_management_issues\": false\n", + " }\n", + "}\n", + "```\n", + "\n", + "- **urgency**: Determine the urgency level of the request (`high`, `medium`, or `low`) based on keywords and context.\n", + "- **sentiment**: Determine the emotional tone of the message (`negative`, `neutral`, or `positive`).\n", + "- **categories**: For each category, set the value to `true` if the input relates to that category, otherwise `false`. Only one or more categories should be `true` based on the content of the input.\n", + "\n", + "The output must be a valid JSON string on a single line with no additional text or formatting.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 14: Generate a JSON object that categorizes, assesses sentiment, and determines the urgency of a given input message related to facility management services. Follow these steps:\n", + "\n", + "1. **Categorization**: \n", + " - Analyze the input message to determine which of the following categories apply: \n", + " `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. \n", + " - Return a dictionary where each category is a key, and the corresponding value is a boolean (`true` or `false`) indicating whether the category applies to the input.\n", + "\n", + "2. **Sentiment Analysis**: \n", + " - Evaluate the tone of the message to classify the sentiment as `negative`, `neutral`, or `positive`.\n", + "\n", + "3. **Urgency Assessment**: \n", + " - Based on the content and context of the message, determine the urgency level as `high`, `medium`, or `low`.\n", + "\n", + "4. **Output Format**: \n", + " - Return a **compact JSON string** containing only the keys `\"urgency\"`, `\"sentiment\"`, and `\"categories\"`. \n", + " - The JSON must be on a single line with no extra characters, formatting, or encapsulation (e.g., no ```json...```). \n", + " - Ensure the JSON is valid and contains no unnecessary whitespace or newlines.\n", + "\n", + "Example Output Format: \n", + "`{\"urgency\":\"medium\",\"sentiment\":\"neutral\",\"categories\":{\"emergency_repair_services\":false,\"routine_maintenance_requests\":false,\"quality_and_safety_concerns\":true,\"specialized_cleaning_services\":false,\"general_inquiries\":false,\"sustainability_and_environmental_practices\":false,\"training_and_support_requests\":false,\"cleaning_services_scheduling\":false,\"customer_feedback_and_complaints\":true,\"facility_management_issues\":false}}`\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 15: Analyze the provided input message and generate a JSON object with the following structure:\n", + "- \"urgency\": one of `high`, `medium`, or `low` based on the message's priority level.\n", + "- \"sentiment\": one of `negative`, `neutral`, or `positive` based on the emotional tone of the message.\n", + "- \"categories\": a dictionary where each key is a category from the list (`emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`) and the value is a boolean (`true` or `false`) indicating whether the category applies to the input.\n", + "\n", + "The output must be a valid JSON string on a single line with no additional text or formatting.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 16: Generate a precise and structured JSON output that categorizes, assesses sentiment, and determines the urgency of a customer service email. The JSON must adhere strictly to the following format and rules:\n", + "\n", + "**JSON Structure:**\n", + "- The JSON must include three top-level keys: `\"urgency\"`, `\"sentiment\"`, and `\"categories\"`.\n", + "- `\"urgency\"` must be one of the following strings: `\"high\"`, `\"medium\"`, or `\"low\"`. Determine this based on the explicit urgency indicators in the email (e.g., words like \"immediate,\" \"urgent,\" \"as soon as possible,\" or phrases indicating pressing conditions).\n", + "- `\"sentiment\"` must be one of the following strings: `\"negative\"`, `\"neutral\"`, or `\"positive\"`. Assess this by analyzing the overall tone of the email, including the use of positive or negative language, expressions of satisfaction or dissatisfaction, and the presence of complaints or praise.\n", + "- `\"categories\"` must be a dictionary where each key is one of the following category names: \n", + " ```\n", + " \"emergency_repair_services\",\n", + " \"routine_maintenance_requests\",\n", + " \"quality_and_safety_concerns\",\n", + " \"specialized_cleaning_services\",\n", + " \"general_inquiries\",\n", + " \"sustainability_and_environmental_practices\",\n", + " \"training_and_support_requests\",\n", + " \"cleaning_services_scheduling\",\n", + " \"customer_feedback_and_complaints\",\n", + " \"facility_management_issues\"\n", + " ```\n", + " The corresponding value for each category must be a boolean (`true` or `false`) indicating whether the email content aligns with that category. Only one or more categories should be `true` if they are explicitly relevant to the email’s primary concern(s). All irrelevant categories must be set to `false`.\n", + "\n", + "**Critical Requirements:**\n", + "- **Accuracy is paramount**: Misclassifying urgency or sentiment can lead to delayed responses or inappropriate handling of customer concerns, resulting in escalated issues or loss of trust. Similarly, incorrect categorization may route the email to the wrong department, causing further delays and frustration.\n", + "- **Analyze explicitly**: Base your determinations solely on the content of the email. Look for clear indicators of urgency (e.g., time-sensitive language, descriptions of discomfort or danger), sentiment (e.g., positive remarks, complaints, neutral statements), and category relevance (e.g., mentions of specific services, issues, or requests).\n", + "- **Output Format**: The response must be a valid JSON string with no additional text, formatting, or encapsulation (e.g., no ```json...```). The JSON must be compact, with no unnecessary whitespace or newlines, and must be on a single line.\n", + "\n", + "**Example Output:**\n", + "```{\"urgency\": \"high\", \"sentiment\": \"negative\", \"categories\": {\"emergency_repair_services\": true, \"routine_maintenance_requests\": false, \"quality_and_safety_concerns\": false, \"specialized_cleaning_services\": false, \"general_inquiries\": false, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": false, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": true, \"facility_management_issues\": false}}```\n", + "\n", + "**Task:**\n", + "Carefully read the provided customer service email and generate the JSON output as described above. Ensure that your analysis is thorough and that the output strictly adheres to the specified format and rules. Remember, the consequences of inaccuracies are significantβ€”incorrect urgency levels may lead to delayed responses, and misclassified categories may result in misrouting, both of which can severely impact customer satisfaction and operational efficiency.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 17: ## Task \n", + "You are an AI system designed to **classify and analyze customer inquiries or support tickets** related to facility management services. Your goal is to extract structured information from an input message and return it in a specific JSON format.\n", + "\n", + "## Context \n", + "You will receive an input that is a formal, professional email or message addressing a facility management service request. The input may include a subject line and a body text. The message may express various levels of urgency, sentiment, and may pertain to one or more predefined categories.\n", + "\n", + "The JSON output must include the following keys:\n", + "- `\"urgency\"`: Must be one of `high`, `medium`, or `low`.\n", + "- `\"sentiment\"`: Must be one of `negative`, `neutral`, or `positive`.\n", + "- `\"categories\"`: A dictionary where each key is a category from the list: \n", + " `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. \n", + " The corresponding value for each category must be a boolean (`true`/`false`) indicating whether the category is the best match for the input.\n", + "\n", + "## Instructions \n", + "1. **Analyze the Input**: Carefully read the input message, including the subject line and body text, to understand the content, tone, and intent.\n", + "2. **Determine Urgency**: \n", + " - If the message indicates an immediate need, time-sensitive issue, or potential disruption, classify the urgency as `high`. \n", + " - If the message requests action but does not indicate immediate need, classify the urgency as `medium`. \n", + " - If the message is informational, exploratory, or indicates no immediate need, classify the urgency as `low`.\n", + "3. **Determine Sentiment**: \n", + " - If the message expresses dissatisfaction, frustration, or negative emotions, classify the sentiment as `negative`. \n", + " - If the message is neutral, factual, or inquisitive without strong positive or negative emotions, classify the sentiment as `neutral`. \n", + " - If the message expresses satisfaction, gratitude, or positive emotions, classify the sentiment as `positive`.\n", + "4. **Classify Categories**: \n", + " - For each category in the list, decide whether the input message pertains to that category. \n", + " - Set the corresponding value in the `\"categories\"` dictionary to `true` if the category is relevant, and `false` otherwise. \n", + " - Only one category may be `true` unless the input explicitly pertains to multiple categories; in such cases, set all applicable categories to `true`.\n", + "5. **Output Format**: \n", + " - Return a valid JSON string containing only the keys `\"urgency\"`, `\"sentiment\"`, and `\"categories\"`. \n", + " - The JSON must be compact with no unnecessary whitespace or newlines. \n", + " - The JSON must be on a single line with no extra characters or formatting (e.g., no ```json...```).\n", + "\n", + "## Examples \n", + "Use the provided examples to guide your analysis and ensure consistency in classification.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 18: Analyze the provided customer service email to classify it into categories, determine its sentiment, and assess its urgency. Return a JSON object with the following structure:\n", + "\n", + "```json\n", + "{\"urgency\": \"high|medium|low\", \"sentiment\": \"negative|neutral|positive\", \"categories\": {\"emergency_repair_services\": false, \"routine_maintenance_requests\": false, \"quality_and_safety_concerns\": false, \"specialized_cleaning_services\": false, \"general_inquiries\": false, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": false, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": false, \"facility_management_issues\": false}}\n", + "```\n", "\n", - "2025/07/02 20:22:48 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "- **urgency**: Determine if the email indicates an urgent issue (`high`), a moderate need (`medium`), or no immediate concern (`low`).\n", + "- **sentiment**: Assess the emotional tone of the email as `negative` (frustration, complaint), `neutral` (informative, inquiry), or `positive` (praise, gratitude).\n", + "- **categories**: For each category in the dictionary, set its value to `true` if the email content matches the category, otherwise `false`. Categories include issues like scheduling, complaints, training requests, maintenance, etc.\n", "\n", - "2025/07/02 20:22:48 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Restored GroundedProposer, current GroundedProposer class=\n", - "2025/07/02 20:22:48 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.custom_adapters.custom_chat_adapter: Initializing CustomChatAdapter with enable_json_fallback=False\n", - "2025/07/02 20:22:49 INFO dspy.teleprompt.mipro_optimizer_v2: ==> STEP 3: FINDING OPTIMAL PROMPT PARAMETERS <==\n", - "2025/07/02 20:22:49 INFO dspy.teleprompt.mipro_optimizer_v2: We will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination using Bayesian Optimization.\n", + "The output must be a valid JSON string on a single line with no additional text or formatting.\n", "\n", - "2025/07/02 20:22:49 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 1 / 37 - Full Evaluation of Default Program ==\n" + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: 19: You are an expert in analyzing customer service emails and classifying them into specific categories, assessing sentiment, and determining urgency. Your task is to process an incoming email and generate a structured JSON output with the following keys:\n", + "\n", + "1. **\"urgency\"**: Determine the urgency level of the request. It must be one of `high`, `medium`, or `low`. Consider the tone, context, and explicit mentions of urgency in the email to make this determination.\n", + " \n", + "2. **\"sentiment\"**: Assess the sentiment of the email. It must be one of `negative`, `neutral`, or `positive`. Analyze the tone, language, and overall attitude expressed by the sender to classify the sentiment accurately.\n", + "\n", + "3. **\"categories\"**: A dictionary where each key is a category from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. The corresponding value for each category must be a boolean (`True`/`False`) indicating whether the category is the best match for the input. Only one or more categories should be `True` based on the content of the email; all others should be `False`.\n", + "\n", + "**Steps to Follow:**\n", + "1. **Read the Email Carefully**: Understand the main purpose and context of the email. Identify any explicit mentions of urgency, sentiment indicators, and keywords related to the predefined categories.\n", + " \n", + "2. **Determine Urgency**:\n", + " - **High**: If the email mentions urgent issues, immediate action required, or critical problems affecting operations or safety.\n", + " - **Medium**: If the email discusses important but not immediately critical issues, or requests that need timely attention.\n", + " - **Low**: If the email is informational, general inquiry, or does not indicate any immediate need for action.\n", + "\n", + "3. **Assess Sentiment**:\n", + " - **Negative**: If the email expresses dissatisfaction, complaints, frustration, or negative feelings.\n", + " - **Neutral**: If the email is factual, informative, or does not express strong emotions.\n", + " - **Positive**: If the email expresses satisfaction, gratitude, or positive feelings.\n", + "\n", + "4. **Classify Categories**:\n", + " - Review the list of categories and match the content of the email to the most relevant ones. Consider the main topic, requests, and any specific issues mentioned.\n", + " - Set the corresponding boolean values to `True` for the categories that best match the input. All other categories should be `False`.\n", + "\n", + "5. **Generate JSON Output**:\n", + " - Construct a JSON object with the keys \"urgency\", \"sentiment\", and \"categories\".\n", + " - Ensure the JSON is valid, compact, and on a single line with no extra formatting or text.\n", + "\n", + "**Example Outputs:**\n", + "- For an email requesting general information about HVAC maintenance best practices: \n", + " ```{\"urgency\": \"low\", \"sentiment\": \"neutral\", \"categories\": {\"emergency_repair_services\": false, \"routine_maintenance_requests\": false, \"quality_and_safety_concerns\": false, \"specialized_cleaning_services\": false, \"general_inquiries\": true, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": true, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": false, \"facility_management_issues\": false}}```\n", + "- For an email complaining about ongoing maintenance issues: \n", + " ```{\"urgency\": \"high\", \"sentiment\": \"negative\", \"categories\": {\"emergency_repair_services\": false, \"routine_maintenance_requests\": true, \"quality_and_safety_concerns\": true, \"specialized_cleaning_services\": false, \"general_inquiries\": false, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": false, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": true, \"facility_management_issues\": true}}```\n", + "\n", + "**Important**: The output must be a valid JSON string with no additional text, formatting, or encapsulation (e.g., no ```json...```). The JSON must be compact with no unnecessary whitespace or newlines.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "\n", + "2026/01/29 18:24:08 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Restored GroundedProposer, current GroundedProposer class=\n", + "2026/01/29 18:24:08 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.custom_adapters.custom_chat_adapter: Initializing CustomChatAdapter with enable_json_fallback=False\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: ==> STEP 3: FINDING OPTIMAL PROMPT PARAMETERS <==\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: We will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination using Bayesian Optimization.\n", + "\n", + "2026/01/29 18:24:08 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 1 / 37 - Full Evaluation of Default Program ==\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 35.20 / 50 (70.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:28<00:00, 1.75it/s]" + "Average Metric: 41.37 / 50 (82.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:28<00:00, 1.74it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:23:17 INFO dspy.evaluate.evaluate: Average Metric: 35.199999999999996 / 50 (70.4%)\n", - "2025/07/02 20:23:17 INFO dspy.teleprompt.mipro_optimizer_v2: Default program score: 70.4\n", + "2026/01/29 18:24:36 INFO dspy.evaluate.evaluate: Average Metric: 41.36666666666666 / 50 (82.7%)\n", + "2026/01/29 18:24:36 INFO dspy.teleprompt.mipro_optimizer_v2: Default program score: 82.73\n", "\n", - "/local/home/shenov/workplace/NovaPromptOptimizer/src/NovaPromptOptimizer/samples/facility-support-analyzer/.venv/lib/python3.12/site-packages/optuna/_experimental.py:32: ExperimentalWarning: Argument ``multivariate`` is an experimental feature. The interface can change in the future.\n", - " warnings.warn(\n", - "2025/07/02 20:23:17 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 2 / 37 - Minibatch ==\n" + "/Users/shenov/Library/Python/3.9/lib/python/site-packages/optuna/_experimental.py:33: ExperimentalWarning: Argument ``multivariate`` is an experimental feature. The interface can change in the future.\n", + " optuna_warn(\n", + "2026/01/29 18:24:36 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 2 / 37 - Minibatch ==\n" ] }, { @@ -814,22 +1383,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.33 / 35 (72.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.35it/s]" + "Average Metric: 28.60 / 35 (81.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.69it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:23:43 INFO dspy.evaluate.evaluate: Average Metric: 25.333333333333332 / 35 (72.4%)\n", - "2025/07/02 20:23:43 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.38 on minibatch of size 35 with parameters ['Predictor 0: Instruction 12', 'Predictor 0: Few-Shot Set 6'].\n", - "2025/07/02 20:23:43 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38]\n", - "2025/07/02 20:23:43 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4]\n", - "2025/07/02 20:23:43 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.4\n", - "2025/07/02 20:23:43 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 18:24:57 INFO dspy.evaluate.evaluate: Average Metric: 28.599999999999994 / 35 (81.7%)\n", + "2026/01/29 18:24:57 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 81.71 on minibatch of size 35 with parameters ['Predictor 0: Instruction 12', 'Predictor 0: Few-Shot Set 6'].\n", + "2026/01/29 18:24:57 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71]\n", + "2026/01/29 18:24:57 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73]\n", + "2026/01/29 18:24:57 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:24:57 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:23:43 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 3 / 37 - Minibatch ==\n" + "2026/01/29 18:24:57 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 3 / 37 - Minibatch ==\n" ] }, { @@ -837,22 +1406,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.03 / 35 (68.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.48it/s]" + "Average Metric: 27.67 / 35 (79.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:19<00:00, 1.75it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:24:07 INFO dspy.evaluate.evaluate: Average Metric: 24.03333333333333 / 35 (68.7%)\n", - "2025/07/02 20:24:07 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 68.67 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 4'].\n", - "2025/07/02 20:24:07 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67]\n", - "2025/07/02 20:24:07 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4]\n", - "2025/07/02 20:24:07 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.4\n", - "2025/07/02 20:24:07 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 18:25:17 INFO dspy.evaluate.evaluate: Average Metric: 27.66666666666666 / 35 (79.0%)\n", + "2026/01/29 18:25:17 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 79.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 18:25:17 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05]\n", + "2026/01/29 18:25:17 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73]\n", + "2026/01/29 18:25:17 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:25:17 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:24:07 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 4 / 37 - Minibatch ==\n" + "2026/01/29 18:25:17 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 4 / 37 - Minibatch ==\n" ] }, { @@ -860,22 +1429,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.13 / 35 (71.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:27<00:00, 1.27it/s]" + "Average Metric: 25.43 / 35 (72.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.70it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:24:34 INFO dspy.evaluate.evaluate: Average Metric: 25.133333333333333 / 35 (71.8%)\n", - "2025/07/02 20:24:34 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 71.81 on minibatch of size 35 with parameters ['Predictor 0: Instruction 3', 'Predictor 0: Few-Shot Set 13'].\n", - "2025/07/02 20:24:34 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81]\n", - "2025/07/02 20:24:34 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4]\n", - "2025/07/02 20:24:34 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.4\n", - "2025/07/02 20:24:34 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 18:25:38 INFO dspy.evaluate.evaluate: Average Metric: 25.433333333333326 / 35 (72.7%)\n", + "2026/01/29 18:25:38 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.67 on minibatch of size 35 with parameters ['Predictor 0: Instruction 3', 'Predictor 0: Few-Shot Set 13'].\n", + "2026/01/29 18:25:38 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67]\n", + "2026/01/29 18:25:38 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73]\n", + "2026/01/29 18:25:38 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:25:38 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:24:34 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 5 / 37 - Minibatch ==\n" + "2026/01/29 18:25:38 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 5 / 37 - Minibatch ==\n" ] }, { @@ -883,22 +1452,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 23.70 / 35 (67.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:26<00:00, 1.35it/s]" + "Average Metric: 29.43 / 35 (84.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.61it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:25:00 INFO dspy.evaluate.evaluate: Average Metric: 23.7 / 35 (67.7%)\n", - "2025/07/02 20:25:00 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 67.71 on minibatch of size 35 with parameters ['Predictor 0: Instruction 9', 'Predictor 0: Few-Shot Set 7'].\n", - "2025/07/02 20:25:00 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71]\n", - "2025/07/02 20:25:00 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4]\n", - "2025/07/02 20:25:00 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.4\n", - "2025/07/02 20:25:00 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 18:25:59 INFO dspy.evaluate.evaluate: Average Metric: 29.433333333333334 / 35 (84.1%)\n", + "2026/01/29 18:25:59 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 84.1 on minibatch of size 35 with parameters ['Predictor 0: Instruction 9', 'Predictor 0: Few-Shot Set 7'].\n", + "2026/01/29 18:25:59 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1]\n", + "2026/01/29 18:25:59 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73]\n", + "2026/01/29 18:25:59 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:25:59 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:25:00 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 6 / 37 - Minibatch ==\n" + "2026/01/29 18:25:59 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 6 / 37 - Minibatch ==\n" ] }, { @@ -906,23 +1475,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 22.80 / 35 (65.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:24<00:00, 1.40it/s]" + "Average Metric: 30.60 / 35 (87.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.66it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:25:25 INFO dspy.evaluate.evaluate: Average Metric: 22.8 / 35 (65.1%)\n", - "2025/07/02 20:25:25 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 65.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 0', 'Predictor 0: Few-Shot Set 9'].\n", - "2025/07/02 20:25:25 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14]\n", - "2025/07/02 20:25:25 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4]\n", - "2025/07/02 20:25:25 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.4\n", - "2025/07/02 20:25:25 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 18:26:21 INFO dspy.evaluate.evaluate: Average Metric: 30.6 / 35 (87.4%)\n", + "2026/01/29 18:26:21 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 87.43 on minibatch of size 35 with parameters ['Predictor 0: Instruction 0', 'Predictor 0: Few-Shot Set 9'].\n", + "2026/01/29 18:26:21 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43]\n", + "2026/01/29 18:26:21 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73]\n", + "2026/01/29 18:26:21 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:26:21 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:25:25 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 7 / 37 - Full Evaluation =====\n", - "2025/07/02 20:25:25 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 72.38) from minibatch trials...\n" + "2026/01/29 18:26:21 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 7 / 37 - Full Evaluation =====\n", + "2026/01/29 18:26:21 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 87.43) from minibatch trials...\n" ] }, { @@ -930,21 +1499,20 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 36.60 / 50 (73.2%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:11<00:00, 4.36it/s]" + "Average Metric: 41.00 / 50 (82.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:31<00:00, 1.61it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:25:37 INFO dspy.evaluate.evaluate: Average Metric: 36.6 / 50 (73.2%)\n", - "2025/07/02 20:25:37 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 73.2\n", - "2025/07/02 20:25:37 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2]\n", - "2025/07/02 20:25:37 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:25:37 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:25:37 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 18:26:52 INFO dspy.evaluate.evaluate: Average Metric: 41.0 / 50 (82.0%)\n", + "2026/01/29 18:26:52 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0]\n", + "2026/01/29 18:26:52 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:26:52 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 18:26:52 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:25:37 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 8 / 37 - Minibatch ==\n" + "2026/01/29 18:26:52 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 8 / 37 - Minibatch ==\n" ] }, { @@ -952,22 +1520,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.67 / 35 (70.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:24<00:00, 1.44it/s]" + "Average Metric: 26.27 / 35 (75.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:19<00:00, 1.82it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:26:01 INFO dspy.evaluate.evaluate: Average Metric: 24.666666666666664 / 35 (70.5%)\n", - "2025/07/02 20:26:01 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 70.48 on minibatch of size 35 with parameters ['Predictor 0: Instruction 10', 'Predictor 0: Few-Shot Set 15'].\n", - "2025/07/02 20:26:01 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48]\n", - "2025/07/02 20:26:01 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2]\n", - "2025/07/02 20:26:01 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:26:01 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 18:27:11 INFO dspy.evaluate.evaluate: Average Metric: 26.266666666666662 / 35 (75.0%)\n", + "2026/01/29 18:27:11 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 75.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 10', 'Predictor 0: Few-Shot Set 15'].\n", + "2026/01/29 18:27:11 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05]\n", + "2026/01/29 18:27:11 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0]\n", + "2026/01/29 18:27:11 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:27:11 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:26:01 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 9 / 37 - Minibatch ==\n" + "2026/01/29 18:27:11 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 9 / 37 - Minibatch ==\n" ] }, { @@ -975,22 +1543,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.37 / 35 (72.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:28<00:00, 1.23it/s]" + "Average Metric: 27.10 / 35 (77.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.68it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:26:29 INFO dspy.evaluate.evaluate: Average Metric: 25.366666666666667 / 35 (72.5%)\n", - "2025/07/02 20:26:29 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.48 on minibatch of size 35 with parameters ['Predictor 0: Instruction 6', 'Predictor 0: Few-Shot Set 17'].\n", - "2025/07/02 20:26:29 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48]\n", - "2025/07/02 20:26:29 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2]\n", - "2025/07/02 20:26:29 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:26:29 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 18:27:32 INFO dspy.evaluate.evaluate: Average Metric: 27.09999999999999 / 35 (77.4%)\n", + "2026/01/29 18:27:32 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 77.43 on minibatch of size 35 with parameters ['Predictor 0: Instruction 6', 'Predictor 0: Few-Shot Set 17'].\n", + "2026/01/29 18:27:32 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43]\n", + "2026/01/29 18:27:32 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0]\n", + "2026/01/29 18:27:32 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:27:32 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:26:29 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 10 / 37 - Minibatch ==\n" + "2026/01/29 18:27:32 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 10 / 37 - Minibatch ==\n" ] }, { @@ -998,22 +1566,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 23.23 / 35 (66.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:24<00:00, 1.46it/s]" + "Average Metric: 28.83 / 35 (82.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.70it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:26:53 INFO dspy.evaluate.evaluate: Average Metric: 23.233333333333334 / 35 (66.4%)\n", - "2025/07/02 20:26:53 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 66.38 on minibatch of size 35 with parameters ['Predictor 0: Instruction 18', 'Predictor 0: Few-Shot Set 9'].\n", - "2025/07/02 20:26:53 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38]\n", - "2025/07/02 20:26:53 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2]\n", - "2025/07/02 20:26:53 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:26:53 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:27:52 INFO dspy.evaluate.evaluate: Average Metric: 28.833333333333325 / 35 (82.4%)\n", + "2026/01/29 18:27:52 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 82.38 on minibatch of size 35 with parameters ['Predictor 0: Instruction 18', 'Predictor 0: Few-Shot Set 9'].\n", + "2026/01/29 18:27:52 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38]\n", + "2026/01/29 18:27:52 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0]\n", + "2026/01/29 18:27:52 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:27:52 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:26:53 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 11 / 37 - Minibatch ==\n" + "2026/01/29 18:27:52 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 11 / 37 - Minibatch ==\n" ] }, { @@ -1021,22 +1589,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.13 / 35 (71.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.46it/s]" + "Average Metric: 29.93 / 35 (85.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.68it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:27:17 INFO dspy.evaluate.evaluate: Average Metric: 25.133333333333333 / 35 (71.8%)\n", - "2025/07/02 20:27:18 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 71.81 on minibatch of size 35 with parameters ['Predictor 0: Instruction 12', 'Predictor 0: Few-Shot Set 19'].\n", - "2025/07/02 20:27:18 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81]\n", - "2025/07/02 20:27:18 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2]\n", - "2025/07/02 20:27:18 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:27:18 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:28:13 INFO dspy.evaluate.evaluate: Average Metric: 29.933333333333323 / 35 (85.5%)\n", + "2026/01/29 18:28:13 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 85.52 on minibatch of size 35 with parameters ['Predictor 0: Instruction 0', 'Predictor 0: Few-Shot Set 9'].\n", + "2026/01/29 18:28:13 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52]\n", + "2026/01/29 18:28:13 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0]\n", + "2026/01/29 18:28:13 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:28:13 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:27:18 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 12 / 37 - Minibatch ==\n" + "2026/01/29 18:28:13 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 12 / 37 - Minibatch ==\n" ] }, { @@ -1044,23 +1612,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.17 / 35 (71.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:26<00:00, 1.34it/s]" + "Average Metric: 29.53 / 35 (84.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.73it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:27:44 INFO dspy.evaluate.evaluate: Average Metric: 25.166666666666668 / 35 (71.9%)\n", - "2025/07/02 20:27:44 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 71.9 on minibatch of size 35 with parameters ['Predictor 0: Instruction 6', 'Predictor 0: Few-Shot Set 10'].\n", - "2025/07/02 20:27:44 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9]\n", - "2025/07/02 20:27:44 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2]\n", - "2025/07/02 20:27:44 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:27:44 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:28:34 INFO dspy.evaluate.evaluate: Average Metric: 29.53333333333333 / 35 (84.4%)\n", + "2026/01/29 18:28:34 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 84.38 on minibatch of size 35 with parameters ['Predictor 0: Instruction 0', 'Predictor 0: Few-Shot Set 9'].\n", + "2026/01/29 18:28:34 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38]\n", + "2026/01/29 18:28:34 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0]\n", + "2026/01/29 18:28:34 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 82.73\n", + "2026/01/29 18:28:34 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:27:44 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 13 / 37 - Full Evaluation =====\n", - "2025/07/02 20:27:44 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 72.48) from minibatch trials...\n" + "2026/01/29 18:28:34 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 13 / 37 - Full Evaluation =====\n", + "2026/01/29 18:28:34 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 84.1) from minibatch trials...\n" ] }, { @@ -1068,20 +1636,21 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 35.53 / 50 (71.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:10<00:00, 4.75it/s]" + "Average Metric: 42.67 / 50 (85.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:31<00:00, 1.61it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:27:54 INFO dspy.evaluate.evaluate: Average Metric: 35.53333333333333 / 50 (71.1%)\n", - "2025/07/02 20:27:54 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07]\n", - "2025/07/02 20:27:54 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:27:54 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:27:54 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 18:29:05 INFO dspy.evaluate.evaluate: Average Metric: 42.666666666666664 / 50 (85.3%)\n", + "2026/01/29 18:29:05 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 85.33\n", + "2026/01/29 18:29:05 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33]\n", + "2026/01/29 18:29:05 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.33\n", + "2026/01/29 18:29:05 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 18:29:05 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:27:54 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 14 / 37 - Minibatch ==\n" + "2026/01/29 18:29:05 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 14 / 37 - Minibatch ==\n" ] }, { @@ -1089,22 +1658,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.43 / 35 (69.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:27<00:00, 1.29it/s]" + "Average Metric: 26.63 / 35 (76.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.70it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:28:21 INFO dspy.evaluate.evaluate: Average Metric: 24.433333333333334 / 35 (69.8%)\n", - "2025/07/02 20:28:21 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 69.81 on minibatch of size 35 with parameters ['Predictor 0: Instruction 13', 'Predictor 0: Few-Shot Set 12'].\n", - "2025/07/02 20:28:21 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81]\n", - "2025/07/02 20:28:21 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07]\n", - "2025/07/02 20:28:21 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:28:21 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:29:25 INFO dspy.evaluate.evaluate: Average Metric: 26.633333333333333 / 35 (76.1%)\n", + "2026/01/29 18:29:25 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 76.1 on minibatch of size 35 with parameters ['Predictor 0: Instruction 14', 'Predictor 0: Few-Shot Set 1'].\n", + "2026/01/29 18:29:25 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1]\n", + "2026/01/29 18:29:25 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33]\n", + "2026/01/29 18:29:25 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.33\n", + "2026/01/29 18:29:25 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:28:21 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 15 / 37 - Minibatch ==\n" + "2026/01/29 18:29:25 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 15 / 37 - Minibatch ==\n" ] }, { @@ -1112,22 +1681,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.37 / 35 (69.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:26<00:00, 1.31it/s]" + "Average Metric: 25.20 / 35 (72.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.66it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:28:48 INFO dspy.evaluate.evaluate: Average Metric: 24.366666666666667 / 35 (69.6%)\n", - "2025/07/02 20:28:48 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 69.62 on minibatch of size 35 with parameters ['Predictor 0: Instruction 5', 'Predictor 0: Few-Shot Set 17'].\n", - "2025/07/02 20:28:48 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62]\n", - "2025/07/02 20:28:48 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07]\n", - "2025/07/02 20:28:48 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:28:48 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:29:46 INFO dspy.evaluate.evaluate: Average Metric: 25.2 / 35 (72.0%)\n", + "2026/01/29 18:29:46 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.0 on minibatch of size 35 with parameters ['Predictor 0: Instruction 13', 'Predictor 0: Few-Shot Set 11'].\n", + "2026/01/29 18:29:46 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0]\n", + "2026/01/29 18:29:46 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33]\n", + "2026/01/29 18:29:46 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.33\n", + "2026/01/29 18:29:46 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:28:48 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 16 / 37 - Minibatch ==\n" + "2026/01/29 18:29:46 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 16 / 37 - Minibatch ==\n" ] }, { @@ -1135,22 +1704,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 22.70 / 35 (64.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.36it/s]" + "Average Metric: 26.20 / 35 (74.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:19<00:00, 1.84it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:29:14 INFO dspy.evaluate.evaluate: Average Metric: 22.7 / 35 (64.9%)\n", - "2025/07/02 20:29:14 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 64.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 12', 'Predictor 0: Few-Shot Set 12'].\n", - "2025/07/02 20:29:14 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86]\n", - "2025/07/02 20:29:14 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07]\n", - "2025/07/02 20:29:14 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:29:14 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:30:05 INFO dspy.evaluate.evaluate: Average Metric: 26.2 / 35 (74.9%)\n", + "2026/01/29 18:30:05 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 74.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 11', 'Predictor 0: Few-Shot Set 18'].\n", + "2026/01/29 18:30:05 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86]\n", + "2026/01/29 18:30:05 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33]\n", + "2026/01/29 18:30:05 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.33\n", + "2026/01/29 18:30:05 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:29:14 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 17 / 37 - Minibatch ==\n" + "2026/01/29 18:30:05 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 17 / 37 - Minibatch ==\n" ] }, { @@ -1158,22 +1727,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 22.97 / 35 (65.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.38it/s]" + "Average Metric: 30.57 / 35 (87.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:19<00:00, 1.80it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:29:39 INFO dspy.evaluate.evaluate: Average Metric: 22.966666666666665 / 35 (65.6%)\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 65.62 on minibatch of size 35 with parameters ['Predictor 0: Instruction 15', 'Predictor 0: Few-Shot Set 16'].\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62]\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07]\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:30:25 INFO dspy.evaluate.evaluate: Average Metric: 30.566666666666652 / 35 (87.3%)\n", + "2026/01/29 18:30:25 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 87.33 on minibatch of size 35 with parameters ['Predictor 0: Instruction 17', 'Predictor 0: Few-Shot Set 19'].\n", + "2026/01/29 18:30:25 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33]\n", + "2026/01/29 18:30:25 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33]\n", + "2026/01/29 18:30:25 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.33\n", + "2026/01/29 18:30:25 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 18 / 37 - Minibatch ==\n" + "2026/01/29 18:30:25 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 18 / 37 - Minibatch ==\n" ] }, { @@ -1181,23 +1750,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.17 / 35 (74.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:00<00:00, 2602.71it/s]" + "Average Metric: 28.67 / 35 (81.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.66it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:29:39 INFO dspy.evaluate.evaluate: Average Metric: 26.166666666666668 / 35 (74.8%)\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 74.76 on minibatch of size 35 with parameters ['Predictor 0: Instruction 6', 'Predictor 0: Few-Shot Set 17'].\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76]\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07]\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.2\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:30:46 INFO dspy.evaluate.evaluate: Average Metric: 28.66666666666666 / 35 (81.9%)\n", + "2026/01/29 18:30:46 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 81.9 on minibatch of size 35 with parameters ['Predictor 0: Instruction 17', 'Predictor 0: Few-Shot Set 8'].\n", + "2026/01/29 18:30:46 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9]\n", + "2026/01/29 18:30:46 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33]\n", + "2026/01/29 18:30:46 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.33\n", + "2026/01/29 18:30:46 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 19 / 37 - Full Evaluation =====\n", - "2025/07/02 20:29:39 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 71.9) from minibatch trials...\n" + "2026/01/29 18:30:46 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 19 / 37 - Full Evaluation =====\n", + "2026/01/29 18:30:46 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 87.33) from minibatch trials...\n" ] }, { @@ -1205,21 +1774,21 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 36.83 / 50 (73.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:13<00:00, 3.72it/s]" + "Average Metric: 42.70 / 50 (85.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:28<00:00, 1.73it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:29:53 INFO dspy.evaluate.evaluate: Average Metric: 36.833333333333336 / 50 (73.7%)\n", - "2025/07/02 20:29:53 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 73.67\n", - "2025/07/02 20:29:53 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67]\n", - "2025/07/02 20:29:53 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.67\n", - "2025/07/02 20:29:53 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:29:53 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 18:31:15 INFO dspy.evaluate.evaluate: Average Metric: 42.699999999999996 / 50 (85.4%)\n", + "2026/01/29 18:31:15 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 85.4\n", + "2026/01/29 18:31:15 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4]\n", + "2026/01/29 18:31:15 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.4\n", + "2026/01/29 18:31:15 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 18:31:15 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:29:53 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 20 / 37 - Minibatch ==\n" + "2026/01/29 18:31:15 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 20 / 37 - Minibatch ==\n" ] }, { @@ -1227,22 +1796,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.67 / 35 (70.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:26<00:00, 1.34it/s]" + "Average Metric: 28.20 / 35 (80.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.66it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:30:19 INFO dspy.evaluate.evaluate: Average Metric: 24.666666666666664 / 35 (70.5%)\n", - "2025/07/02 20:30:19 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 70.48 on minibatch of size 35 with parameters ['Predictor 0: Instruction 17', 'Predictor 0: Few-Shot Set 1'].\n", - "2025/07/02 20:30:19 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48]\n", - "2025/07/02 20:30:19 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67]\n", - "2025/07/02 20:30:19 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.67\n", - "2025/07/02 20:30:19 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:31:36 INFO dspy.evaluate.evaluate: Average Metric: 28.199999999999992 / 35 (80.6%)\n", + "2026/01/29 18:31:36 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 80.57 on minibatch of size 35 with parameters ['Predictor 0: Instruction 15', 'Predictor 0: Few-Shot Set 3'].\n", + "2026/01/29 18:31:36 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57]\n", + "2026/01/29 18:31:36 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4]\n", + "2026/01/29 18:31:36 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.4\n", + "2026/01/29 18:31:36 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:30:19 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 21 / 37 - Minibatch ==\n" + "2026/01/29 18:31:36 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 21 / 37 - Minibatch ==\n" ] }, { @@ -1250,22 +1819,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.90 / 35 (71.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:27<00:00, 1.25it/s]" + "Average Metric: 30.20 / 35 (86.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:19<00:00, 1.77it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:30:47 INFO dspy.evaluate.evaluate: Average Metric: 24.9 / 35 (71.1%)\n", - "2025/07/02 20:30:47 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 71.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 19', 'Predictor 0: Few-Shot Set 3'].\n", - "2025/07/02 20:30:47 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14]\n", - "2025/07/02 20:30:47 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67]\n", - "2025/07/02 20:30:47 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.67\n", - "2025/07/02 20:30:47 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:31:56 INFO dspy.evaluate.evaluate: Average Metric: 30.19999999999999 / 35 (86.3%)\n", + "2026/01/29 18:31:56 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 86.29 on minibatch of size 35 with parameters ['Predictor 0: Instruction 10', 'Predictor 0: Few-Shot Set 19'].\n", + "2026/01/29 18:31:56 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29]\n", + "2026/01/29 18:31:56 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4]\n", + "2026/01/29 18:31:56 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.4\n", + "2026/01/29 18:31:56 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:30:47 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 22 / 37 - Minibatch ==\n" + "2026/01/29 18:31:56 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 22 / 37 - Minibatch ==\n" ] }, { @@ -1273,22 +1842,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.87 / 35 (76.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:50<00:00, 1.46s/it]" + "Average Metric: 29.90 / 35 (85.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.62it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:31:38 INFO dspy.evaluate.evaluate: Average Metric: 26.866666666666667 / 35 (76.8%)\n", - "2025/07/02 20:31:38 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 76.76 on minibatch of size 35 with parameters ['Predictor 0: Instruction 11', 'Predictor 0: Few-Shot Set 5'].\n", - "2025/07/02 20:31:38 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76]\n", - "2025/07/02 20:31:38 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67]\n", - "2025/07/02 20:31:38 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.67\n", - "2025/07/02 20:31:38 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:32:18 INFO dspy.evaluate.evaluate: Average Metric: 29.89999999999999 / 35 (85.4%)\n", + "2026/01/29 18:32:18 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 85.43 on minibatch of size 35 with parameters ['Predictor 0: Instruction 5', 'Predictor 0: Few-Shot Set 19'].\n", + "2026/01/29 18:32:18 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43]\n", + "2026/01/29 18:32:18 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4]\n", + "2026/01/29 18:32:18 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.4\n", + "2026/01/29 18:32:18 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:31:38 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 23 / 37 - Minibatch ==\n" + "2026/01/29 18:32:18 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 23 / 37 - Minibatch ==\n" ] }, { @@ -1296,22 +1865,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.07 / 35 (71.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:08<00:00, 3.90it/s]" + "Average Metric: 29.57 / 35 (84.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.66it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:31:47 INFO dspy.evaluate.evaluate: Average Metric: 25.066666666666666 / 35 (71.6%)\n", - "2025/07/02 20:31:47 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 71.62 on minibatch of size 35 with parameters ['Predictor 0: Instruction 11', 'Predictor 0: Few-Shot Set 5'].\n", - "2025/07/02 20:31:47 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62]\n", - "2025/07/02 20:31:47 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67]\n", - "2025/07/02 20:31:47 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.67\n", - "2025/07/02 20:31:47 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:32:39 INFO dspy.evaluate.evaluate: Average Metric: 29.566666666666663 / 35 (84.5%)\n", + "2026/01/29 18:32:39 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 84.48 on minibatch of size 35 with parameters ['Predictor 0: Instruction 16', 'Predictor 0: Few-Shot Set 19'].\n", + "2026/01/29 18:32:39 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48]\n", + "2026/01/29 18:32:39 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4]\n", + "2026/01/29 18:32:39 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.4\n", + "2026/01/29 18:32:39 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:31:47 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 24 / 37 - Minibatch ==\n" + "2026/01/29 18:32:39 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 24 / 37 - Minibatch ==\n" ] }, { @@ -1319,23 +1888,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.60 / 35 (73.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.48it/s]" + "Average Metric: 29.70 / 35 (84.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.68it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:32:11 INFO dspy.evaluate.evaluate: Average Metric: 25.599999999999998 / 35 (73.1%)\n", - "2025/07/02 20:32:11 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 73.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 2', 'Predictor 0: Few-Shot Set 5'].\n", - "2025/07/02 20:32:11 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14]\n", - "2025/07/02 20:32:11 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67]\n", - "2025/07/02 20:32:11 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 73.67\n", - "2025/07/02 20:32:11 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:32:59 INFO dspy.evaluate.evaluate: Average Metric: 29.69999999999999 / 35 (84.9%)\n", + "2026/01/29 18:32:59 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 84.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 2', 'Predictor 0: Few-Shot Set 5'].\n", + "2026/01/29 18:32:59 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86]\n", + "2026/01/29 18:32:59 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4]\n", + "2026/01/29 18:32:59 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 85.4\n", + "2026/01/29 18:33:00 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:32:11 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 25 / 37 - Full Evaluation =====\n", - "2025/07/02 20:32:11 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 74.19) from minibatch trials...\n" + "2026/01/29 18:33:00 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 25 / 37 - Full Evaluation =====\n", + "2026/01/29 18:33:00 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 86.29) from minibatch trials...\n" ] }, { @@ -1343,21 +1912,21 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 37.00 / 50 (74.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:01<00:00, 28.03it/s]" + "Average Metric: 43.27 / 50 (86.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:29<00:00, 1.69it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:32:12 INFO dspy.evaluate.evaluate: Average Metric: 37.0 / 50 (74.0%)\n", - "2025/07/02 20:32:12 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 74.0\n", - "2025/07/02 20:32:12 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0]\n", - "2025/07/02 20:32:12 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:32:12 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:32:12 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 18:33:29 INFO dspy.evaluate.evaluate: Average Metric: 43.26666666666666 / 50 (86.5%)\n", + "2026/01/29 18:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 86.53\n", + "2026/01/29 18:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53]\n", + "2026/01/29 18:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.53\n", + "2026/01/29 18:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 18:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:32:12 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 26 / 37 - Minibatch ==\n" + "2026/01/29 18:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 26 / 37 - Minibatch ==\n" ] }, { @@ -1365,22 +1934,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.57 / 35 (70.2%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.37it/s]" + "Average Metric: 27.67 / 35 (79.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:22<00:00, 1.58it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:32:38 INFO dspy.evaluate.evaluate: Average Metric: 24.566666666666666 / 35 (70.2%)\n", - "2025/07/02 20:32:38 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 70.19 on minibatch of size 35 with parameters ['Predictor 0: Instruction 11', 'Predictor 0: Few-Shot Set 9'].\n", - "2025/07/02 20:32:38 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19]\n", - "2025/07/02 20:32:38 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0]\n", - "2025/07/02 20:32:38 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:32:38 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:33:51 INFO dspy.evaluate.evaluate: Average Metric: 27.666666666666664 / 35 (79.0%)\n", + "2026/01/29 18:33:51 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 79.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 6', 'Predictor 0: Few-Shot Set 10'].\n", + "2026/01/29 18:33:51 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05]\n", + "2026/01/29 18:33:51 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53]\n", + "2026/01/29 18:33:51 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.53\n", + "2026/01/29 18:33:51 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:32:38 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 27 / 37 - Minibatch ==\n" + "2026/01/29 18:33:51 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 27 / 37 - Minibatch ==\n" ] }, { @@ -1388,22 +1957,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.30 / 35 (75.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.37it/s]" + "Average Metric: 26.77 / 35 (76.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.72it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:33:03 INFO dspy.evaluate.evaluate: Average Metric: 26.3 / 35 (75.1%)\n", - "2025/07/02 20:33:04 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 75.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 11', 'Predictor 0: Few-Shot Set 11'].\n", - "2025/07/02 20:33:04 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19, 75.14]\n", - "2025/07/02 20:33:04 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0]\n", - "2025/07/02 20:33:04 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:33:04 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:34:12 INFO dspy.evaluate.evaluate: Average Metric: 26.76666666666666 / 35 (76.5%)\n", + "2026/01/29 18:34:12 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 76.48 on minibatch of size 35 with parameters ['Predictor 0: Instruction 10', 'Predictor 0: Few-Shot Set 10'].\n", + "2026/01/29 18:34:12 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05, 76.48]\n", + "2026/01/29 18:34:12 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53]\n", + "2026/01/29 18:34:12 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.53\n", + "2026/01/29 18:34:12 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:33:04 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 28 / 37 - Minibatch ==\n" + "2026/01/29 18:34:12 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 28 / 37 - Minibatch ==\n" ] }, { @@ -1411,22 +1980,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.37 / 35 (69.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.36it/s]" + "Average Metric: 28.17 / 35 (80.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.63it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:33:29 INFO dspy.evaluate.evaluate: Average Metric: 24.366666666666667 / 35 (69.6%)\n", - "2025/07/02 20:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 69.62 on minibatch of size 35 with parameters ['Predictor 0: Instruction 7', 'Predictor 0: Few-Shot Set 14'].\n", - "2025/07/02 20:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19, 75.14, 69.62]\n", - "2025/07/02 20:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0]\n", - "2025/07/02 20:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:34:33 INFO dspy.evaluate.evaluate: Average Metric: 28.166666666666657 / 35 (80.5%)\n", + "2026/01/29 18:34:33 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 80.48 on minibatch of size 35 with parameters ['Predictor 0: Instruction 19', 'Predictor 0: Few-Shot Set 14'].\n", + "2026/01/29 18:34:33 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05, 76.48, 80.48]\n", + "2026/01/29 18:34:33 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53]\n", + "2026/01/29 18:34:33 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.53\n", + "2026/01/29 18:34:33 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:33:29 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 29 / 37 - Minibatch ==\n" + "2026/01/29 18:34:33 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 29 / 37 - Minibatch ==\n" ] }, { @@ -1434,22 +2003,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.27 / 35 (75.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.36it/s]" + "Average Metric: 28.63 / 35 (81.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:24<00:00, 1.43it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:33:55 INFO dspy.evaluate.evaluate: Average Metric: 26.266666666666666 / 35 (75.0%)\n", - "2025/07/02 20:33:55 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 75.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 15', 'Predictor 0: Few-Shot Set 11'].\n", - "2025/07/02 20:33:55 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19, 75.14, 69.62, 75.05]\n", - "2025/07/02 20:33:55 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0]\n", - "2025/07/02 20:33:55 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:33:55 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:34:58 INFO dspy.evaluate.evaluate: Average Metric: 28.63333333333333 / 35 (81.8%)\n", + "2026/01/29 18:34:58 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 81.81 on minibatch of size 35 with parameters ['Predictor 0: Instruction 17', 'Predictor 0: Few-Shot Set 2'].\n", + "2026/01/29 18:34:58 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05, 76.48, 80.48, 81.81]\n", + "2026/01/29 18:34:58 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53]\n", + "2026/01/29 18:34:58 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.53\n", + "2026/01/29 18:34:58 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:33:55 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 30 / 37 - Minibatch ==\n" + "2026/01/29 18:34:58 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 30 / 37 - Minibatch ==\n" ] }, { @@ -1457,23 +2026,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.33 / 35 (69.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:27<00:00, 1.25it/s]" + "Average Metric: 28.97 / 35 (82.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.65it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:34:23 INFO dspy.evaluate.evaluate: Average Metric: 24.333333333333332 / 35 (69.5%)\n", - "2025/07/02 20:34:23 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 69.52 on minibatch of size 35 with parameters ['Predictor 0: Instruction 1', 'Predictor 0: Few-Shot Set 11'].\n", - "2025/07/02 20:34:23 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19, 75.14, 69.62, 75.05, 69.52]\n", - "2025/07/02 20:34:23 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0]\n", - "2025/07/02 20:34:23 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:34:23 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:35:19 INFO dspy.evaluate.evaluate: Average Metric: 28.966666666666654 / 35 (82.8%)\n", + "2026/01/29 18:35:19 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 82.76 on minibatch of size 35 with parameters ['Predictor 0: Instruction 19', 'Predictor 0: Few-Shot Set 12'].\n", + "2026/01/29 18:35:19 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05, 76.48, 80.48, 81.81, 82.76]\n", + "2026/01/29 18:35:19 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53]\n", + "2026/01/29 18:35:19 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.53\n", + "2026/01/29 18:35:19 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:34:23 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 31 / 37 - Full Evaluation =====\n", - "2025/07/02 20:34:23 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 75.14) from minibatch trials...\n" + "2026/01/29 18:35:19 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 31 / 37 - Full Evaluation =====\n", + "2026/01/29 18:35:19 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 85.43) from minibatch trials...\n" ] }, { @@ -1481,20 +2050,21 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 36.20 / 50 (72.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:11<00:00, 4.44it/s]" + "Average Metric: 43.37 / 50 (86.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:29<00:00, 1.71it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:34:34 INFO dspy.evaluate.evaluate: Average Metric: 36.199999999999996 / 50 (72.4%)\n", - "2025/07/02 20:34:34 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0, 72.4]\n", - "2025/07/02 20:34:34 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:34:34 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:34:34 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 18:35:48 INFO dspy.evaluate.evaluate: Average Metric: 43.36666666666666 / 50 (86.7%)\n", + "2026/01/29 18:35:48 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 86.73\n", + "2026/01/29 18:35:48 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53, 86.73]\n", + "2026/01/29 18:35:48 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.73\n", + "2026/01/29 18:35:48 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 18:35:48 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:34:34 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 32 / 37 - Minibatch ==\n" + "2026/01/29 18:35:48 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 32 / 37 - Minibatch ==\n" ] }, { @@ -1502,22 +2072,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 22.37 / 35 (63.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:27<00:00, 1.25it/s]" + "Average Metric: 29.77 / 35 (85.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.63it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:35:02 INFO dspy.evaluate.evaluate: Average Metric: 22.366666666666667 / 35 (63.9%)\n", - "2025/07/02 20:35:02 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 63.9 on minibatch of size 35 with parameters ['Predictor 0: Instruction 4', 'Predictor 0: Few-Shot Set 0'].\n", - "2025/07/02 20:35:02 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19, 75.14, 69.62, 75.05, 69.52, 63.9]\n", - "2025/07/02 20:35:02 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0, 72.4]\n", - "2025/07/02 20:35:02 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:35:02 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:36:09 INFO dspy.evaluate.evaluate: Average Metric: 29.766666666666655 / 35 (85.0%)\n", + "2026/01/29 18:36:09 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 85.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 5', 'Predictor 0: Few-Shot Set 2'].\n", + "2026/01/29 18:36:09 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05, 76.48, 80.48, 81.81, 82.76, 85.05]\n", + "2026/01/29 18:36:09 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53, 86.73]\n", + "2026/01/29 18:36:09 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.73\n", + "2026/01/29 18:36:09 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:35:02 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 33 / 37 - Minibatch ==\n" + "2026/01/29 18:36:09 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 33 / 37 - Minibatch ==\n" ] }, { @@ -1525,22 +2095,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.47 / 35 (69.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.35it/s]" + "Average Metric: 27.00 / 35 (77.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.74it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:35:28 INFO dspy.evaluate.evaluate: Average Metric: 24.466666666666665 / 35 (69.9%)\n", - "2025/07/02 20:35:28 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 69.9 on minibatch of size 35 with parameters ['Predictor 0: Instruction 15', 'Predictor 0: Few-Shot Set 7'].\n", - "2025/07/02 20:35:28 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19, 75.14, 69.62, 75.05, 69.52, 63.9, 69.9]\n", - "2025/07/02 20:35:28 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0, 72.4]\n", - "2025/07/02 20:35:28 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:35:28 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:36:30 INFO dspy.evaluate.evaluate: Average Metric: 26.999999999999996 / 35 (77.1%)\n", + "2026/01/29 18:36:30 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 77.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 4', 'Predictor 0: Few-Shot Set 5'].\n", + "2026/01/29 18:36:30 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05, 76.48, 80.48, 81.81, 82.76, 85.05, 77.14]\n", + "2026/01/29 18:36:30 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53, 86.73]\n", + "2026/01/29 18:36:30 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.73\n", + "2026/01/29 18:36:30 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:35:28 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 34 / 37 - Minibatch ==\n" + "2026/01/29 18:36:30 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 34 / 37 - Minibatch ==\n" ] }, { @@ -1548,22 +2118,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.93 / 35 (74.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:30<00:00, 1.13it/s]" + "Average Metric: 27.63 / 35 (79.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:20<00:00, 1.68it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:35:59 INFO dspy.evaluate.evaluate: Average Metric: 25.933333333333334 / 35 (74.1%)\n", - "2025/07/02 20:35:59 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 74.1 on minibatch of size 35 with parameters ['Predictor 0: Instruction 14', 'Predictor 0: Few-Shot Set 11'].\n", - "2025/07/02 20:35:59 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19, 75.14, 69.62, 75.05, 69.52, 63.9, 69.9, 74.1]\n", - "2025/07/02 20:35:59 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0, 72.4]\n", - "2025/07/02 20:35:59 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:35:59 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:36:50 INFO dspy.evaluate.evaluate: Average Metric: 27.633333333333326 / 35 (79.0%)\n", + "2026/01/29 18:36:50 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 78.95 on minibatch of size 35 with parameters ['Predictor 0: Instruction 7', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 18:36:50 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05, 76.48, 80.48, 81.81, 82.76, 85.05, 77.14, 78.95]\n", + "2026/01/29 18:36:50 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53, 86.73]\n", + "2026/01/29 18:36:50 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.73\n", + "2026/01/29 18:36:50 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:35:59 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 35 / 37 - Minibatch ==\n" + "2026/01/29 18:36:50 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 35 / 37 - Minibatch ==\n" ] }, { @@ -1571,22 +2141,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.80 / 35 (73.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:26<00:00, 1.32it/s]" + "Average Metric: 28.00 / 35 (80.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.66it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:36:26 INFO dspy.evaluate.evaluate: Average Metric: 25.8 / 35 (73.7%)\n", - "2025/07/02 20:36:26 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 73.71 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 11'].\n", - "2025/07/02 20:36:26 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19, 75.14, 69.62, 75.05, 69.52, 63.9, 69.9, 74.1, 73.71]\n", - "2025/07/02 20:36:26 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0, 72.4]\n", - "2025/07/02 20:36:26 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:36:26 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:37:12 INFO dspy.evaluate.evaluate: Average Metric: 27.99999999999999 / 35 (80.0%)\n", + "2026/01/29 18:37:12 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 80.0 on minibatch of size 35 with parameters ['Predictor 0: Instruction 5', 'Predictor 0: Few-Shot Set 5'].\n", + "2026/01/29 18:37:12 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05, 76.48, 80.48, 81.81, 82.76, 85.05, 77.14, 78.95, 80.0]\n", + "2026/01/29 18:37:12 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53, 86.73]\n", + "2026/01/29 18:37:12 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.73\n", + "2026/01/29 18:37:12 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:36:26 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 36 / 37 - Minibatch ==\n" + "2026/01/29 18:37:12 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 36 / 37 - Minibatch ==\n" ] }, { @@ -1594,23 +2164,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 22.57 / 35 (64.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:27<00:00, 1.27it/s]" + "Average Metric: 28.13 / 35 (80.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.65it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:36:53 INFO dspy.evaluate.evaluate: Average Metric: 22.566666666666666 / 35 (64.5%)\n", - "2025/07/02 20:36:53 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 64.48 on minibatch of size 35 with parameters ['Predictor 0: Instruction 13', 'Predictor 0: Few-Shot Set 8'].\n", - "2025/07/02 20:36:53 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.38, 68.67, 71.81, 67.71, 65.14, 70.48, 72.48, 66.38, 71.81, 71.9, 69.81, 69.62, 64.86, 65.62, 74.76, 70.48, 71.14, 76.76, 71.62, 73.14, 70.19, 75.14, 69.62, 75.05, 69.52, 63.9, 69.9, 74.1, 73.71, 64.48]\n", - "2025/07/02 20:36:53 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0, 72.4]\n", - "2025/07/02 20:36:53 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:36:53 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 18:37:33 INFO dspy.evaluate.evaluate: Average Metric: 28.133333333333326 / 35 (80.4%)\n", + "2026/01/29 18:37:33 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 80.38 on minibatch of size 35 with parameters ['Predictor 0: Instruction 2', 'Predictor 0: Few-Shot Set 16'].\n", + "2026/01/29 18:37:33 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [81.71, 79.05, 72.67, 84.1, 87.43, 75.05, 77.43, 82.38, 85.52, 84.38, 76.1, 72.0, 74.86, 87.33, 81.9, 80.57, 86.29, 85.43, 84.48, 84.86, 79.05, 76.48, 80.48, 81.81, 82.76, 85.05, 77.14, 78.95, 80.0, 80.38]\n", + "2026/01/29 18:37:33 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53, 86.73]\n", + "2026/01/29 18:37:33 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.73\n", + "2026/01/29 18:37:33 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:36:53 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 37 / 37 - Full Evaluation =====\n", - "2025/07/02 20:36:53 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 75.05) from minibatch trials...\n" + "2026/01/29 18:37:33 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 37 / 37 - Full Evaluation =====\n", + "2026/01/29 18:37:33 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 85.05) from minibatch trials...\n" ] }, { @@ -1618,20 +2188,20 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 36.47 / 50 (72.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:13<00:00, 3.84it/s]" + "Average Metric: 41.13 / 50 (82.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:29<00:00, 1.71it/s]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:37:06 INFO dspy.evaluate.evaluate: Average Metric: 36.46666666666667 / 50 (72.9%)\n", - "2025/07/02 20:37:06 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.4, 73.2, 71.07, 73.67, 74.0, 72.4, 72.93]\n", - "2025/07/02 20:37:06 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 74.0\n", - "2025/07/02 20:37:06 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:37:06 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 18:38:02 INFO dspy.evaluate.evaluate: Average Metric: 41.13333333333333 / 50 (82.3%)\n", + "2026/01/29 18:38:02 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [82.73, 82.0, 85.33, 85.4, 86.53, 86.73, 82.27]\n", + "2026/01/29 18:38:02 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 86.73\n", + "2026/01/29 18:38:02 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 18:38:02 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:37:06 INFO dspy.teleprompt.mipro_optimizer_v2: Returning best identified program with score 74.0!\n" + "2026/01/29 18:38:02 INFO dspy.teleprompt.mipro_optimizer_v2: Returning best identified program with score 86.73!\n" ] }, { @@ -1660,7 +2230,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:37:07 INFO amzn_nova_prompt_optimizer.core.input_adapters.prompt_adapter: \n", + "2026/01/29 18:38:02 INFO amzn_nova_prompt_optimizer.core.input_adapters.prompt_adapter: \n", "Standardized Prompt:\n" ] }, @@ -1673,14 +2243,14 @@ " \"variables\": [\n", " \"input\"\n", " ],\n", - " \"template\": \"Extract and return a json with the following keys and values from the input: [{{input}}]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\\n\\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces.\",\n", + " \"template\": \"The input provided to you is: {{input}}\",\n", " \"metadata\": {\n", " \"format\": \"text\"\n", " }\n", " },\n", " \"system_prompt\": {\n", " \"variables\": [],\n", - " \"template\": \"Analyze the customer service email to classify its content accurately. Return a JSON object with \\\"urgency\\\" (high/medium/low), \\\"sentiment\\\" (negative/neutral/positive), and \\\"categories\\\" (boolean flags for predefined types). Ensure compliance with data privacy standards by omitting personal identifiers. Follow the format strictly: `{\\\"urgency\\\": \\\"...\\\", \\\"sentiment\\\": \\\"...\\\", \\\"categories\\\": {\\\"...\\\": true/false, ...}}`.\",\n", + " \"template\": \"Generate a JSON object from the provided customer support email input. The JSON must contain three keys: \\\"urgency\\\", \\\"sentiment\\\", and \\\"categories\\\".\\n\\n**Key Requirements:**\\n1. **\\\"urgency\\\"**: Must be one of `high`, `medium`, or `low`, based on how time-sensitive the request is.\\n2. **\\\"sentiment\\\"**: Must be one of `negative`, `neutral`, or `positive`, reflecting the emotional tone of the email.\\n3. **\\\"categories\\\"**: Must be a dictionary where:\\n - The keys are from this fixed list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`.\\n - Each value is a boolean (`true` or `false`) indicating whether the category best matches the content of the email. Only categories that are clearly relevant should be set to `true`; all others should be `false`.\\n\\n**Steps to Follow:**\\n1. **Read the email carefully** to understand the main request, tone, and urgency.\\n2. **Determine urgency**:\\n - Use `high` if the issue requires immediate attention (e.g., an emergency or something causing significant disruption).\\n - Use `medium` for issues that need prompt attention but are not emergencies.\\n - Use `low` for non-urgent requests that can be addressed at a convenient time.\\n3. **Determine sentiment**:\\n - Use `positive` if the email expresses satisfaction, gratitude, or a constructive tone.\\n - Use `neutral` if the email is factual, informative, or lacks strong emotional language.\\n - Use `negative` if the email expresses dissatisfaction, frustration, or complaint.\\n4. **Identify categories**:\\n - For each category in the list, decide whether it accurately describes the main focus of the email.\\n - Set the corresponding value to `true` if the category applies; otherwise, set it to `false`.\\n - Only one or a few categories should be `true`\\u2014choose the most relevant ones based on the email content.\\n5. **Output the JSON**:\\n - The final output must be a **single-line JSON string** with no additional text, formatting, or encapsulation (e.g., no ```json...```).\\n - Ensure the JSON is compact with no unnecessary whitespace or newlines.\\n\\n**Example Output Format:**\\n```{\\\"urgency\\\": \\\"medium\\\", \\\"sentiment\\\": \\\"positive\\\", \\\"categories\\\": {\\\"emergency_repair_services\\\": false, \\\"routine_maintenance_requests\\\": true, \\\"quality_and_safety_concerns\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"general_inquiries\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"training_and_support_requests\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"facility_management_issues\\\": false}}```\\n\\nUse this format to process the input email and return the JSON object.\",\n", " \"metadata\": {\n", " \"format\": \"text\"\n", " }\n", @@ -1688,20 +2258,20 @@ " \"few_shot\": {\n", " \"examples\": [\n", " {\n", - " \"input\": \"Extract and return a json with the following keys and values from the input: [Subject: Urgent Request for Training and Support\\n\\nHi ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I manage a residential complex that has been utilizing ProCare Facility Solutions for our maintenance and cleaning needs for the past year. Your services have been instrumental in keeping our environment safe and well-maintained.\\n\\nHowever, I am reaching out with an urgent request. We are in immediate need of comprehensive training for our in-house maintenance team. Given the complexity of our facility's systems, particularly the HVAC and electrical components, it's crucial that our team is well-versed in best practices and troubleshooting techniques. \\n\\nWe have encountered several issues recently that have highlighted gaps in our current knowledge and capabilities. While we have managed to address these problems temporarily, a more sustainable solution is necessary. We need a detailed training program that covers routine maintenance, emergency repairs, and energy efficiency practices.\\n\\nCould you please provide information on the available training programs and how quickly we can schedule a session? Time is of the essence, as we aim to prevent any further disruptions to our facility's operations.\\n\\nThank you for your prompt attention to this matter. I look forward to your swift response.\\n\\nBest regards,\\n[Sender]]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\\n\\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces.\",\n", - " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": true, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"neutral\\\", \\\"urgency\\\": \\\"high\\\"}\"\n", + " \"input\": \"The input provided to you is: Subject: Inquiry About HVAC Maintenance Best Practices\\n\\nHi ProCare Support Team,\\n\\nI hope this message finds you well. My name is Alex, and I'm a novice electrician currently learning the ropes of working with HVAC units. I've recently started gaining hands-on experience and have been exploring various aspects of facility maintenance.\\n\\nI wanted to reach out to you with a few general inquiries regarding best practices for maintaining HVAC systems. Given your expertise in facility management and maintenance, I believe your insights would be incredibly valuable to me as I continue to develop my skills.\\n\\nSo far, I've been following basic maintenance routines like checking filters and ensuring proper airflow, but I would love to know if there are any specific tips or advanced techniques that I should be aware of. Additionally, are there any common pitfalls or mistakes that I should avoid while working with HVAC units?\\n\\nI haven't encountered any major issues yet, but I want to make sure I'm on the right track and doing everything correctly from the start. Any guidance or resources you could provide would be greatly appreciated. Also, if there are any training programs or support resources available for someone in my position, I would be very interested in learning more about them.\\n\\nThank you for your time and assistance. I look forward to hearing from you soon.\\n\\nBest regards, \\nAlex\",\n", + " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": true, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": true}, \\\"sentiment\\\": \\\"neutral\\\", \\\"urgency\\\": \\\"low\\\"}\"\n", " },\n", " {\n", - " \"input\": \"Extract and return a json with the following keys and values from the input: [Subject: Immediate Assistance Required for Facility Management Issue\\n\\nDear ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I am currently residing in one of the residential complexes managed by ProCare Facility Solutions. I am reaching out to you with an urgent matter that requires immediate attention.\\n\\nOver the past few days, I have noticed significant issues with the facility management in my building. Specifically, there have been recurring problems with the HVAC system, which has resulted in inconsistent heating and cooling. This has made it quite challenging to maintain a comfortable living environment, especially given the current weather conditions.\\n\\nI have already attempted to address this issue by contacting the building's maintenance staff directly, but unfortunately, the problem persists. Given the urgency of the situation, I am requesting that a qualified technician be dispatched as soon as possible to resolve this matter.\\n\\nYour prompt assistance in this regard would be greatly appreciated, as it is crucial for me to have a stable and comfortable living environment to focus on my studies.\\n\\nThank you for your understanding and swift action.\\n\\nBest regards,\\n[Sender]]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\\n\\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces.\",\n", - " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": true, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"negative\\\", \\\"urgency\\\": \\\"high\\\"}\"\n", + " \"input\": \"The input provided to you is: Subject: Request for Training and Support on Latest Facility Management Trends\\n\\nHi ProCare Support Team,\\n\\nHope you're all doing great! My name is Alex, and I've been following ProCare Facility Solutions for a while now. I really appreciate the top-notch services you provide, especially your commitment to sustainability and quality.\\n\\nI'm reaching out because I think there's a fantastic opportunity for us to stay ahead of the curve with some of the latest online trends in facility management. As someone who grew up with the internet, I've noticed a lot of new tools and practices emerging that could really benefit our operations. I believe integrating these trends could enhance our efficiency and overall service quality.\\n\\nI've done some preliminary research and tried to implement a few of these trends myself, but I think a more structured training program from your end would be incredibly beneficial. Specifically, I'm interested in learning more about the latest best practices in energy efficiency and environmental impact reduction, as well as any new technologies that could streamline our maintenance and cleaning processes.\\n\\nCould you please provide some guidance or set up a training session to help us get up to speed with these new trends? I think it would be a great addition to our current practices and help us maintain our high standards.\\n\\nThanks a lot for your time and assistance. Looking forward to hearing from you soon!\\n\\nBest regards,\\nAlex\",\n", + " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": true, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": true, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": true, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"positive\\\", \\\"urgency\\\": \\\"low\\\"}\"\n", " },\n", " {\n", - " \"input\": \"Extract and return a json with the following keys and values from the input: [Subject: Scheduling Cleaning Services for My Home\\n\\nHi ProCare Support Team,\\n\\nI hope this message finds you well. My name is Alex, and I\\u2019m a high school student currently living in a residential complex that uses your services. I\\u2019ve heard great things about ProCare Facility Solutions from my neighbors and have seen firsthand the excellent work your team does.\\n\\nI\\u2019m reaching out because I need to schedule a cleaning service for my home. With school and college applications taking up most of my time, I haven\\u2019t been able to keep up with the cleaning as much as I\\u2019d like. I\\u2019m looking for a regular cleaning schedule, preferably weekly, to help maintain a clean and healthy environment.\\n\\nI haven\\u2019t taken any steps yet to resolve this, as I wanted to get in touch with you directly to understand the best options available. Could you please guide me on how to set up a cleaning schedule and what the next steps would be?\\n\\nThank you for your assistance. I look forward to hearing from you soon.\\n\\nBest regards,\\nAlex]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\\n\\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces.\",\n", - " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": true, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"neutral\\\", \\\"urgency\\\": \\\"low\\\"}\"\n", + " \"input\": \"The input provided to you is: Subject: Immediate Attention Required for Sustainability Practices\\n\\nDear ProCare Support Team,\\n\\nI hope this message finds you well, though I must admit, I am quite frustrated as I write this. My name is [Sender], and I have been utilizing your services for my dairy farm's facility management for the past year. While I initially chose ProCare Facility Solutions for your reputed expertise and commitment to quality, recent experiences have left me questioning that decision.\\n\\nAs a dairy farmer who deeply values sustainable practices, I was particularly drawn to your promise of implementing best practices for energy efficiency and environmental impact reduction. However, I have noticed a significant lack of follow-through in this area. Despite multiple assurances, there has been no visible effort to incorporate eco-friendly solutions into the maintenance and cleaning services provided to my farm.\\n\\nI have already reached out to your team on two separate occasions to address this issue, but the responses have been unsatisfactory and vague at best. I was assured that someone would look into it, yet here we are, with no tangible progress or updates.\\n\\nI am requesting immediate and concrete action to rectify this situation. Specifically, I need a detailed plan outlining how ProCare intends to integrate sustainable practices into the services provided to my farm. This includes the use of eco-friendly cleaning products, energy-efficient maintenance solutions, and any other measures that align with your advertised commitment to sustainability.\\n\\nI trust that you will treat this matter with the urgency it deserves and provide a satisfactory resolution promptly. Failure to do so will force me to reconsider my association with ProCare Facility Solutions.\\n\\nLooking forward to your prompt response.\\n\\nBest regards,\\n[Sender]\",\n", + " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": true, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": true, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": true, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": true, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"negative\\\", \\\"urgency\\\": \\\"high\\\"}\"\n", " },\n", " {\n", - " \"input\": \"Extract and return a json with the following keys and values from the input: [Subject: Immediate Attention Required: Emergency Repair Needed\\n\\nHey ProCare Support Team,\\n\\nThis is [Sender], and I have to say, I'm not impressed. I've been using your services for a while now, and I expected better. Right now, I'm dealing with a major issue that needs your immediate attention.\\n\\nThe HVAC system in my building has completely failed, and it's causing a lot of problems. The temperature is unbearable, and it's affecting everyone here. This isn't just an inconvenience; it's a serious problem that needs to be fixed right away. I've tried resetting the system and checking the circuit breakers, but nothing has worked.\\n\\nI need your team to come out and fix this immediately. This kind of failure is unacceptable, and I expect a prompt response. If this isn't resolved quickly, I might have to reconsider using your services in the future.\\n\\nLooking forward to your swift action.\\n\\n[Sender]]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\\n\\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces.\",\n", - " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": true, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": true, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"negative\\\", \\\"urgency\\\": \\\"high\\\"}\"\n", + " \"input\": \"The input provided to you is: Subject: Scheduled Maintenance Request for Bathroom Plumbing\\n\\nDear ProCare Facility Solutions Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I have been utilizing your services for my residential property for some time now. I have always appreciated the quality and reliability of your maintenance services.\\n\\nI am writing to inform you of a minor issue that has recently come up. There seems to be a small problem with the plumbing in my bathroom. While it is not an urgent matter, I would like to have it addressed at your earliest convenience to prevent any potential complications.\\n\\nI have not taken any steps to resolve the issue myself, as I trust your team\\u2019s expertise in handling such matters. Could you please arrange for a technician to visit my home and take a look at the problem as part of your scheduled maintenance services?\\n\\nThank you for your attention to this matter. I look forward to your prompt response.\\n\\nBest regards,\\n[Sender]\",\n", + " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": true, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"neutral\\\", \\\"urgency\\\": \\\"medium\\\"}\"\n", " }\n", " ],\n", " \"format\": \"converse\"\n", @@ -1732,7 +2302,37 @@ "name": "stdout", "output_type": "stream", "text": [ - "Analyze the customer service email to classify its content accurately. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and \"categories\" (boolean flags for predefined types). Ensure compliance with data privacy standards by omitting personal identifiers. Follow the format strictly: `{\"urgency\": \"...\", \"sentiment\": \"...\", \"categories\": {\"...\": true/false, ...}}`.\n" + "Generate a JSON object from the provided customer support email input. The JSON must contain three keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "\n", + "**Key Requirements:**\n", + "1. **\"urgency\"**: Must be one of `high`, `medium`, or `low`, based on how time-sensitive the request is.\n", + "2. **\"sentiment\"**: Must be one of `negative`, `neutral`, or `positive`, reflecting the emotional tone of the email.\n", + "3. **\"categories\"**: Must be a dictionary where:\n", + " - The keys are from this fixed list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`.\n", + " - Each value is a boolean (`true` or `false`) indicating whether the category best matches the content of the email. Only categories that are clearly relevant should be set to `true`; all others should be `false`.\n", + "\n", + "**Steps to Follow:**\n", + "1. **Read the email carefully** to understand the main request, tone, and urgency.\n", + "2. **Determine urgency**:\n", + " - Use `high` if the issue requires immediate attention (e.g., an emergency or something causing significant disruption).\n", + " - Use `medium` for issues that need prompt attention but are not emergencies.\n", + " - Use `low` for non-urgent requests that can be addressed at a convenient time.\n", + "3. **Determine sentiment**:\n", + " - Use `positive` if the email expresses satisfaction, gratitude, or a constructive tone.\n", + " - Use `neutral` if the email is factual, informative, or lacks strong emotional language.\n", + " - Use `negative` if the email expresses dissatisfaction, frustration, or complaint.\n", + "4. **Identify categories**:\n", + " - For each category in the list, decide whether it accurately describes the main focus of the email.\n", + " - Set the corresponding value to `true` if the category applies; otherwise, set it to `false`.\n", + " - Only one or a few categories should be `true`β€”choose the most relevant ones based on the email content.\n", + "5. **Output the JSON**:\n", + " - The final output must be a **single-line JSON string** with no additional text, formatting, or encapsulation (e.g., no ```json...```).\n", + " - Ensure the JSON is compact with no unnecessary whitespace or newlines.\n", + "\n", + "**Example Output Format:**\n", + "```{\"urgency\": \"medium\", \"sentiment\": \"positive\", \"categories\": {\"emergency_repair_services\": false, \"routine_maintenance_requests\": true, \"quality_and_safety_concerns\": false, \"specialized_cleaning_services\": false, \"general_inquiries\": false, \"sustainability_and_environmental_practices\": false, \"training_and_support_requests\": false, \"cleaning_services_scheduling\": false, \"customer_feedback_and_complaints\": false, \"facility_management_issues\": false}}```\n", + "\n", + "Use this format to process the input email and return the JSON object.\n" ] } ], @@ -1758,12 +2358,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Extract and return a json with the following keys and values from the input: [{{input}}]\n", - "- \"urgency\" as one of `high`, `medium`, `low`\n", - "- \"sentiment\" as one of `negative`, `neutral`, `positive`\n", - "- \"categories\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\n", - "\n", - "Your complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces.\n" + "The input provided to you is: {{input}}\n" ] } ], @@ -1807,7 +2402,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'input': 'Extract and return a json with the following keys and values from the input: [Subject: Urgent Request for Training and Support\\n\\nHi ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I manage a residential complex that has been utilizing ProCare Facility Solutions for our maintenance and cleaning needs for the past year. Your services have been instrumental in keeping our environment safe and well-maintained.\\n\\nHowever, I am reaching out with an urgent request. We are in immediate need of comprehensive training for our in-house maintenance team. Given the complexity of our facility\\'s systems, particularly the HVAC and electrical components, it\\'s crucial that our team is well-versed in best practices and troubleshooting techniques. \\n\\nWe have encountered several issues recently that have highlighted gaps in our current knowledge and capabilities. While we have managed to address these problems temporarily, a more sustainable solution is necessary. We need a detailed training program that covers routine maintenance, emergency repairs, and energy efficiency practices.\\n\\nCould you please provide information on the available training programs and how quickly we can schedule a session? Time is of the essence, as we aim to prevent any further disruptions to our facility\\'s operations.\\n\\nThank you for your prompt attention to this matter. I look forward to your swift response.\\n\\nBest regards,\\n[Sender]]\\n- \"urgency\" as one of `high`, `medium`, `low`\\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\\n- \"categories\" as a dictionary with boolean values for: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\\n\\nYour complete message should be a valid json string that can be read directly and only contain the keys mentioned in the list above. Never enclose it in ```json...```, no newlines, no unnecessary whitespaces.', 'output': '{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": true, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"neutral\", \"urgency\": \"high\"}'}\n" + "{'input': \"The input provided to you is: Subject: Inquiry About HVAC Maintenance Best Practices\\n\\nHi ProCare Support Team,\\n\\nI hope this message finds you well. My name is Alex, and I'm a novice electrician currently learning the ropes of working with HVAC units. I've recently started gaining hands-on experience and have been exploring various aspects of facility maintenance.\\n\\nI wanted to reach out to you with a few general inquiries regarding best practices for maintaining HVAC systems. Given your expertise in facility management and maintenance, I believe your insights would be incredibly valuable to me as I continue to develop my skills.\\n\\nSo far, I've been following basic maintenance routines like checking filters and ensuring proper airflow, but I would love to know if there are any specific tips or advanced techniques that I should be aware of. Additionally, are there any common pitfalls or mistakes that I should avoid while working with HVAC units?\\n\\nI haven't encountered any major issues yet, but I want to make sure I'm on the right track and doing everything correctly from the start. Any guidance or resources you could provide would be greatly appreciated. Also, if there are any training programs or support resources available for someone in my position, I would be very interested in learning more about them.\\n\\nThank you for your time and assistance. I look forward to hearing from you soon.\\n\\nBest regards, \\nAlex\", 'output': '{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": true, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": true}, \"sentiment\": \"neutral\", \"urgency\": \"low\"}'}\n" ] } ], @@ -1848,18 +2443,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:40:54 INFO amzn_nova_prompt_optimizer.core.evaluation: Cache miss - Running new inference on Dataset\n", - "Running inference: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100/100 [00:45<00:00, 2.18it/s]\n", - "2025/07/02 20:41:40 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Batch Evaluation on Dataset, using `batch_apply` metric\n", - "2025/07/02 20:41:40 INFO amzn_nova_prompt_optimizer.core.evaluation: Using cached inference results\n", - "2025/07/02 20:41:40 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Evaluation on Dataset, using `apply` metric\n" + "2026/01/29 18:38:02 INFO amzn_nova_prompt_optimizer.core.evaluation: Cache miss - Running new inference on Dataset\n", + "Running inference: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100/100 [00:50<00:00, 1.97it/s]\n", + "2026/01/29 18:38:53 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Batch Evaluation on Dataset, using `batch_apply` metric\n", + "2026/01/29 18:38:53 INFO amzn_nova_prompt_optimizer.core.evaluation: Using cached inference results\n", + "2026/01/29 18:38:53 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Evaluation on Dataset, using `apply` metric\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Nova Prompt Optimizer = {'is_valid_json': 1.0, 'correct_categories': 0.95, 'correct_sentiment': 0.65, 'correct_urgency': 0.77, 'total': 0.79}\n" + "Nova Prompt Optimizer = {'is_valid_json': 1.0, 'correct_categories': 0.912, 'correct_sentiment': 0.75, 'correct_urgency': 0.9, 'total': 0.8540000000000006}\n" ] } ], @@ -1881,9 +2476,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.12 (.venv)", + "display_name": "Python 3", "language": "python", - "name": "python312venv" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1895,7 +2490,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.11" + "version": "3.9.6" } }, "nbformat": 4, diff --git a/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/few_shot.json b/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/few_shot.json index 41e3204..1b10c6c 100644 --- a/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/few_shot.json +++ b/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/few_shot.json @@ -3,7 +3,7 @@ "role": "user", "content": [ { - "text": "Extract and return a json with the following keys and values from the input provided: [Subject: Request for Post-Renovation Cleaning Services\n\nDear ProCare Support Team,\n\nI hope this message finds you well. My name is [Sender], and I have been utilizing ProCare Facility Solutions for the maintenance and management of my commercial property for the past few years. Your services have always been reliable and efficient, which is why I am reaching out to you today.\n\nI am writing to request assistance with a specialized cleaning service for our office building. We have recently undergone some renovations, and there is a significant amount of dust and debris that needs to be addressed. Additionally, the carpets and windows require a thorough cleaning to restore them to their original condition.\n\nI have already scheduled a routine cleaning, but I believe this situation requires more specialized attention. Given the nature of the work needed, I would appreciate it if you could arrange for a team that specializes in deep cleaning and post-renovation cleanup.\n\nCould you please provide me with the available dates and any additional information required to proceed with this request? Your prompt assistance in this matter would be greatly appreciated, as we aim to have the office fully operational and presentable as soon as possible.\n\nThank you for your attention to this matter. I look forward to your response.\n\nBest regards,\n\n[Sender]]\n- \"urgency\" as one of `high`, `medium`, `low`\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\n- \"categories\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`" + "text": "Subject: Scheduled Maintenance Request for Bathroom Plumbing\n\nDear ProCare Facility Solutions Support Team,\n\nI hope this message finds you well. My name is [Sender], and I have been utilizing your services for my residential property for some time now. I have always appreciated the quality and reliability of your maintenance services.\n\nI am writing to inform you of a minor issue that has recently come up. There seems to be a small problem with the plumbing in my bathroom. While it is not an urgent matter, I would like to have it addressed at your earliest convenience to prevent any potential complications.\n\nI have not taken any steps to resolve the issue myself, as I trust your team\u2019s expertise in handling such matters. Could you please arrange for a technician to visit my home and take a look at the problem as part of your scheduled maintenance services?\n\nThank you for your attention to this matter. I look forward to your prompt response.\n\nBest regards,\n[Sender]" } ] }, @@ -11,7 +11,7 @@ "role": "assistant", "content": [ { - "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": true, \"specialized_cleaning_services\": true, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"neutral\", \"urgency\": \"medium\"}" + "text": " {\"categories\": {\"routine_maintenance_requests\": true, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": true, \"general_inquiries\": false}, \"sentiment\": \"neutral\", \"urgency\": \"medium\"}" } ] }, @@ -19,7 +19,7 @@ "role": "user", "content": [ { - "text": "Extract and return a json with the following keys and values from the input provided: [Subject: Urgent HVAC System Repair Needed\n\nDear ProCare Support Team,\n\nI hope this message finds you well. My name is [Sender], and I am the curator at [Museum Name]. We have been utilizing ProCare Facility Solutions for our maintenance needs for the past year, and I have always appreciated the professionalism and quality of your services.\n\nI am writing to inform you of an urgent issue we are currently facing with our HVAC system. Over the past few days, we have noticed irregularities in the temperature control within our exhibit halls. This is particularly concerning as we have several sensitive classical art pieces that require a stable climate to ensure their preservation.\n\nWe have attempted to adjust the settings manually and have conducted a basic inspection of the system, but the problem persists. Given the importance of maintaining an optimal environment for our collections, I would appreciate it if you could send a technician to address this issue immediately.\n\nThank you for your prompt attention to this matter. I look forward to your swift response and resolution.\n\nBest regards,\n\n[Sender] \n[Museum Name] \n[Contact Information]]\n- \"urgency\" as one of `high`, `medium`, `low`\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\n- \"categories\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`" + "text": "Subject: Follow-Up on HVAC Maintenance Issue\n\nDear ProCare Facility Solutions Support Team,\n\nI hope this message finds you well. My name is Dr. [Sender], a retired professor and a long-time resident of [Residential Complex Name]. I have been utilizing your maintenance services for some time now and generally appreciate the professionalism and thoroughness your team brings to the table.\n\nHowever, I would like to bring to your attention a recent issue I encountered with the routine maintenance of the HVAC system in my apartment. While the technician was courteous and seemed knowledgeable, the problem with the system persists. Despite the service visit, the HVAC unit continues to make an unusual noise, which is quite disruptive.\n\nI have not taken any further steps beyond the initial service call, as I wanted to first communicate my concerns directly with your support team. I would appreciate it if you could arrange for a follow-up visit to address this issue more comprehensively.\n\nThank you for your attention to this matter. I look forward to your prompt response and resolution.\n\nBest regards,\nDr. [Sender]" } ] }, @@ -27,7 +27,7 @@ "role": "assistant", "content": [ { - "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": true, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": true, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"neutral\", \"urgency\": \"high\"}" + "text": " {\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": true, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": true, \"general_inquiries\": false}, \"sentiment\": \"negative\", \"urgency\": \"medium\"}" } ] }, @@ -35,7 +35,7 @@ "role": "user", "content": [ { - "text": "Extract and return a json with the following keys and values from the input provided: [Subject: Routine Maintenance Request for Theater Facility\n\nDear ProCare Support Team,\n\nI hope this message finds you well. My name is [Sender], and I am a theater producer who has been working closely with your team to ensure our facility remains in top condition. I have always admired the innovative methods your team employs, which have significantly contributed to the smooth operation of our workshops and classes.\n\nI am writing to request routine maintenance for our theater facility. Specifically, we have noticed that the HVAC system is not performing optimally, and there are minor plumbing issues in the restrooms that need attention. These issues are not urgent but do require timely intervention to prevent any disruption to our scheduled activities.\n\nWe have not taken any steps to address these issues internally, as we prefer to rely on your expertise to ensure everything is handled correctly and efficiently.\n\nCould you please arrange for a maintenance visit at your earliest convenience? We would appreciate it if the visit could be scheduled within the next week to ensure everything is in order before our next major event.\n\nThank you for your attention to this matter. We look forward to your prompt response and continued excellent service.\n\nBest regards,\n\n[Sender]]\n- \"urgency\" as one of `high`, `medium`, `low`\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\n- \"categories\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`" + "text": "Subject: Request for Eco-Friendly Deep Cleaning Services\n\nHi ProCare Support Team,\n\nI hope this message finds you well. My name is [Sender], and I\u2019ve been a client of ProCare Facility Solutions for a while now. As someone who\u2019s spent years in the fast lane, both on and off the track, I appreciate the importance of precision and attention to detail, which is why I\u2019ve always trusted your services.\n\nI\u2019m reaching out today because I need some specialized cleaning services for my property. Specifically, I\u2019m looking for a deep cleaning of my garage and workshop area. These spaces have accumulated quite a bit of grime and dust over time, and I\u2019d like to get them back to a pristine condition. Given the nature of the work I do there, it\u2019s crucial that the cleaning is thorough and uses eco-friendly products.\n\nI haven\u2019t taken any steps to address this issue yet, as I wanted to consult with the experts first. I\u2019m hoping you can provide a customized cleaning plan that fits my needs and schedule.\n\nCould you please let me know the next steps and any details you need from my end to get this sorted? I\u2019m looking forward to your assistance in making my workspace spotless again.\n\nThanks in advance for your help.\n\nBest regards,\n[Sender]" } ] }, @@ -43,7 +43,7 @@ "role": "assistant", "content": [ { - "text": "{\"categories\": {\"routine_maintenance_requests\": true, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"positive\", \"urgency\": \"medium\"}" + "text": " {\"categories\": {\"specialized_cleaning_services\": true, \"sustainability_and_environmental_practices\": true, \"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"cleaning_services_scheduling\": false, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"positive\", \"urgency\": \"medium\"}" } ] }, @@ -51,7 +51,7 @@ "role": "user", "content": [ { - "text": "Extract and return a json with the following keys and values from the input provided: [Subject: Scheduling Cleaning Services for Upcoming Event\n\nHi ProCare Support Team,\n\nI hope this message finds you well. My name is [Sender], and I\u2019ve been a loyal customer of ProCare Facility Solutions for the past year. I manage the performance analysis for the New York Red Bulls, and I must say, your services have been instrumental in maintaining our office environment, allowing us to focus on our work without any distractions.\n\nI\u2019m reaching out to schedule a cleaning service for our office space. We have an important event coming up next month, and I want to ensure that everything is in pristine condition for our guests. Given the nature of our work, a clean and organized environment is crucial for our productivity and overall morale.\n\nI haven\u2019t taken any steps yet to schedule this service, as I wanted to get in touch with your team directly to ensure we get the best possible arrangement. Ideally, we would need a thorough cleaning a few days before the event, including window washing and carpet maintenance.\n\nCould you please assist me in setting up a suitable cleaning schedule? I\u2019m confident that your team will handle this with the same excellence and attention to detail that we\u2019ve come to expect from ProCare.\n\nThank you so much for your help. Looking forward to your response.\n\nBest regards,\n[Sender]]\n- \"urgency\" as one of `high`, `medium`, `low`\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\n- \"categories\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`" + "text": "Subject: Urgent Assistance Needed for HVAC System Issue\n\nHi ProCare Support Team,\n\nI hope this message finds you well. My name is [Sender], and I\u2019ve been a satisfied client of ProCare Facility Solutions for the past year. Your team has always provided exceptional service, and I truly appreciate the dedication and professionalism you bring to maintaining my property.\n\nHowever, I\u2019m currently facing an urgent issue that requires immediate attention. Over the past few days, I\u2019ve noticed a significant drop in the efficiency of the HVAC system in my home. Despite the routine maintenance checks, the system seems to be struggling to maintain a consistent temperature, which is crucial for my training and recovery as a professional athlete.\n\nI\u2019ve already tried adjusting the thermostat and checking the filters, but the problem persists. Given the high stakes of my athletic performance, I need this issue resolved as quickly as possible to ensure my living environment remains optimal for my needs.\n\nCould you please arrange for an emergency repair at the earliest convenience? Your prompt assistance in this matter would be greatly appreciated, as it directly impacts my daily routine and overall well-being.\n\nThank you for your understanding and swift action. I look forward to hearing from you soon.\n\nBest regards,\n[Sender]" } ] }, @@ -59,7 +59,7 @@ "role": "assistant", "content": [ { - "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": true, \"specialized_cleaning_services\": true, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"positive\", \"urgency\": \"medium\"}" + "text": "{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": true, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"positive\", \"urgency\": \"high\"}" } ] } diff --git a/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/system_prompt.txt b/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/system_prompt.txt index 237abe9..0321933 100644 --- a/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/system_prompt.txt +++ b/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/system_prompt.txt @@ -1 +1,19 @@ -Analyze the input text and classify it into a JSON object with "urgency", "sentiment", and "categories" keys. Ensure all categories are covered with boolean values, and the output is a valid JSON string without extra formatting. \ No newline at end of file +Task: Analyze and categorize the incoming text to determine urgency, sentiment, and relevant categories. + +Context: +- The input will be analyzed to determine urgency, sentiment, and relevant categories. +- Urgency must be classified as one of `high`, `medium`, or `low`. +- Sentiment must be classified as one of `negative`, `neutral`, or `positive`. +- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag. + +Instructions: +- Carefully analyze the input to accurately determine urgency and sentiment. +- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False. +- The output must be a valid JSON string containing only the specified keys: "urgency", "sentiment", and "categories". +- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting. + +Response Format: +- The response MUST be a valid JSON string. +- The JSON MUST contain exactly the keys: "urgency", "sentiment", and "categories". +- The JSON MUST be on a single line with no extra spaces or line breaks. +- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure. \ No newline at end of file diff --git a/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/user_prompt.txt b/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/user_prompt.txt index c0d0c8f..a88c866 100644 --- a/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/user_prompt.txt +++ b/samples/facility-support-analyzer/user_prompt_only/optimized_prompt/user_prompt.txt @@ -1,4 +1 @@ -Extract and return a json with the following keys and values from the input provided: [{{ input }}] -- "urgency" as one of `high`, `medium`, `low` -- "sentiment" as one of `negative`, `neutral`, `positive` -- "categories" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues` \ No newline at end of file +{{ input }} \ No newline at end of file diff --git a/samples/facility-support-analyzer/user_prompt_only/sample_user_prompt_template_optimization.ipynb b/samples/facility-support-analyzer/user_prompt_only/sample_user_prompt_template_optimization.ipynb index f030dc9..b155d87 100644 --- a/samples/facility-support-analyzer/user_prompt_only/sample_user_prompt_template_optimization.ipynb +++ b/samples/facility-support-analyzer/user_prompt_only/sample_user_prompt_template_optimization.ipynb @@ -10,26 +10,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "8954cb65-5eb1-4e76-92bb-331be63f7e73", "metadata": {}, "outputs": [], "source": [ "import os\n", "# Setup your AWS Access Key and Secret Key as environment variables.\n", - "os.environ[\"AWS_ACCESS_KEY_ID\"]\n", - "os.environ[\"AWS_SECRET_ACCESS_KEY\"] " + "#os.environ[\"AWS_ACCESS_KEY_ID\"]\n", + "#os.environ[\"AWS_SECRET_ACCESS_KEY\"] \n", + "\n", + "import sys\n", + "sys.path.insert(0, '../../../src')" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "51fc0d85", "metadata": {}, "outputs": [], "source": [ - "# Setup Nova Model\n", - "NOVA_MODEL_ID = \"us.amazon.nova-pro-v1:0\"" + "# Setup Nova Model (For SageMaker Inference this is just a label since the endpoint is the model)\n", + "NOVA_MODEL_ID = \"nova-micro\"" ] }, { @@ -44,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 38, "id": "da047488-0220-4e49-b8d0-361c5afcdfb4", "metadata": {}, "outputs": [], @@ -74,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "621dc01d-6896-4a37-ad64-63f27c597f75", "metadata": {}, "outputs": [ @@ -82,16 +85,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:45:28 INFO amzn_nova_prompt_optimizer.core.input_adapters.prompt_adapter: System Prompt not set, initializing as empty string...\n" + "2026/01/29 19:33:26 INFO amzn_nova_prompt_optimizer.core.input_adapters.prompt_adapter: System Prompt not set, initializing as empty string...\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -121,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "f093a481-31a0-4871-99a0-670a60d67b1d", "metadata": {}, "outputs": [], @@ -227,14 +230,26 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "ea654ddd-59d0-495e-8e56-29fe0ed6dd7b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026/01/29 21:37:25 INFO amzn_nova_prompt_optimizer.core.inference.sagemaker_adapter: Initialized SageMaker adapter for endpoint: jezhu-micro-rai-fixed-Jan21-endpoint\n" + ] + } + ], "source": [ - "from amzn_nova_prompt_optimizer.core.inference.adapter import BedrockInferenceAdapter\n", + "from amzn_nova_prompt_optimizer.core.inference import SageMakerInferenceAdapter\n", "\n", - "inference_adapter = BedrockInferenceAdapter(region_name=\"us-east-1\")" + "inference_adapter = SageMakerInferenceAdapter(\n", + " endpoint_name=\"jezhu-micro-rai-fixed-Jan21-endpoint\",\n", + " region_name=\"us-west-2\",\n", + " rate_limit=10\n", + ")" ] }, { @@ -257,7 +272,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 40, "id": "9f158430-ed74-470c-8657-2b49b57ae79f", "metadata": {}, "outputs": [], @@ -277,18 +292,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 04:35:03 INFO amzn_nova_prompt_optimizer.core.evaluation: Cache miss - Running new inference on Dataset\n", - "Running inference: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100/100 [00:45<00:00, 2.21it/s]\n", - "2025/07/02 04:35:48 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Batch Evaluation on Dataset, using `batch_apply` metric\n", - "2025/07/02 04:35:48 INFO amzn_nova_prompt_optimizer.core.evaluation: Using cached inference results\n", - "2025/07/02 04:35:48 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Evaluation on Dataset, using `apply` metric\n" + "2026/01/29 19:33:26 INFO amzn_nova_prompt_optimizer.core.evaluation: Cache miss - Running new inference on Dataset\n", + "Running inference: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100/100 [01:30<00:00, 1.11it/s]\n", + "2026/01/29 19:34:57 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Batch Evaluation on Dataset, using `batch_apply` metric\n", + "2026/01/29 19:34:57 INFO amzn_nova_prompt_optimizer.core.evaluation: Using cached inference results\n", + "2026/01/29 19:34:57 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Evaluation on Dataset, using `apply` metric\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Original Prompt Evaluation Score = {'is_valid_json': 0.27, 'correct_categories': 0.251, 'correct_sentiment': 0.13, 'correct_urgency': 0.2, 'total': 0.19366666666666668}\n" + "Original Prompt Evaluation Score = {'is_valid_json': 1.0, 'correct_categories': 0.917, 'correct_sentiment': 0.64, 'correct_urgency': 0.66, 'total': 0.7390000000000001}\n" ] } ], @@ -310,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "id": "8241fc92", "metadata": {}, "outputs": [], @@ -337,7 +352,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 22, "id": "d7c2d973", "metadata": {}, "outputs": [ @@ -345,16 +360,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:45:38 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_meta_prompter.nova_mp_optimizer: Optimizing prompt using Nova Meta Prompter\n", - "2025/07/02 20:45:44 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using us.amazon.nova-pro-v1:0 for Evaluation\n", - "2025/07/02 20:45:44 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using us.amazon.nova-premier-v1:0 for Prompting\n", - "2025/07/02 20:45:44 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.custom_adapters.custom_chat_adapter: Initializing CustomChatAdapter with enable_json_fallback=False\n", - "2025/07/02 20:45:44 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using Nova tips for MIPROv2 optimization\n", - "2025/07/02 20:45:44 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 20:45:45 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer: No meta_prompt_inference_adapter provided. Creating default BedrockInferenceAdapter with Nova 2.0 Lite for meta-prompting.\n", + "2026/01/29 20:45:45 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer: Using separate inference adapters: Meta-prompting with BedrockInferenceAdapter, Task optimization with SageMakerInferenceAdapter\n", + "2026/01/29 20:45:45 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_meta_prompter.nova_mp_optimizer: Optimizing prompt using Nova Meta Prompter with Model: us.amazon.nova-2-lite-v1:0\n", + "2026/01/29 20:45:47 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using us.amazon.nova-micro-v1:0 for Evaluation\n", + "2026/01/29 20:45:47 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using us.amazon.nova-2-lite-v1:0 for Prompting\n", + "2026/01/29 20:45:47 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.custom_adapters.custom_chat_adapter: Initializing CustomChatAdapter with enable_json_fallback=False\n", + "2026/01/29 20:45:47 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Using Nova tips for MIPROv2 optimization\n", + "2026/01/29 20:45:47 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "==> STEP 1: BOOTSTRAP FEWSHOT EXAMPLES <==\n", - "2025/07/02 20:45:44 INFO dspy.teleprompt.mipro_optimizer_v2: These will be used as few-shot example candidates for our program and for creating instructions.\n", + "2026/01/29 20:45:47 INFO dspy.teleprompt.mipro_optimizer_v2: These will be used as few-shot example candidates for our program and for creating instructions.\n", "\n", - "2025/07/02 20:45:44 INFO dspy.teleprompt.mipro_optimizer_v2: Bootstrapping N=20 sets of demonstrations...\n" + "2026/01/29 20:45:47 INFO dspy.teleprompt.mipro_optimizer_v2: Bootstrapping N=20 sets of demonstrations...\n" ] }, { @@ -370,7 +387,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 8%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 4/50 [00:06<01:09, 1.51s/it]\n" + " 8%|β–Š | 4/50 [00:12<02:27, 3.20s/it]\n" ] }, { @@ -385,7 +402,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 8%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 4/50 [00:05<01:04, 1.41s/it]\n" + " 8%|β–Š | 4/50 [00:12<02:19, 3.04s/it]\n" ] }, { @@ -400,7 +417,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 6%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 3/50 [00:04<01:06, 1.41s/it]\n" + " 6%|β–Œ | 3/50 [00:09<02:30, 3.20s/it]\n" ] }, { @@ -415,7 +432,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 4%|β–ˆβ–ˆβ–ˆβ–Š | 2/50 [00:03<01:18, 1.63s/it]\n" + " 4%|▍ | 2/50 [00:06<02:31, 3.15s/it]\n" ] }, { @@ -430,7 +447,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 2%|β–ˆβ–‰ | 1/50 [00:01<01:05, 1.34s/it]\n" + " 2%|▏ | 1/50 [00:03<02:43, 3.33s/it]\n" ] }, { @@ -445,7 +462,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 6%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 3/50 [00:04<01:04, 1.37s/it]\n" + " 6%|β–Œ | 3/50 [00:09<02:27, 3.15s/it]\n" ] }, { @@ -460,7 +477,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 8%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 4/50 [00:05<01:02, 1.37s/it]\n" + " 8%|β–Š | 4/50 [00:12<02:22, 3.09s/it]\n" ] }, { @@ -475,7 +492,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 6%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 3/50 [00:03<01:02, 1.33s/it]\n" + " 6%|β–Œ | 3/50 [00:09<02:24, 3.07s/it]\n" ] }, { @@ -490,7 +507,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 2%|β–ˆβ–‰ | 1/50 [00:01<01:10, 1.45s/it]\n" + " 2%|▏ | 1/50 [00:03<02:37, 3.22s/it]\n" ] }, { @@ -505,7 +522,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 6%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 3/50 [00:04<01:08, 1.45s/it]\n" + " 6%|β–Œ | 3/50 [00:09<02:22, 3.03s/it]\n" ] }, { @@ -520,7 +537,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 6%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 3/50 [00:04<01:05, 1.39s/it]\n" + " 6%|β–Œ | 3/50 [00:09<02:22, 3.02s/it]\n" ] }, { @@ -535,7 +552,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 4%|β–ˆβ–ˆβ–ˆβ–Š | 2/50 [00:02<01:04, 1.34s/it]\n" + " 4%|▍ | 2/50 [00:06<02:33, 3.19s/it]\n" ] }, { @@ -550,7 +567,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 4%|β–ˆβ–ˆβ–ˆβ–Š | 2/50 [00:02<01:02, 1.30s/it]\n" + " 4%|▍ | 2/50 [00:06<02:30, 3.13s/it]\n" ] }, { @@ -565,7 +582,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 8%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 4/50 [00:05<01:06, 1.45s/it]\n" + " 8%|β–Š | 4/50 [00:12<02:24, 3.14s/it]\n" ] }, { @@ -580,7 +597,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 8%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‹ | 4/50 [00:05<00:59, 1.30s/it]\n" + " 8%|β–Š | 4/50 [00:12<02:20, 3.06s/it]\n" ] }, { @@ -595,7 +612,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 6%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–Š | 3/50 [00:03<00:59, 1.27s/it]\n" + " 6%|β–Œ | 3/50 [00:09<02:29, 3.17s/it]\n" ] }, { @@ -610,7 +627,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 2%|β–ˆβ–‰ | 1/50 [00:01<01:07, 1.37s/it]\n" + " 2%|▏ | 1/50 [00:03<02:48, 3.43s/it]\n" ] }, { @@ -625,13 +642,13 @@ "name": "stderr", "output_type": "stream", "text": [ - " 2%|β–ˆβ–‰ | 1/50 [00:01<01:14, 1.51s/it]\n", - "2025/07/02 20:46:51 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Entering patched_propose_instructions, patching GroundedProposer with NovaGroundedProposer\n", - "2025/07/02 20:46:51 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Patched GroundedProposer, current GroundedProposer class=\n", - "2025/07/02 20:46:51 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + " 2%|▏ | 1/50 [00:03<02:33, 3.13s/it]\n", + "2026/01/29 20:48:18 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Entering patched_propose_instructions, patching GroundedProposer with NovaGroundedProposer\n", + "2026/01/29 20:48:18 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Patched GroundedProposer, current GroundedProposer class=\n", + "2026/01/29 20:48:18 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "==> STEP 2: PROPOSE INSTRUCTION CANDIDATES <==\n", - "2025/07/02 20:46:51 INFO dspy.teleprompt.mipro_optimizer_v2: We will use the few-shot examples from the previous step, a generated dataset summary, a summary of the program code, and a randomly selected prompting tip to propose instructions.\n", - "2025/07/02 20:46:51 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_grounded_proposer: Initializing NovaGroundedProposer\n" + "2026/01/29 20:48:18 INFO dspy.teleprompt.mipro_optimizer_v2: We will use the few-shot examples from the previous step, a generated dataset summary, a summary of the program code, and a randomly selected prompting tip to propose instructions.\n", + "2026/01/29 20:48:18 INFO amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_grounded_proposer: Initializing NovaGroundedProposer\n" ] }, { @@ -645,7 +662,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:47:10 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 20:49:12 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "Proposing N=20 instructions...\n", "\n" ] @@ -680,148 +697,457 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: Proposed Instructions for Predictor 0:\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 0: **Task:**\n", - "Extract and return a JSON with specified keys and values based on the input.\n", - "\n", - "**Context:**\n", - "- The JSON must include \"urgency\", \"sentiment\", and \"categories\".\n", - "- \"urgency\" can be `high`, `medium`, or `low`.\n", - "- \"sentiment\" can be `negative`, `neutral`, or `positive`.\n", - "- \"categories\" is a dictionary with boolean values indicating if each category matches the input.\n", - "\n", - "**Instructions:**\n", - "- MUST include all specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", - "- \"categories\" MUST include all listed support category tags with boolean values.\n", - "- The JSON string MUST be valid and readable directly.\n", - "- DO NOT enclose the JSON in ```json...```.\n", - "- DO NOT include newlines or unnecessary whitespaces.\n", - "\n", - "**Response Format:**\n", - "- The response MUST be a single-line JSON string.\n", - "- MUST adhere to the specified format and include all required keys and values.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 1: As a customer service analyst, your task is to process the given email and generate a JSON response that accurately reflects its content. Identify the urgency level (high, medium, low), sentiment (negative, neutral, positive), and relevant categories from the predefined list. Ensure the JSON is concise, valid, and includes all required keys with appropriate boolean values for categories.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 2: Analyze the provided customer service email and generate a JSON object containing the urgency level (\"high\", \"medium\", \"low\"), sentiment (\"positive\", \"neutral\", \"negative\"), and a boolean dictionary for predefined categories. Ensure all categories are addressed, and the JSON is valid. For example, detect urgency cues like \"immediate\" or \"as soon as possible,\" sentiment from tone, and categorize based on keywords (e.g., \"safety\" for quality_and_safety_concerns).\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 3: Given an input email, analyze its content to determine the urgency, sentiment, and relevant categories. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and a \"categories\" dictionary with boolean values for each predefined category. Ensure all keys are present and the JSON is valid.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 4: Analyze the input email to extract relevant information and return a JSON object with the following keys: \"urgency\", \"sentiment\", and \"categories\". Determine the urgency level (`high`, `medium`, `low`) based on keywords like \"urgent\" or \"as soon as possible\". Assess sentiment (`negative`, `neutral`, `positive`) from the tone and language used. For \"categories\", assign boolean values to each predefined category based on the email's content. Ensure all keys are present and the JSON is valid.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 5: Analyze the customer service inquiry to determine its urgency, sentiment, and relevant categories. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and \"categories\" (boolean values for each predefined category). Ensure all keys are present, and the JSON is valid without extra formatting. If the inquiry mentions multiple topics, assign true to all relevant categories. For example, if both scheduling and safety are discussed, mark both categories as true.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 6: Analyze the input text and classify it into a JSON object with \"urgency\", \"sentiment\", and \"categories\" keys. Ensure all categories are covered with boolean values, and the output is a valid JSON string without extra formatting.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 7: **\n", - "Analyze the provided customer service email and generate a precise JSON output containing urgency, sentiment, and category classifications. This output will directly impact service prioritization and resource allocation.\n", - "\n", - "**Requirements:**\n", - "- Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (positive/neutral/negative), and \"categories\" (boolean flags for all predefined topics).\n", - "- Ensure 100% accuracy in classification, as errors could lead to delayed responses or misallocated resources.\n", - "- Maintain strict JSON formatting without errors or extraneous characters.\n", - "\n", - "**Example:**\n", - "Input: \"Subject: Urgent Safety Issue...[details]\"\n", - "Output: {\"urgency\":\"high\",\"sentiment\":\"negative\",\"categories\":{\"safety\":true,\"maintenance\":false,...}}\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 8: **\n", - "Analyze the customer service inquiry and generate a structured JSON summary that accurately reflects the email's urgency, sentiment, and relevant categories. This summary is crucial for prioritizing responses and ensuring customer satisfaction, especially in high-stakes environments like healthcare or emergency services.\n", - "\n", - "**Instructions:**\n", - "- Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and \"categories\" (all predefined tags with boolean values).\n", - "- Ensure 100% accuracy in classification, as misinterpretation could lead to delayed responses or unresolved critical issues.\n", - "- Maintain strict JSON format without errors or extraneous characters.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 9: Analyze the provided email to classify its content accurately. Generate a JSON object with three keys: \"urgency\" (values: \"high\", \"medium\", \"low\"), \"sentiment\" (values: \"positive\", \"neutral\", \"negative\"), and \"categories\" (a dictionary with boolean flags for: \"routine_maintenance_requests\", \"customer_feedback_and_complaints\", \"training_and_support_requests\", \"quality_and_safety_concerns\", \"sustainability_and_environmental_practices\", \"cleaning_services_scheduling\", \"specialized_cleaning_services\", \"emergency_repair_services\", \"facility_management_issues\", \"general_inquiries\"). Ensure no extraneous spaces or line breaks. For example, if an email requests urgent cleaning after renovations, set \"urgency\":\"high\", enable \"specialized_cleaning_services\":true, and leave unrelated categories false.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 10: Given an email text, analyze its content to generate a JSON object with three main components: urgency, sentiment, and categories. For urgency, detect keywords or phrases indicating time sensitivity (e.g., \"ASAP\" for high, \"next week\" for medium). Sentiment should reflect the email's tone (e.g., complaints as negative, praise as positive). Categories require matching explicit mentions of service types (e.g., \"HVAC maintenance\" for routine_maintenance_requests) against a predefined list. Ensure boolean values in categories strictly correspond to presence/absence of each topic. The output must be a minified JSON string with exact field names and no formatting errors.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 11: Analyze the provided email content to generate a JSON object containing \"urgency\", \"sentiment\", and \"categories\" fields. Ensure compliance with data privacy standards by omitting any personally identifiable information. The JSON must strictly follow the format: `{\"urgency\": \"[high/medium/low]\", \"sentiment\": \"[positive/neutral/negative]\", \"categories\": {\"[category_name]\": [true/false], ...}}`. Include all predefined categories with boolean flags, and maintain valid JSON syntax without extra whitespace.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 12: Analyze the provided customer service email to extract structured information. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (positive/neutral/negative), and a \"categories\" dictionary containing boolean flags for: routine_maintenance_requests, customer_feedback_and_complaints, training_and_support_requests, quality_and_safety_concerns, sustainability_and_environmental_practices, cleaning_services_scheduling, specialized_cleaning_services, emergency_repair_services, facility_management_issues, and general_inquiries. Use contextual clues like problem severity (\"urgent\", \"ASAP\") for urgency, emotional tone for sentiment, and explicit mentions of services/issues for categories. Ensure no invalid JSON syntax and maintain single-line output.\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 13: Analyze the input email and classify it into the appropriate categories. Determine the sentiment and urgency levels. Return a JSON object with \"categories\" (boolean for each category), \"sentiment\" (positive, neutral, negative), and \"urgency\" (high, medium, low). Ensure all keys are present and the JSON is valid.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 14: Analyze the provided customer service inquiry and generate a JSON object containing \"urgency\" (high/medium/low), \"sentiment\" (positive/neutral/negative), and \"categories\" (boolean flags for each predefined category). Consider the email's tone, keywords, and context to accurately determine classifications. If ambiguity exists, default to neutral sentiment and medium urgency unless explicit urgency indicators are present. Ensure all category keys are included with appropriate boolean values.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 15: Analyze the input email to classify the service request, determine sentiment, and assess urgency. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (positive/neutral/negative), and \"categories\" (boolean values for predefined tags). Ensure all keys are present and the JSON is valid.\n", - "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 16: **\n", - "Analyze the customer service email to generate a precise JSON output containing urgency, sentiment, and category classifications. Errors in categorization could lead to delayed responses for urgent issues or misallocation of resources. Incorrect sentiment analysis might affect customer satisfaction tracking, while wrong urgency levels could compromise service quality.\n", - "\n", - "**Requirements:**\n", - "- Return valid JSON with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and \"categories\" (boolean values for all predefined tags).\n", - "- Ensure 100% accuracy in category matching. For example, sustainability concerns must activate `\"sustainability_and_environmental_practices\":true` even if paired with complaints.\n", - "- Maintain strict JSON formatting without extra whitespace or line breaks.\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: Proposed Instructions for Predictor 0:\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 0: Task: Extract and return specific information from the provided input.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 1: ]\n", + "Task: Analyze and categorize incoming support requests based on content and context.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 2: ]\n", + "Task: Analyze and categorize incoming support requests based on content and context.\n", + "\n", + "Context:\n", + "- The input will be a support request in text format.\n", + "- Your goal is to determine the urgency, sentiment, and relevant categories for the request.\n", + "- Urgency must be classified as `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully read the input to accurately determine the urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to `True`, while setting all others to `False`.\n", + "- The output must be a structured JSON object containing exactly the keys: `urgency`, `sentiment`, and `categories`.\n", + "- The JSON object must be formatted as a compact string with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: `urgency`, `sentiment`, and `categories`.\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 3: ]\n", + "Task: Analyze and categorize incoming messages based on content, sentiment, and urgency.\n", + "\n", + "Context:\n", + "- The input will be a message that needs to be analyzed to determine its urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully analyze the input message to accurately determine its urgency and sentiment.\n", + "- Match the input message to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a structured JSON object containing the urgency, sentiment, and categories.\n", + "- The JSON object must be formatted as a compact string with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: `urgency`, `sentiment`, and `categories`.\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 4: ]\n", + "Task: Analyze and categorize incoming support requests.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully read the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 5: ]\n", + "Task: Analyze and categorize customer inquiries for ProCare Facility Solutions.\n", + "\n", + "Context:\n", + "- The input will be a customer inquiry that needs to be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully analyze the input to accurately determine the urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 6: ]\n", + "Task: Analyze and categorize incoming support requests.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: `urgency`, `sentiment`, and `categories`.\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: `urgency`, `sentiment`, and `categories`.\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 7: ]\n", + "Task: Analyze and categorize the incoming text to extract urgency, sentiment, and relevant categories.\n", + "\n", + "Context:\n", + "- The input will be a detailed text message related to facility management services.\n", + "- Urgency must be classified as `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `customer_feedback_and_complaints`, `quality_and_safety_concerns`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `specialized_cleaning_services`, `facility_management_issues`, `general_inquiries`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 8: ]\n", + "Task: Analyze and categorize the incoming text to determine urgency, sentiment, and relevant categories.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 9: ]\n", + "Task: Analyze and categorize incoming facility management inquiries.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 10: ]\n", + "Task: Analyze and categorize the incoming customer communication based on content, sentiment, and urgency.\n", + "\n", + "Context:\n", + "- The input will be a detailed customer communication that includes various aspects such as requests, feedback, and inquiries.\n", + "- The goal is to determine the urgency level (`high`, `medium`, `low`), the overall sentiment (`negative`, `neutral`, `positive`), and categorize the communication into relevant topics from the predefined list.\n", + "\n", + "Instructions:\n", + "- Carefully analyze the input to identify the specific topics mentioned (e.g., sustainability practices, feedback, safety concerns, routine maintenance requests).\n", + "- Determine the urgency level of the request based on the tone and content of the communication.\n", + "- Assess the overall sentiment expressed in the communication.\n", + "- Categorize the communication by mapping it to the most appropriate predefined categories, setting the relevant category to `True` and all others to `False`.\n", + "- Generate a structured response in JSON format that includes the urgency, sentiment, and categories.\n", + "- Ensure the JSON output is compact, containing only the necessary keys: \"urgency\", \"sentiment\", and \"categories\", with no additional text or formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 11: ]\n", + "Task: Analyze and categorize incoming communications to determine urgency, sentiment, and relevant categories.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "Compliance:\n", + "- Ensure all communications are handled in accordance with company policy and any relevant data protection regulations.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 12: ]\n", + "Task: Analyze the provided input to determine its urgency, sentiment, and relevant categories.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully read and understand the input to accurately determine its urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 13: ]\n", + "Task: Analyze and categorize the incoming message based on its content and context.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully read the input message to accurately determine its urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 14: ]\n", + "Task: Analyze and categorize incoming support requests based on urgency, sentiment, and relevant categories.\n", + "\n", + "Context:\n", + "- The input will be a detailed support request message.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully analyze the input to accurately determine the urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 15: ]\n", + "Task: Analyze and categorize incoming facility management requests.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: `urgency`, `sentiment`, and `categories`.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: `urgency`, `sentiment`, and `categories`.\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 16: ]\n", + "Task: Analyze and categorize the incoming client inquiries to determine urgency, sentiment, and relevant categories.\n", + "\n", + "Context:\n", + "- The input will be a client's message to ProCare Facility Solutions.\n", + "- The goal is to accurately classify the urgency (`high`, `medium`, `low`), sentiment (`negative`, `neutral`, `positive`), and categorize the request into one of the predefined categories: `emergency_repair_services`, `routine_maintenance_requests`, `training_and_support_requests`, `sustainability_and_environmental_practices`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`, `general_inquiries`.\n", + "\n", + "Instructions:\n", + "- Carefully analyze the input text to determine the urgency level of the request.\n", + "- Assess the sentiment expressed in the message to gauge the client's mood.\n", + "- Match the content of the message to the most appropriate category from the predefined list, setting its value to `True`, and all others to `False`.\n", + "- The output must be a compact JSON string containing exactly the keys: `urgency`, `sentiment`, and `categories`, with `categories` being a dictionary where only the relevant category is set to `True`.\n", + "- Ensure the JSON string is on a single line with no extra spaces or line breaks.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: `urgency`, `sentiment`, and `categories`.\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "Example:\n", + "Input: Subject: Inquiry About Training Programs for Facility Management\n", + "Output: {\"urgency\":\"low\",\"sentiment\":\"positive\",\"categories\":{\"emergency_repair_services\":false,\"routine_maintenance_requests\":false,\"training_and_support_requests\":true,\"sustainability_and_environmental_practices\":false,\"cleaning_services_scheduling\":false,\"customer_feedback_and_complaints\":false,\"facility_management_issues\":false,\"general_inquiries\":false}}\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 17: ]\n", + "Task: Analyze and categorize customer inquiries\n", + "\n", + "Context:\n", + "- The input will be a customer inquiry or message.\n", + "- The goal is to determine the urgency, sentiment, and relevant categories of the request.\n", + "\n", + "Instructions:\n", + "- Analyze the input to accurately determine the urgency level: `high`, `medium`, or `low`.\n", + "- Perform sentiment analysis to classify the emotional tone as `negative`, `neutral`, or `positive`.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False. The categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`.\n", + "- The output must be a valid JSON string containing exactly the keys: `urgency`, `sentiment`, and `categories`.\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", "\n", - "**Example Failure Impact:**\n", - "Misclassifying an emergency repair request as low urgency could result in facility downtime or safety hazards. Overlooking negative sentiment in feedback might hide systemic issues.\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: `urgency`, `sentiment`, and `categories`.\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", + "\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 18: ]\n", + "Task: Analyze and categorize incoming support requests.\n", "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 17: **\n", - "Analyze the provided customer service email and generate a JSON output containing urgency, sentiment, and category classifications.\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", "\n", - "**Context:**\n", - "- The input is a customer support email requiring classification into predefined categories.\n", - "- Use the email's content to determine if it matches any of the 10 support categories (e.g., emergency repairs, routine maintenance).\n", - "- Assess sentiment (positive/neutral/negative) based on language cues.\n", - "- Determine urgency (high/medium/low) from explicit mentions or implied context.\n", + "Instructions:\n", + "- Analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", "\n", - "**Instructions:**\n", - "1. Read the email body and subject line carefully.\n", - "2. Identify keywords/phrases related to urgency (e.g., \"urgent\", \"ASAP\") and sentiment (e.g., \"appreciate\", \"concerned\").\n", - "3. Match content to categories like `emergency_repair_services` or `routine_maintenance_requests`.\n", - "4. Return a JSON object with:\n", - " - \"urgency\": \"high\"/\"medium\"/\"low\"\n", - " - \"sentiment\": \"positive\"/\"neutral\"/\"negative\"\n", - " - \"categories\": Dictionary with 10 boolean-valued keys (e.g., {\"specialized_cleaning_services\":true})\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", "\n", - "**Example Output:**\n", - "{\"urgency\":\"medium\",\"sentiment\":\"neutral\",\"categories\":{\"routine_maintenance_requests\":false,\"specialized_cleaning_services\":true}}\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: 19: ]\n", + "Task: Analyze and categorize incoming support requests based on content and context.\n", "\n", - "**Validation:**\n", - "- Ensure all 10 categories are present in the JSON.\n", - "- Maintain proper boolean capitalization (true/false).\n", - "- No extraneous characters or formatting outside the JSON structure.\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `customer_feedback_and_complaints`, `training_and_support_requests`, `quality_and_safety_concerns`, `sustainability_and_environmental_practices`, `cleaning_services_scheduling`, `specialized_cleaning_services`, `facility_management_issues`, `general_inquiries`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 18: Analyze the input text to classify the urgency, sentiment, and relevant categories. Return a JSON object with \"urgency\" (high/medium/low), \"sentiment\" (negative/neutral/positive), and \"categories\" (boolean values for each predefined category). Ensure the output is a valid JSON string without extra formatting.\n", + "Instructions:\n", + "- Analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: 19: Analyze the provided customer service email and generate a JSON object with the following keys: \"urgency\" (high, medium, or low), \"sentiment\" (negative, neutral, or positive), and \"categories\" (a dictionary with boolean values for each category). Ensure all categories are included and accurately reflect the email's content. If the email's intent is unclear, prioritize categories based on explicit mentions and context clues.\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n", "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:49:38 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Restored GroundedProposer, current GroundedProposer class=\n", - "2025/07/02 20:49:38 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.custom_adapters.custom_chat_adapter: Initializing CustomChatAdapter with enable_json_fallback=False\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: ==> STEP 3: FINDING OPTIMAL PROMPT PARAMETERS <==\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: We will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination using Bayesian Optimization.\n", + "2026/01/29 20:54:52 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.miprov2_optimizer: Restored GroundedProposer, current GroundedProposer class=\n", + "2026/01/29 20:54:52 INFO amzn_nova_prompt_optimizer.core.optimizers.miprov2.custom_adapters.custom_chat_adapter: Initializing CustomChatAdapter with enable_json_fallback=False\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: ==> STEP 3: FINDING OPTIMAL PROMPT PARAMETERS <==\n", + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: We will evaluate the program over a series of trials with different combinations of instructions and few-shot examples to find the optimal combination using Bayesian Optimization.\n", "\n", - "2025/07/02 20:49:38 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 1 / 37 - Full Evaluation of Default Program ==\n" + "2026/01/29 20:54:52 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 1 / 37 - Full Evaluation of Default Program ==\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 35.17 / 50 (70.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:36<00:00, 1.37it/s]" + "Average Metric: 35.53 / 50 (71.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [01:12<00:00, 1.44s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:50:15 INFO dspy.evaluate.evaluate: Average Metric: 35.166666666666664 / 50 (70.3%)\n", - "2025/07/02 20:50:15 INFO dspy.teleprompt.mipro_optimizer_v2: Default program score: 70.33\n", + "2026/01/29 20:56:04 INFO dspy.evaluate.evaluate: Average Metric: 35.53333333333333 / 50 (71.1%)\n", + "2026/01/29 20:56:04 INFO dspy.teleprompt.mipro_optimizer_v2: Default program score: 71.07\n", "\n", - "/local/home/shenov/workplace/NovaPromptOptimizer/src/NovaPromptOptimizer/samples/facility-support-analyzer/.venv/lib/python3.12/site-packages/optuna/_experimental.py:32: ExperimentalWarning: Argument ``multivariate`` is an experimental feature. The interface can change in the future.\n", - " warnings.warn(\n", - "2025/07/02 20:50:15 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 2 / 37 - Minibatch ==\n" + "2026/01/29 20:56:04 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 2 / 37 - Minibatch ==\n" ] }, { @@ -829,22 +1155,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.50 / 35 (72.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:22<00:00, 1.55it/s]" + "Average Metric: 25.77 / 35 (73.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:00<00:00, 1.72s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:50:37 INFO dspy.evaluate.evaluate: Average Metric: 25.5 / 35 (72.9%)\n", - "2025/07/02 20:50:37 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 12', 'Predictor 0: Few-Shot Set 6'].\n", - "2025/07/02 20:50:37 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86]\n", - "2025/07/02 20:50:37 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33]\n", - "2025/07/02 20:50:37 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.33\n", - "2025/07/02 20:50:37 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 20:57:04 INFO dspy.evaluate.evaluate: Average Metric: 25.76666666666667 / 35 (73.6%)\n", + "2026/01/29 20:57:04 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 73.62 on minibatch of size 35 with parameters ['Predictor 0: Instruction 12', 'Predictor 0: Few-Shot Set 6'].\n", + "2026/01/29 20:57:04 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62]\n", + "2026/01/29 20:57:04 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07]\n", + "2026/01/29 20:57:04 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 71.07\n", + "2026/01/29 20:57:04 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:50:37 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 3 / 37 - Minibatch ==\n" + "2026/01/29 20:57:04 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 3 / 37 - Minibatch ==\n" ] }, { @@ -852,22 +1178,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.17 / 35 (71.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:22<00:00, 1.54it/s]" + "Average Metric: 29.07 / 35 (83.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:59<00:00, 1.69s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:51:00 INFO dspy.evaluate.evaluate: Average Metric: 25.166666666666664 / 35 (71.9%)\n", - "2025/07/02 20:51:00 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 71.9 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 4'].\n", - "2025/07/02 20:51:00 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9]\n", - "2025/07/02 20:51:00 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33]\n", - "2025/07/02 20:51:00 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.33\n", - "2025/07/02 20:51:00 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 20:58:03 INFO dspy.evaluate.evaluate: Average Metric: 29.06666666666666 / 35 (83.0%)\n", + "2026/01/29 20:58:03 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 83.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 20:58:03 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05]\n", + "2026/01/29 20:58:03 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07]\n", + "2026/01/29 20:58:03 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 71.07\n", + "2026/01/29 20:58:03 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:51:00 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 4 / 37 - Minibatch ==\n" + "2026/01/29 20:58:03 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 4 / 37 - Minibatch ==\n" ] }, { @@ -875,22 +1201,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 27.13 / 35 (77.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:22<00:00, 1.57it/s]" + "Average Metric: 25.50 / 35 (72.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:02<00:00, 1.78s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:51:22 INFO dspy.evaluate.evaluate: Average Metric: 27.133333333333333 / 35 (77.5%)\n", - "2025/07/02 20:51:22 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 77.52 on minibatch of size 35 with parameters ['Predictor 0: Instruction 3', 'Predictor 0: Few-Shot Set 13'].\n", - "2025/07/02 20:51:22 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52]\n", - "2025/07/02 20:51:22 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33]\n", - "2025/07/02 20:51:22 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.33\n", - "2025/07/02 20:51:22 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 20:59:06 INFO dspy.evaluate.evaluate: Average Metric: 25.5 / 35 (72.9%)\n", + "2026/01/29 20:59:06 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 3', 'Predictor 0: Few-Shot Set 13'].\n", + "2026/01/29 20:59:06 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86]\n", + "2026/01/29 20:59:06 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07]\n", + "2026/01/29 20:59:06 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 71.07\n", + "2026/01/29 20:59:06 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:51:22 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 5 / 37 - Minibatch ==\n" + "2026/01/29 20:59:06 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 5 / 37 - Minibatch ==\n" ] }, { @@ -898,22 +1224,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.40 / 35 (72.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.49it/s]" + "Average Metric: 28.43 / 35 (81.2%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:59<00:00, 1.69s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:51:46 INFO dspy.evaluate.evaluate: Average Metric: 25.4 / 35 (72.6%)\n", - "2025/07/02 20:51:46 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.57 on minibatch of size 35 with parameters ['Predictor 0: Instruction 9', 'Predictor 0: Few-Shot Set 7'].\n", - "2025/07/02 20:51:46 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57]\n", - "2025/07/02 20:51:46 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33]\n", - "2025/07/02 20:51:46 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.33\n", - "2025/07/02 20:51:46 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 21:00:05 INFO dspy.evaluate.evaluate: Average Metric: 28.43333333333333 / 35 (81.2%)\n", + "2026/01/29 21:00:05 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 81.24 on minibatch of size 35 with parameters ['Predictor 0: Instruction 9', 'Predictor 0: Few-Shot Set 7'].\n", + "2026/01/29 21:00:05 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24]\n", + "2026/01/29 21:00:05 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07]\n", + "2026/01/29 21:00:05 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 71.07\n", + "2026/01/29 21:00:05 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:51:46 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 6 / 37 - Minibatch ==\n" + "2026/01/29 21:00:05 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 6 / 37 - Minibatch ==\n" ] }, { @@ -921,23 +1247,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.07 / 35 (71.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.46it/s]" + "Average Metric: 26.30 / 35 (75.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:10<00:00, 2.02s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:52:10 INFO dspy.evaluate.evaluate: Average Metric: 25.066666666666666 / 35 (71.6%)\n", - "2025/07/02 20:52:10 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 71.62 on minibatch of size 35 with parameters ['Predictor 0: Instruction 0', 'Predictor 0: Few-Shot Set 9'].\n", - "2025/07/02 20:52:10 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62]\n", - "2025/07/02 20:52:10 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33]\n", - "2025/07/02 20:52:10 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 70.33\n", - "2025/07/02 20:52:10 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 21:01:16 INFO dspy.evaluate.evaluate: Average Metric: 26.300000000000004 / 35 (75.1%)\n", + "2026/01/29 21:01:16 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 75.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 0', 'Predictor 0: Few-Shot Set 9'].\n", + "2026/01/29 21:01:16 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14]\n", + "2026/01/29 21:01:16 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07]\n", + "2026/01/29 21:01:16 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 71.07\n", + "2026/01/29 21:01:16 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:52:10 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 7 / 37 - Full Evaluation =====\n", - "2025/07/02 20:52:10 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 77.52) from minibatch trials...\n" + "2026/01/29 21:01:16 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 7 / 37 - Full Evaluation =====\n", + "2026/01/29 21:01:16 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 83.05) from minibatch trials...\n" ] }, { @@ -945,21 +1271,21 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 38.53 / 50 (77.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:11<00:00, 4.45it/s]" + "Average Metric: 40.63 / 50 (81.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [01:20<00:00, 1.62s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:52:21 INFO dspy.evaluate.evaluate: Average Metric: 38.53333333333333 / 50 (77.1%)\n", - "2025/07/02 20:52:21 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 77.07\n", - "2025/07/02 20:52:21 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07]\n", - "2025/07/02 20:52:21 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 77.07\n", - "2025/07/02 20:52:21 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:52:21 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 21:02:37 INFO dspy.evaluate.evaluate: Average Metric: 40.633333333333326 / 50 (81.3%)\n", + "2026/01/29 21:02:37 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 81.27\n", + "2026/01/29 21:02:37 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27]\n", + "2026/01/29 21:02:37 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:02:37 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 21:02:37 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:52:21 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 8 / 37 - Minibatch ==\n" + "2026/01/29 21:02:37 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 8 / 37 - Minibatch ==\n" ] }, { @@ -967,22 +1293,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.93 / 35 (74.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:24<00:00, 1.41it/s]" + "Average Metric: 26.63 / 35 (76.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:00<00:00, 1.72s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:52:46 INFO dspy.evaluate.evaluate: Average Metric: 25.933333333333334 / 35 (74.1%)\n", - "2025/07/02 20:52:46 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 74.1 on minibatch of size 35 with parameters ['Predictor 0: Instruction 10', 'Predictor 0: Few-Shot Set 15'].\n", - "2025/07/02 20:52:46 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1]\n", - "2025/07/02 20:52:46 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07]\n", - "2025/07/02 20:52:46 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 77.07\n", - "2025/07/02 20:52:46 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 21:03:37 INFO dspy.evaluate.evaluate: Average Metric: 26.633333333333326 / 35 (76.1%)\n", + "2026/01/29 21:03:37 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 76.1 on minibatch of size 35 with parameters ['Predictor 0: Instruction 10', 'Predictor 0: Few-Shot Set 15'].\n", + "2026/01/29 21:03:37 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1]\n", + "2026/01/29 21:03:37 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27]\n", + "2026/01/29 21:03:37 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:03:37 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:52:46 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 9 / 37 - Minibatch ==\n" + "2026/01/29 21:03:37 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 9 / 37 - Minibatch ==\n" ] }, { @@ -990,22 +1316,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.70 / 35 (76.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:30<00:00, 1.14it/s]" + "Average Metric: 23.70 / 35 (67.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:11<00:00, 2.05s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:53:17 INFO dspy.evaluate.evaluate: Average Metric: 26.7 / 35 (76.3%)\n", - "2025/07/02 20:53:17 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 76.29 on minibatch of size 35 with parameters ['Predictor 0: Instruction 6', 'Predictor 0: Few-Shot Set 17'].\n", - "2025/07/02 20:53:17 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29]\n", - "2025/07/02 20:53:17 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07]\n", - "2025/07/02 20:53:17 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 77.07\n", - "2025/07/02 20:53:17 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", + "2026/01/29 21:04:49 INFO dspy.evaluate.evaluate: Average Metric: 23.700000000000003 / 35 (67.7%)\n", + "2026/01/29 21:04:49 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 67.71 on minibatch of size 35 with parameters ['Predictor 0: Instruction 6', 'Predictor 0: Few-Shot Set 17'].\n", + "2026/01/29 21:04:49 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71]\n", + "2026/01/29 21:04:49 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27]\n", + "2026/01/29 21:04:49 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:04:49 INFO dspy.teleprompt.mipro_optimizer_v2: =========================================\n", "\n", "\n", - "2025/07/02 20:53:17 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 10 / 37 - Minibatch ==\n" + "2026/01/29 21:04:49 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 10 / 37 - Minibatch ==\n" ] }, { @@ -1013,22 +1339,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.30 / 35 (69.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.52it/s]" + "Average Metric: 25.30 / 35 (72.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:59<00:00, 1.71s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:53:40 INFO dspy.evaluate.evaluate: Average Metric: 24.3 / 35 (69.4%)\n", - "2025/07/02 20:53:40 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 69.43 on minibatch of size 35 with parameters ['Predictor 0: Instruction 18', 'Predictor 0: Few-Shot Set 9'].\n", - "2025/07/02 20:53:40 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43]\n", - "2025/07/02 20:53:40 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07]\n", - "2025/07/02 20:53:40 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 77.07\n", - "2025/07/02 20:53:40 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:05:49 INFO dspy.evaluate.evaluate: Average Metric: 25.3 / 35 (72.3%)\n", + "2026/01/29 21:05:49 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.29 on minibatch of size 35 with parameters ['Predictor 0: Instruction 18', 'Predictor 0: Few-Shot Set 9'].\n", + "2026/01/29 21:05:49 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29]\n", + "2026/01/29 21:05:49 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27]\n", + "2026/01/29 21:05:49 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:05:49 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:53:40 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 11 / 37 - Minibatch ==\n" + "2026/01/29 21:05:49 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 11 / 37 - Minibatch ==\n" ] }, { @@ -1036,22 +1362,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.20 / 35 (72.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.46it/s]" + "Average Metric: 28.50 / 35 (81.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:57<00:00, 1.65s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:54:04 INFO dspy.evaluate.evaluate: Average Metric: 25.2 / 35 (72.0%)\n", - "2025/07/02 20:54:04 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.0 on minibatch of size 35 with parameters ['Predictor 0: Instruction 7', 'Predictor 0: Few-Shot Set 13'].\n", - "2025/07/02 20:54:04 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0]\n", - "2025/07/02 20:54:04 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07]\n", - "2025/07/02 20:54:04 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 77.07\n", - "2025/07/02 20:54:04 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:06:47 INFO dspy.evaluate.evaluate: Average Metric: 28.49999999999999 / 35 (81.4%)\n", + "2026/01/29 21:06:47 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 81.43 on minibatch of size 35 with parameters ['Predictor 0: Instruction 3', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 21:06:47 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43]\n", + "2026/01/29 21:06:47 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27]\n", + "2026/01/29 21:06:47 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:06:47 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:54:04 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 12 / 37 - Minibatch ==\n" + "2026/01/29 21:06:47 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 12 / 37 - Minibatch ==\n" ] }, { @@ -1059,23 +1385,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.53 / 35 (75.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.46it/s]" + "Average Metric: 27.83 / 35 (79.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:58<00:00, 1.68s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:54:28 INFO dspy.evaluate.evaluate: Average Metric: 26.53333333333333 / 35 (75.8%)\n", - "2025/07/02 20:54:28 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 75.81 on minibatch of size 35 with parameters ['Predictor 0: Instruction 3', 'Predictor 0: Few-Shot Set 5'].\n", - "2025/07/02 20:54:28 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81]\n", - "2025/07/02 20:54:28 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07]\n", - "2025/07/02 20:54:28 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 77.07\n", - "2025/07/02 20:54:28 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:07:45 INFO dspy.evaluate.evaluate: Average Metric: 27.833333333333325 / 35 (79.5%)\n", + "2026/01/29 21:07:45 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 79.52 on minibatch of size 35 with parameters ['Predictor 0: Instruction 13', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 21:07:45 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52]\n", + "2026/01/29 21:07:45 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27]\n", + "2026/01/29 21:07:45 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:07:45 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:54:28 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 13 / 37 - Full Evaluation =====\n", - "2025/07/02 20:54:28 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 76.29) from minibatch trials...\n" + "2026/01/29 21:07:45 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 13 / 37 - Full Evaluation =====\n", + "2026/01/29 21:07:45 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 81.43) from minibatch trials...\n" ] }, { @@ -1083,21 +1409,20 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 40.03 / 50 (80.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:11<00:00, 4.33it/s]" + "Average Metric: 40.40 / 50 (80.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [01:18<00:00, 1.57s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:54:39 INFO dspy.evaluate.evaluate: Average Metric: 40.03333333333333 / 50 (80.1%)\n", - "2025/07/02 20:54:39 INFO dspy.teleprompt.mipro_optimizer_v2: \u001b[92mNew best full eval score!\u001b[0m Score: 80.07\n", - "2025/07/02 20:54:39 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07]\n", - "2025/07/02 20:54:39 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:54:39 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:54:39 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 21:09:04 INFO dspy.evaluate.evaluate: Average Metric: 40.39999999999999 / 50 (80.8%)\n", + "2026/01/29 21:09:04 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8]\n", + "2026/01/29 21:09:04 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:09:04 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 21:09:04 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:54:39 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 14 / 37 - Minibatch ==\n" + "2026/01/29 21:09:04 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 14 / 37 - Minibatch ==\n" ] }, { @@ -1105,22 +1430,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 27.03 / 35 (77.2%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.50it/s]" + "Average Metric: 26.57 / 35 (75.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:59<00:00, 1.71s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:55:03 INFO dspy.evaluate.evaluate: Average Metric: 27.03333333333333 / 35 (77.2%)\n", - "2025/07/02 20:55:03 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 77.24 on minibatch of size 35 with parameters ['Predictor 0: Instruction 13', 'Predictor 0: Few-Shot Set 13'].\n", - "2025/07/02 20:55:03 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24]\n", - "2025/07/02 20:55:03 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07]\n", - "2025/07/02 20:55:03 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:55:03 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:10:03 INFO dspy.evaluate.evaluate: Average Metric: 26.56666666666666 / 35 (75.9%)\n", + "2026/01/29 21:10:03 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 75.9 on minibatch of size 35 with parameters ['Predictor 0: Instruction 16', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 21:10:03 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9]\n", + "2026/01/29 21:10:03 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8]\n", + "2026/01/29 21:10:03 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:10:03 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:55:03 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 15 / 37 - Minibatch ==\n" + "2026/01/29 21:10:03 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 15 / 37 - Minibatch ==\n" ] }, { @@ -1128,22 +1453,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.90 / 35 (76.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.50it/s]" + "Average Metric: 28.80 / 35 (82.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:56<00:00, 1.61s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:55:26 INFO dspy.evaluate.evaluate: Average Metric: 26.9 / 35 (76.9%)\n", - "2025/07/02 20:55:26 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 76.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 6', 'Predictor 0: Few-Shot Set 1'].\n", - "2025/07/02 20:55:26 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86]\n", - "2025/07/02 20:55:26 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07]\n", - "2025/07/02 20:55:26 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:55:26 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:11:00 INFO dspy.evaluate.evaluate: Average Metric: 28.79999999999999 / 35 (82.3%)\n", + "2026/01/29 21:11:00 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 82.29 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 21:11:00 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29]\n", + "2026/01/29 21:11:00 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8]\n", + "2026/01/29 21:11:00 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:11:00 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:55:26 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 16 / 37 - Minibatch ==\n" + "2026/01/29 21:11:00 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 16 / 37 - Minibatch ==\n" ] }, { @@ -1151,22 +1476,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.70 / 35 (76.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:28<00:00, 1.22it/s]" + "Average Metric: 25.57 / 35 (73.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:59<00:00, 1.71s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:55:55 INFO dspy.evaluate.evaluate: Average Metric: 26.7 / 35 (76.3%)\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 76.29 on minibatch of size 35 with parameters ['Predictor 0: Instruction 2', 'Predictor 0: Few-Shot Set 6'].\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29]\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07]\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:11:59 INFO dspy.evaluate.evaluate: Average Metric: 25.566666666666656 / 35 (73.0%)\n", + "2026/01/29 21:11:59 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 73.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 15'].\n", + "2026/01/29 21:11:59 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05]\n", + "2026/01/29 21:11:59 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8]\n", + "2026/01/29 21:11:59 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:11:59 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 17 / 37 - Minibatch ==\n" + "2026/01/29 21:11:59 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 17 / 37 - Minibatch ==\n" ] }, { @@ -1174,22 +1499,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 27.73 / 35 (79.2%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:00<00:00, 1377.01it/s]" + "Average Metric: 28.90 / 35 (82.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:56<00:00, 1.61s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:55:55 INFO dspy.evaluate.evaluate: Average Metric: 27.733333333333334 / 35 (79.2%)\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 79.24 on minibatch of size 35 with parameters ['Predictor 0: Instruction 3', 'Predictor 0: Few-Shot Set 13'].\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24]\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07]\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:12:56 INFO dspy.evaluate.evaluate: Average Metric: 28.899999999999988 / 35 (82.6%)\n", + "2026/01/29 21:12:56 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 82.57 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 21:12:56 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57]\n", + "2026/01/29 21:12:56 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8]\n", + "2026/01/29 21:12:56 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:12:56 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:55:55 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 18 / 37 - Minibatch ==\n" + "2026/01/29 21:12:56 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 18 / 37 - Minibatch ==\n" ] }, { @@ -1197,23 +1522,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.57 / 35 (73.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.39it/s]" + "Average Metric: 27.23 / 35 (77.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:59<00:00, 1.71s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:56:20 INFO dspy.evaluate.evaluate: Average Metric: 25.566666666666666 / 35 (73.0%)\n", - "2025/07/02 20:56:20 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 73.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 6', 'Predictor 0: Few-Shot Set 6'].\n", - "2025/07/02 20:56:20 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05]\n", - "2025/07/02 20:56:20 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07]\n", - "2025/07/02 20:56:20 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:56:20 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:13:56 INFO dspy.evaluate.evaluate: Average Metric: 27.233333333333327 / 35 (77.8%)\n", + "2026/01/29 21:13:56 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 77.81 on minibatch of size 35 with parameters ['Predictor 0: Instruction 14', 'Predictor 0: Few-Shot Set 11'].\n", + "2026/01/29 21:13:56 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81]\n", + "2026/01/29 21:13:56 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8]\n", + "2026/01/29 21:13:56 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:13:56 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:56:20 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 19 / 37 - Full Evaluation =====\n", - "2025/07/02 20:56:20 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 77.24) from minibatch trials...\n" + "2026/01/29 21:13:56 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 19 / 37 - Full Evaluation =====\n", + "2026/01/29 21:13:56 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 81.24) from minibatch trials...\n" ] }, { @@ -1221,20 +1546,20 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 38.57 / 50 (77.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:10<00:00, 4.97it/s]" + "Average Metric: 39.67 / 50 (79.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [01:21<00:00, 1.64s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:56:30 INFO dspy.evaluate.evaluate: Average Metric: 38.56666666666666 / 50 (77.1%)\n", - "2025/07/02 20:56:30 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13]\n", - "2025/07/02 20:56:30 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:56:30 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:56:30 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 21:15:18 INFO dspy.evaluate.evaluate: Average Metric: 39.66666666666666 / 50 (79.3%)\n", + "2026/01/29 21:15:18 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33]\n", + "2026/01/29 21:15:18 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:15:18 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 21:15:18 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:56:30 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 20 / 37 - Minibatch ==\n" + "2026/01/29 21:15:18 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 20 / 37 - Minibatch ==\n" ] }, { @@ -1242,22 +1567,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 27.60 / 35 (78.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.47it/s]" + "Average Metric: 28.40 / 35 (81.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:00<00:00, 1.72s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:56:54 INFO dspy.evaluate.evaluate: Average Metric: 27.599999999999998 / 35 (78.9%)\n", - "2025/07/02 20:56:54 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 78.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 17', 'Predictor 0: Few-Shot Set 1'].\n", - "2025/07/02 20:56:54 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86]\n", - "2025/07/02 20:56:54 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13]\n", - "2025/07/02 20:56:54 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:56:54 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:16:18 INFO dspy.evaluate.evaluate: Average Metric: 28.39999999999999 / 35 (81.1%)\n", + "2026/01/29 21:16:18 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 81.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 17', 'Predictor 0: Few-Shot Set 1'].\n", + "2026/01/29 21:16:18 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14]\n", + "2026/01/29 21:16:18 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33]\n", + "2026/01/29 21:16:18 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:16:18 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:56:54 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 21 / 37 - Minibatch ==\n" + "2026/01/29 21:16:18 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 21 / 37 - Minibatch ==\n" ] }, { @@ -1265,22 +1590,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.57 / 35 (73.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.49it/s]" + "Average Metric: 26.93 / 35 (77.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:00<00:00, 1.73s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:57:17 INFO dspy.evaluate.evaluate: Average Metric: 25.566666666666666 / 35 (73.0%)\n", - "2025/07/02 20:57:17 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 73.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 16', 'Predictor 0: Few-Shot Set 18'].\n", - "2025/07/02 20:57:17 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05]\n", - "2025/07/02 20:57:17 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13]\n", - "2025/07/02 20:57:17 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:57:17 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:17:18 INFO dspy.evaluate.evaluate: Average Metric: 26.933333333333334 / 35 (77.0%)\n", + "2026/01/29 21:17:18 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 76.95 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 2'].\n", + "2026/01/29 21:17:18 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95]\n", + "2026/01/29 21:17:18 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33]\n", + "2026/01/29 21:17:18 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:17:18 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:57:17 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 22 / 37 - Minibatch ==\n" + "2026/01/29 21:17:18 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 22 / 37 - Minibatch ==\n" ] }, { @@ -1288,22 +1613,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 27.70 / 35 (79.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:22<00:00, 1.59it/s]" + "Average Metric: 28.27 / 35 (80.8%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:56<00:00, 1.62s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:57:39 INFO dspy.evaluate.evaluate: Average Metric: 27.7 / 35 (79.1%)\n", - "2025/07/02 20:57:39 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 79.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 4', 'Predictor 0: Few-Shot Set 1'].\n", - "2025/07/02 20:57:39 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14]\n", - "2025/07/02 20:57:39 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13]\n", - "2025/07/02 20:57:39 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:57:39 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:18:15 INFO dspy.evaluate.evaluate: Average Metric: 28.266666666666655 / 35 (80.8%)\n", + "2026/01/29 21:18:15 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 80.76 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 21:18:15 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76]\n", + "2026/01/29 21:18:15 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33]\n", + "2026/01/29 21:18:15 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:18:15 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:57:39 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 23 / 37 - Minibatch ==\n" + "2026/01/29 21:18:15 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 23 / 37 - Minibatch ==\n" ] }, { @@ -1311,22 +1636,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.20 / 35 (72.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:21<00:00, 1.61it/s]" + "Average Metric: 27.53 / 35 (78.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:59<00:00, 1.70s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:58:01 INFO dspy.evaluate.evaluate: Average Metric: 25.2 / 35 (72.0%)\n", - "2025/07/02 20:58:01 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.0 on minibatch of size 35 with parameters ['Predictor 0: Instruction 4', 'Predictor 0: Few-Shot Set 4'].\n", - "2025/07/02 20:58:01 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0]\n", - "2025/07/02 20:58:01 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13]\n", - "2025/07/02 20:58:01 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:58:01 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:19:14 INFO dspy.evaluate.evaluate: Average Metric: 27.533333333333328 / 35 (78.7%)\n", + "2026/01/29 21:19:14 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 78.67 on minibatch of size 35 with parameters ['Predictor 0: Instruction 15', 'Predictor 0: Few-Shot Set 5'].\n", + "2026/01/29 21:19:14 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67]\n", + "2026/01/29 21:19:14 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33]\n", + "2026/01/29 21:19:14 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:19:14 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:58:01 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 24 / 37 - Minibatch ==\n" + "2026/01/29 21:19:14 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 24 / 37 - Minibatch ==\n" ] }, { @@ -1334,23 +1659,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.50 / 35 (72.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:24<00:00, 1.45it/s]" + "Average Metric: 27.80 / 35 (79.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:58<00:00, 1.68s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:58:25 INFO dspy.evaluate.evaluate: Average Metric: 25.5 / 35 (72.9%)\n", - "2025/07/02 20:58:25 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 15', 'Predictor 0: Few-Shot Set 16'].\n", - "2025/07/02 20:58:25 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86]\n", - "2025/07/02 20:58:25 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13]\n", - "2025/07/02 20:58:25 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:58:25 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:20:13 INFO dspy.evaluate.evaluate: Average Metric: 27.799999999999994 / 35 (79.4%)\n", + "2026/01/29 21:20:13 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 79.43 on minibatch of size 35 with parameters ['Predictor 0: Instruction 2', 'Predictor 0: Few-Shot Set 4'].\n", + "2026/01/29 21:20:13 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43]\n", + "2026/01/29 21:20:13 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33]\n", + "2026/01/29 21:20:13 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:20:13 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:58:25 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 25 / 37 - Full Evaluation =====\n", - "2025/07/02 20:58:25 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 79.14) from minibatch trials...\n" + "2026/01/29 21:20:13 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 25 / 37 - Full Evaluation =====\n", + "2026/01/29 21:20:13 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 81.14) from minibatch trials...\n" ] }, { @@ -1358,20 +1683,20 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 39.20 / 50 (78.4%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:09<00:00, 5.15it/s]" + "Average Metric: 38.43 / 50 (76.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [01:32<00:00, 1.85s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:58:35 INFO dspy.evaluate.evaluate: Average Metric: 39.199999999999996 / 50 (78.4%)\n", - "2025/07/02 20:58:35 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4]\n", - "2025/07/02 20:58:35 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:58:35 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 20:58:35 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 21:21:46 INFO dspy.evaluate.evaluate: Average Metric: 38.43333333333333 / 50 (76.9%)\n", + "2026/01/29 21:21:46 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87]\n", + "2026/01/29 21:21:46 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:21:46 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 21:21:46 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 20:58:35 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 26 / 37 - Minibatch ==\n" + "2026/01/29 21:21:46 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 26 / 37 - Minibatch ==\n" ] }, { @@ -1379,22 +1704,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 22.73 / 35 (65.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:28<00:00, 1.24it/s]" + "Average Metric: 25.37 / 35 (72.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:01<00:00, 1.76s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:59:03 INFO dspy.evaluate.evaluate: Average Metric: 22.733333333333334 / 35 (65.0%)\n", - "2025/07/02 20:59:03 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 64.95 on minibatch of size 35 with parameters ['Predictor 0: Instruction 5', 'Predictor 0: Few-Shot Set 12'].\n", - "2025/07/02 20:59:03 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95]\n", - "2025/07/02 20:59:03 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4]\n", - "2025/07/02 20:59:03 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:59:03 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:22:47 INFO dspy.evaluate.evaluate: Average Metric: 25.366666666666667 / 35 (72.5%)\n", + "2026/01/29 21:22:47 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 72.48 on minibatch of size 35 with parameters ['Predictor 0: Instruction 19', 'Predictor 0: Few-Shot Set 14'].\n", + "2026/01/29 21:22:47 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48]\n", + "2026/01/29 21:22:47 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87]\n", + "2026/01/29 21:22:47 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:22:47 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:59:03 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 27 / 37 - Minibatch ==\n" + "2026/01/29 21:22:47 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 27 / 37 - Minibatch ==\n" ] }, { @@ -1402,22 +1727,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.30 / 35 (75.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:22<00:00, 1.54it/s]" + "Average Metric: 24.97 / 35 (71.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:58<00:00, 1.66s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:59:26 INFO dspy.evaluate.evaluate: Average Metric: 26.3 / 35 (75.1%)\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 75.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 19', 'Predictor 0: Few-Shot Set 10'].\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95, 75.14]\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4]\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:23:46 INFO dspy.evaluate.evaluate: Average Metric: 24.96666666666666 / 35 (71.3%)\n", + "2026/01/29 21:23:46 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 71.33 on minibatch of size 35 with parameters ['Predictor 0: Instruction 1', 'Predictor 0: Few-Shot Set 16'].\n", + "2026/01/29 21:23:46 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48, 71.33]\n", + "2026/01/29 21:23:46 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87]\n", + "2026/01/29 21:23:46 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:23:46 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 28 / 37 - Minibatch ==\n" + "2026/01/29 21:23:46 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 28 / 37 - Minibatch ==\n" ] }, { @@ -1425,22 +1750,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.43 / 35 (75.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:00<00:00, 1232.02it/s]" + "Average Metric: 27.07 / 35 (77.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:00<00:00, 1.72s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:59:26 INFO dspy.evaluate.evaluate: Average Metric: 26.433333333333334 / 35 (75.5%)\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 75.52 on minibatch of size 35 with parameters ['Predictor 0: Instruction 4', 'Predictor 0: Few-Shot Set 1'].\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95, 75.14, 75.52]\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4]\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:24:46 INFO dspy.evaluate.evaluate: Average Metric: 27.06666666666667 / 35 (77.3%)\n", + "2026/01/29 21:24:46 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 77.33 on minibatch of size 35 with parameters ['Predictor 0: Instruction 4', 'Predictor 0: Few-Shot Set 10'].\n", + "2026/01/29 21:24:46 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48, 71.33, 77.33]\n", + "2026/01/29 21:24:46 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87]\n", + "2026/01/29 21:24:46 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:24:46 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:59:26 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 29 / 37 - Minibatch ==\n" + "2026/01/29 21:24:46 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 29 / 37 - Minibatch ==\n" ] }, { @@ -1448,22 +1773,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.60 / 35 (70.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:22<00:00, 1.56it/s]" + "Average Metric: 23.40 / 35 (66.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:57<00:00, 1.66s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 20:59:49 INFO dspy.evaluate.evaluate: Average Metric: 24.599999999999998 / 35 (70.3%)\n", - "2025/07/02 20:59:49 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 70.29 on minibatch of size 35 with parameters ['Predictor 0: Instruction 15', 'Predictor 0: Few-Shot Set 11'].\n", - "2025/07/02 20:59:49 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95, 75.14, 75.52, 70.29]\n", - "2025/07/02 20:59:49 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4]\n", - "2025/07/02 20:59:49 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 20:59:49 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:25:44 INFO dspy.evaluate.evaluate: Average Metric: 23.400000000000002 / 35 (66.9%)\n", + "2026/01/29 21:25:44 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 66.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 7', 'Predictor 0: Few-Shot Set 18'].\n", + "2026/01/29 21:25:44 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48, 71.33, 77.33, 66.86]\n", + "2026/01/29 21:25:44 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87]\n", + "2026/01/29 21:25:44 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:25:44 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 20:59:49 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 30 / 37 - Minibatch ==\n" + "2026/01/29 21:25:44 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 30 / 37 - Minibatch ==\n" ] }, { @@ -1471,23 +1796,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 24.13 / 35 (69.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.50it/s]" + "Average Metric: 29.03 / 35 (83.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:00<00:00, 1.72s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:00:12 INFO dspy.evaluate.evaluate: Average Metric: 24.133333333333333 / 35 (69.0%)\n", - "2025/07/02 21:00:12 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 68.95 on minibatch of size 35 with parameters ['Predictor 0: Instruction 1', 'Predictor 0: Few-Shot Set 11'].\n", - "2025/07/02 21:00:12 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95, 75.14, 75.52, 70.29, 68.95]\n", - "2025/07/02 21:00:12 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4]\n", - "2025/07/02 21:00:12 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 21:00:12 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:26:44 INFO dspy.evaluate.evaluate: Average Metric: 29.033333333333324 / 35 (83.0%)\n", + "2026/01/29 21:26:44 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 82.95 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 12'].\n", + "2026/01/29 21:26:44 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48, 71.33, 77.33, 66.86, 82.95]\n", + "2026/01/29 21:26:44 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87]\n", + "2026/01/29 21:26:44 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:26:44 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 21:00:12 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 31 / 37 - Full Evaluation =====\n", - "2025/07/02 21:00:12 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 78.86) from minibatch trials...\n" + "2026/01/29 21:26:44 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 31 / 37 - Full Evaluation =====\n", + "2026/01/29 21:26:44 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 82.95) from minibatch trials...\n" ] }, { @@ -1495,20 +1820,20 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 39.30 / 50 (78.6%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:10<00:00, 4.63it/s]" + "Average Metric: 37.13 / 50 (74.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [01:17<00:00, 1.56s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:00:23 INFO dspy.evaluate.evaluate: Average Metric: 39.3 / 50 (78.6%)\n", - "2025/07/02 21:00:23 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4, 78.6]\n", - "2025/07/02 21:00:23 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 21:00:23 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 21:00:23 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 21:28:02 INFO dspy.evaluate.evaluate: Average Metric: 37.13333333333332 / 50 (74.3%)\n", + "2026/01/29 21:28:02 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87, 74.27]\n", + "2026/01/29 21:28:02 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:28:02 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 21:28:02 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 21:00:23 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 32 / 37 - Minibatch ==\n" + "2026/01/29 21:28:02 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 32 / 37 - Minibatch ==\n" ] }, { @@ -1516,22 +1841,36 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.27 / 35 (75.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:23<00:00, 1.49it/s]" + "Average Metric: 10.97 / 14 (78.3%): 37%|β–ˆβ–ˆβ–ˆβ–‹ | 13/35 [00:22<00:36, 1.64s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:00:46 INFO dspy.evaluate.evaluate: Average Metric: 26.266666666666666 / 35 (75.0%)\n", - "2025/07/02 21:00:46 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 75.05 on minibatch of size 35 with parameters ['Predictor 0: Instruction 17', 'Predictor 0: Few-Shot Set 8'].\n", - "2025/07/02 21:00:46 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95, 75.14, 75.52, 70.29, 68.95, 75.05]\n", - "2025/07/02 21:00:46 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4, 78.6]\n", - "2025/07/02 21:00:46 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 21:00:46 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:28:25 WARNING amzn_nova_prompt_optimizer.core.inference.sagemaker_adapter: SageMaker ClientError: ModelError - Received server error (0) from primary with message \"Connection reset by peer for the jezhu-micro-rai-fixed-Jan21-endpoint endpoint. Please retry.\". See https://us-west-2.console.aws.amazon.com/cloudwatch/home?region=us-west-2#logEventViewer:group=/aws/sagemaker/Endpoints/jezhu-micro-rai-fixed-Jan21-endpoint in account 792306802243 for more information.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 27.00 / 35 (77.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:58<00:00, 1.66s/it]" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2026/01/29 21:29:00 INFO dspy.evaluate.evaluate: Average Metric: 26.99999999999999 / 35 (77.1%)\n", + "2026/01/29 21:29:00 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 77.14 on minibatch of size 35 with parameters ['Predictor 0: Instruction 15', 'Predictor 0: Few-Shot Set 12'].\n", + "2026/01/29 21:29:00 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48, 71.33, 77.33, 66.86, 82.95, 77.14]\n", + "2026/01/29 21:29:00 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87, 74.27]\n", + "2026/01/29 21:29:00 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:29:00 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 21:00:46 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 33 / 37 - Minibatch ==\n" + "2026/01/29 21:29:00 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 33 / 37 - Minibatch ==\n" ] }, { @@ -1539,22 +1878,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 26.20 / 35 (74.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:22<00:00, 1.54it/s]" + "Average Metric: 26.60 / 35 (76.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:50<00:00, 1.44s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:01:09 INFO dspy.evaluate.evaluate: Average Metric: 26.2 / 35 (74.9%)\n", - "2025/07/02 21:01:09 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 74.86 on minibatch of size 35 with parameters ['Predictor 0: Instruction 14', 'Predictor 0: Few-Shot Set 1'].\n", - "2025/07/02 21:01:09 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95, 75.14, 75.52, 70.29, 68.95, 75.05, 74.86]\n", - "2025/07/02 21:01:09 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4, 78.6]\n", - "2025/07/02 21:01:09 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 21:01:09 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:29:51 INFO dspy.evaluate.evaluate: Average Metric: 26.599999999999998 / 35 (76.0%)\n", + "2026/01/29 21:29:51 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 76.0 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 0'].\n", + "2026/01/29 21:29:51 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48, 71.33, 77.33, 66.86, 82.95, 77.14, 76.0]\n", + "2026/01/29 21:29:51 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87, 74.27]\n", + "2026/01/29 21:29:51 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:29:51 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 21:01:09 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 34 / 37 - Minibatch ==\n" + "2026/01/29 21:29:51 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 34 / 37 - Minibatch ==\n" ] }, { @@ -1562,22 +1901,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 25.87 / 35 (73.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:25<00:00, 1.39it/s]" + "Average Metric: 25.03 / 35 (71.5%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:00<00:00, 1.73s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:01:34 INFO dspy.evaluate.evaluate: Average Metric: 25.866666666666667 / 35 (73.9%)\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 73.9 on minibatch of size 35 with parameters ['Predictor 0: Instruction 11', 'Predictor 0: Few-Shot Set 1'].\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95, 75.14, 75.52, 70.29, 68.95, 75.05, 74.86, 73.9]\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4, 78.6]\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:30:51 INFO dspy.evaluate.evaluate: Average Metric: 25.03333333333333 / 35 (71.5%)\n", + "2026/01/29 21:30:51 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 71.52 on minibatch of size 35 with parameters ['Predictor 0: Instruction 5', 'Predictor 0: Few-Shot Set 12'].\n", + "2026/01/29 21:30:51 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48, 71.33, 77.33, 66.86, 82.95, 77.14, 76.0, 71.52]\n", + "2026/01/29 21:30:51 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87, 74.27]\n", + "2026/01/29 21:30:51 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:30:51 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 35 / 37 - Minibatch ==\n" + "2026/01/29 21:30:51 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 35 / 37 - Minibatch ==\n" ] }, { @@ -1585,22 +1924,22 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 27.07 / 35 (77.3%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:00<00:00, 1187.20it/s]" + "Average Metric: 24.13 / 35 (69.0%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [01:00<00:00, 1.72s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:01:34 INFO dspy.evaluate.evaluate: Average Metric: 27.066666666666666 / 35 (77.3%)\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 77.33 on minibatch of size 35 with parameters ['Predictor 0: Instruction 17', 'Predictor 0: Few-Shot Set 1'].\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95, 75.14, 75.52, 70.29, 68.95, 75.05, 74.86, 73.9, 77.33]\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4, 78.6]\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:31:51 INFO dspy.evaluate.evaluate: Average Metric: 24.133333333333336 / 35 (69.0%)\n", + "2026/01/29 21:31:51 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 68.95 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 6'].\n", + "2026/01/29 21:31:51 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48, 71.33, 77.33, 66.86, 82.95, 77.14, 76.0, 71.52, 68.95]\n", + "2026/01/29 21:31:51 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87, 74.27]\n", + "2026/01/29 21:31:51 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:31:51 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 21:01:34 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 36 / 37 - Minibatch ==\n" + "2026/01/29 21:31:51 INFO dspy.teleprompt.mipro_optimizer_v2: == Trial 36 / 37 - Minibatch ==\n" ] }, { @@ -1608,23 +1947,23 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 28.23 / 35 (80.7%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:22<00:00, 1.53it/s]" + "Average Metric: 27.33 / 35 (78.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 35/35 [00:59<00:00, 1.71s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:01:57 INFO dspy.evaluate.evaluate: Average Metric: 28.233333333333334 / 35 (80.7%)\n", - "2025/07/02 21:01:57 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 80.67 on minibatch of size 35 with parameters ['Predictor 0: Instruction 12', 'Predictor 0: Few-Shot Set 14'].\n", - "2025/07/02 21:01:57 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [72.86, 71.9, 77.52, 72.57, 71.62, 74.1, 76.29, 69.43, 72.0, 75.81, 77.24, 76.86, 76.29, 79.24, 73.05, 78.86, 73.05, 79.14, 72.0, 72.86, 64.95, 75.14, 75.52, 70.29, 68.95, 75.05, 74.86, 73.9, 77.33, 80.67]\n", - "2025/07/02 21:01:57 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4, 78.6]\n", - "2025/07/02 21:01:57 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 21:01:57 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", + "2026/01/29 21:32:51 INFO dspy.evaluate.evaluate: Average Metric: 27.33333333333334 / 35 (78.1%)\n", + "2026/01/29 21:32:51 INFO dspy.teleprompt.mipro_optimizer_v2: Score: 78.1 on minibatch of size 35 with parameters ['Predictor 0: Instruction 8', 'Predictor 0: Few-Shot Set 7'].\n", + "2026/01/29 21:32:51 INFO dspy.teleprompt.mipro_optimizer_v2: Minibatch scores so far: [73.62, 83.05, 72.86, 81.24, 75.14, 76.1, 67.71, 72.29, 81.43, 79.52, 75.9, 82.29, 73.05, 82.57, 77.81, 81.14, 76.95, 80.76, 78.67, 79.43, 72.48, 71.33, 77.33, 66.86, 82.95, 77.14, 76.0, 71.52, 68.95, 78.1]\n", + "2026/01/29 21:32:51 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87, 74.27]\n", + "2026/01/29 21:32:51 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:32:51 INFO dspy.teleprompt.mipro_optimizer_v2: ==========================================\n", "\n", "\n", - "2025/07/02 21:01:57 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 37 / 37 - Full Evaluation =====\n", - "2025/07/02 21:01:57 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 80.67) from minibatch trials...\n" + "2026/01/29 21:32:51 INFO dspy.teleprompt.mipro_optimizer_v2: ===== Trial 37 / 37 - Full Evaluation =====\n", + "2026/01/29 21:32:51 INFO dspy.teleprompt.mipro_optimizer_v2: Doing full eval on next top averaging program (Avg Score: 79.52) from minibatch trials...\n" ] }, { @@ -1632,20 +1971,20 @@ "output_type": "stream", "text": [ "\n", - "Average Metric: 38.43 / 50 (76.9%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [00:09<00:00, 5.19it/s]" + "Average Metric: 40.03 / 50 (80.1%): 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 50/50 [01:22<00:00, 1.64s/it]" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:02:07 INFO dspy.evaluate.evaluate: Average Metric: 38.43333333333333 / 50 (76.9%)\n", - "2025/07/02 21:02:07 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [70.33, 77.07, 80.07, 77.13, 78.4, 78.6, 76.87]\n", - "2025/07/02 21:02:07 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 80.07\n", - "2025/07/02 21:02:07 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", - "2025/07/02 21:02:07 INFO dspy.teleprompt.mipro_optimizer_v2: \n", + "2026/01/29 21:34:13 INFO dspy.evaluate.evaluate: Average Metric: 40.033333333333324 / 50 (80.1%)\n", + "2026/01/29 21:34:13 INFO dspy.teleprompt.mipro_optimizer_v2: Full eval scores so far: [71.07, 81.27, 80.8, 79.33, 76.87, 74.27, 80.07]\n", + "2026/01/29 21:34:13 INFO dspy.teleprompt.mipro_optimizer_v2: Best full score so far: 81.27\n", + "2026/01/29 21:34:13 INFO dspy.teleprompt.mipro_optimizer_v2: =======================\n", + "2026/01/29 21:34:13 INFO dspy.teleprompt.mipro_optimizer_v2: \n", "\n", - "2025/07/02 21:02:07 INFO dspy.teleprompt.mipro_optimizer_v2: Returning best identified program with score 80.07!\n" + "2026/01/29 21:34:13 INFO dspy.teleprompt.mipro_optimizer_v2: Returning best identified program with score 81.27!\n" ] }, { @@ -1661,12 +2000,12 @@ "\n", "nova_prompt_optimizer = NovaPromptOptimizer(prompt_adapter=prompt_adapter, inference_adapter=inference_adapter, dataset_adapter=train_set, metric_adapter=nova_metric_adapter)\n", "\n", - "optimized_prompt_adapter = nova_prompt_optimizer.optimize(mode=\"pro\")" + "optimized_prompt_adapter = nova_prompt_optimizer.optimize(mode=\"micro\")" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 31, "id": "5f1de0fc", "metadata": {}, "outputs": [ @@ -1674,7 +2013,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:02:07 INFO amzn_nova_prompt_optimizer.core.input_adapters.prompt_adapter: \n", + "2026/01/29 21:36:29 INFO amzn_nova_prompt_optimizer.core.input_adapters.prompt_adapter: \n", "Standardized Prompt:\n" ] }, @@ -1687,14 +2026,14 @@ " \"variables\": [\n", " \"input\"\n", " ],\n", - " \"template\": \"Extract and return a json with the following keys and values from the input provided: [{{ input }}]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\",\n", + " \"template\": \"{{ input }}\",\n", " \"metadata\": {\n", " \"format\": \"text\"\n", " }\n", " },\n", " \"system_prompt\": {\n", " \"variables\": [],\n", - " \"template\": \"Analyze the input text and classify it into a JSON object with \\\"urgency\\\", \\\"sentiment\\\", and \\\"categories\\\" keys. Ensure all categories are covered with boolean values, and the output is a valid JSON string without extra formatting.\",\n", + " \"template\": \"]\\nTask: Analyze and categorize the incoming text to determine urgency, sentiment, and relevant categories.\\n\\nContext:\\n- The input will be analyzed to determine urgency, sentiment, and relevant categories.\\n- Urgency must be classified as one of `high`, `medium`, or `low`.\\n- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\\n- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\\n\\nInstructions:\\n- Carefully analyze the input to accurately determine urgency and sentiment.\\n- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\\n- The output must be a valid JSON string containing only the specified keys: \\\"urgency\\\", \\\"sentiment\\\", and \\\"categories\\\".\\n- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\\n\\nResponse Format:\\n- The response MUST be a valid JSON string.\\n- The JSON MUST contain exactly the keys: \\\"urgency\\\", \\\"sentiment\\\", and \\\"categories\\\".\\n- The JSON MUST be on a single line with no extra spaces or line breaks.\\n- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\",\n", " \"metadata\": {\n", " \"format\": \"text\"\n", " }\n", @@ -1702,20 +2041,20 @@ " \"few_shot\": {\n", " \"examples\": [\n", " {\n", - " \"input\": \"Extract and return a json with the following keys and values from the input provided: [Subject: Request for Post-Renovation Cleaning Services\\n\\nDear ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I have been utilizing ProCare Facility Solutions for the maintenance and management of my commercial property for the past few years. Your services have always been reliable and efficient, which is why I am reaching out to you today.\\n\\nI am writing to request assistance with a specialized cleaning service for our office building. We have recently undergone some renovations, and there is a significant amount of dust and debris that needs to be addressed. Additionally, the carpets and windows require a thorough cleaning to restore them to their original condition.\\n\\nI have already scheduled a routine cleaning, but I believe this situation requires more specialized attention. Given the nature of the work needed, I would appreciate it if you could arrange for a team that specializes in deep cleaning and post-renovation cleanup.\\n\\nCould you please provide me with the available dates and any additional information required to proceed with this request? Your prompt assistance in this matter would be greatly appreciated, as we aim to have the office fully operational and presentable as soon as possible.\\n\\nThank you for your attention to this matter. I look forward to your response.\\n\\nBest regards,\\n\\n[Sender]]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\",\n", - " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": true, \\\"specialized_cleaning_services\\\": true, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"neutral\\\", \\\"urgency\\\": \\\"medium\\\"}\"\n", + " \"input\": \"Subject: Scheduled Maintenance Request for Bathroom Plumbing\\n\\nDear ProCare Facility Solutions Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I have been utilizing your services for my residential property for some time now. I have always appreciated the quality and reliability of your maintenance services.\\n\\nI am writing to inform you of a minor issue that has recently come up. There seems to be a small problem with the plumbing in my bathroom. While it is not an urgent matter, I would like to have it addressed at your earliest convenience to prevent any potential complications.\\n\\nI have not taken any steps to resolve the issue myself, as I trust your team\\u2019s expertise in handling such matters. Could you please arrange for a technician to visit my home and take a look at the problem as part of your scheduled maintenance services?\\n\\nThank you for your attention to this matter. I look forward to your prompt response.\\n\\nBest regards,\\n[Sender]\",\n", + " \"output\": \" {\\\"categories\\\": {\\\"routine_maintenance_requests\\\": true, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": true, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"neutral\\\", \\\"urgency\\\": \\\"medium\\\"}\"\n", " },\n", " {\n", - " \"input\": \"Extract and return a json with the following keys and values from the input provided: [Subject: Urgent HVAC System Repair Needed\\n\\nDear ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I am the curator at [Museum Name]. We have been utilizing ProCare Facility Solutions for our maintenance needs for the past year, and I have always appreciated the professionalism and quality of your services.\\n\\nI am writing to inform you of an urgent issue we are currently facing with our HVAC system. Over the past few days, we have noticed irregularities in the temperature control within our exhibit halls. This is particularly concerning as we have several sensitive classical art pieces that require a stable climate to ensure their preservation.\\n\\nWe have attempted to adjust the settings manually and have conducted a basic inspection of the system, but the problem persists. Given the importance of maintaining an optimal environment for our collections, I would appreciate it if you could send a technician to address this issue immediately.\\n\\nThank you for your prompt attention to this matter. I look forward to your swift response and resolution.\\n\\nBest regards,\\n\\n[Sender] \\n[Museum Name] \\n[Contact Information]]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\",\n", - " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": true, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": true, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"neutral\\\", \\\"urgency\\\": \\\"high\\\"}\"\n", + " \"input\": \"Subject: Follow-Up on HVAC Maintenance Issue\\n\\nDear ProCare Facility Solutions Support Team,\\n\\nI hope this message finds you well. My name is Dr. [Sender], a retired professor and a long-time resident of [Residential Complex Name]. I have been utilizing your maintenance services for some time now and generally appreciate the professionalism and thoroughness your team brings to the table.\\n\\nHowever, I would like to bring to your attention a recent issue I encountered with the routine maintenance of the HVAC system in my apartment. While the technician was courteous and seemed knowledgeable, the problem with the system persists. Despite the service visit, the HVAC unit continues to make an unusual noise, which is quite disruptive.\\n\\nI have not taken any further steps beyond the initial service call, as I wanted to first communicate my concerns directly with your support team. I would appreciate it if you could arrange for a follow-up visit to address this issue more comprehensively.\\n\\nThank you for your attention to this matter. I look forward to your prompt response and resolution.\\n\\nBest regards,\\nDr. [Sender]\",\n", + " \"output\": \" {\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": true, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": true, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"negative\\\", \\\"urgency\\\": \\\"medium\\\"}\"\n", " },\n", " {\n", - " \"input\": \"Extract and return a json with the following keys and values from the input provided: [Subject: Routine Maintenance Request for Theater Facility\\n\\nDear ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I am a theater producer who has been working closely with your team to ensure our facility remains in top condition. I have always admired the innovative methods your team employs, which have significantly contributed to the smooth operation of our workshops and classes.\\n\\nI am writing to request routine maintenance for our theater facility. Specifically, we have noticed that the HVAC system is not performing optimally, and there are minor plumbing issues in the restrooms that need attention. These issues are not urgent but do require timely intervention to prevent any disruption to our scheduled activities.\\n\\nWe have not taken any steps to address these issues internally, as we prefer to rely on your expertise to ensure everything is handled correctly and efficiently.\\n\\nCould you please arrange for a maintenance visit at your earliest convenience? We would appreciate it if the visit could be scheduled within the next week to ensure everything is in order before our next major event.\\n\\nThank you for your attention to this matter. We look forward to your prompt response and continued excellent service.\\n\\nBest regards,\\n\\n[Sender]]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\",\n", - " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": true, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"positive\\\", \\\"urgency\\\": \\\"medium\\\"}\"\n", + " \"input\": \"Subject: Request for Eco-Friendly Deep Cleaning Services\\n\\nHi ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I\\u2019ve been a client of ProCare Facility Solutions for a while now. As someone who\\u2019s spent years in the fast lane, both on and off the track, I appreciate the importance of precision and attention to detail, which is why I\\u2019ve always trusted your services.\\n\\nI\\u2019m reaching out today because I need some specialized cleaning services for my property. Specifically, I\\u2019m looking for a deep cleaning of my garage and workshop area. These spaces have accumulated quite a bit of grime and dust over time, and I\\u2019d like to get them back to a pristine condition. Given the nature of the work I do there, it\\u2019s crucial that the cleaning is thorough and uses eco-friendly products.\\n\\nI haven\\u2019t taken any steps to address this issue yet, as I wanted to consult with the experts first. I\\u2019m hoping you can provide a customized cleaning plan that fits my needs and schedule.\\n\\nCould you please let me know the next steps and any details you need from my end to get this sorted? I\\u2019m looking forward to your assistance in making my workspace spotless again.\\n\\nThanks in advance for your help.\\n\\nBest regards,\\n[Sender]\",\n", + " \"output\": \" {\\\"categories\\\": {\\\"specialized_cleaning_services\\\": true, \\\"sustainability_and_environmental_practices\\\": true, \\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"positive\\\", \\\"urgency\\\": \\\"medium\\\"}\"\n", " },\n", " {\n", - " \"input\": \"Extract and return a json with the following keys and values from the input provided: [Subject: Scheduling Cleaning Services for Upcoming Event\\n\\nHi ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I\\u2019ve been a loyal customer of ProCare Facility Solutions for the past year. I manage the performance analysis for the New York Red Bulls, and I must say, your services have been instrumental in maintaining our office environment, allowing us to focus on our work without any distractions.\\n\\nI\\u2019m reaching out to schedule a cleaning service for our office space. We have an important event coming up next month, and I want to ensure that everything is in pristine condition for our guests. Given the nature of our work, a clean and organized environment is crucial for our productivity and overall morale.\\n\\nI haven\\u2019t taken any steps yet to schedule this service, as I wanted to get in touch with your team directly to ensure we get the best possible arrangement. Ideally, we would need a thorough cleaning a few days before the event, including window washing and carpet maintenance.\\n\\nCould you please assist me in setting up a suitable cleaning schedule? I\\u2019m confident that your team will handle this with the same excellence and attention to detail that we\\u2019ve come to expect from ProCare.\\n\\nThank you so much for your help. Looking forward to your response.\\n\\nBest regards,\\n[Sender]]\\n- \\\"urgency\\\" as one of `high`, `medium`, `low`\\n- \\\"sentiment\\\" as one of `negative`, `neutral`, `positive`\\n- \\\"categories\\\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\",\n", - " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": true, \\\"specialized_cleaning_services\\\": true, \\\"emergency_repair_services\\\": false, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"positive\\\", \\\"urgency\\\": \\\"medium\\\"}\"\n", + " \"input\": \"Subject: Urgent Assistance Needed for HVAC System Issue\\n\\nHi ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I\\u2019ve been a satisfied client of ProCare Facility Solutions for the past year. Your team has always provided exceptional service, and I truly appreciate the dedication and professionalism you bring to maintaining my property.\\n\\nHowever, I\\u2019m currently facing an urgent issue that requires immediate attention. Over the past few days, I\\u2019ve noticed a significant drop in the efficiency of the HVAC system in my home. Despite the routine maintenance checks, the system seems to be struggling to maintain a consistent temperature, which is crucial for my training and recovery as a professional athlete.\\n\\nI\\u2019ve already tried adjusting the thermostat and checking the filters, but the problem persists. Given the high stakes of my athletic performance, I need this issue resolved as quickly as possible to ensure my living environment remains optimal for my needs.\\n\\nCould you please arrange for an emergency repair at the earliest convenience? Your prompt assistance in this matter would be greatly appreciated, as it directly impacts my daily routine and overall well-being.\\n\\nThank you for your understanding and swift action. I look forward to hearing from you soon.\\n\\nBest regards,\\n[Sender]\",\n", + " \"output\": \"{\\\"categories\\\": {\\\"routine_maintenance_requests\\\": false, \\\"customer_feedback_and_complaints\\\": false, \\\"training_and_support_requests\\\": false, \\\"quality_and_safety_concerns\\\": false, \\\"sustainability_and_environmental_practices\\\": false, \\\"cleaning_services_scheduling\\\": false, \\\"specialized_cleaning_services\\\": false, \\\"emergency_repair_services\\\": true, \\\"facility_management_issues\\\": false, \\\"general_inquiries\\\": false}, \\\"sentiment\\\": \\\"positive\\\", \\\"urgency\\\": \\\"high\\\"}\"\n", " }\n", " ],\n", " \"format\": \"converse\"\n", @@ -1738,7 +2077,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 32, "id": "4b4b52d7", "metadata": {}, "outputs": [ @@ -1746,7 +2085,26 @@ "name": "stdout", "output_type": "stream", "text": [ - "Analyze the input text and classify it into a JSON object with \"urgency\", \"sentiment\", and \"categories\" keys. Ensure all categories are covered with boolean values, and the output is a valid JSON string without extra formatting.\n" + "]\n", + "Task: Analyze and categorize the incoming text to determine urgency, sentiment, and relevant categories.\n", + "\n", + "Context:\n", + "- The input will be analyzed to determine urgency, sentiment, and relevant categories.\n", + "- Urgency must be classified as one of `high`, `medium`, or `low`.\n", + "- Sentiment must be classified as one of `negative`, `neutral`, or `positive`.\n", + "- Categories must be a dictionary with keys from the list: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`. Each category will have a boolean value indicating whether it is the best matching support category tag.\n", + "\n", + "Instructions:\n", + "- Carefully analyze the input to accurately determine urgency and sentiment.\n", + "- Match the input to the most appropriate category from the provided list and set its value to True, while setting all others to False.\n", + "- The output must be a valid JSON string containing only the specified keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON must be compact with no unnecessary whitespace or newlines and should not be enclosed in triple backticks or any other formatting.\n", + "\n", + "Response Format:\n", + "- The response MUST be a valid JSON string.\n", + "- The JSON MUST contain exactly the keys: \"urgency\", \"sentiment\", and \"categories\".\n", + "- The JSON MUST be on a single line with no extra spaces or line breaks.\n", + "- The response MUST NOT include any additional text, explanations, or formatting outside the JSON structure.\n" ] } ], @@ -1764,7 +2122,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 33, "id": "9795751a", "metadata": {}, "outputs": [ @@ -1772,10 +2130,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Extract and return a json with the following keys and values from the input provided: [{{ input }}]\n", - "- \"urgency\" as one of `high`, `medium`, `low`\n", - "- \"sentiment\" as one of `negative`, `neutral`, `positive`\n", - "- \"categories\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`\n" + "{{ input }}\n" ] } ], @@ -1793,7 +2148,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 34, "id": "4dfa1fdb-7273-4c76-a0ad-e8a0692bfbec", "metadata": {}, "outputs": [ @@ -1811,7 +2166,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 35, "id": "b25e3b16-0f49-41cd-a68b-d6ccbc51b656", "metadata": {}, "outputs": [ @@ -1819,7 +2174,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'input': 'Extract and return a json with the following keys and values from the input provided: [Subject: Request for Post-Renovation Cleaning Services\\n\\nDear ProCare Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I have been utilizing ProCare Facility Solutions for the maintenance and management of my commercial property for the past few years. Your services have always been reliable and efficient, which is why I am reaching out to you today.\\n\\nI am writing to request assistance with a specialized cleaning service for our office building. We have recently undergone some renovations, and there is a significant amount of dust and debris that needs to be addressed. Additionally, the carpets and windows require a thorough cleaning to restore them to their original condition.\\n\\nI have already scheduled a routine cleaning, but I believe this situation requires more specialized attention. Given the nature of the work needed, I would appreciate it if you could arrange for a team that specializes in deep cleaning and post-renovation cleanup.\\n\\nCould you please provide me with the available dates and any additional information required to proceed with this request? Your prompt assistance in this matter would be greatly appreciated, as we aim to have the office fully operational and presentable as soon as possible.\\n\\nThank you for your attention to this matter. I look forward to your response.\\n\\nBest regards,\\n\\n[Sender]]\\n- \"urgency\" as one of `high`, `medium`, `low`\\n- \"sentiment\" as one of `negative`, `neutral`, `positive`\\n- \"categories\" as a dictionary with categories as keys and boolean values indicating if the category matches the input. Categories are: `emergency_repair_services`, `routine_maintenance_requests`, `quality_and_safety_concerns`, `specialized_cleaning_services`, `general_inquiries`, `sustainability_and_environmental_practices`, `training_and_support_requests`, `cleaning_services_scheduling`, `customer_feedback_and_complaints`, `facility_management_issues`', 'output': '{\"categories\": {\"routine_maintenance_requests\": false, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": true, \"specialized_cleaning_services\": true, \"emergency_repair_services\": false, \"facility_management_issues\": false, \"general_inquiries\": false}, \"sentiment\": \"neutral\", \"urgency\": \"medium\"}'}\n" + "{'input': 'Subject: Scheduled Maintenance Request for Bathroom Plumbing\\n\\nDear ProCare Facility Solutions Support Team,\\n\\nI hope this message finds you well. My name is [Sender], and I have been utilizing your services for my residential property for some time now. I have always appreciated the quality and reliability of your maintenance services.\\n\\nI am writing to inform you of a minor issue that has recently come up. There seems to be a small problem with the plumbing in my bathroom. While it is not an urgent matter, I would like to have it addressed at your earliest convenience to prevent any potential complications.\\n\\nI have not taken any steps to resolve the issue myself, as I trust your team’s expertise in handling such matters. Could you please arrange for a technician to visit my home and take a look at the problem as part of your scheduled maintenance services?\\n\\nThank you for your attention to this matter. I look forward to your prompt response.\\n\\nBest regards,\\n[Sender]', 'output': ' {\"categories\": {\"routine_maintenance_requests\": true, \"customer_feedback_and_complaints\": false, \"training_and_support_requests\": false, \"quality_and_safety_concerns\": false, \"sustainability_and_environmental_practices\": false, \"cleaning_services_scheduling\": false, \"specialized_cleaning_services\": false, \"emergency_repair_services\": false, \"facility_management_issues\": true, \"general_inquiries\": false}, \"sentiment\": \"neutral\", \"urgency\": \"medium\"}'}\n" ] } ], @@ -1840,7 +2195,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 41, "id": "e136ccc2", "metadata": {}, "outputs": [], @@ -1852,7 +2207,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 42, "id": "e43b528d", "metadata": {}, "outputs": [ @@ -1860,18 +2215,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025/07/02 21:02:07 INFO amzn_nova_prompt_optimizer.core.evaluation: Cache miss - Running new inference on Dataset\n", - "Running inference: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100/100 [00:44<00:00, 2.27it/s]\n", - "2025/07/02 21:02:51 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Batch Evaluation on Dataset, using `batch_apply` metric\n", - "2025/07/02 21:02:51 INFO amzn_nova_prompt_optimizer.core.evaluation: Using cached inference results\n", - "2025/07/02 21:02:51 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Evaluation on Dataset, using `apply` metric\n" + "2026/01/29 21:37:38 INFO amzn_nova_prompt_optimizer.core.evaluation: Cache miss - Running new inference on Dataset\n", + "Running inference: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 100/100 [01:29<00:00, 1.12it/s]\n", + "2026/01/29 21:39:07 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Batch Evaluation on Dataset, using `batch_apply` metric\n", + "2026/01/29 21:39:07 INFO amzn_nova_prompt_optimizer.core.evaluation: Using cached inference results\n", + "2026/01/29 21:39:07 INFO amzn_nova_prompt_optimizer.core.evaluation: Running Evaluation on Dataset, using `apply` metric\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Nova Prompt Optimization = {'is_valid_json': 1.0, 'correct_categories': 0.9309999999999999, 'correct_sentiment': 0.66, 'correct_urgency': 0.84, 'total': 0.8103333333333333}\n" + "Nova Prompt Optimization = {'is_valid_json': 1.0, 'correct_categories': 0.8990000000000002, 'correct_sentiment': 0.69, 'correct_urgency': 0.84, 'total': 0.8096666666666666}\n" ] } ], @@ -1882,7 +2237,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 43, "id": "98a74bbb-1744-4daa-a17e-89ede77fee6f", "metadata": {}, "outputs": [], @@ -1893,9 +2248,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.12 (.venv)", + "display_name": "Python 3", "language": "python", - "name": "python312venv" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1907,7 +2262,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.11" + "version": "3.9.6" } }, "nbformat": 4, diff --git a/src/amzn_nova_prompt_optimizer/__version__.py b/src/amzn_nova_prompt_optimizer/__version__.py index 8af31f4..1383b0a 100644 --- a/src/amzn_nova_prompt_optimizer/__version__.py +++ b/src/amzn_nova_prompt_optimizer/__version__.py @@ -11,4 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -VERSION = "1.0.53" # pragma: no cover +VERSION = "1.1.0" # pragma: no cover diff --git a/src/amzn_nova_prompt_optimizer/core/inference/__init__.py b/src/amzn_nova_prompt_optimizer/core/inference/__init__.py index acfd449..ec824a4 100644 --- a/src/amzn_nova_prompt_optimizer/core/inference/__init__.py +++ b/src/amzn_nova_prompt_optimizer/core/inference/__init__.py @@ -13,7 +13,21 @@ # limitations under the License. import logging -from amzn_nova_prompt_optimizer.core.inference.adapter import InferenceAdapter +from amzn_nova_prompt_optimizer.core.inference.adapter import ( + InferenceAdapter +) +from amzn_nova_prompt_optimizer.core.inference.bedrock_adapter import ( + BedrockInferenceAdapter +) +from amzn_nova_prompt_optimizer.core.inference.sagemaker_adapter import ( + SageMakerInferenceAdapter +) +from amzn_nova_prompt_optimizer.core.inference.dspy_compatible import ( + DSPyCompatibleInferenceAdapter, + DSPyBedrockAdapter, + DSPySageMakerAdapter, + create_dspy_adapter +) from amzn_nova_prompt_optimizer.core.inference.inference_constants import ( MAX_TOKENS_FIELD, TEMPERATURE_FIELD, TOP_P_FIELD, TOP_K_FIELD ) diff --git a/src/amzn_nova_prompt_optimizer/core/inference/adapter.py b/src/amzn_nova_prompt_optimizer/core/inference/adapter.py index 89dd35b..67b72f3 100644 --- a/src/amzn_nova_prompt_optimizer/core/inference/adapter.py +++ b/src/amzn_nova_prompt_optimizer/core/inference/adapter.py @@ -11,22 +11,38 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging -import random -from abc import ABC, abstractmethod -from typing import Optional, Dict, Any, List -import boto3 -import time -from botocore.exceptions import ClientError +""" +Base Inference Adapter + +This module provides the abstract base class for all inference adapters. +""" -from amzn_nova_prompt_optimizer.core.inference.bedrock_converse import BedrockConverseHandler -from amzn_nova_prompt_optimizer.util.rate_limiter import RateLimiter +from abc import ABC, abstractmethod +from typing import Dict, Any, List -logger = logging.getLogger(__name__) class InferenceAdapter(ABC): + """ + Abstract base class for inference adapters. + + All inference adapters must inherit from this class and implement + the call_model method. This provides a consistent interface for + making inference calls across different backends (Bedrock, SageMaker, etc.). + + Attributes: + region: AWS region or endpoint region + rate_limit: Maximum requests per second + """ + def __init__(self, region: str, rate_limit: int = 2): + """ + Initialize the inference adapter. + + Args: + region: AWS region or endpoint region + rate_limit: Maximum requests per second (default: 2) + """ self.region = region self.rate_limit = rate_limit @@ -34,76 +50,50 @@ def __init__(self, region: str, rate_limit: int = 2): def call_model(self, model_id: str, system_prompt: str, messages: List[Dict[str, str]], inf_config: Dict[str, Any]) -> str: + """ + Call the model for inference. + + This method must be implemented by all subclasses. + + Args: + model_id: Model identifier + system_prompt: System prompt text + messages: List of conversation messages in format: + [{"user": "..."}, {"assistant": "..."}, ...] + inf_config: Inference configuration parameters + + Returns: + Model response text + + Raises: + NotImplementedError: If not implemented by subclass + """ pass - - -class BedrockInferenceAdapter(InferenceAdapter): - def __init__(self, - region_name: str = 'us-east-1', - profile_name: Optional[str] = None, - max_retries: int = 5, - rate_limit: int = 2, - initial_backoff: int = 1): + + def to_dspy_lm(self, model_id: str, **kwargs): """ - Initialize Bedrock Inference Adapter with AWS credentials - + Create a DSPy-compatible wrapper for this adapter. + + This is a convenience method that automatically creates the + appropriate DSPy-compatible adapter based on the adapter type. + Args: - region_name: AWS region name - profile_name: Optional. AWS credential profile name. - max_retries: Maximum number of retries for API calls - rate_limit: Max TPS of the bedrock call this adapter can make. Default to 2. + model_id: Model identifier to use + **kwargs: Additional parameters for the DSPy adapter + + Returns: + DSPy-compatible adapter instance + + Example: + >>> adapter = BedrockInferenceAdapter(region_name="us-east-1") + >>> dspy_lm = adapter.to_dspy_lm("us.amazon.nova-pro-v1:0") + >>> import dspy + >>> dspy.configure(lm=dspy_lm) """ - super().__init__(region=region_name, rate_limit=rate_limit) - self.initial_backoff = initial_backoff - self.max_retries = max_retries - self.rate_limiter = RateLimiter(rate_limit=self.rate_limit) - - # Initialize AWS session with provided credentials - if profile_name: - # Use AWS profile if specified - session = boto3.Session(profile_name=profile_name) - else: - # Fall back to default credentials (environment variables or IAM role) - session = boto3.Session() - - # Create Bedrock client - self.bedrock_client = session.client( - 'bedrock-runtime', - region_name=region_name + from amzn_nova_prompt_optimizer.core.inference.dspy_compatible import ( + create_dspy_adapter ) - self.converse_client = BedrockConverseHandler(self.bedrock_client) - - def call_model(self, model_id: str, system_prompt: str, - messages: List[Dict[str, str]], inf_config: Dict[str, Any]) -> str: - self.rate_limiter.apply_rate_limiting() - return self._call_model_with_retry(model_id, system_prompt, messages, inf_config) + return create_dspy_adapter(self, model_id, **kwargs) - def _call_model_with_retry(self, model_id:str, system_prompt: str, - messages: List[Dict[str, str]], inf_config: Dict[str, Any]) -> str: - retries = 0 - while retries < self.max_retries: - try: - return self.converse_client.call_model(model_id, system_prompt, messages, inf_config) - except ClientError as e: - if e.response['Error']['Code'] == 'ThrottlingException': - wait_time = self._calculate_backoff_time(retries) - logger.debug(f"Throttled. Retrying in {wait_time} seconds... (Attempt {retries + 1}/{self.max_retries})") - time.sleep(wait_time) - retries += 1 - elif e.response['Error']['Code'] == 'ModelErrorException': - wait_time = self._calculate_backoff_time(retries) - logger.debug(f"Encountered ModelErrorException, Retrying in {wait_time} seconds...(Attempt {retries + 1}/{self.max_retries})") - time.sleep(wait_time) - retries += 1 - elif e.response['Error']['Code'] == 'ServiceUnavailableException': - wait_time = self._calculate_backoff_time(retries) - logger.debug(f"Retryable exception: ServiceUnavailableException {model_id}. Retrying in {wait_time} seconds... (Attempt {retries + 1}/{self.max_retries})") - time.sleep(wait_time) - retries += 1 - else: - raise e - raise Exception(f"Max retries ({self.max_retries}) exceeded for model call") - def _calculate_backoff_time(self, retry_count): - # Exponential backoff with jitter - return self.initial_backoff * (2 ** retry_count) + random.uniform(0, 1) +__all__ = ['InferenceAdapter'] diff --git a/src/amzn_nova_prompt_optimizer/core/inference/bedrock_adapter.py b/src/amzn_nova_prompt_optimizer/core/inference/bedrock_adapter.py new file mode 100644 index 0000000..7a1520b --- /dev/null +++ b/src/amzn_nova_prompt_optimizer/core/inference/bedrock_adapter.py @@ -0,0 +1,181 @@ +# Copyright 2025 Amazon Inc + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Bedrock Inference Adapter + +This module provides the BedrockInferenceAdapter for making inference calls +to Amazon Bedrock models using the Converse API. +""" + +import logging +import random +import time +from typing import Optional, Dict, Any, List + +import boto3 +from botocore.exceptions import ClientError + +from amzn_nova_prompt_optimizer.core.inference.adapter import InferenceAdapter +from amzn_nova_prompt_optimizer.core.inference.bedrock_converse import BedrockConverseHandler +from amzn_nova_prompt_optimizer.util.rate_limiter import RateLimiter + +logger = logging.getLogger(__name__) + + +class BedrockInferenceAdapter(InferenceAdapter): + """ + Inference adapter for Amazon Bedrock models. + + This adapter handles communication with Bedrock models using the Converse API, + including automatic retry logic with exponential backoff and rate limiting. + + Attributes: + region: AWS region name + rate_limit: Maximum requests per second + max_retries: Maximum number of retry attempts + initial_backoff: Initial backoff time in seconds for retries + bedrock_client: Boto3 Bedrock runtime client + converse_client: Handler for Bedrock Converse API calls + rate_limiter: Rate limiter instance + """ + + def __init__(self, + region_name: str = 'us-east-1', + profile_name: Optional[str] = None, + max_retries: int = 5, + rate_limit: int = 2, + initial_backoff: int = 1): + """ + Initialize Bedrock Inference Adapter with AWS credentials. + + Args: + region_name: AWS region name (default: 'us-east-1') + profile_name: Optional AWS credential profile name + max_retries: Maximum number of retries for API calls (default: 5) + rate_limit: Max requests per second (default: 2) + initial_backoff: Initial backoff time in seconds (default: 1) + """ + super().__init__(region=region_name, rate_limit=rate_limit) + self.initial_backoff = initial_backoff + self.max_retries = max_retries + self.rate_limiter = RateLimiter(rate_limit=self.rate_limit) + + # Initialize AWS session with provided credentials + if profile_name: + # Use AWS profile if specified + session = boto3.Session(profile_name=profile_name) + else: + # Fall back to default credentials (environment variables or IAM role) + session = boto3.Session() + + # Create Bedrock client + self.bedrock_client = session.client( + 'bedrock-runtime', + region_name=region_name + ) + self.converse_client = BedrockConverseHandler(self.bedrock_client) + + def call_model(self, model_id: str, system_prompt: str, + messages: List[Dict[str, str]], inf_config: Dict[str, Any]) -> str: + """ + Call a Bedrock model with rate limiting and retry logic. + + Args: + model_id: Bedrock model identifier + system_prompt: System prompt text + messages: List of conversation messages + inf_config: Inference configuration parameters + + Returns: + Model response text + + Raises: + Exception: If max retries exceeded or non-retryable error occurs + """ + self.rate_limiter.apply_rate_limiting() + return self._call_model_with_retry(model_id, system_prompt, messages, inf_config) + + def _call_model_with_retry(self, model_id: str, system_prompt: str, + messages: List[Dict[str, str]], inf_config: Dict[str, Any]) -> str: + """ + Call model with automatic retry logic for transient errors. + + Retries on: + - ThrottlingException + - ModelErrorException + - ServiceUnavailableException + + Args: + model_id: Bedrock model identifier + system_prompt: System prompt text + messages: List of conversation messages + inf_config: Inference configuration parameters + + Returns: + Model response text + + Raises: + ClientError: For non-retryable errors + Exception: If max retries exceeded + """ + retries = 0 + while retries < self.max_retries: + try: + return self.converse_client.call_model(model_id, system_prompt, messages, inf_config) + except ClientError as e: + error_code = e.response['Error']['Code'] + + if error_code == 'ThrottlingException': + wait_time = self._calculate_backoff_time(retries) + logger.debug( + f"Throttled. Retrying in {wait_time} seconds... " + f"(Attempt {retries + 1}/{self.max_retries})" + ) + time.sleep(wait_time) + retries += 1 + elif error_code == 'ModelErrorException': + wait_time = self._calculate_backoff_time(retries) + logger.debug( + f"Encountered ModelErrorException, Retrying in {wait_time} seconds... " + f"(Attempt {retries + 1}/{self.max_retries})" + ) + time.sleep(wait_time) + retries += 1 + elif error_code == 'ServiceUnavailableException': + wait_time = self._calculate_backoff_time(retries) + logger.debug( + f"Retryable exception: ServiceUnavailableException {model_id}. " + f"Retrying in {wait_time} seconds... (Attempt {retries + 1}/{self.max_retries})" + ) + time.sleep(wait_time) + retries += 1 + else: + # Non-retryable error + raise e + + raise Exception(f"Max retries ({self.max_retries}) exceeded for model call") + + def _calculate_backoff_time(self, retry_count: int) -> float: + """ + Calculate exponential backoff time with jitter. + + Args: + retry_count: Current retry attempt number + + Returns: + Wait time in seconds + """ + # Exponential backoff with jitter + return self.initial_backoff * (2 ** retry_count) + random.uniform(0, 1) diff --git a/src/amzn_nova_prompt_optimizer/core/inference/dspy_compatible.py b/src/amzn_nova_prompt_optimizer/core/inference/dspy_compatible.py new file mode 100644 index 0000000..e5d9b94 --- /dev/null +++ b/src/amzn_nova_prompt_optimizer/core/inference/dspy_compatible.py @@ -0,0 +1,333 @@ +# Copyright 2025 Amazon Inc + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +DSPy-Compatible Inference Adapter + +This module provides the bridge between DSPy's LM interface and the +InferenceAdapter interface used throughout the Nova Prompt Optimizer. +""" + +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional +import logging + +logger = logging.getLogger(__name__) + + +class DSPyCompatibleInferenceAdapter(ABC): + """ + Abstract base class that makes any InferenceAdapter compatible with DSPy's LM interface. + + This adapter acts as a bridge between DSPy's expected interface and the + InferenceAdapter interface used throughout the Nova Prompt Optimizer. + + Attributes: + inference_adapter: The underlying InferenceAdapter instance + model_id: The model identifier to use for inference + history: List of previous calls (required by DSPy) + cache: Whether to cache responses (required by DSPy) + kwargs: Additional model-specific kwargs (required by DSPy) + """ + + def __init__( + self, + inference_adapter: 'InferenceAdapter', + model_id: str, + cache: bool = False, + **kwargs + ): + """ + Initialize the DSPy-compatible adapter. + + Args: + inference_adapter: The InferenceAdapter instance to wrap + model_id: The model identifier for inference + cache: Whether to enable response caching + **kwargs: Additional model-specific parameters + """ + self.inference_adapter = inference_adapter + self.model_id = model_id + self.history = [] # Required by DSPy for tracking calls + self.cache = cache + + # Initialize kwargs with defaults that DSPy expects + self.kwargs = { + 'temperature': 1.0, + 'max_tokens': 1000, + 'top_p': 1.0, + **kwargs # User-provided kwargs override defaults + } + self.model = model_id # Alias for DSPy compatibility + + def __call__(self, messages: List[Dict[str, str]], **lm_kwargs) -> List[str]: + """ + Main interface method called by DSPy. + + Args: + messages: List of message dicts in DSPy format: + [{"role": "system", "content": "..."}, + {"role": "user", "content": "..."}, + {"role": "assistant", "content": "..."}] + **lm_kwargs: Additional parameters (temperature, max_tokens, etc.) + + Returns: + List of completion strings (DSPy expects list for n>1 generations) + """ + # Convert DSPy format to InferenceAdapter format + system_prompt, conversation_messages = self._convert_messages_to_adapter_format(messages) + + # Build inference config + inf_config = self._build_inference_config(lm_kwargs) + + # Call the underlying inference adapter + try: + response = self.inference_adapter.call_model( + model_id=self.model_id, + system_prompt=system_prompt, + messages=conversation_messages, + inf_config=inf_config + ) + + # Track in history (DSPy requirement) + self.history.append({ + "messages": messages, + "kwargs": lm_kwargs, + "response": response + }) + + # Return as list (DSPy expects list for multiple generations) + return [response] + + except Exception as e: + logger.error(f"Error calling inference adapter: {e}") + raise + + async def acall(self, messages: List[Dict[str, str]], **lm_kwargs) -> List[str]: + """ + Async version of __call__. + + Args: + messages: List of message dicts in DSPy format + **lm_kwargs: Additional parameters + + Returns: + List of completion strings + """ + # For now, delegate to synchronous version + # Can be overridden by subclasses for true async support + return self.__call__(messages, **lm_kwargs) + + def _convert_messages_to_adapter_format( + self, + messages: List[Dict[str, str]] + ) -> tuple: + """ + Convert DSPy message format to InferenceAdapter format. + + DSPy format: + [{"role": "system", "content": "..."}, + {"role": "user", "content": "..."}, + {"role": "assistant", "content": "..."}] + + InferenceAdapter format: + system_prompt: str + messages: [{"user": "..."}, {"assistant": "..."}, ...] + + Args: + messages: Messages in DSPy format + + Returns: + Tuple of (system_prompt, conversation_messages) + """ + system_prompt = "" + conversation_messages = [] + + for msg in messages: + role = msg.get("role", "") + content = msg.get("content", "") + + if role == "system": + # Accumulate system prompts (there might be multiple) + if system_prompt: + system_prompt += "\n\n" + content + else: + system_prompt = content + elif role == "user": + conversation_messages.append({"user": content}) + elif role == "assistant": + conversation_messages.append({"assistant": content}) + else: + logger.warning(f"Unknown message role: {role}") + + return system_prompt, conversation_messages + + @abstractmethod + def _build_inference_config(self, lm_kwargs: Dict[str, Any]) -> Dict[str, Any]: + """ + Build inference config from DSPy kwargs. + + This method should be implemented by subclasses to handle + backend-specific parameter mapping. + + Args: + lm_kwargs: Kwargs from DSPy (temperature, max_tokens, etc.) + + Returns: + Inference config dict for the specific backend + """ + pass + + def copy(self, **kwargs): + """ + Create a copy of this adapter with updated parameters. + Required by some DSPy optimizers. + + Args: + **kwargs: Parameters to update + + Returns: + New instance with updated parameters + """ + # Extract special parameters + model_id = kwargs.pop('model_id', self.model_id) + cache = kwargs.pop('cache', self.cache) + + # Merge remaining kwargs + new_kwargs = {**self.kwargs, **kwargs} + + return self.__class__( + inference_adapter=self.inference_adapter, + model_id=model_id, + cache=cache, + **new_kwargs + ) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(model_id={self.model_id})" + + +class DSPyBedrockAdapter(DSPyCompatibleInferenceAdapter): + """ + DSPy-compatible adapter for Bedrock inference. + + This adapter wraps BedrockInferenceAdapter to make it compatible + with DSPy's LM interface. + """ + + def _build_inference_config(self, lm_kwargs: Dict[str, Any]) -> Dict[str, Any]: + """ + Build Bedrock-specific inference config. + + Maps DSPy parameters to Bedrock Converse API parameters. + Merges call-time kwargs with initialization kwargs (self.kwargs). + + Args: + lm_kwargs: DSPy kwargs passed at call time + + Returns: + Bedrock inference config + """ + from amzn_nova_prompt_optimizer.core.inference import ( + MAX_TOKENS_FIELD, TEMPERATURE_FIELD, TOP_P_FIELD, TOP_K_FIELD + ) + + # Merge self.kwargs (defaults) with lm_kwargs (call-time overrides) + merged_kwargs = {**self.kwargs, **lm_kwargs} + + return { + MAX_TOKENS_FIELD: merged_kwargs.get("max_tokens", 5000), + TEMPERATURE_FIELD: merged_kwargs.get("temperature", 0.0), + TOP_P_FIELD: merged_kwargs.get("top_p", 1.0), + TOP_K_FIELD: merged_kwargs.get("top_k", 1) + } + + +class DSPySageMakerAdapter(DSPyCompatibleInferenceAdapter): + """ + DSPy-compatible adapter for SageMaker inference. + + This adapter wraps SageMakerInferenceAdapter to make it compatible + with DSPy's LM interface. + """ + + def _build_inference_config(self, lm_kwargs: Dict[str, Any]) -> Dict[str, Any]: + """ + Build SageMaker-specific inference config. + + Maps DSPy parameters to SageMaker endpoint parameters. + Different SageMaker endpoints may expect different formats. + Merges call-time kwargs with initialization kwargs (self.kwargs). + + Args: + lm_kwargs: DSPy kwargs passed at call time + + Returns: + SageMaker inference config + """ + # Merge self.kwargs (defaults) with lm_kwargs (call-time overrides) + merged_kwargs = {**self.kwargs, **lm_kwargs} + + # SageMaker endpoints often use different parameter names + # This is a common format, but may need customization + temperature = merged_kwargs.get("temperature", 0.0) + + return { + "max_new_tokens": merged_kwargs.get("max_tokens", 5000), + "temperature": temperature, + "top_p": merged_kwargs.get("top_p", 1.0), + "top_k": merged_kwargs.get("top_k", 1), + # Additional SageMaker-specific parameters + "do_sample": temperature > 0, + "return_full_text": False + } + + +def create_dspy_adapter( + inference_adapter: 'InferenceAdapter', + model_id: str, + **kwargs +) -> DSPyCompatibleInferenceAdapter: + """ + Factory function to create the appropriate DSPy-compatible adapter. + + Automatically detects the type of InferenceAdapter and returns + the corresponding DSPy-compatible wrapper. + + Args: + inference_adapter: The InferenceAdapter instance + model_id: Model identifier + **kwargs: Additional parameters + + Returns: + Appropriate DSPyCompatibleInferenceAdapter instance + + Example: + >>> bedrock_adapter = BedrockInferenceAdapter(region_name="us-east-1") + >>> dspy_lm = create_dspy_adapter(bedrock_adapter, "us.amazon.nova-pro-v1:0") + >>> dspy.configure(lm=dspy_lm) + """ + adapter_type = type(inference_adapter).__name__ + + if "Bedrock" in adapter_type: + return DSPyBedrockAdapter(inference_adapter, model_id, **kwargs) + elif "SageMaker" in adapter_type: + return DSPySageMakerAdapter(inference_adapter, model_id, **kwargs) + else: + # Default to Bedrock-style config for unknown adapters + logger.warning( + f"Unknown adapter type: {adapter_type}. " + f"Using DSPyBedrockAdapter as default." + ) + return DSPyBedrockAdapter(inference_adapter, model_id, **kwargs) diff --git a/src/amzn_nova_prompt_optimizer/core/inference/sagemaker_adapter.py b/src/amzn_nova_prompt_optimizer/core/inference/sagemaker_adapter.py new file mode 100644 index 0000000..a04a73d --- /dev/null +++ b/src/amzn_nova_prompt_optimizer/core/inference/sagemaker_adapter.py @@ -0,0 +1,378 @@ +# Copyright 2025 Amazon Inc + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +SageMaker Inference Adapter + +This module provides an InferenceAdapter implementation for calling +LLMs deployed on AWS SageMaker endpoints. +""" + +import json +import logging +import random +import time +from typing import Any, Dict, List, Optional + +import boto3 +from botocore.exceptions import ClientError + +from amzn_nova_prompt_optimizer.core.inference.adapter import InferenceAdapter +from amzn_nova_prompt_optimizer.util.rate_limiter import RateLimiter + +logger = logging.getLogger(__name__) + + +class SageMakerInferenceAdapter(InferenceAdapter): + """ + SageMaker inference adapter for calling SageMaker endpoints. + + This adapter supports calling LLMs deployed on SageMaker endpoints, + with automatic DSPy compatibility through the to_dspy_lm() method. + """ + + def __init__( + self, + endpoint_name: str, + region_name: str = 'us-east-1', + profile_name: Optional[str] = None, + max_retries: int = 5, + rate_limit: int = 2, + initial_backoff: int = 1, + content_type: str = "application/json", + accept: str = "application/json" + ): + """ + Initialize SageMaker inference adapter. + + Args: + endpoint_name: Name of the SageMaker endpoint + region_name: AWS region name + profile_name: Optional AWS profile name + max_retries: Maximum number of retries + rate_limit: Max TPS for endpoint calls + initial_backoff: Initial backoff time for retries + content_type: Content type for requests + accept: Accept type for responses + """ + super().__init__(region=region_name, rate_limit=rate_limit) + self.endpoint_name = endpoint_name + self.profile_name = profile_name + self.max_retries = max_retries + self.initial_backoff = initial_backoff + self.content_type = content_type + self.accept = accept + + # Initialize rate limiter + self.rate_limiter = RateLimiter(rate_limit=self.rate_limit) + + # Initialize SageMaker client + if profile_name: + session = boto3.Session(profile_name=profile_name) + else: + session = boto3.Session() + + self.sagemaker_runtime = session.client( + 'sagemaker-runtime', + region_name=region_name + ) + + logger.info(f"Initialized SageMaker adapter for endpoint: {endpoint_name}") + + def test_connection(self) -> bool: + """ + Test if the SageMaker endpoint is accessible and responding. + + Returns: + True if endpoint is accessible, False otherwise + """ + try: + # Try a simple test payload with 'messages' format + # Parameters should be at root level, not nested + test_payload = { + "messages": [{"role": "user", "content": "Test"}], + "max_tokens": 10, + "temperature": 0.0 + } + + logger.info(f"Testing connection to endpoint: {self.endpoint_name}") + response = self.sagemaker_runtime.invoke_endpoint( + EndpointName=self.endpoint_name, + ContentType=self.content_type, + Accept=self.accept, + Body=json.dumps(test_payload) + ) + + logger.info(f"βœ“ Endpoint {self.endpoint_name} is accessible") + return True + + except ClientError as e: + error_code = e.response['Error']['Code'] + error_message = e.response['Error'].get('Message', 'No message') + logger.error( + f"βœ— Endpoint {self.endpoint_name} error: {error_code} - {error_message}" + ) + return False + except Exception as e: + logger.error( + f"βœ— Endpoint {self.endpoint_name} error: {type(e).__name__}: {str(e)}" + ) + return False + + def call_model( + self, + model_id: str, + system_prompt: str, + messages: List[Dict[str, str]], + inf_config: Dict[str, Any] + ) -> str: + """ + Call SageMaker endpoint with the given parameters. + + Args: + model_id: Model identifier (can be used for routing if needed) + system_prompt: System prompt string + messages: List of conversation messages + inf_config: Inference configuration + + Returns: + Model response as string + """ + self.rate_limiter.apply_rate_limiting() + return self._call_model_with_retry( + model_id, system_prompt, messages, inf_config + ) + + def _call_model_with_retry( + self, + model_id: str, + system_prompt: str, + messages: List[Dict[str, str]], + inf_config: Dict[str, Any] + ) -> str: + """ + Call SageMaker endpoint with retry logic. + + Args: + model_id: Model identifier + system_prompt: System prompt + messages: Conversation messages + inf_config: Inference config + + Returns: + Model response + """ + retries = 0 + last_error = None + + while retries < self.max_retries: + try: + # Format the payload for SageMaker endpoint + payload = self._format_payload( + system_prompt, messages, inf_config + ) + + logger.debug(f"Calling SageMaker endpoint: {self.endpoint_name}") + + # Call SageMaker endpoint + response = self.sagemaker_runtime.invoke_endpoint( + EndpointName=self.endpoint_name, + ContentType=self.content_type, + Accept=self.accept, + Body=json.dumps(payload) + ) + + # Parse response + result = json.loads(response['Body'].read().decode()) + return self._extract_text_from_response(result) + + except ClientError as e: + error_code = e.response['Error']['Code'] + error_message = e.response['Error'].get('Message', 'No message') + last_error = e + + logger.warning( + f"SageMaker ClientError: {error_code} - {error_message}" + ) + + if error_code in ['ThrottlingException', 'ModelError', 'ServiceUnavailable']: + wait_time = self._calculate_backoff_time(retries) + logger.debug( + f"Retryable error: {error_code}. " + f"Retrying in {wait_time}s... " + f"(Attempt {retries + 1}/{self.max_retries})" + ) + time.sleep(wait_time) + retries += 1 + else: + # Non-retryable error + logger.error( + f"Non-retryable SageMaker error: {error_code} - {error_message}" + ) + raise e + + except Exception as e: + # Catch all other exceptions (JSON decode errors, etc.) + last_error = e + logger.warning( + f"SageMaker error (attempt {retries + 1}/{self.max_retries}): " + f"{type(e).__name__}: {str(e)}" + ) + wait_time = self._calculate_backoff_time(retries) + time.sleep(wait_time) + retries += 1 + + # Max retries exceeded - provide detailed error + error_msg = ( + f"Max retries ({self.max_retries}) exceeded for SageMaker endpoint '{self.endpoint_name}'. " + f"Last error: {type(last_error).__name__}: {str(last_error)}" + ) + logger.error(error_msg) + raise Exception(error_msg) + + def _format_payload( + self, + system_prompt: str, + messages: List[Dict[str, str]], + inf_config: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Format the payload for SageMaker endpoint. + + Different SageMaker endpoints may expect different formats. + This implementation uses the OpenAI-compatible format with 'messages' + and parameters at the root level. + Override this method for custom endpoint formats. + + Args: + system_prompt: System prompt + messages: Conversation messages + inf_config: Inference config + + Returns: + Payload dict for SageMaker endpoint + """ + # Build chat messages in OpenAI format + chat_messages = [] + + if system_prompt: + chat_messages.append({ + "role": "system", + "content": system_prompt + }) + + for msg in messages: + if "user" in msg: + chat_messages.append({ + "role": "user", + "content": msg["user"] + }) + elif "assistant" in msg: + chat_messages.append({ + "role": "assistant", + "content": msg["assistant"] + }) + + # OpenAI-compatible format: parameters at root level + # Use max_tokens instead of max_new_tokens for compatibility + payload = { + "messages": chat_messages, + "max_tokens": inf_config.get("max_new_tokens", inf_config.get("max_tokens", 5000)), + "temperature": inf_config.get("temperature", 0.0), + "top_p": inf_config.get("top_p", 1.0), + "top_k": inf_config.get("top_k", 1) + } + + return payload + + def _extract_text_from_response(self, response: Dict[str, Any]) -> str: + """ + Extract text from SageMaker endpoint response. + + Different endpoints return different response formats. + This handles common formats including OpenAI-compatible responses. + Override for custom formats. + + Args: + response: Response dict from SageMaker + + Returns: + Extracted text + """ + # OpenAI-compatible format: {"choices": [{"message": {"content": "..."}}]} + if "choices" in response and len(response["choices"]) > 0: + choice = response["choices"][0] + if "message" in choice and "content" in choice["message"]: + return choice["message"]["content"] + elif "text" in choice: + return choice["text"] + + # Common response formats + if isinstance(response, list) and len(response) > 0: + # Format: [{"generated_text": "..."}] + if "generated_text" in response[0]: + return response[0]["generated_text"] + + if "generated_text" in response: + # Format: {"generated_text": "..."} + return response["generated_text"] + + if "outputs" in response: + # Format: {"outputs": "..."} + return response["outputs"] + + if isinstance(response, str): + # Format: "..." + return response + + # Fallback: return JSON string + logger.warning( + f"Unknown SageMaker response format. Returning JSON string." + ) + return json.dumps(response) + + def _calculate_backoff_time(self, retry_count: int) -> float: + """Calculate exponential backoff time with jitter.""" + return self.initial_backoff * (2 ** retry_count) + random.uniform(0, 1) + + def to_dspy_lm(self, model_id: str, **kwargs): + """ + Create a DSPy-compatible wrapper for this adapter. + + For SageMaker, we override the model_id to use the endpoint_name + since SageMaker endpoints don't use traditional model IDs. + + Args: + model_id: Model identifier (ignored for SageMaker, endpoint_name is used instead) + **kwargs: Additional parameters for the DSPy adapter + + Returns: + DSPy-compatible adapter instance with endpoint_name as model_id + + Example: + >>> adapter = SageMakerInferenceAdapter(endpoint_name="my-endpoint") + >>> # task_model_id is ignored, endpoint_name is used instead + >>> dspy_lm = adapter.to_dspy_lm("any-model-id") + >>> import dspy + >>> dspy.configure(lm=dspy_lm) + """ + from amzn_nova_prompt_optimizer.core.inference.dspy_compatible import ( + create_dspy_adapter + ) + # Override model_id with endpoint_name for SageMaker + logger.info( + f"SageMaker adapter: Using endpoint_name '{self.endpoint_name}' " + f"as model_id (provided model_id '{model_id}' is ignored)" + ) + return create_dspy_adapter(self, self.endpoint_name, **kwargs) diff --git a/src/amzn_nova_prompt_optimizer/core/optimizers/miprov2/custom_adapters/custom_chat_adapter.py b/src/amzn_nova_prompt_optimizer/core/optimizers/miprov2/custom_adapters/custom_chat_adapter.py index ad58203..d641a1e 100644 --- a/src/amzn_nova_prompt_optimizer/core/optimizers/miprov2/custom_adapters/custom_chat_adapter.py +++ b/src/amzn_nova_prompt_optimizer/core/optimizers/miprov2/custom_adapters/custom_chat_adapter.py @@ -26,7 +26,7 @@ class CustomChatAdapter(ChatAdapter): - def __init__(self, user_prompt_template:str, callbacks: Optional[list[BaseCallback]] = None, + def __init__(self, user_prompt_template: str, callbacks: Optional[list] = None, enable_json_fallback: bool = False, **kwargs): super().__init__(callbacks) logger.info(f"Initializing CustomChatAdapter with enable_json_fallback={enable_json_fallback}") @@ -36,11 +36,11 @@ def __init__(self, user_prompt_template:str, callbacks: Optional[list[BaseCallba def __call__( self, lm: LM, - lm_kwargs: dict[str, Any], + lm_kwargs: dict, signature: Type[Signature], - demos: list[dict[str, Any]], - inputs: dict[str, Any], - ) -> list[dict[str, Any]]: + demos: list, + inputs: dict, + ) -> list: try: return Adapter.__call__(self, lm, lm_kwargs, signature, demos, inputs) except Exception as e: @@ -70,7 +70,7 @@ def format_task_description(self, signature: Type[Signature]) -> str: def format_user_message_content( self, signature: Type[Signature], - inputs: dict[str, Any], + inputs: dict, prefix: str = "", suffix: str = "", main_request: bool = False, @@ -94,8 +94,8 @@ def replace_variable(match): def format_assistant_message_content( self, signature: Type[Signature], - outputs: dict[str, Any], - missing_field_message: str | None = None, + outputs: dict, + missing_field_message: Optional[str] = None, ) -> str: assistant_message_content = "\n".join( str(outputs.get(k, missing_field_message)) @@ -103,7 +103,7 @@ def format_assistant_message_content( ) return assistant_message_content - def parse(self, signature: Type[Signature], completion: str) -> dict[str, Any]: + def parse(self, signature: Type[Signature], completion: str) -> dict: try: return {k: completion for k in signature.output_fields} # simplistic assumption except Exception as e: diff --git a/src/amzn_nova_prompt_optimizer/core/optimizers/miprov2/miprov2_optimizer.py b/src/amzn_nova_prompt_optimizer/core/optimizers/miprov2/miprov2_optimizer.py index 7844496..c064ac2 100644 --- a/src/amzn_nova_prompt_optimizer/core/optimizers/miprov2/miprov2_optimizer.py +++ b/src/amzn_nova_prompt_optimizer/core/optimizers/miprov2/miprov2_optimizer.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import os import textwrap from typing import Type, Optional, List, Dict @@ -32,7 +31,7 @@ from amzn_nova_prompt_optimizer.core.optimizers.miprov2.custom_adapters.custom_chat_adapter import CustomChatAdapter DEFAULT_TASK_MODEL = "us.amazon.nova-pro-v1:0" -DEFAULT_PROMPT_MODEL = "us.amazon.nova-premier-v1:0" +DEFAULT_PROMPT_MODEL = "us.amazon.nova-2-lite-v1:0" logger = logging.getLogger(__name__) @@ -244,16 +243,17 @@ def optimize(self, elif num_trials is None: raise ValueError("num_trials must be specified when num_candidates is provided") - # Set AWS region - if self.inference_adapter.region: - os.environ["AWS_REGION_NAME"] = self.inference_adapter.region - else: - os.environ["AWS_REGION_NAME"] = 'us-west-2' - - # Setup dspy.LM - task_lm = RateLimitedLM(dspy.LM(f'bedrock/{task_model_id}'), rate_limit=self.inference_adapter.rate_limit) + # Setup DSPy-compatible LM using InferenceAdapter + task_lm = RateLimitedLM( + self.inference_adapter.to_dspy_lm(task_model_id), + rate_limit=self.inference_adapter.rate_limit + ) logger.info(f"Using {task_model_id} for Evaluation") - prompt_lm = RateLimitedLM(dspy.LM(f'bedrock/{prompter_model_id}'), rate_limit=self.inference_adapter.rate_limit) + + prompt_lm = RateLimitedLM( + self.inference_adapter.to_dspy_lm(prompter_model_id), + rate_limit=self.inference_adapter.rate_limit + ) logger.info(f"Using {prompter_model_id} for Prompting") # Configure DSPy @@ -365,16 +365,17 @@ def optimize(self, elif num_trials is None: raise ValueError("num_trials must be specified when num_candidates is provided") - # Set AWS region - if self.inference_adapter.region: - os.environ["AWS_REGION_NAME"] = self.inference_adapter.region - else: - os.environ["AWS_REGION_NAME"] = 'us-west-2' - - # Setup dspy.LM - task_lm = RateLimitedLM(dspy.LM(f'bedrock/{task_model_id}'), rate_limit=self.inference_adapter.rate_limit) + # Setup DSPy-compatible LM using InferenceAdapter + task_lm = RateLimitedLM( + self.inference_adapter.to_dspy_lm(task_model_id), + rate_limit=self.inference_adapter.rate_limit + ) logger.info(f"Using {task_model_id} for Evaluation") - prompt_lm = RateLimitedLM(dspy.LM(f'bedrock/{prompter_model_id}'), rate_limit=self.inference_adapter.rate_limit) + + prompt_lm = RateLimitedLM( + self.inference_adapter.to_dspy_lm(prompter_model_id), + rate_limit=self.inference_adapter.rate_limit + ) logger.info(f"Using {prompter_model_id} for Prompting") # Configure DSPy diff --git a/src/amzn_nova_prompt_optimizer/core/optimizers/nova_meta_prompter/nova_mp_optimizer.py b/src/amzn_nova_prompt_optimizer/core/optimizers/nova_meta_prompter/nova_mp_optimizer.py index 229d47b..ef540a5 100644 --- a/src/amzn_nova_prompt_optimizer/core/optimizers/nova_meta_prompter/nova_mp_optimizer.py +++ b/src/amzn_nova_prompt_optimizer/core/optimizers/nova_meta_prompter/nova_mp_optimizer.py @@ -29,7 +29,7 @@ from typing import Optional, List logger = logging.getLogger(__name__) -DEFAULT_PROMPTER_MODEL_ID = "us.amazon.nova-premier-v1:0" +DEFAULT_PROMPTER_MODEL_ID = "us.amazon.nova-2-lite-v1:0" class NovaMPOptimizationAdapter(OptimizationAdapter): def __init__(self, prompt_adapter: PromptAdapter, diff --git a/src/amzn_nova_prompt_optimizer/core/optimizers/nova_prompt_optimizer/nova_prompt_optimizer.py b/src/amzn_nova_prompt_optimizer/core/optimizers/nova_prompt_optimizer/nova_prompt_optimizer.py index ad997f4..d424874 100644 --- a/src/amzn_nova_prompt_optimizer/core/optimizers/nova_prompt_optimizer/nova_prompt_optimizer.py +++ b/src/amzn_nova_prompt_optimizer/core/optimizers/nova_prompt_optimizer/nova_prompt_optimizer.py @@ -26,8 +26,8 @@ NOVA_PROMPT_OPTIMIZER_MODE: Dict[str, Dict[str, Any]] = { "micro": { - "meta_prompt_model_id": "us.amazon.nova-premier-v1:0", - "prompter_model_id": "us.amazon.nova-premier-v1:0", + "meta_prompt_model_id": "us.amazon.nova-2-lite-v1:0", + "prompter_model_id": "us.amazon.nova-2-lite-v1:0", "task_model_id": "us.amazon.nova-micro-v1:0", "num_candidates": 20, "num_trials": 30, @@ -35,8 +35,8 @@ "max_labeled_demos": 4 }, "lite": { - "meta_prompt_model_id": "us.amazon.nova-premier-v1:0", - "prompter_model_id": "us.amazon.nova-premier-v1:0", + "meta_prompt_model_id": "us.amazon.nova-2-lite-v1:0", + "prompter_model_id": "us.amazon.nova-2-lite-v1:0", "task_model_id": "us.amazon.nova-lite-v1:0", "num_candidates": 20, "num_trials": 30, @@ -44,18 +44,18 @@ "max_labeled_demos": 4 }, "pro": { - "meta_prompt_model_id": "us.amazon.nova-premier-v1:0", - "prompter_model_id": "us.amazon.nova-premier-v1:0", + "meta_prompt_model_id": "us.amazon.nova-2-lite-v1:0", + "prompter_model_id": "us.amazon.nova-2-lite-v1:0", "task_model_id": "us.amazon.nova-pro-v1:0", "num_candidates": 20, "num_trials": 30, "max_bootstrapped_demos": 4, "max_labeled_demos": 4 }, - "premier": { - "meta_prompt_model_id": "us.amazon.nova-premier-v1:0", - "prompter_model_id": "us.amazon.nova-premier-v1:0", - "task_model_id": "us.amazon.nova-premier-v1:0", + "lite-2": { + "meta_prompt_model_id": "us.amazon.nova-2-lite-v1:0", + "prompter_model_id": "us.amazon.nova-2-lite-v1:0", + "task_model_id": "us.amazon.nova-2-lite-v1:0", "num_candidates": 20, "num_trials": 30, "max_bootstrapped_demos": 4, @@ -68,20 +68,72 @@ class NovaPromptOptimizer(OptimizationAdapter): """ NovaPromptOptimizer is a combination of Meta Prompting and MIPROv2 for Nova Models that yields a stable prompt optimization result. + + This optimizer supports using separate inference adapters for: + - Meta-prompting phase (generating optimized prompts) + - Task model optimization phase (evaluating prompts on the target model) + + This allows you to use Bedrock for meta-prompting while optimizing a SageMaker endpoint. """ - def __init__(self, prompt_adapter: PromptAdapter, + def __init__(self, + prompt_adapter: PromptAdapter, inference_adapter: InferenceAdapter, dataset_adapter: DatasetAdapter, - metric_adapter: MetricAdapter): + metric_adapter: MetricAdapter, + meta_prompt_inference_adapter: Optional[InferenceAdapter] = None): + """ + Initialize NovaPromptOptimizer. + + Args: + prompt_adapter: Adapter for managing prompts + inference_adapter: Adapter for task model inference (used in MIPROv2 optimization) + dataset_adapter: Adapter for dataset management + metric_adapter: Adapter for evaluation metrics + meta_prompt_inference_adapter: Optional separate adapter for meta-prompting phase. + If not provided, creates a default BedrockInferenceAdapter with Nova 2.0 Lite + for meta-prompting, while using inference_adapter for task optimization. + This allows using Bedrock for meta-prompting while optimizing a SageMaker endpoint. + """ # Call parent class's __init__ super().__init__(prompt_adapter, inference_adapter, dataset_adapter, metric_adapter) self.prompt_adapter = prompt_adapter self.inference_adapter = inference_adapter self.dataset_adapter = dataset_adapter self.metric_adapter = metric_adapter - self.meta_prompt_optimization_adapter = NovaMPOptimizationAdapter(prompt_adapter, inference_adapter) + + # Use separate adapter for meta-prompting if provided, otherwise create default Bedrock adapter + if meta_prompt_inference_adapter is not None: + self.meta_prompt_inference_adapter = meta_prompt_inference_adapter + else: + # Import here to avoid circular dependency + from amzn_nova_prompt_optimizer.core.inference import BedrockInferenceAdapter + + # Create default Bedrock adapter for meta-prompting with Nova 2.0 Lite + logger.info( + "No meta_prompt_inference_adapter provided. " + "Creating default BedrockInferenceAdapter with Nova 2.0 Lite for meta-prompting." + ) + self.meta_prompt_inference_adapter = BedrockInferenceAdapter( + region_name=getattr(inference_adapter, 'region', 'us-east-1'), + rate_limit=5 + ) + + self.meta_prompt_optimization_adapter = NovaMPOptimizationAdapter( + prompt_adapter, + self.meta_prompt_inference_adapter + ) def optimize(self, mode: str = "pro", custom_params = None) -> PromptAdapter: + """ + Optimize the prompt using Nova Meta Prompter followed by MIPROv2. + + Args: + mode: Optimization mode ('micro', 'lite', 'pro', 'lite-2', or 'custom') + custom_params: Custom parameters for 'custom' mode + + Returns: + Optimized PromptAdapter + """ if mode == "custom": if not custom_params: raise ValueError("Custom mode requires custom_params dictionary") @@ -99,10 +151,19 @@ def optimize(self, mode: str = "pro", custom_params = None) -> PromptAdapter: meta_prompt_model_id = config.pop("meta_prompt_model_id") optimization_params = config - if not self.inference_adapter: raise ValueError("Inference Adapter not passed. " "Initialize and Pass Inference Adapter to use this Optimizer") + + # Log which adapters are being used + if self.meta_prompt_inference_adapter is not self.inference_adapter: + logger.info( + f"Using separate inference adapters: " + f"Meta-prompting with {type(self.meta_prompt_inference_adapter).__name__}, " + f"Task optimization with {type(self.inference_adapter).__name__}" + ) + + # Phase 1: Meta-prompting (uses meta_prompt_inference_adapter) if meta_prompt_model_id: intermediate_prompt_adapter = ( self.meta_prompt_optimization_adapter.optimize(prompter_model_id=meta_prompt_model_id)) @@ -113,6 +174,7 @@ def optimize(self, mode: str = "pro", custom_params = None) -> PromptAdapter: logger.info("[Nova Prompt Optimizer] No Dataset or No metric provided, running only Nova Meta Prompter") return intermediate_prompt_adapter + # Phase 2: MIPROv2 optimization (uses main inference_adapter for task model) nova_prompt_optimizer = NovaMIPROv2OptimizationAdapter( prompt_adapter=intermediate_prompt_adapter, dataset_adapter=self.dataset_adapter, diff --git a/tests/core/inference/test_dspy_compatible_adapter.py b/tests/core/inference/test_dspy_compatible_adapter.py new file mode 100644 index 0000000..c8133ff --- /dev/null +++ b/tests/core/inference/test_dspy_compatible_adapter.py @@ -0,0 +1,461 @@ +# Copyright 2025 Amazon Inc + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for DSPy-Compatible Inference Adapters +""" + +import unittest +from unittest.mock import Mock, MagicMock + +from amzn_nova_prompt_optimizer.core.inference import ( + DSPyCompatibleInferenceAdapter, + DSPyBedrockAdapter, + DSPySageMakerAdapter, + create_dspy_adapter, + BedrockInferenceAdapter, + MAX_TOKENS_FIELD, + TEMPERATURE_FIELD, + TOP_P_FIELD, + TOP_K_FIELD +) + + +class TestDSPyCompatibleInferenceAdapter(unittest.TestCase): + """Test the base DSPy-compatible adapter.""" + + def setUp(self): + """Set up test fixtures.""" + self.mock_inference_adapter = Mock() + self.mock_inference_adapter.call_model.return_value = "Test response" + self.mock_inference_adapter.rate_limit = 5 + + self.adapter = DSPyBedrockAdapter( + inference_adapter=self.mock_inference_adapter, + model_id="test-model" + ) + + def test_initialization(self): + """Test adapter initialization.""" + self.assertEqual(self.adapter.model_id, "test-model") + self.assertEqual(self.adapter.model, "test-model") + self.assertFalse(self.adapter.cache) + self.assertEqual(self.adapter.history, []) + + # Verify default kwargs are set (required by DSPy MIPROv2) + self.assertIn('temperature', self.adapter.kwargs) + self.assertIn('max_tokens', self.adapter.kwargs) + self.assertIn('top_p', self.adapter.kwargs) + self.assertEqual(self.adapter.kwargs['temperature'], 1.0) + self.assertEqual(self.adapter.kwargs['max_tokens'], 1000) + self.assertEqual(self.adapter.kwargs['top_p'], 1.0) + + def test_message_conversion_simple(self): + """Test simple DSPy message format conversion.""" + dspy_messages = [ + {"role": "system", "content": "You are helpful"}, + {"role": "user", "content": "Hello"} + ] + + system_prompt, messages = self.adapter._convert_messages_to_adapter_format( + dspy_messages + ) + + self.assertEqual(system_prompt, "You are helpful") + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0], {"user": "Hello"}) + + def test_message_conversion_conversation(self): + """Test conversation DSPy message format conversion.""" + dspy_messages = [ + {"role": "system", "content": "You are helpful"}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "user", "content": "How are you?"} + ] + + system_prompt, messages = self.adapter._convert_messages_to_adapter_format( + dspy_messages + ) + + self.assertEqual(system_prompt, "You are helpful") + self.assertEqual(len(messages), 3) + self.assertEqual(messages[0], {"user": "Hello"}) + self.assertEqual(messages[1], {"assistant": "Hi"}) + self.assertEqual(messages[2], {"user": "How are you?"}) + + def test_multiple_system_prompts(self): + """Test handling multiple system prompts.""" + dspy_messages = [ + {"role": "system", "content": "First instruction"}, + {"role": "system", "content": "Second instruction"}, + {"role": "user", "content": "Hello"} + ] + + system_prompt, messages = self.adapter._convert_messages_to_adapter_format( + dspy_messages + ) + + self.assertEqual(system_prompt, "First instruction\n\nSecond instruction") + self.assertEqual(len(messages), 1) + + def test_empty_system_prompt(self): + """Test handling no system prompt.""" + dspy_messages = [ + {"role": "user", "content": "Hello"} + ] + + system_prompt, messages = self.adapter._convert_messages_to_adapter_format( + dspy_messages + ) + + self.assertEqual(system_prompt, "") + self.assertEqual(len(messages), 1) + + def test_call_method(self): + """Test the __call__ method.""" + dspy_messages = [ + {"role": "system", "content": "Be helpful"}, + {"role": "user", "content": "Test question"} + ] + + result = self.adapter( + messages=dspy_messages, + temperature=0.5, + max_tokens=100 + ) + + # Verify call_model was called + self.mock_inference_adapter.call_model.assert_called_once() + + # Verify result format + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + self.assertEqual(result[0], "Test response") + + # Verify history tracking + self.assertEqual(len(self.adapter.history), 1) + self.assertEqual(self.adapter.history[0]["response"], "Test response") + + def test_call_with_conversation(self): + """Test calling with conversation history.""" + dspy_messages = [ + {"role": "system", "content": "Be helpful"}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "user", "content": "How are you?"} + ] + + result = self.adapter(messages=dspy_messages) + + # Verify call was made + self.mock_inference_adapter.call_model.assert_called_once() + + # Verify arguments + call_args = self.mock_inference_adapter.call_model.call_args + self.assertEqual(call_args[1]["system_prompt"], "Be helpful") + self.assertEqual(len(call_args[1]["messages"]), 3) + + def test_copy_method(self): + """Test the copy method.""" + new_adapter = self.adapter.copy(model_id="new-model", cache=True) + + self.assertEqual(new_adapter.model_id, "new-model") + self.assertTrue(new_adapter.cache) + self.assertIsNot(new_adapter, self.adapter) + # Verify it shares the same underlying adapter + self.assertIs(new_adapter.inference_adapter, self.adapter.inference_adapter) + + def test_repr(self): + """Test string representation.""" + repr_str = repr(self.adapter) + self.assertIn("DSPyBedrockAdapter", repr_str) + self.assertIn("test-model", repr_str) + + +class TestDSPyBedrockAdapter(unittest.TestCase): + """Test Bedrock-specific DSPy adapter.""" + + def setUp(self): + """Set up test fixtures.""" + self.mock_bedrock_adapter = Mock(spec=BedrockInferenceAdapter) + self.mock_bedrock_adapter.call_model.return_value = "Bedrock response" + self.mock_bedrock_adapter.rate_limit = 2 + + self.adapter = DSPyBedrockAdapter( + inference_adapter=self.mock_bedrock_adapter, + model_id="us.amazon.nova-pro-v1:0" + ) + + def test_bedrock_config_format(self): + """Test Bedrock-specific config format.""" + config = self.adapter._build_inference_config({ + "temperature": 1.0, + "max_tokens": 5000, + "top_p": 0.9, + "top_k": 50 + }) + + self.assertIn(MAX_TOKENS_FIELD, config) + self.assertIn(TEMPERATURE_FIELD, config) + self.assertIn(TOP_P_FIELD, config) + self.assertIn(TOP_K_FIELD, config) + self.assertEqual(config[MAX_TOKENS_FIELD], 5000) + self.assertEqual(config[TEMPERATURE_FIELD], 1.0) + self.assertEqual(config[TOP_P_FIELD], 0.9) + self.assertEqual(config[TOP_K_FIELD], 50) + + def test_bedrock_config_defaults(self): + """Test Bedrock config with default values from self.kwargs.""" + config = self.adapter._build_inference_config({}) + + # Should use defaults from self.kwargs (set in __init__) + self.assertEqual(config[MAX_TOKENS_FIELD], 1000) # From self.kwargs + self.assertEqual(config[TEMPERATURE_FIELD], 1.0) # From self.kwargs + self.assertEqual(config[TOP_P_FIELD], 1.0) # From self.kwargs + self.assertEqual(config[TOP_K_FIELD], 1) # Hardcoded fallback + + def test_bedrock_config_uses_init_kwargs(self): + """Test that config uses kwargs from initialization.""" + # Create adapter with custom kwargs at init + adapter = DSPyBedrockAdapter( + self.mock_bedrock_adapter, + "us.amazon.nova-pro-v1:0", + temperature=0.7, + max_tokens=2000 + ) + + # Call without any kwargs - should use init kwargs + config = adapter._build_inference_config({}) + + self.assertEqual(config[TEMPERATURE_FIELD], 0.7) + self.assertEqual(config[MAX_TOKENS_FIELD], 2000) + self.assertEqual(config[TOP_P_FIELD], 1.0) # Default + + def test_bedrock_config_call_kwargs_override_init(self): + """Test that call-time kwargs override init kwargs.""" + # Create adapter with custom kwargs at init + adapter = DSPyBedrockAdapter( + self.mock_bedrock_adapter, + "us.amazon.nova-pro-v1:0", + temperature=0.7, + max_tokens=2000 + ) + + # Call with different kwargs - should override + config = adapter._build_inference_config({ + "temperature": 0.9, + "top_p": 0.8 + }) + + self.assertEqual(config[TEMPERATURE_FIELD], 0.9) # Overridden + self.assertEqual(config[MAX_TOKENS_FIELD], 2000) # From init + self.assertEqual(config[TOP_P_FIELD], 0.8) # Overridden + + def test_bedrock_call(self): + """Test calling Bedrock through DSPy interface.""" + messages = [ + {"role": "system", "content": "System prompt"}, + {"role": "user", "content": "User message"} + ] + + result = self.adapter(messages, temperature=0.0, max_tokens=1000) + + # Verify call was made + self.mock_bedrock_adapter.call_model.assert_called_once() + + # Verify arguments + call_args = self.mock_bedrock_adapter.call_model.call_args + self.assertEqual(call_args[1]["model_id"], "us.amazon.nova-pro-v1:0") + self.assertEqual(call_args[1]["system_prompt"], "System prompt") + + # Verify result + self.assertEqual(result, ["Bedrock response"]) + + +class TestDSPySageMakerAdapter(unittest.TestCase): + """Test SageMaker-specific DSPy adapter.""" + + def setUp(self): + """Set up test fixtures.""" + self.mock_sagemaker_adapter = Mock() + self.mock_sagemaker_adapter.call_model.return_value = "SageMaker response" + self.mock_sagemaker_adapter.rate_limit = 10 + + self.adapter = DSPySageMakerAdapter( + inference_adapter=self.mock_sagemaker_adapter, + model_id="llama-3-70b" + ) + + def test_sagemaker_config_format(self): + """Test SageMaker-specific config format.""" + config = self.adapter._build_inference_config({ + "temperature": 0.8, + "max_tokens": 3000, + "top_p": 0.95, + "top_k": 40 + }) + + self.assertIn("max_new_tokens", config) + self.assertIn("temperature", config) + self.assertIn("do_sample", config) + self.assertIn("return_full_text", config) + self.assertEqual(config["max_new_tokens"], 3000) + self.assertEqual(config["temperature"], 0.8) + self.assertTrue(config["do_sample"]) # temperature > 0 + + def test_sagemaker_do_sample_false(self): + """Test do_sample is False when temperature is 0.""" + config = self.adapter._build_inference_config({"temperature": 0.0}) + self.assertFalse(config["do_sample"]) + + def test_sagemaker_config_uses_init_kwargs(self): + """Test that config uses kwargs from initialization.""" + # Create adapter with custom kwargs at init + adapter = DSPySageMakerAdapter( + self.mock_sagemaker_adapter, + "llama-3-70b", + temperature=0.7, + max_tokens=2000 + ) + + # Call without any kwargs - should use init kwargs + config = adapter._build_inference_config({}) + + self.assertEqual(config["temperature"], 0.7) + self.assertEqual(config["max_new_tokens"], 2000) + self.assertTrue(config["do_sample"]) # temperature > 0 + + def test_sagemaker_config_call_kwargs_override_init(self): + """Test that call-time kwargs override init kwargs.""" + # Create adapter with custom kwargs at init + adapter = DSPySageMakerAdapter( + self.mock_sagemaker_adapter, + "llama-3-70b", + temperature=0.7, + max_tokens=2000 + ) + + # Call with different kwargs - should override + config = adapter._build_inference_config({ + "temperature": 0.0, + "top_p": 0.8 + }) + + self.assertEqual(config["temperature"], 0.0) # Overridden + self.assertEqual(config["max_new_tokens"], 2000) # From init + self.assertEqual(config["top_p"], 0.8) # Overridden + self.assertFalse(config["do_sample"]) # temperature is 0 + + def test_sagemaker_do_sample_true(self): + """Test do_sample is True when temperature > 0.""" + config = self.adapter._build_inference_config({"temperature": 0.5}) + self.assertTrue(config["do_sample"]) + + def test_sagemaker_config_defaults(self): + """Test SageMaker config with default values from self.kwargs.""" + config = self.adapter._build_inference_config({}) + + # Should use defaults from self.kwargs (set in __init__) + self.assertEqual(config["max_new_tokens"], 1000) # From self.kwargs + self.assertEqual(config["temperature"], 1.0) # From self.kwargs + self.assertTrue(config["do_sample"]) # temperature > 0 + self.assertFalse(config["return_full_text"]) + + +class TestCreateDSPyAdapter(unittest.TestCase): + """Test the factory function.""" + + def test_bedrock_detection(self): + """Test automatic detection of Bedrock adapter.""" + mock_adapter = Mock(spec=BedrockInferenceAdapter) + mock_adapter.__class__.__name__ = "BedrockInferenceAdapter" + + result = create_dspy_adapter(mock_adapter, "test-model") + + self.assertIsInstance(result, DSPyBedrockAdapter) + self.assertEqual(result.model_id, "test-model") + + def test_sagemaker_detection(self): + """Test automatic detection of SageMaker adapter.""" + mock_adapter = Mock() + mock_adapter.__class__.__name__ = "SageMakerInferenceAdapter" + + result = create_dspy_adapter(mock_adapter, "test-model") + + self.assertIsInstance(result, DSPySageMakerAdapter) + self.assertEqual(result.model_id, "test-model") + + def test_unknown_adapter_fallback(self): + """Test fallback for unknown adapter types.""" + mock_adapter = Mock() + mock_adapter.__class__.__name__ = "UnknownAdapter" + + with self.assertLogs('amzn_nova_prompt_optimizer.core.inference.dspy_compatible', level='WARNING') as log: + result = create_dspy_adapter(mock_adapter, "test-model") + + # Should fallback to Bedrock adapter + self.assertIsInstance(result, DSPyBedrockAdapter) + self.assertTrue(any("Unknown adapter type" in message for message in log.output)) + + def test_factory_with_kwargs(self): + """Test factory function with additional kwargs.""" + mock_adapter = Mock(spec=BedrockInferenceAdapter) + mock_adapter.__class__.__name__ = "BedrockInferenceAdapter" + + result = create_dspy_adapter( + mock_adapter, + "test-model", + cache=True, + custom_param="value" + ) + + self.assertTrue(result.cache) + self.assertEqual(result.kwargs.get("custom_param"), "value") + + +class TestErrorHandling(unittest.TestCase): + """Test error handling in DSPy adapters.""" + + def test_call_model_error_propagation(self): + """Test that errors from call_model are propagated.""" + mock_adapter = Mock() + mock_adapter.call_model.side_effect = Exception("API Error") + + dspy_adapter = DSPyBedrockAdapter(mock_adapter, "test-model") + + with self.assertRaises(Exception) as context: + dspy_adapter([{"role": "user", "content": "test"}]) + + self.assertIn("API Error", str(context.exception)) + + def test_unknown_role_warning(self): + """Test warning for unknown message roles.""" + mock_adapter = Mock() + mock_adapter.call_model.return_value = "response" + + dspy_adapter = DSPyBedrockAdapter(mock_adapter, "test-model") + + messages = [ + {"role": "unknown", "content": "test"} + ] + + with self.assertLogs('amzn_nova_prompt_optimizer.core.inference.dspy_compatible', level='WARNING') as log: + dspy_adapter._convert_messages_to_adapter_format(messages) + + self.assertTrue(any("Unknown message role" in message for message in log.output)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/inference/test_dspy_integration.py b/tests/core/inference/test_dspy_integration.py new file mode 100644 index 0000000..fbff2cc --- /dev/null +++ b/tests/core/inference/test_dspy_integration.py @@ -0,0 +1,297 @@ +# Copyright 2025 Amazon Inc + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Integration tests for DSPy compatibility +""" + +import unittest +from unittest.mock import Mock, patch + +from amzn_nova_prompt_optimizer.core.inference import ( + BedrockInferenceAdapter, + SageMakerInferenceAdapter, + DSPyBedrockAdapter, + DSPySageMakerAdapter, + create_dspy_adapter +) + + +# Simple RateLimitedLM implementation for testing +class RateLimitedLM: + """Simple rate-limited LM wrapper for testing.""" + + def __init__(self, model, rate_limit=2): + self.wrapped_model = model + self.rate_limit = rate_limit + + def __call__(self, *args, **kwargs): + return self.wrapped_model(*args, **kwargs) + + def __getattr__(self, name): + return getattr(self.wrapped_model, name) + + +class MockInferenceAdapter: + """Mock inference adapter for testing without API calls.""" + + def __init__(self, responses=None): + self.region = "us-east-1" + self.rate_limit = 100 + self.responses = responses or ["Mock response"] + self.call_count = 0 + self.call_history = [] + + def call_model(self, model_id, system_prompt, messages, inf_config): + """Return mock response.""" + self.call_history.append({ + "model_id": model_id, + "system_prompt": system_prompt, + "messages": messages, + "inf_config": inf_config + }) + + response = self.responses[self.call_count % len(self.responses)] + self.call_count += 1 + return response + + def to_dspy_lm(self, model_id, **kwargs): + """Create DSPy-compatible wrapper.""" + return DSPyBedrockAdapter(self, model_id, **kwargs) + + +class TestDSPyIntegration(unittest.TestCase): + """Integration tests with DSPy.""" + + def test_mock_adapter_basic(self): + """Test basic usage with mock adapter.""" + mock = MockInferenceAdapter(responses=["Response 1", "Response 2"]) + + dspy_lm = mock.to_dspy_lm("mock-model") + + # First call + result1 = dspy_lm([{"role": "user", "content": "Test 1"}]) + self.assertEqual(result1, ["Response 1"]) + + # Second call + result2 = dspy_lm([{"role": "user", "content": "Test 2"}]) + self.assertEqual(result2, ["Response 2"]) + + # Verify call history + self.assertEqual(mock.call_count, 2) + self.assertEqual(len(mock.call_history), 2) + + def test_with_rate_limited_lm(self): + """Test integration with RateLimitedLM.""" + mock = MockInferenceAdapter(responses=["Response"]) + + dspy_lm = mock.to_dspy_lm("mock-model") + rate_limited = RateLimitedLM(dspy_lm, rate_limit=5) + + # Should work without errors + messages = [{"role": "user", "content": "Test"}] + result = rate_limited(messages) + + self.assertEqual(result, ["Response"]) + self.assertEqual(mock.call_count, 1) + + def test_conversation_history(self): + """Test with conversation history.""" + mock = MockInferenceAdapter() + dspy_lm = mock.to_dspy_lm("mock-model") + + messages = [ + {"role": "system", "content": "Be helpful"}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "user", "content": "How are you?"} + ] + + result = dspy_lm(messages) + + self.assertEqual(result, ["Mock response"]) + + # Verify the adapter received correct format + call = mock.call_history[0] + self.assertEqual(call["system_prompt"], "Be helpful") + self.assertEqual(len(call["messages"]), 3) + + def test_parameter_passing(self): + """Test parameter passing through DSPy interface.""" + mock = MockInferenceAdapter() + dspy_lm = mock.to_dspy_lm("mock-model") + + messages = [{"role": "user", "content": "Test"}] + result = dspy_lm( + messages, + temperature=0.7, + max_tokens=1000, + top_p=0.9, + top_k=50 + ) + + self.assertEqual(result, ["Mock response"]) + + # Verify parameters were passed and transformed by DSPyBedrockAdapter + call = mock.call_history[0] + config = call["inf_config"] + # DSPyBedrockAdapter transforms max_tokens -> maxTokens + from amzn_nova_prompt_optimizer.core.inference import ( + MAX_TOKENS_FIELD, TEMPERATURE_FIELD, TOP_P_FIELD, TOP_K_FIELD + ) + self.assertEqual(config.get(TEMPERATURE_FIELD), 0.7) + self.assertEqual(config.get(MAX_TOKENS_FIELD), 1000) + self.assertEqual(config.get(TOP_P_FIELD), 0.9) + self.assertEqual(config.get(TOP_K_FIELD), 50) + + def test_history_tracking(self): + """Test that DSPy adapter tracks history.""" + mock = MockInferenceAdapter() + dspy_lm = mock.to_dspy_lm("mock-model") + + # Make multiple calls + dspy_lm([{"role": "user", "content": "Test 1"}]) + dspy_lm([{"role": "user", "content": "Test 2"}]) + dspy_lm([{"role": "user", "content": "Test 3"}]) + + # Verify history + self.assertEqual(len(dspy_lm.history), 3) + self.assertEqual(dspy_lm.history[0]["response"], "Mock response") + + def test_copy_method(self): + """Test copy method creates independent instance.""" + mock = MockInferenceAdapter() + dspy_lm1 = mock.to_dspy_lm("model-1") + dspy_lm2 = dspy_lm1.copy(model_id="model-2") + + # Should be different instances + self.assertIsNot(dspy_lm1, dspy_lm2) + + # Should have different model IDs + self.assertEqual(dspy_lm1.model_id, "model-1") + self.assertEqual(dspy_lm2.model_id, "model-2") + + # Should share same underlying adapter + self.assertIs(dspy_lm1.inference_adapter, dspy_lm2.inference_adapter) + + +class TestFactoryFunction(unittest.TestCase): + """Test the factory function with different adapters.""" + + def test_factory_with_mock(self): + """Test factory function with mock adapter.""" + mock = MockInferenceAdapter() + dspy_lm = create_dspy_adapter(mock, "test-model") + + self.assertIsInstance(dspy_lm, DSPyBedrockAdapter) + self.assertEqual(dspy_lm.model_id, "test-model") + + @patch('boto3.Session') + def test_factory_with_bedrock(self, mock_session): + """Test factory function with Bedrock adapter.""" + mock_runtime = Mock() + mock_session.return_value.client.return_value = mock_runtime + + bedrock_adapter = BedrockInferenceAdapter(region_name="us-east-1") + dspy_lm = create_dspy_adapter(bedrock_adapter, "us.amazon.nova-pro-v1:0") + + self.assertIsInstance(dspy_lm, DSPyBedrockAdapter) + self.assertEqual(dspy_lm.model_id, "us.amazon.nova-pro-v1:0") + + @patch('boto3.Session') + def test_factory_with_sagemaker(self, mock_session): + """Test factory function with SageMaker adapter.""" + mock_runtime = Mock() + mock_session.return_value.client.return_value = mock_runtime + + sagemaker_adapter = SageMakerInferenceAdapter( + endpoint_name="test-endpoint" + ) + dspy_lm = create_dspy_adapter(sagemaker_adapter, "llama-3-70b") + + self.assertIsInstance(dspy_lm, DSPySageMakerAdapter) + self.assertEqual(dspy_lm.model_id, "llama-3-70b") + + +class TestEndToEnd(unittest.TestCase): + """End-to-end integration tests.""" + + def test_complete_workflow(self): + """Test complete workflow from adapter creation to inference.""" + # Create mock adapter + mock = MockInferenceAdapter(responses=[ + "Response 1", + "Response 2", + "Response 3" + ]) + + # Create DSPy-compatible wrapper + dspy_lm = mock.to_dspy_lm("test-model") + + # Wrap with rate limiting + rate_limited_lm = RateLimitedLM(dspy_lm, rate_limit=10) + + # Make calls + results = [] + for i in range(3): + messages = [ + {"role": "system", "content": "Be helpful"}, + {"role": "user", "content": f"Question {i+1}"} + ] + result = rate_limited_lm(messages, temperature=0.0) + results.append(result[0]) + + # Verify results + self.assertEqual(results, ["Response 1", "Response 2", "Response 3"]) + self.assertEqual(mock.call_count, 3) + + # Verify all calls had correct format + for call in mock.call_history: + self.assertEqual(call["system_prompt"], "Be helpful") + self.assertEqual(len(call["messages"]), 1) + self.assertIn("user", call["messages"][0]) + + def test_switching_backends(self): + """Test switching between different backends.""" + # Create different mock adapters with proper class names + class BedrockInferenceAdapter(MockInferenceAdapter): + pass + + class SageMakerInferenceAdapter(MockInferenceAdapter): + pass + + bedrock_mock = BedrockInferenceAdapter(responses=["Bedrock response"]) + sagemaker_mock = SageMakerInferenceAdapter(responses=["SageMaker response"]) + + # Create DSPy adapters + bedrock_lm = create_dspy_adapter(bedrock_mock, "nova-model") + sagemaker_lm = create_dspy_adapter(sagemaker_mock, "llama-model") + + # Verify correct adapter types + self.assertIsInstance(bedrock_lm, DSPyBedrockAdapter) + self.assertIsInstance(sagemaker_lm, DSPySageMakerAdapter) + + # Make calls + messages = [{"role": "user", "content": "Test"}] + + bedrock_result = bedrock_lm(messages) + sagemaker_result = sagemaker_lm(messages) + + # Verify different responses + self.assertEqual(bedrock_result, ["Bedrock response"]) + self.assertEqual(sagemaker_result, ["SageMaker response"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/inference/test_inference_adapter.py b/tests/core/inference/test_inference_adapter.py index 62cbcfe..68904d0 100644 --- a/tests/core/inference/test_inference_adapter.py +++ b/tests/core/inference/test_inference_adapter.py @@ -1,7 +1,7 @@ import unittest import time -from amzn_nova_prompt_optimizer.core.inference.adapter import BedrockInferenceAdapter +from amzn_nova_prompt_optimizer.core.inference.bedrock_adapter import BedrockInferenceAdapter from amzn_nova_prompt_optimizer.core.inference.bedrock_converse import BedrockConverseHandler from unittest.mock import Mock, patch @@ -21,7 +21,7 @@ def setUp(self): self.test_max_retries = 5 # Create patcher for boto3.Session - self.session_patcher = patch('amzn_nova_prompt_optimizer.core.inference.adapter.boto3.Session') + self.session_patcher = patch('amzn_nova_prompt_optimizer.core.inference.bedrock_adapter.boto3.Session') self.mock_session_class = self.session_patcher.start() # Setup mock session and client diff --git a/tests/core/inference/test_sagemaker_adapter.py b/tests/core/inference/test_sagemaker_adapter.py new file mode 100644 index 0000000..b6ba241 --- /dev/null +++ b/tests/core/inference/test_sagemaker_adapter.py @@ -0,0 +1,353 @@ +# Copyright 2025 Amazon Inc + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for SageMaker Inference Adapter +""" + +import json +import unittest +from unittest.mock import Mock, MagicMock, patch + +from botocore.exceptions import ClientError + +from amzn_nova_prompt_optimizer.core.inference import SageMakerInferenceAdapter + + +class TestSageMakerInferenceAdapter(unittest.TestCase): + """Test SageMaker inference adapter.""" + + @patch('boto3.Session') + def setUp(self, mock_session): + """Set up test fixtures.""" + self.mock_runtime = Mock() + mock_session.return_value.client.return_value = self.mock_runtime + + self.adapter = SageMakerInferenceAdapter( + endpoint_name="test-endpoint", + region_name="us-east-1", + rate_limit=5 + ) + + def test_initialization(self): + """Test adapter initialization.""" + self.assertEqual(self.adapter.endpoint_name, "test-endpoint") + self.assertEqual(self.adapter.region, "us-east-1") + self.assertEqual(self.adapter.rate_limit, 5) + self.assertEqual(self.adapter.max_retries, 5) + self.assertEqual(self.adapter.content_type, "application/json") + + def test_payload_formatting(self): + """Test payload formatting for SageMaker.""" + system_prompt = "You are helpful" + messages = [ + {"user": "Hello"}, + {"assistant": "Hi"}, + {"user": "How are you?"} + ] + inf_config = { + "max_new_tokens": 1000, + "temperature": 0.7, + "top_p": 0.9, + "top_k": 50 + } + + payload = self.adapter._format_payload( + system_prompt, messages, inf_config + ) + + # Check OpenAI-compatible format + self.assertIn("messages", payload) + self.assertIn("max_tokens", payload) + self.assertIn("temperature", payload) + self.assertIn("top_p", payload) + self.assertIn("top_k", payload) + + # Check parameter values (max_tokens instead of max_new_tokens) + self.assertEqual(payload["max_tokens"], 1000) + self.assertEqual(payload["temperature"], 0.7) + self.assertEqual(payload["top_p"], 0.9) + self.assertEqual(payload["top_k"], 50) + + # Check message structure + self.assertEqual(len(payload["messages"]), 4) + self.assertEqual(payload["messages"][0]["role"], "system") + self.assertEqual(payload["messages"][0]["content"], "You are helpful") + self.assertEqual(payload["messages"][1]["role"], "user") + self.assertEqual(payload["messages"][1]["content"], "Hello") + self.assertEqual(payload["messages"][2]["role"], "assistant") + self.assertEqual(payload["messages"][2]["content"], "Hi") + self.assertEqual(payload["messages"][3]["role"], "user") + self.assertEqual(payload["messages"][3]["content"], "How are you?") + + def test_payload_formatting_no_system_prompt(self): + """Test payload formatting without system prompt.""" + messages = [{"user": "Hello"}] + inf_config = {"max_new_tokens": 100} + + payload = self.adapter._format_payload("", messages, inf_config) + + # Check OpenAI-compatible format without system prompt + self.assertIn("messages", payload) + self.assertEqual(len(payload["messages"]), 1) + self.assertEqual(payload["messages"][0]["role"], "user") + self.assertEqual(payload["messages"][0]["content"], "Hello") + + def test_response_extraction_list_format(self): + """Test extracting text from list response format.""" + response = [{"generated_text": "Test response"}] + + text = self.adapter._extract_text_from_response(response) + + self.assertEqual(text, "Test response") + + def test_response_extraction_dict_format(self): + """Test extracting text from dict response format.""" + response = {"generated_text": "Test response"} + + text = self.adapter._extract_text_from_response(response) + + self.assertEqual(text, "Test response") + + def test_response_extraction_outputs_format(self): + """Test extracting text from outputs format.""" + response = {"outputs": "Test response"} + + text = self.adapter._extract_text_from_response(response) + + self.assertEqual(text, "Test response") + + def test_response_extraction_string_format(self): + """Test extracting text from string format.""" + response = "Test response" + + text = self.adapter._extract_text_from_response(response) + + self.assertEqual(text, "Test response") + + def test_response_extraction_unknown_format(self): + """Test extracting text from unknown format.""" + response = {"unknown_field": "value"} + + with self.assertLogs('amzn_nova_prompt_optimizer.core.inference.sagemaker_adapter', level='WARNING') as log: + text = self.adapter._extract_text_from_response(response) + + # Should return JSON string + self.assertIn("unknown_field", text) + self.assertTrue(any("Unknown SageMaker response format" in message for message in log.output)) + + @patch('time.sleep') + def test_successful_call(self, mock_sleep): + """Test successful model call.""" + # Mock successful response + self.mock_runtime.invoke_endpoint.return_value = { + "Body": Mock( + read=Mock( + return_value=b'{"generated_text": "Success"}' + ) + ) + } + + result = self.adapter.call_model( + model_id="test", + system_prompt="Test", + messages=[{"user": "Test"}], + inf_config={"max_new_tokens": 100} + ) + + self.assertEqual(result, "Success") + self.mock_runtime.invoke_endpoint.assert_called_once() + mock_sleep.assert_not_called() + + @patch('time.sleep') + def test_retry_on_throttling(self, mock_sleep): + """Test retry logic on throttling.""" + # Mock throttling error then success + self.mock_runtime.invoke_endpoint.side_effect = [ + ClientError( + {"Error": {"Code": "ThrottlingException"}}, + "invoke_endpoint" + ), + { + "Body": Mock( + read=Mock( + return_value=b'{"generated_text": "Success"}' + ) + ) + } + ] + + result = self.adapter.call_model( + model_id="test", + system_prompt="Test", + messages=[{"user": "Test"}], + inf_config={"max_new_tokens": 100} + ) + + # Should have retried + self.assertEqual(self.mock_runtime.invoke_endpoint.call_count, 2) + self.assertEqual(result, "Success") + mock_sleep.assert_called_once() + + @patch('time.sleep') + def test_retry_on_model_error(self, mock_sleep): + """Test retry logic on model error.""" + # Mock model error then success + self.mock_runtime.invoke_endpoint.side_effect = [ + ClientError( + {"Error": {"Code": "ModelError"}}, + "invoke_endpoint" + ), + { + "Body": Mock( + read=Mock( + return_value=b'{"generated_text": "Success"}' + ) + ) + } + ] + + result = self.adapter.call_model( + model_id="test", + system_prompt="Test", + messages=[{"user": "Test"}], + inf_config={"max_new_tokens": 100} + ) + + self.assertEqual(self.mock_runtime.invoke_endpoint.call_count, 2) + self.assertEqual(result, "Success") + + @patch('time.sleep') + def test_max_retries_exceeded(self, mock_sleep): + """Test max retries exceeded.""" + # Mock continuous throttling + self.mock_runtime.invoke_endpoint.side_effect = ClientError( + {"Error": {"Code": "ThrottlingException"}}, + "invoke_endpoint" + ) + + with self.assertRaises(Exception) as context: + self.adapter.call_model( + model_id="test", + system_prompt="Test", + messages=[{"user": "Test"}], + inf_config={"max_new_tokens": 100} + ) + + self.assertIn("Max retries", str(context.exception)) + self.assertEqual(self.mock_runtime.invoke_endpoint.call_count, 5) + + @patch('time.sleep') + def test_non_retryable_error(self, mock_sleep): + """Test non-retryable error is raised immediately.""" + # Mock non-retryable error + self.mock_runtime.invoke_endpoint.side_effect = ClientError( + {"Error": {"Code": "ValidationException"}}, + "invoke_endpoint" + ) + + with self.assertRaises(ClientError): + self.adapter.call_model( + model_id="test", + system_prompt="Test", + messages=[{"user": "Test"}], + inf_config={"max_new_tokens": 100} + ) + + # Should not retry + self.assertEqual(self.mock_runtime.invoke_endpoint.call_count, 1) + mock_sleep.assert_not_called() + + def test_backoff_calculation(self): + """Test exponential backoff calculation.""" + # Test backoff increases exponentially + backoff_0 = self.adapter._calculate_backoff_time(0) + backoff_1 = self.adapter._calculate_backoff_time(1) + backoff_2 = self.adapter._calculate_backoff_time(2) + + # Should increase (with jitter, so approximate) + self.assertGreater(backoff_1, backoff_0) + self.assertGreater(backoff_2, backoff_1) + + # Check rough exponential growth + self.assertGreater(backoff_2, 3.0) # 1 * 2^2 = 4, minus jitter + + def test_to_dspy_lm(self): + """Test creating DSPy-compatible wrapper uses endpoint_name.""" + dspy_lm = self.adapter.to_dspy_lm("test-model") + + from amzn_nova_prompt_optimizer.core.inference import DSPySageMakerAdapter + self.assertIsInstance(dspy_lm, DSPySageMakerAdapter) + # For SageMaker, model_id should be the endpoint_name, not the provided model_id + self.assertEqual(dspy_lm.model_id, "test-endpoint") + + def test_to_dspy_lm_overrides_model_id_with_endpoint_name(self): + """Test that to_dspy_lm overrides model_id with endpoint_name for SageMaker.""" + # The task_model_id passed should be ignored, endpoint_name should be used instead + dspy_lm = self.adapter.to_dspy_lm("any-model-id-ignored") + + from amzn_nova_prompt_optimizer.core.inference import DSPySageMakerAdapter + self.assertIsInstance(dspy_lm, DSPySageMakerAdapter) + # Should use endpoint_name, not the provided model_id + self.assertEqual(dspy_lm.model_id, "test-endpoint") + self.assertNotEqual(dspy_lm.model_id, "any-model-id-ignored") + + +class TestSageMakerAdapterCustomization(unittest.TestCase): + """Test customization of SageMaker adapter.""" + + @patch('boto3.Session') + def test_custom_content_type(self, mock_session): + """Test custom content type.""" + mock_runtime = Mock() + mock_session.return_value.client.return_value = mock_runtime + + adapter = SageMakerInferenceAdapter( + endpoint_name="test-endpoint", + content_type="text/plain", + accept="text/plain" + ) + + self.assertEqual(adapter.content_type, "text/plain") + self.assertEqual(adapter.accept, "text/plain") + + @patch('boto3.Session') + def test_custom_retries(self, mock_session): + """Test custom max retries.""" + mock_runtime = Mock() + mock_session.return_value.client.return_value = mock_runtime + + adapter = SageMakerInferenceAdapter( + endpoint_name="test-endpoint", + max_retries=10 + ) + + self.assertEqual(adapter.max_retries, 10) + + @patch('boto3.Session') + def test_custom_rate_limit(self, mock_session): + """Test custom rate limit.""" + mock_runtime = Mock() + mock_session.return_value.client.return_value = mock_runtime + + adapter = SageMakerInferenceAdapter( + endpoint_name="test-endpoint", + rate_limit=20 + ) + + self.assertEqual(adapter.rate_limit, 20) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/core/optimizers/miprov2/test_miprov2_optimizer.py b/tests/core/optimizers/miprov2/test_miprov2_optimizer.py index ef75225..cc0ab26 100644 --- a/tests/core/optimizers/miprov2/test_miprov2_optimizer.py +++ b/tests/core/optimizers/miprov2/test_miprov2_optimizer.py @@ -1,4 +1,3 @@ -import os import unittest from unittest.mock import Mock, patch, PropertyMock, MagicMock @@ -318,9 +317,6 @@ def test_optimize_without_nova_tips(self, mock_configure, mock_miprov2, mock_lm) train_split=0.7 ) - # Assert AWS REGION is default to us-west-2 - self.assertEqual(os.getenv('AWS_REGION_NAME'), 'us-west-2') - # Verify that MIPROv2 was initialized correctly mock_miprov2.assert_called_once() @@ -553,9 +549,6 @@ def test_optimize_with_nova_tips(self, mock_grounded_proposer, mock_nova_grounde train_split=0.7 ) - # Assert AWS REGION has been set - self.assertEqual(os.getenv('AWS_REGION_NAME'), 'us-east-1') - # Verify that MIPROv2 was initialized correctly mock_miprov2.assert_called_once() diff --git a/tests/core/optimizers/nova_meta_prompter/test_nova_meta_prompter.py b/tests/core/optimizers/nova_meta_prompter/test_nova_meta_prompter.py index f9d7859..90319ab 100644 --- a/tests/core/optimizers/nova_meta_prompter/test_nova_meta_prompter.py +++ b/tests/core/optimizers/nova_meta_prompter/test_nova_meta_prompter.py @@ -120,10 +120,8 @@ def test_optimize_max_retries(self): self.prompt_adapter.fetch_system_template.return_value = "System template {var1}" self.prompt_adapter.fetch_user_template.return_value = "User template {var2}" - # Create a real method instead of mocking _format_prompt_with_variables - self.optimizer._format_prompt_with_variables = staticmethod( - lambda p, v: p if p else "" - ) + # Mock _format_prompt_with_variables as a regular method + self.optimizer._format_prompt_with_variables = lambda p, v: p if p else "" # Mock the inference adapter to return invalid responses self.inference_adapter.call_model.side_effect = ["Invalid", "Invalid"] diff --git a/tests/core/optimizers/nova_prompt_optimizer/test_nova_prompt_optimizer.py b/tests/core/optimizers/nova_prompt_optimizer/test_nova_prompt_optimizer.py index e9b1c3e..1c750ef 100644 --- a/tests/core/optimizers/nova_prompt_optimizer/test_nova_prompt_optimizer.py +++ b/tests/core/optimizers/nova_prompt_optimizer/test_nova_prompt_optimizer.py @@ -113,7 +113,7 @@ def test_optimize_with_all_adapters_default_mode(self, mock_miprov2_class): # Verify meta prompt optimization was called self.nova_prompt_optimizer.meta_prompt_optimization_adapter.optimize.assert_called_once_with( - prompter_model_id="us.amazon.nova-premier-v1:0" + prompter_model_id="us.amazon.nova-2-lite-v1:0" ) # Verify MIPROv2 was initialized and called @@ -125,7 +125,7 @@ def test_optimize_with_all_adapters_default_mode(self, mock_miprov2_class): ) mock_miprov2_instance.optimize.assert_called_once_with( - prompter_model_id="us.amazon.nova-premier-v1:0", + prompter_model_id="us.amazon.nova-2-lite-v1:0", task_model_id="us.amazon.nova-pro-v1:0", num_candidates=20, num_trials=30, @@ -153,11 +153,11 @@ def test_optimize_with_lite_mode(self, mock_miprov2_class): # Verify meta prompt optimization was called self.nova_prompt_optimizer.meta_prompt_optimization_adapter.optimize.assert_called_once_with( - prompter_model_id="us.amazon.nova-premier-v1:0" + prompter_model_id="us.amazon.nova-2-lite-v1:0" ) mock_miprov2_instance.optimize.assert_called_once_with( - prompter_model_id="us.amazon.nova-premier-v1:0", + prompter_model_id="us.amazon.nova-2-lite-v1:0", task_model_id="us.amazon.nova-lite-v1:0", num_candidates=20, num_trials=30, @@ -249,6 +249,79 @@ def test_optimize_custom_mode_no_params(self): self.assertIn("Custom mode requires custom_params dictionary", str(context.exception)) + @patch('amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer.NovaMIPROv2OptimizationAdapter') + def test_optimize_with_separate_meta_prompt_adapter(self, mock_miprov2_class): + """Test using separate inference adapters for meta-prompting and task optimization.""" + # Create separate mock adapters + meta_prompt_adapter = Mock(spec=InferenceAdapter) + task_adapter = Mock(spec=InferenceAdapter) + + # Create optimizer with separate adapters + optimizer = NovaPromptOptimizer( + prompt_adapter=self.prompt_adapter, + inference_adapter=task_adapter, + dataset_adapter=self.dataset_adapter, + metric_adapter=self.metric_adapter, + meta_prompt_inference_adapter=meta_prompt_adapter + ) + + # Verify the adapters are set correctly + self.assertIs(optimizer.inference_adapter, task_adapter) + self.assertIs(optimizer.meta_prompt_inference_adapter, meta_prompt_adapter) + self.assertIs(optimizer.meta_prompt_optimization_adapter.inference_adapter, meta_prompt_adapter) + + # Mock the optimization process + mock_intermediate_prompt_adapter = Mock(spec=PromptAdapter) + mock_final_prompt_adapter = Mock(spec=PromptAdapter) + + optimizer.meta_prompt_optimization_adapter.optimize = Mock( + return_value=mock_intermediate_prompt_adapter + ) + + mock_miprov2_instance = Mock() + mock_miprov2_instance.optimize.return_value = mock_final_prompt_adapter + mock_miprov2_class.return_value = mock_miprov2_instance + + # Run optimization + result = optimizer.optimize(mode="pro") + + # Verify meta-prompt adapter was used for meta-prompting + optimizer.meta_prompt_optimization_adapter.optimize.assert_called_once() + + # Verify task adapter was passed to MIPROv2 + mock_miprov2_class.assert_called_once() + call_kwargs = mock_miprov2_class.call_args[1] + self.assertIs(call_kwargs['inference_adapter'], task_adapter) + + self.assertIs(result, mock_final_prompt_adapter) + + @patch('amzn_nova_prompt_optimizer.core.inference.BedrockInferenceAdapter') + def test_separate_adapter_defaults_to_bedrock(self, mock_bedrock_class): + """Test that meta_prompt_inference_adapter defaults to BedrockInferenceAdapter if not provided.""" + mock_bedrock_instance = Mock(spec=InferenceAdapter) + mock_bedrock_class.return_value = mock_bedrock_instance + + optimizer = NovaPromptOptimizer( + prompt_adapter=self.prompt_adapter, + inference_adapter=self.inference_adapter, + dataset_adapter=self.dataset_adapter, + metric_adapter=self.metric_adapter + ) + + # Should create a default BedrockInferenceAdapter for meta-prompting + mock_bedrock_class.assert_called_once() + call_kwargs = mock_bedrock_class.call_args[1] + self.assertEqual(call_kwargs['rate_limit'], 5) + + # Should use the created Bedrock adapter for meta-prompting + self.assertIs(optimizer.meta_prompt_inference_adapter, mock_bedrock_instance) + + # Should use the provided adapter for task optimization + self.assertIs(optimizer.inference_adapter, self.inference_adapter) + + # Meta-prompt optimization adapter should use the Bedrock adapter + self.assertIs(optimizer.meta_prompt_optimization_adapter.inference_adapter, mock_bedrock_instance) + def test_optimize_custom_mode_missing_required_keys(self): incomplete_params = { "task_model_id": "custom-model", @@ -283,7 +356,7 @@ def test_optimize_invalid_mode_defaults_to_pro(self, mock_logger, mock_miprov2_c # Verify meta prompt optimization was called self.nova_prompt_optimizer.meta_prompt_optimization_adapter.optimize.assert_called_once_with( - prompter_model_id="us.amazon.nova-premier-v1:0" + prompter_model_id="us.amazon.nova-2-lite-v1:0" ) # Verify MIPROv2 was initialized and called @@ -295,7 +368,7 @@ def test_optimize_invalid_mode_defaults_to_pro(self, mock_logger, mock_miprov2_c ) mock_miprov2_instance.optimize.assert_called_once_with( - prompter_model_id="us.amazon.nova-premier-v1:0", + prompter_model_id="us.amazon.nova-2-lite-v1:0", task_model_id="us.amazon.nova-pro-v1:0", num_candidates=20, num_trials=30, @@ -308,12 +381,12 @@ def test_optimize_invalid_mode_defaults_to_pro(self, mock_logger, mock_miprov2_c @patch('amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer.NovaMIPROv2OptimizationAdapter') def test_optimize_all_modes(self, mock_miprov2_class): - modes = ["micro", "lite", "pro", "premier"] + modes = ["micro", "lite", "pro", "lite-2"] expected_task_models = { "micro": "us.amazon.nova-micro-v1:0", "lite": "us.amazon.nova-lite-v1:0", "pro": "us.amazon.nova-pro-v1:0", - "premier": "us.amazon.nova-premier-v1:0" + "lite-2": "us.amazon.nova-2-lite-v1:0" } for mode in modes: @@ -332,7 +405,7 @@ def test_optimize_all_modes(self, mock_miprov2_class): result = self.nova_prompt_optimizer.optimize(mode=mode) mock_miprov2_instance.optimize.assert_called_with( - prompter_model_id="us.amazon.nova-premier-v1:0", + prompter_model_id="us.amazon.nova-2-lite-v1:0", task_model_id=expected_task_models[mode], num_candidates=20, num_trials=30, diff --git a/tests/integration/test_separate_adapters_integration.py b/tests/integration/test_separate_adapters_integration.py new file mode 100644 index 0000000..d2bddfc --- /dev/null +++ b/tests/integration/test_separate_adapters_integration.py @@ -0,0 +1,269 @@ +""" +Integration test for separate inference adapters feature. + +This test verifies that the NovaPromptOptimizer correctly uses separate +adapters for meta-prompting and task optimization phases. +""" + +import unittest +from unittest.mock import Mock, patch, call + +from amzn_nova_prompt_optimizer.core.inference import InferenceAdapter +from amzn_nova_prompt_optimizer.core.input_adapters.dataset_adapter import DatasetAdapter +from amzn_nova_prompt_optimizer.core.input_adapters.metric_adapter import MetricAdapter +from amzn_nova_prompt_optimizer.core.input_adapters.prompt_adapter import PromptAdapter +from amzn_nova_prompt_optimizer.core.optimizers import NovaPromptOptimizer + + +class TestSeparateAdaptersIntegration(unittest.TestCase): + """Integration tests for separate inference adapters.""" + + def setUp(self): + """Set up test fixtures.""" + # Create mock adapters + self.meta_prompt_adapter = Mock(spec=InferenceAdapter) + self.task_adapter = Mock(spec=InferenceAdapter) + + # Create mock prompt adapter + self.prompt_adapter = Mock(spec=PromptAdapter) + self.prompt_adapter.fetch.return_value = { + "user_prompt": {"content": "Test {{input}}", "variables": ["input"]}, + "system_prompt": {"content": "You are helpful"} + } + self.prompt_adapter.fetch_system_template.return_value = "You are helpful" + self.prompt_adapter.fetch_user_template.return_value = "Test {{input}}" + + # Create mock dataset adapter + self.dataset_adapter = Mock(spec=DatasetAdapter) + self.dataset_adapter.input_columns = ["input"] + self.dataset_adapter.output_columns = ["output"] + self.dataset_adapter.fetch.return_value = [ + {"inputs": {"input": "test1"}, "outputs": {"output": "result1"}}, + {"inputs": {"input": "test2"}, "outputs": {"output": "result2"}} + ] + + # Create mock metric adapter + self.metric_adapter = Mock(spec=MetricAdapter) + self.metric_adapter.apply.return_value = 1.0 + self.metric_adapter.batch_apply.return_value = 1.0 + + @patch('amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer.NovaMIPROv2OptimizationAdapter') + def test_separate_adapters_used_correctly(self, mock_mipro_class): + """Test that separate adapters are used in correct phases.""" + + # Setup mocks + mock_intermediate_prompt = Mock(spec=PromptAdapter) + mock_final_prompt = Mock(spec=PromptAdapter) + + mock_mipro_instance = Mock() + mock_mipro_instance.optimize.return_value = mock_final_prompt + mock_mipro_class.return_value = mock_mipro_instance + + # Create optimizer with separate adapters + optimizer = NovaPromptOptimizer( + prompt_adapter=self.prompt_adapter, + inference_adapter=self.task_adapter, + dataset_adapter=self.dataset_adapter, + metric_adapter=self.metric_adapter, + meta_prompt_inference_adapter=self.meta_prompt_adapter + ) + + # Verify initialization + self.assertIs(optimizer.inference_adapter, self.task_adapter) + self.assertIs(optimizer.meta_prompt_inference_adapter, self.meta_prompt_adapter) + + # Verify meta-prompt optimization adapter uses meta-prompt adapter + self.assertIs( + optimizer.meta_prompt_optimization_adapter.inference_adapter, + self.meta_prompt_adapter + ) + + # Mock the meta-prompt optimization + optimizer.meta_prompt_optimization_adapter.optimize = Mock( + return_value=mock_intermediate_prompt + ) + + # Run optimization + result = optimizer.optimize(mode="lite") + + # Verify meta-prompt adapter was used for meta-prompting + optimizer.meta_prompt_optimization_adapter.optimize.assert_called_once() + + # Verify task adapter was used for MIPROv2 + mock_mipro_class.assert_called_once() + mipro_call_kwargs = mock_mipro_class.call_args[1] + self.assertIs(mipro_call_kwargs['inference_adapter'], self.task_adapter) + + # Verify result + self.assertIs(result, mock_final_prompt) + + @patch('amzn_nova_prompt_optimizer.core.inference.BedrockInferenceAdapter') + @patch('amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer.NovaMIPROv2OptimizationAdapter') + def test_single_adapter_creates_default_bedrock(self, mock_mipro_class, mock_bedrock_class): + """Test that default BedrockInferenceAdapter is created when only one adapter is provided.""" + + # Setup mocks + mock_bedrock_instance = Mock(spec=InferenceAdapter) + mock_bedrock_class.return_value = mock_bedrock_instance + + mock_intermediate_prompt = Mock(spec=PromptAdapter) + mock_final_prompt = Mock(spec=PromptAdapter) + + mock_mipro_instance = Mock() + mock_mipro_instance.optimize.return_value = mock_final_prompt + mock_mipro_class.return_value = mock_mipro_instance + + # Create optimizer with single adapter (creates default Bedrock for meta-prompting) + optimizer = NovaPromptOptimizer( + prompt_adapter=self.prompt_adapter, + inference_adapter=self.task_adapter, + dataset_adapter=self.dataset_adapter, + metric_adapter=self.metric_adapter + # No meta_prompt_inference_adapter provided + ) + + # Verify Bedrock adapter was created for meta-prompting + mock_bedrock_class.assert_called_once() + + # Verify different adapters are used + self.assertIs(optimizer.inference_adapter, self.task_adapter) + self.assertIs(optimizer.meta_prompt_inference_adapter, mock_bedrock_instance) + + # Verify meta-prompt optimization adapter uses Bedrock adapter + self.assertIs( + optimizer.meta_prompt_optimization_adapter.inference_adapter, + mock_bedrock_instance + ) + + # Mock the meta-prompt optimization + optimizer.meta_prompt_optimization_adapter.optimize = Mock( + return_value=mock_intermediate_prompt + ) + + # Run optimization + result = optimizer.optimize(mode="pro") + + # Verify Bedrock adapter was used for meta-prompting + optimizer.meta_prompt_optimization_adapter.optimize.assert_called_once() + + # Verify task adapter was used for MIPROv2 + mock_mipro_class.assert_called_once() + mipro_call_kwargs = mock_mipro_class.call_args[1] + self.assertIs(mipro_call_kwargs['inference_adapter'], self.task_adapter) + + # Verify result + self.assertIs(result, mock_final_prompt) + + @patch('amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer.NovaMIPROv2OptimizationAdapter') + @patch('amzn_nova_prompt_optimizer.core.optimizers.nova_prompt_optimizer.nova_prompt_optimizer.NovaMPOptimizationAdapter') + def test_custom_mode_with_separate_adapters(self, mock_mp_class, mock_mipro_class): + """Test custom mode with separate adapters and custom parameters.""" + + # Setup mocks + mock_mp_instance = Mock() + mock_intermediate_prompt = Mock(spec=PromptAdapter) + mock_mp_instance.optimize.return_value = mock_intermediate_prompt + mock_mp_class.return_value = mock_mp_instance + + mock_mipro_instance = Mock() + mock_final_prompt = Mock(spec=PromptAdapter) + mock_mipro_instance.optimize.return_value = mock_final_prompt + mock_mipro_class.return_value = mock_mipro_instance + + # Create optimizer with separate adapters + optimizer = NovaPromptOptimizer( + prompt_adapter=self.prompt_adapter, + inference_adapter=self.task_adapter, + dataset_adapter=self.dataset_adapter, + metric_adapter=self.metric_adapter, + meta_prompt_inference_adapter=self.meta_prompt_adapter + ) + + # Custom parameters + custom_params = { + "meta_prompt_model_id": "us.amazon.nova-2-lite-v1:0", + "task_model_id": "custom-task-model", + "num_candidates": 10, + "num_trials": 15, + "max_bootstrapped_demos": 2, + "max_labeled_demos": 3 + } + + # Run optimization with custom params + result = optimizer.optimize(mode="custom", custom_params=custom_params) + + # Verify meta-prompt adapter was used with correct model ID + mock_mp_instance.optimize.assert_called_once_with( + prompter_model_id="us.amazon.nova-2-lite-v1:0" + ) + + # Verify task adapter was used with correct parameters + mock_mipro_instance.optimize.assert_called_once_with( + task_model_id="custom-task-model", + num_candidates=10, + num_trials=15, + max_bootstrapped_demos=2, + max_labeled_demos=3, + enable_json_fallback=False + ) + + # Verify result + self.assertIs(result, mock_final_prompt) + + def test_adapter_types_preserved(self): + """Test that adapter types are preserved correctly.""" + + # Create optimizer with separate adapters + optimizer = NovaPromptOptimizer( + prompt_adapter=self.prompt_adapter, + inference_adapter=self.task_adapter, + dataset_adapter=self.dataset_adapter, + metric_adapter=self.metric_adapter, + meta_prompt_inference_adapter=self.meta_prompt_adapter + ) + + # Verify adapters are stored correctly + self.assertIsInstance(optimizer.inference_adapter, InferenceAdapter) + self.assertIsInstance(optimizer.meta_prompt_inference_adapter, InferenceAdapter) + self.assertIsNot(optimizer.inference_adapter, optimizer.meta_prompt_inference_adapter) + + # Verify meta-prompt optimization adapter uses correct adapter + self.assertIs( + optimizer.meta_prompt_optimization_adapter.inference_adapter, + self.meta_prompt_adapter + ) + + +class TestRealWorldScenarios(unittest.TestCase): + """Test real-world usage scenarios.""" + + def test_bedrock_sagemaker_combination(self): + """Test typical scenario: Bedrock for meta-prompting, SageMaker for task.""" + + # This test verifies the typical use case structure + # In real usage, these would be actual adapter instances + + meta_adapter = Mock(spec=InferenceAdapter) + task_adapter = Mock(spec=InferenceAdapter) + + prompt_adapter = Mock(spec=PromptAdapter) + dataset_adapter = Mock(spec=DatasetAdapter) + metric_adapter = Mock(spec=MetricAdapter) + + # Create optimizer + optimizer = NovaPromptOptimizer( + prompt_adapter=prompt_adapter, + inference_adapter=task_adapter, + dataset_adapter=dataset_adapter, + metric_adapter=metric_adapter, + meta_prompt_inference_adapter=meta_adapter + ) + + # Verify configuration - adapters are correctly assigned + self.assertIs(optimizer.meta_prompt_inference_adapter, meta_adapter) + self.assertIs(optimizer.inference_adapter, task_adapter) + self.assertIsNot(optimizer.meta_prompt_inference_adapter, optimizer.inference_adapter) + + +if __name__ == '__main__': + unittest.main()