From 48672e27a030f5d6fb934a9a02fd5bfb7e1643fa Mon Sep 17 00:00:00 2001 From: v-mradi Date: Mon, 3 Nov 2025 20:56:48 +0200 Subject: [PATCH 1/7] Adding samples for all new evaluators --- .../AI_Judge_Evaluator_Fluency.ipynb | 843 +++++++++++ .../AI_Judge_Evaluator_Groundedness.ipynb | 937 ++++++++++++ ...AI_Judge_Evaluator_Intent_Resolution.ipynb | 323 ++++- .../AI_Judge_Evaluator_Relevance.ipynb | 328 +++++ .../AI_Judge_Evaluator_Task_Adherence.ipynb | 344 ++++- .../AI_Judge_Evaluator_Task_Completion.ipynb | 793 ++++++++++ ...Evaluator_Task_Navigation_Efficiency.ipynb | 601 ++++++++ ...I_Judge_Evaluator_Tool_Call_Accuracy.ipynb | 972 ++++++++++++- ...AI_Judge_Evaluator_Tool_Call_Success.ipynb | 995 +++++++++++++ ..._Judge_Evaluator_Tool_Input_Accuracy.ipynb | 581 ++++++++ ...ge_Evaluator_Tool_Output_Utilization.ipynb | 1082 ++++++++++++++ .../AI_Judge_Evaluator_Tool_Selection.ipynb | 1278 +++++++++++++++++ ...dge_Evaluators_Response_Completeness.ipynb | 161 ++- 13 files changed, 9055 insertions(+), 183 deletions(-) create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb new file mode 100644 index 00000000..37edf785 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb @@ -0,0 +1,843 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "21599c07", + "metadata": {}, + "source": [ + "# Fluency Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use the Fluency evaluator to assess the linguistic quality of AI-generated responses. The evaluator measures how well generated text conforms to grammatical rules, syntactic structures, and appropriate vocabulary usage.\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "id": "7add5ec1", + "metadata": {}, + "source": [ + "The Fluency evaluator assesses the extent to which generated text conforms to grammatical rules, syntactic structures, and appropriate vocabulary usage, resulting in linguistically correct responses.\n", + "\n", + "Fluency scores range from 1 to 5:\n", + "\n", + "
\n",
+    "Score 1: Very Poor - The response is incomprehensible with severe grammatical errors and improper vocabulary.\n",
+    "Score 2: Poor - The response has frequent grammatical errors and awkward phrasing that hinder understanding.\n",
+    "Score 3: Fair - The response is understandable but contains noticeable grammatical errors or awkward expressions.\n",
+    "Score 4: Good - The response is mostly fluent with minor grammatical issues that don't significantly impact readability.\n",
+    "Score 5: Excellent - The response is perfectly fluent with proper grammar, syntax, and vocabulary usage.\n",
+    "
\n", + "\n", + "The evaluation requires one of the following input patterns:\n", + "\n", + "**Pattern 1: Single Response Evaluation**\n", + "- Response: The text response to be evaluated for fluency. (string)\n", + "\n", + "**Pattern 2: Conversation Evaluation**\n", + "- Conversation: A multi-turn conversation where each turn is evaluated for fluency. (List[dict])" + ] + }, + { + "cell_type": "markdown", + "id": "2abd4d05", + "metadata": {}, + "source": [ + "### Initialize Fluency Evaluator" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "bfd097f8", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import FluencyEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "fluency_evaluator = FluencyEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "4793759a", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "46dc8414", + "metadata": {}, + "source": [ + "#### Response as String (str) - Excellent Fluency" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "3c2df39a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'fluency': 4.0,\n", + " 'fluency_completion_tokens': 140,\n", + " 'fluency_finish_reason': 'stop',\n", + " 'fluency_model': 'gpt-4.1-2025-04-14',\n", + " 'fluency_prompt_tokens': 948,\n", + " 'fluency_reason': 'The response is well-articulated, coherent, and uses '\n", + " 'varied sentence structures and vocabulary appropriate for '\n", + " 'the context, with no significant errors.',\n", + " 'fluency_result': 'pass',\n", + " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", + " '\\\\\"The weather in Seattle is currently partly cloudy '\n", + " 'with a temperature of 15\\\\\\\\u00b0C. The forecast '\n", + " 'indicates that conditions will remain stable '\n", + " 'throughout the day, with a gentle breeze from the '\n", + " 'northwest.\\\\\"}\"}]',\n", + " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", + " 'step by step: The response is grammatically correct '\n", + " 'and uses appropriate vocabulary for a weather '\n", + " 'report. Sentences are well-structured and coherent, '\n", + " 'with logical flow between the current conditions '\n", + " 'and the forecast. There are no noticeable errors, '\n", + " 'and the text is easy to read and understand. While '\n", + " 'the vocabulary is not highly sophisticated, it is '\n", + " 'varied and precise for the context. The sentence '\n", + " 'structure is somewhat complex, especially in the '\n", + " 'second sentence, which uses a subordinate '\n", + " 'clause.\\\\nThe response is '\n", + " 'well-articulated, coherent, and uses varied '\n", + " 'sentence structures and vocabulary appropriate for '\n", + " 'the context, with no significant '\n", + " 'errors.\\\\n4\"}]',\n", + " 'fluency_threshold': 3,\n", + " 'fluency_total_tokens': 1088,\n", + " 'gpt_fluency': 4.0}\n" + ] + } + ], + "source": [ + "# Excellent fluency - proper grammar, syntax, and vocabulary\n", + "response_excellent = \"The weather in Seattle is currently partly cloudy with a temperature of 15°C. The forecast indicates that conditions will remain stable throughout the day, with a gentle breeze from the northwest.\"\n", + "\n", + "result = fluency_evaluator(response=response_excellent)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "5ca4dfac", + "metadata": {}, + "source": [ + "#### Response as String (str) - Good Fluency with Minor Issues" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "8dcf34b8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'fluency': 3.0,\n", + " 'fluency_completion_tokens': 127,\n", + " 'fluency_finish_reason': 'stop',\n", + " 'fluency_model': 'gpt-4.1-2025-04-14',\n", + " 'fluency_prompt_tokens': 937,\n", + " 'fluency_reason': 'The response is clear and coherent but contains some '\n", + " 'grammatical mistakes and lacks sentence complexity, '\n", + " 'fitting the definition of Competent Fluency.',\n", + " 'fluency_result': 'pass',\n", + " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", + " '\\\\\"The meeting has been schedule for tomorrow at 2 '\n", + " 'PM. Please make sure to reviewing the attached '\n", + " 'documents before attending the session.\\\\\"}\"}]',\n", + " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", + " 'step by step: The response communicates the main '\n", + " 'idea clearly: a meeting is scheduled and documents '\n", + " 'should be reviewed beforehand. However, there are '\n", + " 'noticeable grammatical errors (\\\\\"has been '\n", + " 'schedule\\\\\" should be \\\\\"has been scheduled\\\\\" and '\n", + " '\\\\\"to reviewing\\\\\" should be \\\\\"to review\\\\\"). '\n", + " 'Vocabulary is adequate but not varied, and sentence '\n", + " 'structure is simple. Despite the errors, the '\n", + " 'message is coherent and understandable with minimal '\n", + " 'effort.\\\\nThe response is clear and '\n", + " 'coherent but contains some grammatical mistakes and '\n", + " 'lacks sentence complexity, fitting the definition '\n", + " 'of Competent Fluency.\\\\n3\"}]',\n", + " 'fluency_threshold': 3,\n", + " 'fluency_total_tokens': 1064,\n", + " 'gpt_fluency': 3.0}\n" + ] + } + ], + "source": [ + "# Good fluency - mostly fluent with minor grammatical issues\n", + "response_good = \"The meeting has been schedule for tomorrow at 2 PM. Please make sure to reviewing the attached documents before attending the session.\"\n", + "\n", + "result = fluency_evaluator(response=response_good)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "a89fc139", + "metadata": {}, + "source": [ + "#### Response as String (str) - Fair Fluency" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c30fc6bd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'fluency': 2.0,\n", + " 'fluency_completion_tokens': 146,\n", + " 'fluency_finish_reason': 'stop',\n", + " 'fluency_model': 'gpt-4.1-2025-04-14',\n", + " 'fluency_prompt_tokens': 934,\n", + " 'fluency_reason': 'The response has frequent grammatical errors and awkward '\n", + " 'phrasing, but the main idea is still partially clear.',\n", + " 'fluency_result': 'fail',\n", + " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", + " '\\\\\"The order you placing yesterday is being process '\n", + " 'and will shipped soon. Tracking information send to '\n", + " 'your email when available.\\\\\"}\"}]',\n", + " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", + " 'step by step: The response contains several '\n", + " 'grammatical errors (\\\\\"The order you placing '\n", + " 'yesterday,\\\\\" \\\\\"is being process,\\\\\" \\\\\"will '\n", + " 'shipped soon,\\\\\" \\\\\"Tracking information send to '\n", + " 'your email\\\\\"), which make the sentences awkward '\n", + " 'and difficult to read. Vocabulary is very basic and '\n", + " 'there is little sentence complexity. Despite the '\n", + " 'errors, the main idea can be understood with some '\n", + " 'effort, but the phrasing is repetitive and awkward. '\n", + " 'This fits the description of Basic Fluency, where '\n", + " 'frequent errors and limited vocabulary are present, '\n", + " 'but the message is partially '\n", + " 'understandable.\\\\nThe response has '\n", + " 'frequent grammatical errors and awkward phrasing, '\n", + " 'but the main idea is still partially '\n", + " 'clear.\\\\n2\"}]',\n", + " 'fluency_threshold': 3,\n", + " 'fluency_total_tokens': 1080,\n", + " 'gpt_fluency': 2.0}\n" + ] + } + ], + "source": [ + "# Fair fluency - understandable but contains noticeable errors\n", + "response_fair = \"The order you placing yesterday is being process and will shipped soon. Tracking information send to your email when available.\"\n", + "\n", + "result = fluency_evaluator(response=response_fair)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "59709c65", + "metadata": {}, + "source": [ + "#### Response as String (str) - Poor Fluency" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d9c624ab", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'fluency': 1.0,\n", + " 'fluency_completion_tokens': 108,\n", + " 'fluency_finish_reason': 'stop',\n", + " 'fluency_model': 'gpt-4.1-2025-04-14',\n", + " 'fluency_prompt_tokens': 932,\n", + " 'fluency_reason': 'The response is mostly incoherent and contains pervasive '\n", + " 'grammatical errors, fitting the definition of Emergent '\n", + " 'Fluency.',\n", + " 'fluency_result': 'fail',\n", + " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", + " '\\\\\"Email draft attach is. You review and giving '\n", + " 'feedback must. Important very for project success it '\n", + " 'being.\\\\\"}\"}]',\n", + " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", + " 'step by step: The response contains fragmented '\n", + " 'sentences and awkward phrasing (\\\\\"Email draft '\n", + " 'attach is. You review and giving feedback must. '\n", + " 'Important very for project success it being.\\\\\"). '\n", + " 'There are pervasive grammatical errors, and the '\n", + " 'vocabulary is extremely limited. The message is '\n", + " 'difficult to understand and lacks coherence, making '\n", + " 'it largely incomprehensible.\\\\nThe '\n", + " 'response is mostly incoherent and contains '\n", + " 'pervasive grammatical errors, fitting the '\n", + " 'definition of Emergent '\n", + " 'Fluency.\\\\n1\"}]',\n", + " 'fluency_threshold': 3,\n", + " 'fluency_total_tokens': 1040,\n", + " 'gpt_fluency': 1.0}\n" + ] + } + ], + "source": [ + "# Poor fluency - frequent grammatical errors and awkward phrasing\n", + "response_poor = \"Email draft attach is. You review and giving feedback must. Important very for project success it being.\"\n", + "\n", + "result = fluency_evaluator(response=response_poor)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "e8934a3c", + "metadata": {}, + "source": [ + "#### Response as String (str) - Technical Content" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "36e92f0b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'fluency': 4.0,\n", + " 'fluency_completion_tokens': 150,\n", + " 'fluency_finish_reason': 'stop',\n", + " 'fluency_model': 'gpt-4.1-2025-04-14',\n", + " 'fluency_prompt_tokens': 949,\n", + " 'fluency_reason': 'The response is well-articulated, clear, and uses varied '\n", + " 'technical vocabulary with good grammar and logical flow, '\n", + " 'fitting the definition of proficient fluency.',\n", + " 'fluency_result': 'pass',\n", + " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": \\\\\"To '\n", + " 'resolve the authentication error, please verify that '\n", + " 'your API credentials are correctly configured in the '\n", + " 'environment variables. Additionally, ensure that the '\n", + " 'service principal has the necessary permissions '\n", + " 'assigned at the subscription level.\\\\\"}\"}]',\n", + " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", + " 'step by step: The response is grammatically correct '\n", + " 'and uses appropriate technical vocabulary (\\\\\"API '\n", + " 'credentials,\\\\\" \\\\\"environment variables,\\\\\" '\n", + " '\\\\\"service principal,\\\\\" \\\\\"permissions,\\\\\" '\n", + " '\\\\\"subscription level\\\\\"). Sentences are '\n", + " 'well-structured and connected logically. There are '\n", + " 'no noticeable errors, and the message is clear and '\n", + " 'coherent. The vocabulary is varied within the '\n", + " 'technical context, and the text flows smoothly. '\n", + " 'However, the sentence structure and vocabulary, '\n", + " 'while proficient, do not reach the level of '\n", + " 'exceptional fluency or stylistic '\n", + " 'sophistication.\\\\nThe response is '\n", + " 'well-articulated, clear, and uses varied technical '\n", + " 'vocabulary with good grammar and logical flow, '\n", + " 'fitting the definition of proficient '\n", + " 'fluency.\\\\n4\"}]',\n", + " 'fluency_threshold': 3,\n", + " 'fluency_total_tokens': 1099,\n", + " 'gpt_fluency': 4.0}\n" + ] + } + ], + "source": [ + "# Technical response with excellent fluency\n", + "response_technical = \"To resolve the authentication error, please verify that your API credentials are correctly configured in the environment variables. Additionally, ensure that the service principal has the necessary permissions assigned at the subscription level.\"\n", + "\n", + "result = fluency_evaluator(response=response_technical)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "b26dd4bb", + "metadata": {}, + "source": [ + "#### Conversation as List[dict] - Multi-turn Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9ccaff56", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'evaluation_per_turn': {'fluency': [3.0, 3.0, 4.0],\n", + " 'fluency_completion_tokens': [119, 121, 136],\n", + " 'fluency_finish_reason': ['stop', 'stop', 'stop'],\n", + " 'fluency_model': ['gpt-4.1-2025-04-14',\n", + " 'gpt-4.1-2025-04-14',\n", + " 'gpt-4.1-2025-04-14'],\n", + " 'fluency_prompt_tokens': [938, 938, 937],\n", + " 'fluency_reason': ['The response is clear, correct, '\n", + " 'and coherent, but it does not '\n", + " 'demonstrate complex sentence '\n", + " 'structure or advanced vocabulary.',\n", + " 'The response is clear and '\n", + " 'coherent with only a minor '\n", + " 'grammatical error, fitting the '\n", + " 'definition of Competent Fluency.',\n", + " 'The response is well-articulated, '\n", + " 'coherent, and uses varied '\n", + " 'vocabulary and sentence '\n", + " 'structure, fitting the definition '\n", + " 'of Proficient Fluency.'],\n", + " 'fluency_result': ['pass', 'pass', 'pass'],\n", + " 'fluency_sample_input': ['[{\"role\": \"user\", '\n", + " '\"content\": '\n", + " '\"{\\\\\"response\\\\\": \\\\\"Of '\n", + " \"course! I'd be happy to \"\n", + " 'help you understand this '\n", + " \"week's weather forecast. \"\n", + " 'The outlook shows a mix of '\n", + " 'sunny and cloudy '\n", + " 'conditions.\\\\\"}\"}]',\n", + " '[{\"role\": \"user\", '\n", + " '\"content\": '\n", + " '\"{\\\\\"response\\\\\": '\n", + " '\\\\\"Temperatures will '\n", + " 'ranging from 12\\\\\\\\u00b0C '\n", + " 'to 18\\\\\\\\u00b0C throughout '\n", + " 'the week, with the warmest '\n", + " 'days being Wednesday and '\n", + " 'Thursday.\\\\\"}\"}]',\n", + " '[{\"role\": \"user\", '\n", + " '\"content\": '\n", + " '\"{\\\\\"response\\\\\": \\\\\"Yes, '\n", + " 'it would be advisable to '\n", + " 'carry an umbrella, '\n", + " 'particularly on Friday when '\n", + " 'there is a 60% chance of '\n", + " 'precipitation.\\\\\"}\"}]'],\n", + " 'fluency_sample_output': ['[{\"role\": \"assistant\", '\n", + " '\"content\": \"Let\\'s '\n", + " 'think step by step: The '\n", + " 'response is grammatically '\n", + " 'correct and uses a polite, '\n", + " 'clear tone. Vocabulary is '\n", + " 'appropriate, though not '\n", + " 'particularly advanced or '\n", + " 'varied. Sentences are '\n", + " 'well-structured but '\n", + " 'relatively simple. The '\n", + " 'message is coherent and '\n", + " 'easily understood, with no '\n", + " 'awkward phrasing or '\n", + " 'errors. However, it lacks '\n", + " 'complexity and '\n", + " 'sophistication in sentence '\n", + " 'structure and vocabulary, '\n", + " 'which are required for '\n", + " 'higher '\n", + " 'scores.\\\\nThe '\n", + " 'response is clear, '\n", + " 'correct, and coherent, but '\n", + " 'it does not demonstrate '\n", + " 'complex sentence structure '\n", + " 'or advanced '\n", + " 'vocabulary.\\\\n3\"}]',\n", + " '[{\"role\": \"assistant\", '\n", + " '\"content\": \"Let\\'s '\n", + " 'think step by step: The '\n", + " 'response is mostly clear '\n", + " 'and communicates the '\n", + " 'temperature range and the '\n", + " 'warmest days. There is a '\n", + " 'minor grammatical error '\n", + " '(\\\\\"will ranging\\\\\" should '\n", + " 'be \\\\\"will range\\\\\" or '\n", + " '\\\\\"will be ranging\\\\\"), '\n", + " 'but the rest of the '\n", + " 'sentence is correct and '\n", + " 'easily understood. '\n", + " 'Vocabulary is adequate, '\n", + " 'and sentence structure is '\n", + " 'simple but coherent. The '\n", + " 'message is clear with '\n", + " 'minimal effort required to '\n", + " 'understand.\\\\nThe '\n", + " 'response is clear and '\n", + " 'coherent with only a minor '\n", + " 'grammatical error, fitting '\n", + " 'the definition of '\n", + " 'Competent '\n", + " 'Fluency.\\\\n3\"}]',\n", + " '[{\"role\": \"assistant\", '\n", + " '\"content\": \"Let\\'s '\n", + " 'think step by step: The '\n", + " 'response is grammatically '\n", + " 'correct and uses '\n", + " 'appropriate vocabulary '\n", + " '(\\\\\"advisable,\\\\\" '\n", + " '\\\\\"particularly,\\\\\" '\n", + " '\\\\\"precipitation\\\\\"). The '\n", + " 'sentence is '\n", + " 'well-structured and flows '\n", + " 'smoothly. There is a clear '\n", + " 'connection between the '\n", + " 'advice and the reason '\n", + " 'provided. There are no '\n", + " 'noticeable errors, and the '\n", + " 'message is coherent and '\n", + " 'easy to understand. The '\n", + " 'vocabulary is varied but '\n", + " 'not highly sophisticated, '\n", + " 'and the sentence structure '\n", + " 'is complex but not '\n", + " 'exceptionally '\n", + " 'so.\\\\nThe '\n", + " 'response is '\n", + " 'well-articulated, '\n", + " 'coherent, and uses varied '\n", + " 'vocabulary and sentence '\n", + " 'structure, fitting the '\n", + " 'definition of Proficient '\n", + " 'Fluency.\\\\n4\"}]'],\n", + " 'fluency_threshold': [3, 3, 3],\n", + " 'fluency_total_tokens': [1057, 1059, 1073],\n", + " 'gpt_fluency': [3.0, 3.0, 4.0]},\n", + " 'fluency': 3.3333333333333335,\n", + " 'fluency_completion_tokens': 125.33333333333333,\n", + " 'fluency_prompt_tokens': 937.6666666666666,\n", + " 'fluency_threshold': 3.0,\n", + " 'fluency_total_tokens': 1063.0,\n", + " 'gpt_fluency': 3.3333333333333335}\n" + ] + } + ], + "source": [ + "# Multi-turn conversation - evaluates fluency of each assistant turn\n", + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you help me understand the weather forecast for this week?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Of course! I'd be happy to help you understand this week's weather forecast. The outlook shows a mix of sunny and cloudy conditions.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What about the temperature?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Temperatures will ranging from 12°C to 18°C throughout the week, with the warmest days being Wednesday and Thursday.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Should I bring an umbrella?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Yes, it would be advisable to carry an umbrella, particularly on Friday when there is a 60% chance of precipitation.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = fluency_evaluator(conversation=conversation)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "0ba89e73", + "metadata": {}, + "source": [ + "#### Conversation as List[dict] - Agent with Tool Calls" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "2658f610", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'fluency': 4.0,\n", + " 'fluency_completion_tokens': 123,\n", + " 'fluency_finish_reason': 'stop',\n", + " 'fluency_model': 'gpt-4.1-2025-04-14',\n", + " 'fluency_prompt_tokens': 929,\n", + " 'fluency_reason': 'The response demonstrates proficient fluency with good '\n", + " 'grammar, clear structure, and logical flow, but does not '\n", + " 'reach the level of exceptional fluency due to its '\n", + " 'simplicity.',\n", + " 'fluency_result': 'pass',\n", + " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", + " '\\\\\"I\\'ll fetch the current weather information for '\n", + " 'Seattle and send you an email with the '\n", + " 'details.\\\\\"}\"}]',\n", + " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", + " 'step by step: The response is grammatically correct '\n", + " 'and uses appropriate vocabulary. The sentence '\n", + " 'structure is clear and coherent, with no errors. '\n", + " 'The message is easily understood and flows '\n", + " 'smoothly, but it is relatively simple and lacks '\n", + " 'complex or varied sentence structures. There is no '\n", + " 'sophisticated vocabulary or nuanced expression, but '\n", + " 'the response is well-articulated and '\n", + " 'logical.\\\\nThe response demonstrates '\n", + " 'proficient fluency with good grammar, clear '\n", + " 'structure, and logical flow, but does not reach the '\n", + " 'level of exceptional fluency due to its '\n", + " 'simplicity.\\\\n4\"}]',\n", + " 'fluency_threshold': 3,\n", + " 'fluency_total_tokens': 1052,\n", + " 'gpt_fluency': 4.0}\n" + ] + } + ], + "source": [ + "# Conversation with tool calls - evaluates fluency of assistant responses\n", + "conversation_with_tools = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you send me an email with the weather report for Seattle?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I'll fetch the current weather information for Seattle and send you an email with the details.\"\n", + " },\n", + " {\n", + " \"role\": \"tool\",\n", + " \"content\": \"Temperature: 15°C, Condition: Partly cloudy, Wind: 8 mph NW\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I have successfully sent you an email containing the weather report for Seattle. The current conditions are partly cloudy with a temperature of 15°C and light winds from the northwest.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = fluency_evaluator(conversation=conversation_with_tools)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "ff71d193", + "metadata": {}, + "source": [ + "#### Conversation as List[dict] - Mixed Fluency Quality" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "c0d8bf46", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'evaluation_per_turn': {'fluency': [2.0, 4.0],\n", + " 'fluency_completion_tokens': [124, 127],\n", + " 'fluency_finish_reason': ['stop', 'stop'],\n", + " 'fluency_model': ['gpt-4.1-2025-04-14',\n", + " 'gpt-4.1-2025-04-14'],\n", + " 'fluency_prompt_tokens': [927, 942],\n", + " 'fluency_reason': ['The response has frequent '\n", + " 'grammatical errors and limited '\n", + " 'vocabulary, but the main idea is '\n", + " 'partially understandable.',\n", + " 'The response is well-written, '\n", + " 'clear, and coherent, with good '\n", + " 'grammar and appropriate '\n", + " 'vocabulary, but lacks the '\n", + " 'complexity and nuance required '\n", + " 'for the highest score.'],\n", + " 'fluency_result': ['fail', 'pass'],\n", + " 'fluency_sample_input': ['[{\"role\": \"user\", '\n", + " '\"content\": '\n", + " '\"{\\\\\"response\\\\\": \\\\\"Your '\n", + " 'order is being process '\n", + " 'currently and will be ship '\n", + " 'very soon to you.\\\\\"}\"}]',\n", + " '[{\"role\": \"user\", '\n", + " '\"content\": '\n", + " '\"{\\\\\"response\\\\\": \\\\\"The '\n", + " 'estimated delivery date for '\n", + " 'your order is March 15, '\n", + " '2025. You will receive '\n", + " 'tracking information via '\n", + " 'email once the package has '\n", + " 'been dispatched.\\\\\"}\"}]'],\n", + " 'fluency_sample_output': ['[{\"role\": \"assistant\", '\n", + " '\"content\": \"Let\\'s '\n", + " 'think step by step: The '\n", + " 'response communicates a '\n", + " 'simple idea about an order '\n", + " 'being processed and '\n", + " 'shipped. However, there '\n", + " 'are noticeable grammatical '\n", + " 'errors (\\\\\"being '\n", + " 'process\\\\\" should be '\n", + " '\\\\\"being processed\\\\\", '\n", + " '\\\\\"will be ship\\\\\" should '\n", + " 'be \\\\\"will be shipped\\\\\"), '\n", + " 'and the sentence structure '\n", + " 'is awkward. Vocabulary is '\n", + " 'very basic, and the '\n", + " 'sentences are short and '\n", + " 'not well-constructed. '\n", + " 'Despite these issues, the '\n", + " 'main message can be '\n", + " 'understood with some '\n", + " 'effort.\\\\nThe '\n", + " 'response has frequent '\n", + " 'grammatical errors and '\n", + " 'limited vocabulary, but '\n", + " 'the main idea is partially '\n", + " 'understandable.\\\\n2\"}]',\n", + " '[{\"role\": \"assistant\", '\n", + " '\"content\": \"Let\\'s '\n", + " 'think step by step: The '\n", + " 'response is grammatically '\n", + " 'correct and uses clear, '\n", + " 'precise language. '\n", + " 'Vocabulary is appropriate '\n", + " 'for the context, and '\n", + " 'sentences are '\n", + " 'well-structured and '\n", + " 'coherent. There are no '\n", + " 'errors, and the message is '\n", + " 'easy to understand. '\n", + " 'However, the vocabulary '\n", + " 'and sentence structure are '\n", + " 'straightforward and not '\n", + " 'particularly sophisticated '\n", + " 'or varied, which suggests '\n", + " 'it is not at the highest '\n", + " 'level of '\n", + " 'fluency.\\\\nThe '\n", + " 'response is well-written, '\n", + " 'clear, and coherent, with '\n", + " 'good grammar and '\n", + " 'appropriate vocabulary, '\n", + " 'but lacks the complexity '\n", + " 'and nuance required for '\n", + " 'the highest '\n", + " 'score.\\\\n4\"}]'],\n", + " 'fluency_threshold': [3, 3],\n", + " 'fluency_total_tokens': [1051, 1069],\n", + " 'gpt_fluency': [2.0, 4.0]},\n", + " 'fluency': 3.0,\n", + " 'fluency_completion_tokens': 125.5,\n", + " 'fluency_prompt_tokens': 934.5,\n", + " 'fluency_threshold': 3.0,\n", + " 'fluency_total_tokens': 1060.0,\n", + " 'gpt_fluency': 3.0}\n" + ] + } + ], + "source": [ + "# Conversation with varying fluency levels across turns\n", + "conversation_mixed = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What's the status of my order?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Your order is being process currently and will be ship very soon to you.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"When will it arrive?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"The estimated delivery date for your order is March 15, 2025. You will receive tracking information via email once the package has been dispatched.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = fluency_evaluator(conversation=conversation_mixed)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb new file mode 100644 index 00000000..4830ad40 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb @@ -0,0 +1,937 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6a3f10c0", + "metadata": {}, + "source": [ + "# Groundedness Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use the Groundedness evaluator to assess whether AI-generated responses are grounded in the provided context. The evaluator supports multiple input formats including:\n", + "- Simple response and context evaluation\n", + "- Query, response, and context evaluation\n", + "- Agent responses with tool calls (file_search)\n", + "- Multi-turn conversations\n", + "\n", + "This notebook uses consistent examples with the ToolCallAccuracyEvaluator for better comparison across evaluators.\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook.\n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities.\n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure OpenAI Endpoint to be used for evaluation\n", + "3) **AZURE_OPENAI_API_KEY** - Azure OpenAI Key to be used for evaluation\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure OpenAI API version to be used for evaluation" + ] + }, + { + "cell_type": "markdown", + "id": "fe3af2a1", + "metadata": {}, + "source": [ + "## What is Groundedness?\n", + "\n", + "The Groundedness evaluator assesses the correspondence between claims in an AI-generated response and the source context. It ensures that responses are substantiated by the provided context, preventing hallucinations and unsupported claims.\n", + "\n", + "**Key Points:**\n", + "- Even factually correct responses are considered ungrounded if they can't be verified against the provided context\n", + "- Essential for RAG (Retrieval-Augmented Generation) applications\n", + "- Helps ensure AI responses are trustworthy and verifiable\n", + "\n", + "**Scoring:** Groundedness scores range from 1 to 5, with:\n", + "- **1**: Completely ungrounded - no claims supported by context\n", + "- **2**: Mostly ungrounded - few claims supported\n", + "- **3**: Partially grounded - some claims supported\n", + "- **4**: Mostly grounded - most claims supported\n", + "- **5**: Fully grounded - all claims supported by context" + ] + }, + { + "cell_type": "markdown", + "id": "bd8a00fb", + "metadata": {}, + "source": [ + "## Groundedness Evaluator Input Requirements\n", + "\n", + "The Groundedness evaluator supports multiple input formats:\n", + "\n", + "1. **Basic Context Evaluation:**\n", + " - `response`: The AI response to evaluate (str)\n", + " - `context`: The source context/documents (str)\n", + " - `query`: Optional query for enhanced evaluation (str)\n", + "\n", + "2. **Agent Tool Evaluation:**\n", + " - `query`: The user query (str)\n", + " - `response`: Agent response with tool calls (List[dict])\n", + " - `tool_definitions`: Available tools, only file_search supported (List[dict])\n", + "\n", + "3. **Conversation Evaluation:**\n", + " - `conversation`: Multi-turn conversation with context (Conversation object)" + ] + }, + { + "cell_type": "markdown", + "id": "9aa68c67", + "metadata": {}, + "source": [ + "### Initialize Groundedness Evaluator" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "298d067a", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import GroundednessEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "# Configure the model\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "# Initialize the evaluator\n", + "groundedness_evaluator = GroundednessEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "18b0a169", + "metadata": {}, + "source": [ + "## Sample Evaluations" + ] + }, + { + "cell_type": "markdown", + "id": "b73b1ccd", + "metadata": {}, + "source": [ + "### 1. Well-Grounded Weather Response\n", + "Using weather information consistent with ToolCallAccuracyEvaluator examples." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4e5fa898", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== Well-Grounded Weather Response ===\n", + "{'gpt_groundedness': 4.0,\n", + " 'groundedness': 4.0,\n", + " 'groundedness_completion_tokens': 156,\n", + " 'groundedness_finish_reason': 'stop',\n", + " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", + " 'groundedness_prompt_tokens': 1205,\n", + " 'groundedness_reason': 'The response is correct and covers most key points '\n", + " 'but leaves out some specific details (humidity, wind, '\n", + " 'visibility) present in the context.',\n", + " 'groundedness_result': 'pass',\n", + " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", + " '\\\\\"The current weather in Seattle is rainy with '\n", + " \"a temperature of 14\\\\\\\\u00b0C. It's typical \"\n", + " 'Pacific Northwest weather for this time of year '\n", + " 'with overcast skies and light '\n", + " 'precipitation.\\\\\", \\\\\"context\\\\\": \\\\\"Current '\n", + " 'weather data for Seattle shows rainy conditions '\n", + " 'with a temperature of 14\\\\\\\\u00b0C. \\\\\\\\nThe '\n", + " 'forecast indicates overcast skies with light '\n", + " 'precipitation typical for Pacific Northwest '\n", + " 'weather. \\\\\\\\nHumidity is at 85% with winds '\n", + " 'from the southwest at 12 mph. Visibility is '\n", + " 'reduced to 8 miles due to rain.\\\\\"}\"}]',\n", + " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", + " 'think step by step: The context provides '\n", + " 'several details: rainy conditions, temperature '\n", + " 'of 14\\\\u00b0C, typical Pacific Northwest '\n", + " 'weather, overcast skies, light precipitation, '\n", + " 'humidity at 85%, southwest winds at 12 mph, '\n", + " 'and reduced visibility to 8 miles. The '\n", + " 'response accurately mentions rainy conditions, '\n", + " 'the temperature, typical Pacific Northwest '\n", + " 'weather, overcast skies, and light '\n", + " 'precipitation. However, it omits details about '\n", + " 'humidity, wind speed/direction, and visibility '\n", + " 'reduction. The response is accurate but does '\n", + " 'not include all relevant details from the '\n", + " 'context.\\\\nThe response is correct '\n", + " 'and covers most key points but leaves out some '\n", + " 'specific details (humidity, wind, visibility) '\n", + " 'present in the context.\\\\n4\"}]',\n", + " 'groundedness_threshold': 3,\n", + " 'groundedness_total_tokens': 1361}\n" + ] + } + ], + "source": [ + "# Example of a well-grounded response using weather context\n", + "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", + "The forecast indicates overcast skies with light precipitation typical for Pacific Northwest weather. \n", + "Humidity is at 85% with winds from the southwest at 12 mph. Visibility is reduced to 8 miles due to rain.\"\"\"\n", + "\n", + "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", + "\n", + "result = groundedness_evaluator(response=response, context=context)\n", + "print(\"=== Well-Grounded Weather Response ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "0c03f745", + "metadata": {}, + "source": [ + "### 2. Partially Grounded Response (with hallucination)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "6de6a51e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== Partially Grounded Response ===\n", + "{'gpt_groundedness': 2.0,\n", + " 'groundedness': 2.0,\n", + " 'groundedness_completion_tokens': 139,\n", + " 'groundedness_finish_reason': 'stop',\n", + " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", + " 'groundedness_prompt_tokens': 1185,\n", + " 'groundedness_reason': 'The response includes accurate information from the '\n", + " 'context but also introduces incorrect and unsupported '\n", + " 'details, making it unreliable.',\n", + " 'groundedness_result': 'fail',\n", + " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", + " '\\\\\"The current weather in Seattle is rainy with '\n", + " 'a temperature of 14\\\\\\\\u00b0C. The city is '\n", + " 'experiencing its wettest month in 50 years, and '\n", + " 'the mayor has declared a weather emergency due '\n", + " 'to flooding concerns.\\\\\", \\\\\"context\\\\\": '\n", + " '\\\\\"Current weather data for Seattle shows rainy '\n", + " 'conditions with a temperature of 14\\\\\\\\u00b0C. '\n", + " '\\\\\\\\nThe forecast indicates overcast skies with '\n", + " 'light precipitation typical for Pacific '\n", + " 'Northwest weather.\\\\\"}\"}]',\n", + " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", + " 'think step by step: The context states that '\n", + " 'Seattle is experiencing rainy conditions with '\n", + " 'a temperature of 14\\\\u00b0C, and the forecast '\n", + " 'is for overcast skies with light precipitation '\n", + " 'typical for the region. The response '\n", + " 'accurately repeats the current weather and '\n", + " 'temperature, which is grounded. However, it '\n", + " 'adds that Seattle is experiencing its wettest '\n", + " 'month in 50 years and that the mayor has '\n", + " 'declared a weather emergency due to flooding '\n", + " 'concerns. These claims are not supported or '\n", + " 'mentioned in the context, making them '\n", + " 'incorrect additions.\\\\nThe response '\n", + " 'includes accurate information from the context '\n", + " 'but also introduces incorrect and unsupported '\n", + " 'details, making it '\n", + " 'unreliable.\\\\n2\"}]',\n", + " 'groundedness_threshold': 3,\n", + " 'groundedness_total_tokens': 1324}\n" + ] + } + ], + "source": [ + "# Example of partially grounded response with unsupported claims\n", + "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", + "The forecast indicates overcast skies with light precipitation typical for Pacific Northwest weather.\"\"\"\n", + "\n", + "response = \"The current weather in Seattle is rainy with a temperature of 14°C. The city is experiencing its wettest month in 50 years, and the mayor has declared a weather emergency due to flooding concerns.\"\n", + "\n", + "result = groundedness_evaluator(response=response, context=context)\n", + "print(\"=== Partially Grounded Response ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "9052eb13", + "metadata": {}, + "source": [ + "### 3. Enhanced Evaluation with Weather Query" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c77316c4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== Enhanced Evaluation with Weather Query ===\n", + "{'gpt_groundedness': 4.0,\n", + " 'groundedness': 4.0,\n", + " 'groundedness_completion_tokens': 157,\n", + " 'groundedness_finish_reason': 'stop',\n", + " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", + " 'groundedness_prompt_tokens': 1412,\n", + " 'groundedness_reason': 'The response is accurate and relevant, but it is '\n", + " 'missing the wind speed and direction mentioned in the '\n", + " 'context, making it slightly incomplete.',\n", + " 'groundedness_result': 'pass',\n", + " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"How is the weather in Seattle?\\\\\", '\n", + " '\\\\\"response\\\\\": \\\\\"The weather in Seattle is '\n", + " 'rainy with a temperature of 14\\\\\\\\u00b0C. These '\n", + " 'are typical Pacific Northwest conditions with '\n", + " 'overcast skies.\\\\\", \\\\\"context\\\\\": \\\\\"Weather '\n", + " 'report for Seattle, Washington: Currently '\n", + " 'experiencing rainy weather with temperature at '\n", + " '14\\\\\\\\u00b0C. \\\\\\\\nOvercast conditions with '\n", + " 'light rain are expected to continue. The '\n", + " 'current conditions are typical for the Pacific '\n", + " 'Northwest region \\\\\\\\nduring this season. Wind '\n", + " 'speed is moderate at 12 mph from southwest '\n", + " 'direction.\\\\\"}\"}]',\n", + " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", + " 'think step by step: The query asks about the '\n", + " 'current weather in Seattle. The context '\n", + " 'provides a detailed weather report: rainy, '\n", + " '14\\\\u00b0C, overcast, light rain, typical for '\n", + " 'the region, and moderate wind from the '\n", + " \"southwest at 12 mph. The response states it's \"\n", + " 'rainy, 14\\\\u00b0C, typical Pacific Northwest '\n", + " 'conditions, and overcast skies. The response '\n", + " 'accurately reflects the rain, temperature, '\n", + " 'typicality, and overcast conditions, but omits '\n", + " 'the wind speed and direction. No incorrect '\n", + " 'information is present, but not all details '\n", + " 'are included.\\\\nThe response is '\n", + " 'accurate and relevant, but it is missing the '\n", + " 'wind speed and direction mentioned in the '\n", + " 'context, making it slightly '\n", + " 'incomplete.\\\\n4\"}]',\n", + " 'groundedness_threshold': 3,\n", + " 'groundedness_total_tokens': 1569}\n" + ] + } + ], + "source": [ + "# Example with query for enhanced evaluation - consistent with ToolCallAccuracyEvaluator\n", + "query = \"How is the weather in Seattle?\"\n", + "\n", + "context = \"\"\"Weather report for Seattle, Washington: Currently experiencing rainy weather with temperature at 14°C. \n", + "Overcast conditions with light rain are expected to continue. The current conditions are typical for the Pacific Northwest region \n", + "during this season. Wind speed is moderate at 12 mph from southwest direction.\"\"\"\n", + "\n", + "response = \"The weather in Seattle is rainy with a temperature of 14°C. These are typical Pacific Northwest conditions with overcast skies.\"\n", + "\n", + "result = groundedness_evaluator(response=response, context=context, query=query)\n", + "print(\"=== Enhanced Evaluation with Weather Query ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "fd0a420f", + "metadata": {}, + "source": [ + "### 4. Agent Response with File Search Tool\n", + "This evaluates agent responses that use file_search tools to retrieve weather information - consistent with email/weather theme." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "72b922ee", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== Agent Response with File Search ===\n", + "{'gpt_groundedness': 5.0,\n", + " 'groundedness': 5.0,\n", + " 'groundedness_completion_tokens': 183,\n", + " 'groundedness_finish_reason': 'stop',\n", + " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", + " 'groundedness_prompt_tokens': 1435,\n", + " 'groundedness_reason': 'The response is fully relevant, complete, and '\n", + " 'directly answers the query with specific weather '\n", + " 'information for Seattle, as would be expected from a '\n", + " 'tool call.',\n", + " 'groundedness_result': 'pass',\n", + " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"Can you get me the current weather '\n", + " 'information for Seattle?\\\\\", \\\\\"response\\\\\": '\n", + " '[{\\\\\"role\\\\\": \\\\\"assistant\\\\\", \\\\\"content\\\\\": '\n", + " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", + " '\\\\\"tool_call_id\\\\\": '\n", + " '\\\\\"call_filesearch_weather_123\\\\\", \\\\\"name\\\\\": '\n", + " '\\\\\"file_search\\\\\", \\\\\"arguments\\\\\": '\n", + " '{\\\\\"query\\\\\": \\\\\"current weather Seattle '\n", + " 'temperature conditions\\\\\"}}]}, {\\\\\"role\\\\\": '\n", + " '\\\\\"assistant\\\\\", \\\\\"content\\\\\": \\\\\"Based on the '\n", + " 'weather data, Seattle is currently experiencing '\n", + " 'rainy weather with a temperature of '\n", + " '14\\\\\\\\u00b0C. The conditions include overcast '\n", + " 'skies and light precipitation, which is typical '\n", + " 'for the Pacific Northwest.\\\\\"}], \\\\\"context\\\\\": '\n", + " '\\\\\"<>\\\\\"}\"}]',\n", + " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", + " 'think step by step: The context is empty, so I '\n", + " 'must rely on the tool results and the query. '\n", + " 'The query asks for the current weather '\n", + " 'information for Seattle. The response first '\n", + " 'shows a tool call to search for current '\n", + " 'weather in Seattle, then provides a weather '\n", + " 'report: rainy, 14\\\\u00b0C, overcast skies, '\n", + " 'light precipitation, typical for the Pacific '\n", + " 'Northwest. Since there is no explicit tool '\n", + " 'result shown, I must judge groundedness based '\n", + " 'on whether the response is plausible and '\n", + " 'relevant to the query. The answer is directly '\n", + " 'relevant, provides specific details, and does '\n", + " 'not introduce unrelated or incorrect '\n", + " 'information. There is no evidence of error or '\n", + " 'missing key details for the '\n", + " 'query.\\\\nThe response is fully '\n", + " 'relevant, complete, and directly answers the '\n", + " 'query with specific weather information for '\n", + " 'Seattle, as would be expected from a tool '\n", + " 'call.\\\\n5\"}]',\n", + " 'groundedness_threshold': 3,\n", + " 'groundedness_total_tokens': 1618}\n" + ] + } + ], + "source": [ + "# Example of agent evaluation with file_search tool for weather information\n", + "query = \"Can you get me the current weather information for Seattle?\"\n", + "\n", + "# Simulated agent response with file_search tool call - similar to ToolCallAccuracyEvaluator pattern\n", + "agent_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_filesearch_weather_123\",\n", + " \"name\": \"file_search\",\n", + " \"arguments\": {\"query\": \"current weather Seattle temperature conditions\"}\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"role\": \"tool\",\n", + " \"tool_call_id\": \"call_filesearch_weather_123\", \n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_result\",\n", + " \"tool_result\": {\n", + " \"content\": \"Seattle weather report: Currently rainy with temperature of 14°C. Overcast skies with light precipitation. Typical Pacific Northwest weather with 85% humidity and southwest winds at 12 mph.\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Based on the weather data, Seattle is currently experiencing rainy weather with a temperature of 14°C. The conditions include overcast skies and light precipitation, which is typical for the Pacific Northwest.\"\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"file_search\",\n", + " \"description\": \"Search through uploaded files to find relevant information\",\n", + " \"parameters\": {\n", + " \"type\": \"object\", \n", + " \"properties\": {\n", + " \"query\": {\"type\": \"string\", \"description\": \"Search query\"}\n", + " }\n", + " }\n", + " }\n", + "]\n", + "\n", + "result = groundedness_evaluator(\n", + " query=query, \n", + " response=agent_response, \n", + " tool_definitions=tool_definitions\n", + ")\n", + "print(\"=== Agent Response with File Search ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "6dc31ae0", + "metadata": {}, + "source": [ + "### 5. Conversation Evaluation\n", + "Evaluating groundedness in multi-turn conversations about weather and email requests." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0f37ae51", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== Conversation Evaluation ===\n", + "{'evaluation_per_turn': {'gpt_groundedness': [5.0, 5.0, 5.0],\n", + " 'groundedness': [5.0, 5.0, 5.0],\n", + " 'groundedness_completion_tokens': [107, 161, 130],\n", + " 'groundedness_finish_reason': ['stop', 'stop', 'stop'],\n", + " 'groundedness_model': ['gpt-4.1-2025-04-14',\n", + " 'gpt-4.1-2025-04-14',\n", + " 'gpt-4.1-2025-04-14'],\n", + " 'groundedness_prompt_tokens': [1402, 1409, 1427],\n", + " 'groundedness_reason': ['The response is fully '\n", + " 'correct and complete, '\n", + " 'directly reflecting all the '\n", + " 'information provided in the '\n", + " \"context about Seattle's \"\n", + " 'weather.',\n", + " 'The response is fully '\n", + " 'accurate, complete, and '\n", + " 'directly grounded in the '\n", + " 'provided context, addressing '\n", + " 'the query with all relevant '\n", + " 'details.',\n", + " 'The response is fully '\n", + " 'grounded in the context, '\n", + " 'accurately summarizing the '\n", + " 'weather for both cities as '\n", + " 'described, and directly '\n", + " 'addresses the query.'],\n", + " 'groundedness_result': ['pass', 'pass', 'pass'],\n", + " 'groundedness_sample_input': ['[{\"role\": \"user\", '\n", + " '\"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"Can '\n", + " 'you check the weather '\n", + " 'in Seattle for me?\\\\\", '\n", + " '\\\\\"response\\\\\": '\n", + " '\\\\\"According to the '\n", + " 'current weather data, '\n", + " 'Seattle is '\n", + " 'experiencing rainy '\n", + " 'conditions with a '\n", + " 'temperature of '\n", + " '14\\\\\\\\u00b0C and '\n", + " 'overcast skies.\\\\\", '\n", + " '\\\\\"context\\\\\": '\n", + " '\\\\\"{\\'global_context\\': '\n", + " \"'Weather data shows \"\n", + " 'Seattle currently has '\n", + " 'rainy conditions at '\n", + " '14\\\\\\\\u00b0C with '\n", + " 'overcast skies. London '\n", + " 'shows cloudy weather '\n", + " 'at 8\\\\\\\\u00b0C with '\n", + " 'partly cloudy '\n", + " 'conditions. Both '\n", + " 'cities are '\n", + " 'experiencing typical '\n", + " 'seasonal weather '\n", + " 'patterns.\\'}\\\\\"}\"}]',\n", + " '[{\"role\": \"user\", '\n", + " '\"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"How '\n", + " 'does that compare to '\n", + " 'London?\\\\\", '\n", + " '\\\\\"response\\\\\": '\n", + " '\\\\\"London is currently '\n", + " 'cloudier but drier '\n", + " 'than Seattle, with a '\n", + " 'temperature of '\n", + " '8\\\\\\\\u00b0C and partly '\n", + " 'cloudy conditions. '\n", + " 'Seattle is warmer but '\n", + " 'rainier at '\n", + " '14\\\\\\\\u00b0C.\\\\\", '\n", + " '\\\\\"context\\\\\": '\n", + " '\\\\\"{\\'global_context\\': '\n", + " \"'Weather data shows \"\n", + " 'Seattle currently has '\n", + " 'rainy conditions at '\n", + " '14\\\\\\\\u00b0C with '\n", + " 'overcast skies. London '\n", + " 'shows cloudy weather '\n", + " 'at 8\\\\\\\\u00b0C with '\n", + " 'partly cloudy '\n", + " 'conditions. Both '\n", + " 'cities are '\n", + " 'experiencing typical '\n", + " 'seasonal weather '\n", + " 'patterns.\\'}\\\\\"}\"}]',\n", + " '[{\"role\": \"user\", '\n", + " '\"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"Can '\n", + " 'you email me a summary '\n", + " \"of both cities' \"\n", + " 'weather?\\\\\", '\n", + " '\\\\\"response\\\\\": \\\\\"I '\n", + " 'can provide you with a '\n", + " 'weather summary: '\n", + " 'Seattle has rainy '\n", + " 'weather at '\n", + " '14\\\\\\\\u00b0C with '\n", + " 'overcast skies, while '\n", + " 'London has partly '\n", + " 'cloudy conditions at '\n", + " '8\\\\\\\\u00b0C. However, '\n", + " 'I would need email '\n", + " 'access to send this '\n", + " 'information to '\n", + " 'you.\\\\\", '\n", + " '\\\\\"context\\\\\": '\n", + " '\\\\\"{\\'global_context\\': '\n", + " \"'Weather data shows \"\n", + " 'Seattle currently has '\n", + " 'rainy conditions at '\n", + " '14\\\\\\\\u00b0C with '\n", + " 'overcast skies. London '\n", + " 'shows cloudy weather '\n", + " 'at 8\\\\\\\\u00b0C with '\n", + " 'partly cloudy '\n", + " 'conditions. Both '\n", + " 'cities are '\n", + " 'experiencing typical '\n", + " 'seasonal weather '\n", + " 'patterns.\\'}\\\\\"}\"}]'],\n", + " 'groundedness_sample_output': ['[{\"role\": '\n", + " '\"assistant\", '\n", + " '\"content\": '\n", + " '\"Let\\'s think '\n", + " 'step by step: The '\n", + " 'query asks for the '\n", + " 'current weather in '\n", + " 'Seattle. The context '\n", + " 'provides specific '\n", + " 'details: Seattle has '\n", + " 'rainy conditions, '\n", + " '14\\\\u00b0C '\n", + " 'temperature, and '\n", + " 'overcast skies. The '\n", + " 'response repeats '\n", + " 'these details '\n", + " 'accurately and '\n", + " 'completely, without '\n", + " 'adding unrelated or '\n", + " 'incorrect '\n", + " 'information. All '\n", + " 'relevant details from '\n", + " 'the context are '\n", + " 'included in the '\n", + " 'response.\\\\nThe '\n", + " 'response is fully '\n", + " 'correct and complete, '\n", + " 'directly reflecting '\n", + " 'all the information '\n", + " 'provided in the '\n", + " 'context about '\n", + " \"Seattle's \"\n", + " 'weather.\\\\n5\"}]',\n", + " '[{\"role\": '\n", + " '\"assistant\", '\n", + " '\"content\": '\n", + " '\"Let\\'s think '\n", + " 'step by step: The '\n", + " 'context provides '\n", + " 'current weather data '\n", + " 'for Seattle and '\n", + " 'London, specifying '\n", + " 'that Seattle is rainy '\n", + " 'and 14\\\\u00b0C with '\n", + " 'overcast skies, while '\n", + " 'London is 8\\\\u00b0C '\n", + " 'and partly cloudy. '\n", + " 'The query asks for a '\n", + " 'comparison to London. '\n", + " 'The response '\n", + " 'accurately compares '\n", + " 'the two cities: it '\n", + " 'states London is '\n", + " 'cloudier but drier '\n", + " '(correct, as London '\n", + " 'is partly cloudy and '\n", + " 'not rainy), and '\n", + " 'Seattle is warmer but '\n", + " 'rainier (correct, as '\n", + " 'Seattle is 14\\\\u00b0C '\n", + " 'and rainy). All '\n", + " 'relevant details from '\n", + " 'the context are '\n", + " 'included, and no '\n", + " 'extraneous or '\n", + " 'incorrect information '\n", + " 'is '\n", + " 'added.\\\\nThe '\n", + " 'response is fully '\n", + " 'accurate, complete, '\n", + " 'and directly grounded '\n", + " 'in the provided '\n", + " 'context, addressing '\n", + " 'the query with all '\n", + " 'relevant '\n", + " 'details.\\\\n5\"}]',\n", + " '[{\"role\": '\n", + " '\"assistant\", '\n", + " '\"content\": '\n", + " '\"Let\\'s think '\n", + " 'step by step: The '\n", + " 'query asks for an '\n", + " 'emailed summary of '\n", + " 'the weather in both '\n", + " 'Seattle and London. '\n", + " 'The context provides '\n", + " 'specific weather '\n", + " 'details for both '\n", + " 'cities. The response '\n", + " 'summarizes the '\n", + " 'weather for both '\n", + " 'cities accurately and '\n", + " 'completely, matching '\n", + " 'the context. It also '\n", + " 'notes the need for '\n", + " 'email access, which '\n", + " 'is a relevant '\n", + " 'clarification but '\n", + " 'does not detract from '\n", + " 'the completeness or '\n", + " 'accuracy of the '\n", + " 'summary. No incorrect '\n", + " 'or unrelated '\n", + " 'information is '\n", + " 'introduced.\\\\nThe '\n", + " 'response is fully '\n", + " 'grounded in the '\n", + " 'context, accurately '\n", + " 'summarizing the '\n", + " 'weather for both '\n", + " 'cities as described, '\n", + " 'and directly '\n", + " 'addresses the '\n", + " 'query.\\\\n5\"}]'],\n", + " 'groundedness_threshold': [3, 3, 3],\n", + " 'groundedness_total_tokens': [1509, 1570, 1557]},\n", + " 'gpt_groundedness': 5.0,\n", + " 'groundedness': 5.0,\n", + " 'groundedness_completion_tokens': 132.66666666666666,\n", + " 'groundedness_prompt_tokens': 1412.6666666666667,\n", + " 'groundedness_threshold': 3.0,\n", + " 'groundedness_total_tokens': 1545.3333333333333}\n" + ] + } + ], + "source": [ + "# Example of conversation evaluation - using weather and email theme for consistency\n", + "conversation = {\n", + " \"context\": \"Weather data shows Seattle currently has rainy conditions at 14°C with overcast skies. London shows cloudy weather at 8°C with partly cloudy conditions. Both cities are experiencing typical seasonal weather patterns.\",\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"According to the current weather data, Seattle is experiencing rainy conditions with a temperature of 14°C and overcast skies.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"How does that compare to London?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"London is currently cloudier but drier than Seattle, with a temperature of 8°C and partly cloudy conditions. Seattle is warmer but rainier at 14°C.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Can you email me a summary of both cities' weather?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I can provide you with a weather summary: Seattle has rainy weather at 14°C with overcast skies, while London has partly cloudy conditions at 8°C. However, I would need email access to send this information to you.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = groundedness_evaluator(conversation=conversation)\n", + "print(\"=== Conversation Evaluation ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "dbe6e815", + "metadata": {}, + "source": [ + "### 6. Email Context Evaluation\n", + "Evaluating groundedness when dealing with email-related responses, consistent with ToolCallAccuracyEvaluator." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e46ac0ec", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "=== Email Context Evaluation ===\n", + "{'gpt_groundedness': 5.0,\n", + " 'groundedness': 5.0,\n", + " 'groundedness_completion_tokens': 142,\n", + " 'groundedness_finish_reason': 'stop',\n", + " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", + " 'groundedness_prompt_tokens': 1421,\n", + " 'groundedness_reason': 'The response is fully correct and complete, directly '\n", + " 'addressing the query and including all relevant '\n", + " 'details from the context.',\n", + " 'groundedness_result': 'pass',\n", + " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"Can you send me an email with weather '\n", + " 'information for Seattle?\\\\\", \\\\\"response\\\\\": '\n", + " '\\\\\"I can send you an email with the Seattle '\n", + " 'weather information. The current weather shows '\n", + " 'rainy conditions at 14\\\\\\\\u00b0C with overcast '\n", + " \"skies. I'll prepare this information for your \"\n", + " 'email.\\\\\", \\\\\"context\\\\\": \\\\\"Email service is '\n", + " 'available. Current weather data for Seattle: '\n", + " 'rainy conditions, 14\\\\\\\\u00b0C temperature, '\n", + " '\\\\\\\\novercast skies with light precipitation. '\n", + " 'User email preferences show john@example.com as '\n", + " 'primary contact. \\\\\\\\nEmail system can send '\n", + " 'weather reports with current conditions and '\n", + " 'forecasts.\\\\\"}\"}]',\n", + " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", + " 'think step by step: The query asks for an '\n", + " 'email with Seattle weather information. The '\n", + " 'context confirms that email service is '\n", + " \"available, the user's email is known, and \"\n", + " 'provides current weather details (rainy, '\n", + " '14\\\\u00b0C, overcast, light precipitation). '\n", + " 'The response states it can send an email, '\n", + " 'summarizes the current weather accurately, and '\n", + " 'says it will prepare the information for '\n", + " 'email. All relevant details from the context '\n", + " 'are included, and the response is directly '\n", + " 'tied to the query and context without adding '\n", + " 'unrelated information.\\\\nThe response '\n", + " 'is fully correct and complete, directly '\n", + " 'addressing the query and including all '\n", + " 'relevant details from the '\n", + " 'context.\\\\n5\"}]',\n", + " 'groundedness_threshold': 3,\n", + " 'groundedness_total_tokens': 1563}\n" + ] + } + ], + "source": [ + "# Example with email context - consistent with ToolCallAccuracyEvaluator email scenarios\n", + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "\n", + "context = \"\"\"Email service is available. Current weather data for Seattle: rainy conditions, 14°C temperature, \n", + "overcast skies with light precipitation. User email preferences show john@example.com as primary contact. \n", + "Email system can send weather reports with current conditions and forecasts.\"\"\"\n", + "\n", + "response = \"I can send you an email with the Seattle weather information. The current weather shows rainy conditions at 14°C with overcast skies. I'll prepare this information for your email.\"\n", + "\n", + "result = groundedness_evaluator(response=response, context=context, query=query)\n", + "print(\"=== Email Context Evaluation ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "a617af7b", + "metadata": {}, + "source": [ + "## Understanding the Results\n", + "\n", + "The Groundedness evaluator returns a dictionary with key metrics:\n", + "\n", + "- **`groundedness`**: Main score (1-5) indicating how well-grounded the response is\n", + "- **`groundedness_result`**: Binary result (\"pass\" or \"fail\") based on threshold\n", + "- **`groundedness_threshold`**: Threshold for pass/fail determination (default: 3)\n", + "- **`groundedness_reason`**: Detailed explanation of the grounding assessment\n", + "- **Token usage information**: For monitoring costs and performance\n", + "\n", + "### Interpreting Scores:\n", + "- **Score 5**: Fully grounded - all claims supported by context\n", + "- **Score 4**: Mostly grounded - most claims supported, minor unsupported details\n", + "- **Score 3**: Partially grounded - some claims supported, some not\n", + "- **Score 2**: Mostly ungrounded - few claims supported by context\n", + "- **Score 1**: Completely ungrounded - no claims supported by context\n", + "\n", + "### Best Practices:\n", + "1. **Provide Comprehensive Context**: Include all relevant source material\n", + "2. **Use Clear Queries**: When provided, queries help focus the evaluation\n", + "3. **Monitor Agent Tools**: For agent responses, ensure tool results are properly grounded\n", + "4. **Regular Evaluation**: Test groundedness across different domains and use cases\n", + "\n", + "### Use Cases:\n", + "- **RAG Applications**: Ensure retrieved information supports responses\n", + "- **Customer Support**: Verify responses are based on knowledge base\n", + "- **Agent Systems**: Validate tool-based responses against retrieved data\n", + "- **Content Generation**: Ensure factual accuracy and source attribution\n", + "- **Multi-turn Conversations**: Maintain context grounding across dialogue\n", + "\n", + "### Comparison with Other Evaluators:\n", + "- **vs Relevance**: Groundedness checks factual support, Relevance checks topic alignment\n", + "- **vs Tool Call Accuracy**: Groundedness validates response content, Tool Call Accuracy validates tool usage\n", + "- **Combined Use**: Use all three evaluators for comprehensive agent evaluation" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb index ab69e4fc..e68dde18 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb @@ -61,9 +61,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Class IntentResolutionEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" + ] + } + ], "source": [ "import os\n", "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", @@ -96,9 +104,53 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: What are the opening hours of the Eiffel Tower?\n", + "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Opening hours of the Eiffel Tower are 9:00 AM to 11:00 PM.\n", + "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Opening hours of the Eiffel Tower are 9:00 AM to 11:00 PM.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_intent_resolution': 5.0,\n", + " 'intent_resolution': 5.0,\n", + " 'intent_resolution_completion_tokens': 52,\n", + " 'intent_resolution_finish_reason': 'stop',\n", + " 'intent_resolution_model': 'gpt-4.1-2025-04-14',\n", + " 'intent_resolution_prompt_tokens': 1907,\n", + " 'intent_resolution_reason': 'The user asked for the opening hours of the '\n", + " 'Eiffel Tower. The agent provided a clear and '\n", + " 'direct answer with specific times, fully '\n", + " \"satisfying the user's intent without any notable \"\n", + " 'gaps or errors.',\n", + " 'intent_resolution_result': 'pass',\n", + " 'intent_resolution_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"What are the opening '\n", + " 'hours of the Eiffel Tower?\\\\\", '\n", + " '\\\\\"response\\\\\": \\\\\"Opening hours of the '\n", + " 'Eiffel Tower are 9:00 AM to 11:00 PM.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": null}\"}]',\n", + " 'intent_resolution_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The user asked for '\n", + " 'the opening hours of the Eiffel Tower. '\n", + " 'The agent provided a clear and direct '\n", + " 'answer with specific times, fully '\n", + " \"satisfying the user's intent without any \"\n", + " 'notable gaps or errors.\\\\\",\\\\n '\n", + " '\\\\\"score\\\\\": 5\\\\n}\"}]',\n", + " 'intent_resolution_threshold': 3,\n", + " 'intent_resolution_total_tokens': 1959}\n" + ] + } + ], "source": [ "# Success example. Intent is identified and understood and the response correctly resolves user intent\n", "result = intent_resolution_evaluator(\n", @@ -110,9 +162,55 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: What is the opening hours of the Eiffel Tower?\n", + "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Please check the official website for the up-to-date information on Eiffel Tower opening hours.\n", + "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Please check the official website for the up-to-date information on Eiffel Tower opening hours.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_intent_resolution': 2.0,\n", + " 'intent_resolution': 2.0,\n", + " 'intent_resolution_completion_tokens': 57,\n", + " 'intent_resolution_finish_reason': 'stop',\n", + " 'intent_resolution_model': 'gpt-4.1-2025-04-14',\n", + " 'intent_resolution_prompt_tokens': 1905,\n", + " 'intent_resolution_reason': 'The user asked for the opening hours of the '\n", + " 'Eiffel Tower. The agent did not provide any '\n", + " 'actual information, only redirecting the user to '\n", + " 'the official website, leaving the intent '\n", + " 'unresolved and offering no direct assistance.',\n", + " 'intent_resolution_result': 'fail',\n", + " 'intent_resolution_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"What is the opening '\n", + " 'hours of the Eiffel Tower?\\\\\", '\n", + " '\\\\\"response\\\\\": \\\\\"Please check the '\n", + " 'official website for the up-to-date '\n", + " 'information on Eiffel Tower opening '\n", + " 'hours.\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " 'null}\"}]',\n", + " 'intent_resolution_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The user asked for '\n", + " 'the opening hours of the Eiffel Tower. '\n", + " 'The agent did not provide any actual '\n", + " 'information, only redirecting the user to '\n", + " 'the official website, leaving the intent '\n", + " 'unresolved and offering no direct '\n", + " 'assistance.\\\\\",\\\\n \\\\\"score\\\\\": 2\\\\n}\"}]',\n", + " 'intent_resolution_threshold': 3,\n", + " 'intent_resolution_total_tokens': 1962}\n" + ] + } + ], "source": [ "# Failure example. Even though intent is correctly identified, the response does not resolve the user intent\n", "result = intent_resolution_evaluator(\n", @@ -122,6 +220,94 @@ "pprint(result)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Query and Response as String with Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: What's the status of order #789?\n", + "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Order #789 has been shipped and will arrive on March 18, 2025.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_intent_resolution': 5.0,\n", + " 'intent_resolution': 5.0,\n", + " 'intent_resolution_completion_tokens': 59,\n", + " 'intent_resolution_finish_reason': 'stop',\n", + " 'intent_resolution_model': 'gpt-4.1-2025-04-14',\n", + " 'intent_resolution_prompt_tokens': 1904,\n", + " 'intent_resolution_reason': 'The user asked for the status of order #789. The '\n", + " 'agent provided a clear and complete update, '\n", + " 'stating the order has shipped and specifying the '\n", + " 'expected arrival date, fully resolving the '\n", + " \"user's intent with accurate and relevant \"\n", + " 'information.',\n", + " 'intent_resolution_result': 'pass',\n", + " 'intent_resolution_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"What\\'s the status of '\n", + " 'order #789?\\\\\", \\\\\"response\\\\\": \\\\\"Order '\n", + " '#789 has been shipped and will arrive on '\n", + " 'March 18, 2025.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": {\\\\\"name\\\\\": '\n", + " '\\\\\"get_order\\\\\", \\\\\"description\\\\\": \\\\\"Get '\n", + " 'the details of a specific order.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"order_id\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", + " 'order ID to get the details '\n", + " 'for.\\\\\"}}}}}\"}]',\n", + " 'intent_resolution_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The user asked for '\n", + " 'the status of order #789. The agent '\n", + " 'provided a clear and complete update, '\n", + " 'stating the order has shipped and '\n", + " 'specifying the expected arrival date, '\n", + " \"fully resolving the user's intent with \"\n", + " 'accurate and relevant '\n", + " 'information.\\\\\",\\\\n \\\\\"score\\\\\": '\n", + " '5\\\\n}\"}]',\n", + " 'intent_resolution_threshold': 3,\n", + " 'intent_resolution_total_tokens': 1963}\n" + ] + } + ], + "source": [ + "# Example with a single tool definition dict instead of a list\n", + "query = \"What's the status of order #789?\"\n", + "response = \"Order #789 has been shipped and will arrive on March 18, 2025.\"\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"get_order\",\n", + " \"description\": \"Get the details of a specific order.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"order_id\": {\"type\": \"string\", \"description\": \"The order ID to get the details for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = intent_resolution_evaluator(\n", + " query=query,\n", + " response=response,\n", + " tool_definitions=tool_definition_dict,\n", + ")\n", + "pprint(result)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -131,9 +317,92 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_intent_resolution': 5.0,\n", + " 'intent_resolution': 5.0,\n", + " 'intent_resolution_completion_tokens': 59,\n", + " 'intent_resolution_finish_reason': 'stop',\n", + " 'intent_resolution_model': 'gpt-4.1-2025-04-14',\n", + " 'intent_resolution_prompt_tokens': 1999,\n", + " 'intent_resolution_reason': 'The user requested status updates for their last '\n", + " 'two orders on account #888. The agent provided '\n", + " 'clear, specific information for both orders, '\n", + " 'including shipping and expected delivery dates, '\n", + " \"fully resolving the user's intent with accuracy \"\n", + " 'and completeness.',\n", + " 'intent_resolution_result': 'pass',\n", + " 'intent_resolution_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"User turn 1:\\\\\\\\n Hi, I '\n", + " 'need help with the last 2 orders on my '\n", + " 'account #888. Could you please update me '\n", + " 'on their status?\\\\\\\\n\\\\\\\\n\\\\\", '\n", + " '\\\\\"response\\\\\": \\\\\"Hello! Let me quickly '\n", + " 'look up your account details.\\\\\\\\nThanks '\n", + " 'for your patience. I see two orders on '\n", + " 'your account. Let me fetch the details for '\n", + " 'both.\\\\\\\\nThe order with ID 123 has been '\n", + " 'shipped and is expected to be delivered on '\n", + " 'March 15, 2025. However, the order with ID '\n", + " '124 is delayed and should now arrive by '\n", + " 'March 20, 2025. Is there anything else I '\n", + " 'can help you with?\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", + " '\\\\\"get_orders\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Get the list of orders for a given '\n", + " 'account number.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"account_number\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The account number '\n", + " 'to get the orders for.\\\\\"}}}}, '\n", + " '{\\\\\"name\\\\\": \\\\\"get_order\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Get the details of a '\n", + " 'specific order.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"order_id\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The order ID to get '\n", + " 'the details for.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", + " '\\\\\"initiate_return\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Initiate the return process for an '\n", + " 'order.\\\\\", \\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"order_id\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", + " 'order ID for the return process.\\\\\"}}}}, '\n", + " '{\\\\\"name\\\\\": '\n", + " '\\\\\"update_shipping_address\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Update the shipping '\n", + " 'address for a given account.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"account_number\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", + " 'account number to update.\\\\\"}, '\n", + " '\\\\\"new_address\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", + " 'new shipping address.\\\\\"}}}}]}\"}]',\n", + " 'intent_resolution_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The user requested '\n", + " 'status updates for their last two orders '\n", + " 'on account #888. The agent provided '\n", + " 'clear, specific information for both '\n", + " 'orders, including shipping and expected '\n", + " 'delivery dates, fully resolving the '\n", + " \"user's intent with accuracy and \"\n", + " 'completeness.\\\\\",\\\\n \\\\\"score\\\\\": '\n", + " '5\\\\n}\"}]',\n", + " 'intent_resolution_threshold': 3,\n", + " 'intent_resolution_total_tokens': 2058}\n" + ] + } + ], "source": [ "query = [\n", " {\"role\": \"system\", \"content\": \"You are a friendly and helpful customer service agent.\"},\n", @@ -292,44 +561,11 @@ ")\n", "pprint(result)" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Batch evaluate and visualize results on Azure AI Foundry\n", - "Batch evaluate to leverage asynchronous evaluation on a dataset. \n", - "\n", - "Optionally, you can go to AI Foundry URL for rich Azure AI Foundry data visualization. You can inspect the evaluation scores and reasoning to quickly identify bugs and issues of your agent to fix and improve. Make sure to authenticate to Azure using `az login` in your terminal before running this cell.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.evaluation import evaluate\n", - "\n", - "# This sample files contains the evaluation data in JSONL format. Where each line is a run from agent.\n", - "# This was saved using agent thread and converter.\n", - "file_name = \"evaluation_data.jsonl\"\n", - "\n", - "response = evaluate(\n", - " data=file_name,\n", - " evaluation_name=\"Intent Resolution Evaluation\",\n", - " evaluators={\n", - " \"intent_resolution\": intent_resolution_evaluator,\n", - " },\n", - " azure_ai_project=os.environ[\"AZURE_AI_PROJECT\"],\n", - ")\n", - "pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')" - ] } ], "metadata": { "kernelspec": { - "display_name": "azureai-sample", + "display_name": "samples", "language": "python", "name": "python3" }, @@ -342,7 +578,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.10.19" } }, "nbformat": 4, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb new file mode 100644 index 00000000..f3da5ca5 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb @@ -0,0 +1,328 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "bb0675a7", + "metadata": {}, + "source": [ + "# Relevance Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use the Relevance evaluator to assess the relevance of AI-generated responses. The evaluator supports multiple input formats including:\n", + "- Simple query and response pairs\n", + "- Multi-turn conversations\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 15 minutes running this notebook.\n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities.\n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure OpenAI Endpoint to be used for evaluation\n", + "3) **AZURE_OPENAI_API_KEY** - Azure OpenAI Key to be used for evaluation\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure OpenAI API version to be used for evaluation\n" + ] + }, + { + "cell_type": "markdown", + "id": "1a21aa7e", + "metadata": {}, + "source": [ + "## What is Relevance?\n", + "\n", + "The Relevance evaluator assesses the ability of AI responses to capture the key points and address the user's query appropriately. It measures how well the response relates to and answers the specific question or request.\n", + "\n", + "**Scoring:** Relevance scores range from 1 to 5, with:\n", + "- **1**: Completely irrelevant to the query\n", + "- **2**: Partially relevant but misses key points\n", + "- **3**: Moderately relevant with some key points addressed\n", + "- **4**: Highly relevant with most key points covered\n", + "- **5**: Perfectly relevant, fully addressing the query\n", + "\n", + "High relevance scores indicate the AI system's understanding of the input and capability to produce contextually appropriate outputs." + ] + }, + { + "cell_type": "markdown", + "id": "f0dd0ebf", + "metadata": {}, + "source": [ + "## Relevance Evaluator Input Requirements\n", + "\n", + "The Relevance evaluator supports the following input formats:\n", + "\n", + "1. **Simple Query-Response Evaluation:**\n", + " - `query`: The user's question or request (str)\n", + " - `response`: The AI's response to evaluate (str)\n", + "\n", + "2. **Conversation Evaluation:**\n", + " - `conversation`: Multi-turn conversation with messages list and optional context (Conversation object)\n" + ] + }, + { + "cell_type": "markdown", + "id": "519631da", + "metadata": {}, + "source": [ + "### Initialize Relevance Evaluator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3484e790", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import RelevanceEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "# Configure the model\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "# Initialize the evaluator\n", + "relevance_evaluator = RelevanceEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "4f739700", + "metadata": {}, + "source": [ + "## Sample Evaluations" + ] + }, + { + "cell_type": "markdown", + "id": "83d8d0ec", + "metadata": {}, + "source": [ + "### 1. Highly Relevant Response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43eaf25d", + "metadata": {}, + "outputs": [], + "source": [ + "# Example of a highly relevant response - using weather query similar to ToolCallAccuracyEvaluator\n", + "query = \"How is the weather in Seattle?\"\n", + "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", + "\n", + "result = relevance_evaluator(query=query, response=response)\n", + "print(\"=== Highly Relevant Response ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "89266048", + "metadata": {}, + "source": [ + "### 2. Partially Relevant Response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "69761825", + "metadata": {}, + "outputs": [], + "source": [ + "# Example of a partially relevant response\n", + "query = \"How do I bake a chocolate cake?\"\n", + "response = \"Chocolate is made from cocoa beans and was first discovered by the Mayans. There are many types of chocolate including dark, milk, and white chocolate.\"\n", + "\n", + "result = relevance_evaluator(query=query, response=response)\n", + "print(\"=== Partially Relevant Response ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "aeb6961e", + "metadata": {}, + "source": [ + "### 3. Irrelevant Response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1cb62f5e", + "metadata": {}, + "outputs": [], + "source": [ + "# Example of an irrelevant response\n", + "query = \"What are the symptoms of flu?\"\n", + "response = \"The stock market had significant fluctuations today, with technology stocks leading the decline.\"\n", + "\n", + "result = relevance_evaluator(query=query, response=response)\n", + "print(\"=== Irrelevant Response ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "bd158e43", + "metadata": {}, + "source": [ + "### 4. Conversation Evaluation\n", + "The evaluator also supports evaluating multi-turn conversations using the `conversation` parameter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf17774b", + "metadata": {}, + "outputs": [], + "source": [ + "# Example of conversation evaluation\n", + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"I'm planning a trip to Japan. Can you help me with some recommendations?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I'd be happy to help you plan your trip to Japan! Here are some popular destinations: Tokyo for modern city life and culture, Kyoto for traditional temples and gardens, Osaka for amazing food, and Mount Fuji for scenic beauty. What type of experiences are you most interested in?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"I'm particularly interested in traditional culture and temples.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Perfect! For traditional culture and temples, I highly recommend Kyoto. Visit Fushimi Inari Shrine with its thousands of red torii gates, Kinkaku-ji (Golden Pavilion), and Gion district where you might spot geishas. Nara is also wonderful with its deer park and Todai-ji Temple housing a giant Buddha statue.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = relevance_evaluator(conversation=conversation)\n", + "print(\"=== Conversation Evaluation ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "105c8839", + "metadata": {}, + "source": [ + "### 5. Technical Query Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b73bf46", + "metadata": {}, + "outputs": [], + "source": [ + "# Example of technical query evaluation\n", + "query = \"How do I implement a binary search algorithm in Python?\"\n", + "response = \"\"\"Here's a Python implementation of binary search:\n", + "\n", + "def binary_search(arr, target):\n", + " left, right = 0, len(arr) - 1\n", + " \n", + " while left <= right:\n", + " mid = (left + right) // 2\n", + " \n", + " if arr[mid] == target:\n", + " return mid\n", + " elif arr[mid] < target:\n", + " left = mid + 1\n", + " else:\n", + " right = mid - 1\n", + " \n", + " return -1\n", + "\n", + "This algorithm works on sorted arrays and has O(log n) time complexity.\"\"\"\n", + "\n", + "result = relevance_evaluator(query=query, response=response)\n", + "print(\"=== Technical Query Evaluation ===\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "c98f8d61", + "metadata": {}, + "source": [ + "### 6. Customer Service Scenario" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1604f3d", + "metadata": {}, + "outputs": [], + "source": [ + "# Example of customer service conversation\n", + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"I received a damaged product in my order #12345. What can you do to help?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I'm sorry to hear about the damaged product in order #12345. I can help you with this issue. Let me check your order details and arrange either a replacement or refund. Could you please describe the damage and let me know your preference for resolution?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"The screen of the tablet has a crack. I'd prefer a replacement.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I understand you received a tablet with a cracked screen. I'll immediately process a replacement for you. You'll receive a new tablet within 2-3 business days, and we'll include a prepaid return label for the damaged unit. Is your shipping address still the same as in the original order?\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = relevance_evaluator(conversation=conversation)\n", + "print(\"=== Customer Service Scenario ===\")\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb index 075676b8..2126e759 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb @@ -83,9 +83,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Class TaskAdherenceEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" + ] + } + ], "source": [ "import os\n", "from azure.ai.evaluation import TaskAdherenceEvaluator, AzureOpenAIModelConfiguration\n", @@ -116,9 +124,63 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: What are the best practices for maintaining a healthy rose garden during the summer?\n", + "Agent response could not be parsed, falling back to original response: Make sure to water your roses regularly and trim them occasionally.\n", + "Agent response could not be parsed, falling back to original response: Make sure to water your roses regularly and trim them occasionally.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_task_adherence': 3.0,\n", + " 'task_adherence': 3.0,\n", + " 'task_adherence_completion_tokens': 117,\n", + " 'task_adherence_finish_reason': 'stop',\n", + " 'task_adherence_model': 'gpt-4.1-2025-04-14',\n", + " 'task_adherence_prompt_tokens': 2831,\n", + " 'task_adherence_reason': 'The intended task was to provide best practices for '\n", + " 'maintaining a healthy rose garden in summer. The '\n", + " 'assistant gave only basic advice (watering and '\n", + " 'trimming), omitting key practices like pest '\n", + " 'control, mulching, and fertilization. No system '\n", + " 'constraints or tool requirements were present.',\n", + " 'task_adherence_result': 'pass',\n", + " 'task_adherence_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"What are the best practices for '\n", + " 'maintaining a healthy rose garden during the '\n", + " 'summer?\\\\\", \\\\\"response\\\\\": \\\\\"Make sure to '\n", + " 'water your roses regularly and trim them '\n", + " 'occasionally.\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " 'null}\"}]',\n", + " 'task_adherence_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The intended task was '\n", + " 'to provide best practices for maintaining a '\n", + " 'healthy rose garden in summer. The assistant '\n", + " 'gave only basic advice (watering and '\n", + " 'trimming), omitting key practices like pest '\n", + " 'control, mulching, and fertilization. No '\n", + " 'system constraints or tool requirements were '\n", + " 'present.\\\\\",\\\\n \\\\\"agent_perceived_task\\\\\": '\n", + " '\\\\\"Give basic tips for summer rose garden '\n", + " 'care.\\\\\",\\\\n \\\\\"actual_task\\\\\": \\\\\"Provide '\n", + " 'comprehensive best practices for maintaining '\n", + " 'a healthy rose garden during summer.\\\\\",\\\\n '\n", + " '\\\\\"tools_used\\\\\": [],\\\\n '\n", + " '\\\\\"expected_tool_calls\\\\\": [],\\\\n '\n", + " '\\\\\"score\\\\\": 3\\\\n}\"}]',\n", + " 'task_adherence_threshold': 3,\n", + " 'task_adherence_total_tokens': 2948}\n" + ] + } + ], "source": [ "# Failure example, there's only a vague adherence to the task\n", "result = task_adherence_evaluator(\n", @@ -130,9 +192,77 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: What are the best practices for maintaining a healthy rose garden during the summer?\n", + "Agent response could not be parsed, falling back to original response: For optimal summer care of your rose garden, start by watering deeply early in the morning to ensure the roots are well-hydrated without encouraging fungal growth. Apply a 2-3 inch layer of organic mulch around the base of the plants to conserve moisture and regulate soil temperature. Fertilize with a balanced rose fertilizer every 4 to 6 weeks to support healthy growth. Prune away any dead or diseased wood to promote good air circulation, and inspect regularly for pests such as aphids or spider mites, treating them promptly with an appropriate organic insecticidal soap. Finally, ensure that your roses receive at least 6 hours of direct sunlight daily for robust flowering.\n", + "Agent response could not be parsed, falling back to original response: For optimal summer care of your rose garden, start by watering deeply early in the morning to ensure the roots are well-hydrated without encouraging fungal growth. Apply a 2-3 inch layer of organic mulch around the base of the plants to conserve moisture and regulate soil temperature. Fertilize with a balanced rose fertilizer every 4 to 6 weeks to support healthy growth. Prune away any dead or diseased wood to promote good air circulation, and inspect regularly for pests such as aphids or spider mites, treating them promptly with an appropriate organic insecticidal soap. Finally, ensure that your roses receive at least 6 hours of direct sunlight daily for robust flowering.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_task_adherence': 5.0,\n", + " 'task_adherence': 5.0,\n", + " 'task_adherence_completion_tokens': 118,\n", + " 'task_adherence_finish_reason': 'stop',\n", + " 'task_adherence_model': 'gpt-4.1-2025-04-14',\n", + " 'task_adherence_prompt_tokens': 2955,\n", + " 'task_adherence_reason': 'The assistant provided a thorough, well-structured '\n", + " 'list of best practices for summer rose garden care, '\n", + " \"directly addressing the user's request. No system \"\n", + " 'constraints or tool requirements were present, and '\n", + " 'the response was complete and relevant.',\n", + " 'task_adherence_result': 'pass',\n", + " 'task_adherence_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"What are the best practices for '\n", + " 'maintaining a healthy rose garden during the '\n", + " 'summer?\\\\\", \\\\\"response\\\\\": \\\\\"For optimal '\n", + " 'summer care of your rose garden, start by '\n", + " 'watering deeply early in the morning to '\n", + " 'ensure the roots are well-hydrated without '\n", + " 'encouraging fungal growth. Apply a 2-3 inch '\n", + " 'layer of organic mulch around the base of the '\n", + " 'plants to conserve moisture and regulate soil '\n", + " 'temperature. Fertilize with a balanced rose '\n", + " 'fertilizer every 4 to 6 weeks to support '\n", + " 'healthy growth. Prune away any dead or '\n", + " 'diseased wood to promote good air '\n", + " 'circulation, and inspect regularly for pests '\n", + " 'such as aphids or spider mites, treating them '\n", + " 'promptly with an appropriate organic '\n", + " 'insecticidal soap. Finally, ensure that your '\n", + " 'roses receive at least 6 hours of direct '\n", + " 'sunlight daily for robust flowering.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": null}\"}]',\n", + " 'task_adherence_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The assistant provided '\n", + " 'a thorough, well-structured list of best '\n", + " 'practices for summer rose garden care, '\n", + " \"directly addressing the user's request. No \"\n", + " 'system constraints or tool requirements were '\n", + " 'present, and the response was complete and '\n", + " 'relevant.\\\\\",\\\\n '\n", + " '\\\\\"agent_perceived_task\\\\\": \\\\\"List best '\n", + " 'practices for maintaining a healthy rose '\n", + " 'garden during summer.\\\\\",\\\\n '\n", + " '\\\\\"actual_task\\\\\": \\\\\"Provide best practices '\n", + " 'for maintaining a healthy rose garden during '\n", + " 'the summer, with no system constraints or '\n", + " 'tool requirements.\\\\\",\\\\n \\\\\"tools_used\\\\\": '\n", + " '[],\\\\n \\\\\"expected_tool_calls\\\\\": [],\\\\n '\n", + " '\\\\\"score\\\\\": 5\\\\n}\"}]',\n", + " 'task_adherence_threshold': 3,\n", + " 'task_adherence_total_tokens': 3073}\n" + ] + } + ], "source": [ "# Success example, full adherence to the task\n", "result = task_adherence_evaluator(\n", @@ -142,6 +272,100 @@ "pprint(result)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Query and Response as String with Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: Can you recommend a science fiction book for me?\n", + "Agent response could not be parsed, falling back to original response: I found 'Dune' by Frank Herbert for you. It's a classic science fiction novel set in a distant future.\n", + "Tool definitions could not be parsed, falling back to original definitions: {'name': 'get_book', 'description': 'Retrieve a book recommendation for a specified genre.', 'parameters': {'type': 'object', 'properties': {'genre': {'type': 'string', 'description': 'The genre for which a book recommendation is requested.'}}}}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_task_adherence': 5.0,\n", + " 'task_adherence': 5.0,\n", + " 'task_adherence_completion_tokens': 99,\n", + " 'task_adherence_finish_reason': 'stop',\n", + " 'task_adherence_model': 'gpt-4.1-2025-04-14',\n", + " 'task_adherence_prompt_tokens': 2893,\n", + " 'task_adherence_reason': 'The assistant provided a relevant science fiction '\n", + " \"book recommendation, directly fulfilling the user's \"\n", + " 'request. There were no system constraints or tool '\n", + " 'requirements, so the lack of tool use does not '\n", + " 'impact the score.',\n", + " 'task_adherence_result': 'pass',\n", + " 'task_adherence_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"Can you recommend a science fiction book '\n", + " 'for me?\\\\\", \\\\\"response\\\\\": \\\\\"I found '\n", + " \"'Dune' by Frank Herbert for you. It's a \"\n", + " 'classic science fiction novel set in a '\n", + " 'distant future.\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " '{\\\\\"name\\\\\": \\\\\"get_book\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Retrieve a book '\n", + " 'recommendation for a specified genre.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"genre\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The genre '\n", + " 'for which a book recommendation is '\n", + " 'requested.\\\\\"}}}}}\"}]',\n", + " 'task_adherence_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The assistant provided '\n", + " 'a relevant science fiction book '\n", + " 'recommendation, directly fulfilling the '\n", + " \"user's request. There were no system \"\n", + " 'constraints or tool requirements, so the '\n", + " 'lack of tool use does not impact the '\n", + " 'score.\\\\\",\\\\n \\\\\"agent_perceived_task\\\\\": '\n", + " '\\\\\"recommend a science fiction book\\\\\",\\\\n '\n", + " '\\\\\"actual_task\\\\\": \\\\\"Recommend a science '\n", + " 'fiction book (no system constraints or tool '\n", + " 'requirements)\\\\\",\\\\n \\\\\"tools_used\\\\\": '\n", + " '[],\\\\n \\\\\"expected_tool_calls\\\\\": [],\\\\n '\n", + " '\\\\\"score\\\\\": 5\\\\n}\"}]',\n", + " 'task_adherence_threshold': 3,\n", + " 'task_adherence_total_tokens': 2992}\n" + ] + } + ], + "source": [ + "# Example with a single tool definition dict\n", + "query = \"Can you recommend a science fiction book for me?\"\n", + "response = \"I found 'Dune' by Frank Herbert for you. It's a classic science fiction novel set in a distant future.\"\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"get_book\",\n", + " \"description\": \"Retrieve a book recommendation for a specified genre.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"genre\": {\"type\": \"string\", \"description\": \"The genre for which a book recommendation is requested.\"}\n", + " },\n", + " },\n", + "}\n", + "\n", + "result = task_adherence_evaluator(\n", + " query=query,\n", + " response=response,\n", + " tool_definitions=tool_definition_dict,\n", + ")\n", + "pprint(result)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -151,9 +375,75 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_task_adherence': 5.0,\n", + " 'task_adherence': 5.0,\n", + " 'task_adherence_completion_tokens': 114,\n", + " 'task_adherence_finish_reason': 'stop',\n", + " 'task_adherence_model': 'gpt-4.1-2025-04-14',\n", + " 'task_adherence_prompt_tokens': 2986,\n", + " 'task_adherence_reason': \"The assistant correctly identified the user's \"\n", + " 'request for a historical fiction recommendation, '\n", + " 'used the appropriate tool, and provided a relevant '\n", + " 'book with a brief summary. All system and user '\n", + " 'requirements were met without error.',\n", + " 'task_adherence_result': 'pass',\n", + " 'task_adherence_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"SYSTEM_PROMPT:\\\\\\\\n You are an expert in '\n", + " 'literature and at provid can provide book '\n", + " 'recommendations.\\\\\\\\n\\\\\\\\nUser turn 1:\\\\\\\\n '\n", + " 'I love historical fiction. Can you recommend '\n", + " 'a good book from that genre?\\\\\\\\n\\\\\\\\n\\\\\", '\n", + " '\\\\\"response\\\\\": \\\\\"Let me fetch a '\n", + " 'recommendation for historical '\n", + " 'fiction.\\\\\\\\n[TOOL_CALL] '\n", + " 'get_book(genre=\\\\\\\\\\\\\"historical '\n", + " 'fiction\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] { '\n", + " '\\\\\\\\\\\\\"book\\\\\\\\\\\\\": { \\\\\\\\\\\\\"title\\\\\\\\\\\\\": '\n", + " '\\\\\\\\\\\\\"The Pillars of the Earth\\\\\\\\\\\\\", '\n", + " '\\\\\\\\\\\\\"author\\\\\\\\\\\\\": \\\\\\\\\\\\\"Ken '\n", + " 'Follett\\\\\\\\\\\\\", \\\\\\\\\\\\\"summary\\\\\\\\\\\\\": '\n", + " '\\\\\\\\\\\\\"A captivating tale set in medieval '\n", + " 'England that weaves historical events with '\n", + " 'personal drama.\\\\\\\\\\\\\" } }\\\\\\\\nBased on our '\n", + " \"records, I recommend 'The Pillars of the \"\n", + " \"Earth' by Ken Follett. This novel is an \"\n", + " 'excellent example of historical fiction with '\n", + " 'a rich narrative and well-developed '\n", + " 'characters. Would you like more details or '\n", + " 'another suggestion?\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- get_book: Retrieve '\n", + " 'a book recommendation for a specified genre. '\n", + " '(inputs: genre)\\\\\"}\"}]',\n", + " 'task_adherence_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The assistant '\n", + " \"correctly identified the user's request for \"\n", + " 'a historical fiction recommendation, used '\n", + " 'the appropriate tool, and provided a '\n", + " 'relevant book with a brief summary. All '\n", + " 'system and user requirements were met '\n", + " 'without error.\\\\\",\\\\n '\n", + " '\\\\\"agent_perceived_task\\\\\": \\\\\"recommend a '\n", + " 'good historical fiction book using available '\n", + " 'records and tools\\\\\",\\\\n \\\\\"actual_task\\\\\": '\n", + " '\\\\\"Recommend a good historical fiction book, '\n", + " 'using expertise and available tools as '\n", + " 'needed\\\\\",\\\\n \\\\\"tools_used\\\\\": '\n", + " '[\\\\\"get_book\\\\\"],\\\\n '\n", + " '\\\\\"expected_tool_calls\\\\\": '\n", + " '[\\\\\"get_book\\\\\"],\\\\n \\\\\"score\\\\\": 5\\\\n}\"}]',\n", + " 'task_adherence_threshold': 3,\n", + " 'task_adherence_total_tokens': 3100}\n" + ] + } + ], "source": [ "query = [\n", " {\"role\": \"system\", \"content\": \"You are an expert in literature and at provid can provide book recommendations.\"},\n", @@ -227,44 +517,11 @@ ")\n", "pprint(result)" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Batch evaluate and visualize results on Azure AI Foundry\n", - "Batch evaluate to leverage asynchronous evaluation on a dataset. \n", - "\n", - "Optionally, you can go to AI Foundry URL for rich Azure AI Foundry data visualization. You can inspect the evaluation scores and reasoning to quickly identify bugs and issues of your agent to fix and improve. Make sure to authenticate to Azure using `az login` in your terminal before running this cell." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.evaluation import evaluate\n", - "\n", - "# This sample files contains the evaluation data in JSONL format. Where each line is a run from agent.\n", - "# This was saved using agent thread and converter.\n", - "file_name = \"evaluation_data.jsonl\"\n", - "\n", - "response = evaluate(\n", - " data=file_name,\n", - " evaluation_name=\"Task Adherence Evaluation\",\n", - " evaluators={\n", - " \"task_adherence\": task_adherence_evaluator,\n", - " },\n", - " azure_ai_project=os.environ[\"AZURE_AI_PROJECT\"],\n", - ")\n", - "pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')" - ] } ], "metadata": { "kernelspec": { - "display_name": "azureai-sample", + "display_name": "samples", "language": "python", "name": "python3" }, @@ -277,7 +534,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.10.19" } }, "nbformat": 4, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb new file mode 100644 index 00000000..6da7483d --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb @@ -0,0 +1,793 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cd88c9a4", + "metadata": {}, + "source": [ + "# Task Completion Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use task completion evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing task responses;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "fe5dda1c", + "metadata": {}, + "source": [ + "The Task Completion evaluator assesses whether an AI agent successfully completes the requested task by examining:\n", + "- Whether the task was fully completed\n", + "- Quality of task execution\n", + "- Appropriateness of the response to the original request\n", + "\n", + "The evaluator uses a binary scoring system (0 or 1):\n", + "\n", + " - Score 0: The task was not completed or only partially completed\n", + " - Score 1: The task was successfully and fully completed\n", + "\n", + "This evaluation focuses on measuring whether the agent's response indicates successful completion of the user's request, regardless of the specific methods or tools used to achieve the task." + ] + }, + { + "cell_type": "markdown", + "id": "c1642178", + "metadata": {}, + "source": [ + "Task Completion requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response.\n", + "- Tool Definitions - (Optional) Tool(s) definition used by Agent to answer the query. Providing tool definitions helps the evaluator better understand the context and capabilities available to the agent.\n" + ] + }, + { + "cell_type": "markdown", + "id": "ffad775f", + "metadata": {}, + "source": [ + "### Initialize Task Completion Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "0a5fae33", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Class _TaskCompletionEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" + ] + } + ], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._task_completion import _TaskCompletionEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "\n", + "task_completion = _TaskCompletionEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "af4839bb", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "95305bbd", + "metadata": {}, + "source": [ + "#### Evaluating Simple Task Completion" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "57d4592d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", + "Agent response could not be parsed, falling back to original response: The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\n", + "Agent response could not be parsed, falling back to original response: The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_completion': True,\n", + " 'task_completion_completion_tokens': 107,\n", + " 'task_completion_details': {'completion_gaps': 'None; all relevant weather '\n", + " 'details were provided.',\n", + " 'delivered_outcome': 'Agent delivered a detailed '\n", + " 'weather update with '\n", + " 'temperature, cloud cover, '\n", + " 'wind, humidity, and '\n", + " 'precipitation forecast.',\n", + " 'task_requirements': 'User requested the current '\n", + " 'weather conditions in '\n", + " 'Seattle.'},\n", + " 'task_completion_finish_reason': 'stop',\n", + " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", + " 'task_completion_prompt_tokens': 1771,\n", + " 'task_completion_reason': 'The agent provided a complete and current weather '\n", + " 'report for Seattle, including temperature, '\n", + " 'conditions, wind, humidity, and precipitation '\n", + " 'outlook.',\n", + " 'task_completion_result': 'pass',\n", + " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"How is the weather in Seattle?\\\\\", '\n", + " '\\\\\"response\\\\\": \\\\\"The current weather in '\n", + " 'Seattle is partly cloudy with a temperature '\n", + " \"of 15\\\\\\\\u00b0C (59\\\\\\\\u00b0F). There's a \"\n", + " 'light breeze from the northwest at 8 mph, '\n", + " 'and the humidity is at 68%. No precipitation '\n", + " 'is expected for the rest of the day.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": null}\"}]',\n", + " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The agent provided a '\n", + " 'complete and current weather report for '\n", + " 'Seattle, including temperature, conditions, '\n", + " 'wind, humidity, and precipitation '\n", + " 'outlook.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"task_requirements\\\\\": \\\\\"User requested '\n", + " 'the current weather conditions in '\n", + " 'Seattle.\\\\\",\\\\n \\\\\"delivered_outcome\\\\\": '\n", + " '\\\\\"Agent delivered a detailed weather '\n", + " 'update with temperature, cloud cover, wind, '\n", + " 'humidity, and precipitation '\n", + " 'forecast.\\\\\",\\\\n \\\\\"completion_gaps\\\\\": '\n", + " '\\\\\"None; all relevant weather details were '\n", + " 'provided.\\\\\"\\\\n },\\\\n \\\\\"success\\\\\": '\n", + " 'true\\\\n}\"}]',\n", + " 'task_completion_total_tokens': 1878}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = \"The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\"\n", + "\n", + "# Basic evaluation without tool definitions\n", + "result = task_completion(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "2c66a362", + "metadata": {}, + "source": [ + "#### Task Completion with Tool Context" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "71ea8725", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", + "Agent response could not be parsed, falling back to original response: I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\n", + "Agent response could not be parsed, falling back to original response: I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_completion': True,\n", + " 'task_completion_completion_tokens': 131,\n", + " 'task_completion_details': {'completion_gaps': 'No gaps; all relevant weather '\n", + " 'details were provided as '\n", + " 'requested.',\n", + " 'delivered_outcome': 'The agent delivered a '\n", + " 'concise weather summary: '\n", + " 'partly cloudy, 15°C, light '\n", + " 'breeze, and no rain '\n", + " 'expected today.',\n", + " 'task_requirements': 'The user asked for the '\n", + " 'current weather conditions '\n", + " 'in Seattle, expecting a '\n", + " 'usable summary of '\n", + " 'temperature, sky condition, '\n", + " 'and precipitation.'},\n", + " 'task_completion_finish_reason': 'stop',\n", + " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", + " 'task_completion_prompt_tokens': 1752,\n", + " 'task_completion_reason': 'The agent provided a current weather report for '\n", + " 'Seattle, including cloud cover, temperature, wind, '\n", + " \"and precipitation, fully addressing the user's \"\n", + " 'request.',\n", + " 'task_completion_result': 'pass',\n", + " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"How is the weather in Seattle?\\\\\", '\n", + " '\\\\\"response\\\\\": \\\\\"I\\'ve checked the weather '\n", + " \"for Seattle and found that it's currently \"\n", + " 'partly cloudy with a temperature of '\n", + " \"15\\\\\\\\u00b0C. There's a light breeze and no \"\n", + " 'rain expected today.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", + " 'Fetches the weather information for the '\n", + " 'specified location. (inputs: '\n", + " 'location)\\\\\"}\"}]',\n", + " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The agent provided a '\n", + " 'current weather report for Seattle, '\n", + " 'including cloud cover, temperature, wind, '\n", + " 'and precipitation, fully addressing the '\n", + " 'user\\'s request.\\\\\",\\\\n \\\\\"details\\\\\": '\n", + " '{\\\\n \\\\\"task_requirements\\\\\": \\\\\"The '\n", + " 'user asked for the current weather '\n", + " 'conditions in Seattle, expecting a usable '\n", + " 'summary of temperature, sky condition, and '\n", + " 'precipitation.\\\\\",\\\\n '\n", + " '\\\\\"delivered_outcome\\\\\": \\\\\"The agent '\n", + " 'delivered a concise weather summary: partly '\n", + " 'cloudy, 15\\\\u00b0C, light breeze, and no '\n", + " 'rain expected today.\\\\\",\\\\n '\n", + " '\\\\\"completion_gaps\\\\\": \\\\\"No gaps; all '\n", + " 'relevant weather details were provided as '\n", + " 'requested.\\\\\"\\\\n },\\\\n \\\\\"success\\\\\": '\n", + " 'true\\\\n}\"}]',\n", + " 'task_completion_total_tokens': 1883}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = \"I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\"\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = task_completion(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "1962234d", + "metadata": {}, + "source": [ + "#### Task Completion with Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "9a9cd459", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: What's the current temperature in Boston?\n", + "Agent response could not be parsed, falling back to original response: The current temperature in Boston is 22°C (72°F) with clear skies. It's a beautiful day with low humidity and a gentle breeze.\n", + "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_completion': True,\n", + " 'task_completion_completion_tokens': 98,\n", + " 'task_completion_details': {'completion_gaps': 'None; all requested '\n", + " 'information was provided.',\n", + " 'delivered_outcome': 'Agent delivered the current '\n", + " 'temperature in both Celsius '\n", + " 'and Fahrenheit, plus brief '\n", + " 'weather conditions.',\n", + " 'task_requirements': 'User requested the current '\n", + " 'temperature in Boston.'},\n", + " 'task_completion_finish_reason': 'stop',\n", + " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", + " 'task_completion_prompt_tokens': 1750,\n", + " 'task_completion_reason': 'The agent provided the current temperature in '\n", + " 'Boston, including both Celsius and Fahrenheit, '\n", + " \"which fully answers the user's request.\",\n", + " 'task_completion_result': 'pass',\n", + " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"What\\'s the current temperature in '\n", + " 'Boston?\\\\\", \\\\\"response\\\\\": \\\\\"The current '\n", + " 'temperature in Boston is 22\\\\\\\\u00b0C '\n", + " \"(72\\\\\\\\u00b0F) with clear skies. It's a \"\n", + " 'beautiful day with low humidity and a gentle '\n", + " 'breeze.\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " '{\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", + " 'information for the specified location.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", + " 'weather for.\\\\\"}}}}}\"}]',\n", + " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The agent provided '\n", + " 'the current temperature in Boston, '\n", + " 'including both Celsius and Fahrenheit, '\n", + " \"which fully answers the user's \"\n", + " 'request.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"task_requirements\\\\\": \\\\\"User requested '\n", + " 'the current temperature in '\n", + " 'Boston.\\\\\",\\\\n \\\\\"delivered_outcome\\\\\": '\n", + " '\\\\\"Agent delivered the current temperature '\n", + " 'in both Celsius and Fahrenheit, plus brief '\n", + " 'weather conditions.\\\\\",\\\\n '\n", + " '\\\\\"completion_gaps\\\\\": \\\\\"None; all '\n", + " 'requested information was provided.\\\\\"\\\\n '\n", + " '},\\\\n \\\\\"success\\\\\": true\\\\n}\"}]',\n", + " 'task_completion_total_tokens': 1848}\n" + ] + } + ], + "source": [ + "query = \"What's the current temperature in Boston?\"\n", + "response = \"The current temperature in Boston is 22°C (72°F) with clear skies. It's a beautiful day with low humidity and a gentle breeze.\"\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = task_completion(query=query, response=response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "fc85ade7", + "metadata": {}, + "source": [ + "#### Complex Task with Multiple Steps" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "915768db", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_completion': True,\n", + " 'task_completion_completion_tokens': 133,\n", + " 'task_completion_details': {'completion_gaps': 'No gaps if the email was '\n", + " 'truly sent; otherwise, the '\n", + " 'task would be incomplete.',\n", + " 'delivered_outcome': 'Agent states an email was '\n", + " \"sent with Seattle's weather \"\n", + " 'details (partly cloudy, '\n", + " \"15°C) to the user's \"\n", + " 'registered address.',\n", + " 'task_requirements': 'Send an email to the user '\n", + " 'containing current weather '\n", + " 'information for Seattle.'},\n", + " 'task_completion_finish_reason': 'stop',\n", + " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", + " 'task_completion_prompt_tokens': 1784,\n", + " 'task_completion_reason': 'The agent claims to have sent an email with '\n", + " 'Seattle weather information, including specific '\n", + " 'details. If the email was actually sent as stated, '\n", + " \"the user's request is fully met.\",\n", + " 'task_completion_result': 'pass',\n", + " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"Can you send me an email with weather '\n", + " 'information for Seattle?\\\\\", \\\\\"response\\\\\": '\n", + " '\\\\\"I\\'ll get the current weather information '\n", + " 'for Seattle and then send you an email with '\n", + " 'the details.\\\\\\\\nI have successfully sent '\n", + " 'you an email with the weather information '\n", + " 'for Seattle. The current weather is partly '\n", + " 'cloudy with a temperature of 15\\\\\\\\u00b0C. '\n", + " 'You should receive the email shortly at your '\n", + " 'registered email address.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", + " 'Fetches the weather information for the '\n", + " 'specified location. (inputs: location)\\\\\\\\n- '\n", + " 'send_email: Sends an email with the '\n", + " 'specified subject and body to the recipient. '\n", + " '(inputs: recipient, subject, body)\\\\\"}\"}]',\n", + " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The agent claims to '\n", + " 'have sent an email with Seattle weather '\n", + " 'information, including specific details. If '\n", + " 'the email was actually sent as stated, the '\n", + " 'user\\'s request is fully met.\\\\\",\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"task_requirements\\\\\": \\\\\"Send an email '\n", + " 'to the user containing current weather '\n", + " 'information for Seattle.\\\\\",\\\\n '\n", + " '\\\\\"delivered_outcome\\\\\": \\\\\"Agent states an '\n", + " \"email was sent with Seattle's weather \"\n", + " 'details (partly cloudy, 15\\\\u00b0C) to the '\n", + " 'user\\'s registered address.\\\\\",\\\\n '\n", + " '\\\\\"completion_gaps\\\\\": \\\\\"No gaps if the '\n", + " 'email was truly sent; otherwise, the task '\n", + " 'would be incomplete.\\\\\"\\\\n },\\\\n '\n", + " '\\\\\"success\\\\\": true\\\\n}\"}]',\n", + " 'task_completion_total_tokens': 1917}\n" + ] + } + ], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I'll get the current weather information for Seattle and then send you an email with the details.\",\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is partly cloudy with a temperature of 15°C. You should receive the email shortly at your registered email address.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = task_completion(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "5523dfcf", + "metadata": {}, + "source": [ + "#### Query as Conversation History (List of Messages)\n", + "The evaluator also supports query as a list of messages representing conversation history. This helps evaluate task completion in the context of a full conversation." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2c4772be", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: [{'role': 'system', 'content': 'You are a helpful assistant that can fetch weather information and send emails.'}, {'role': 'user', 'content': 'Hi, I need to plan my day. Can you check the weather in Seattle for me?'}, {'role': 'user', 'content': 'Also, please send me an email summary of the weather so I can reference it later.'}]\n", + "Agent response could not be parsed, falling back to original response: I've checked the weather in Seattle for you. It's currently 15°C and partly cloudy with light winds. I've also sent you an email summary with these details so you can reference them throughout your day. The task has been completed successfully.\n", + "Agent response could not be parsed, falling back to original response: I've checked the weather in Seattle for you. It's currently 15°C and partly cloudy with light winds. I've also sent you an email summary with these details so you can reference them throughout your day. The task has been completed successfully.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_completion': True,\n", + " 'task_completion_completion_tokens': 116,\n", + " 'task_completion_details': {'completion_gaps': 'None; both weather '\n", + " 'information and email summary '\n", + " 'were delivered as requested.',\n", + " 'delivered_outcome': 'Current weather details for '\n", + " 'Seattle and confirmation '\n", + " 'that an email summary was '\n", + " 'sent to the user.',\n", + " 'task_requirements': 'Check Seattle weather and '\n", + " 'send an email summary of '\n", + " 'the weather for later '\n", + " 'reference.'},\n", + " 'task_completion_finish_reason': 'stop',\n", + " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", + " 'task_completion_prompt_tokens': 1845,\n", + " 'task_completion_reason': 'The agent provided the current Seattle weather and '\n", + " 'confirmed sending an email summary, fulfilling '\n", + " 'both user requests. The deliverables are complete '\n", + " 'and actionable.',\n", + " 'task_completion_result': 'pass',\n", + " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '[{\\\\\"role\\\\\": \\\\\"system\\\\\", \\\\\"content\\\\\": '\n", + " '\\\\\"You are a helpful assistant that can '\n", + " 'fetch weather information and send '\n", + " 'emails.\\\\\"}, {\\\\\"role\\\\\": \\\\\"user\\\\\", '\n", + " '\\\\\"content\\\\\": \\\\\"Hi, I need to plan my day. '\n", + " 'Can you check the weather in Seattle for '\n", + " 'me?\\\\\"}, {\\\\\"role\\\\\": \\\\\"user\\\\\", '\n", + " '\\\\\"content\\\\\": \\\\\"Also, please send me an '\n", + " 'email summary of the weather so I can '\n", + " 'reference it later.\\\\\"}], \\\\\"response\\\\\": '\n", + " '\\\\\"I\\'ve checked the weather in Seattle for '\n", + " \"you. It's currently 15\\\\\\\\u00b0C and partly \"\n", + " \"cloudy with light winds. I've also sent you \"\n", + " 'an email summary with these details so you '\n", + " 'can reference them throughout your day. The '\n", + " 'task has been completed successfully.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", + " 'Fetches the weather information for the '\n", + " 'specified location. (inputs: location)\\\\\\\\n- '\n", + " 'send_email: Sends an email with the '\n", + " 'specified subject and body to the recipient. '\n", + " '(inputs: recipient, subject, body)\\\\\"}\"}]',\n", + " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The agent provided '\n", + " 'the current Seattle weather and confirmed '\n", + " 'sending an email summary, fulfilling both '\n", + " 'user requests. The deliverables are '\n", + " 'complete and actionable.\\\\\",\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"task_requirements\\\\\": \\\\\"Check Seattle '\n", + " 'weather and send an email summary of the '\n", + " 'weather for later reference.\\\\\",\\\\n '\n", + " '\\\\\"delivered_outcome\\\\\": \\\\\"Current weather '\n", + " 'details for Seattle and confirmation that '\n", + " 'an email summary was sent to the '\n", + " 'user.\\\\\",\\\\n \\\\\"completion_gaps\\\\\": '\n", + " '\\\\\"None; both weather information and email '\n", + " 'summary were delivered as requested.\\\\\"\\\\n '\n", + " '},\\\\n \\\\\"success\\\\\": true\\\\n}\"}]',\n", + " 'task_completion_total_tokens': 1961}\n" + ] + } + ], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, I need to plan my day. Can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Also, please send me an email summary of the weather so I can reference it later.\"\n", + " }\n", + "]\n", + "\n", + "response = \"I've checked the weather in Seattle for you. It's currently 15°C and partly cloudy with light winds. I've also sent you an email summary with these details so you can reference them throughout your day. The task has been completed successfully.\"\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = task_completion(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "100e943b", + "metadata": {}, + "source": [ + "#### Example of Incomplete Task" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9986fa70", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n", + "Agent response could not be parsed, falling back to original response: I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\n", + "Agent response could not be parsed, falling back to original response: I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_completion': False,\n", + " 'task_completion_completion_tokens': 128,\n", + " 'task_completion_details': {'completion_gaps': 'No email was sent, no '\n", + " 'specific weather information '\n", + " 'included, and the deliverable '\n", + " 'is not actionable or usable '\n", + " 'as requested.',\n", + " 'delivered_outcome': 'General comment about '\n", + " \"Seattle's usual weather, no \"\n", + " 'email sent, no specific '\n", + " 'weather details provided.',\n", + " 'task_requirements': 'Send an email containing '\n", + " 'weather information for '\n", + " 'Seattle.'},\n", + " 'task_completion_finish_reason': 'stop',\n", + " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", + " 'task_completion_prompt_tokens': 1761,\n", + " 'task_completion_reason': 'The agent did not send an email or provide '\n", + " 'specific weather information for Seattle. The '\n", + " 'response only gave a general statement about '\n", + " 'typical weather, failing to deliver the requested '\n", + " 'actionable outcome.',\n", + " 'task_completion_result': 'fail',\n", + " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"Can you send me an email with weather '\n", + " 'information for Seattle?\\\\\", \\\\\"response\\\\\": '\n", + " '\\\\\"I can see that you want weather '\n", + " 'information for Seattle. The weather there '\n", + " 'is usually quite nice this time of year, '\n", + " 'with temperatures ranging from mild to warm '\n", + " 'depending on the season.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", + " 'Fetches the weather information for the '\n", + " 'specified location. (inputs: location)\\\\\\\\n- '\n", + " 'send_email: Sends an email with the '\n", + " 'specified subject and body to the recipient. '\n", + " '(inputs: recipient, subject, body)\\\\\"}\"}]',\n", + " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The agent did not '\n", + " 'send an email or provide specific weather '\n", + " 'information for Seattle. The response only '\n", + " 'gave a general statement about typical '\n", + " 'weather, failing to deliver the requested '\n", + " 'actionable outcome.\\\\\",\\\\n \\\\\"details\\\\\": '\n", + " '{\\\\n \\\\\"task_requirements\\\\\": \\\\\"Send an '\n", + " 'email containing weather information for '\n", + " 'Seattle.\\\\\",\\\\n \\\\\"delivered_outcome\\\\\": '\n", + " '\\\\\"General comment about Seattle\\'s usual '\n", + " 'weather, no email sent, no specific weather '\n", + " 'details provided.\\\\\",\\\\n '\n", + " '\\\\\"completion_gaps\\\\\": \\\\\"No email was '\n", + " 'sent, no specific weather information '\n", + " 'included, and the deliverable is not '\n", + " 'actionable or usable as requested.\\\\\"\\\\n '\n", + " '},\\\\n \\\\\"success\\\\\": false\\\\n}\"}]',\n", + " 'task_completion_total_tokens': 1889}\n" + ] + } + ], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "incomplete_response = \"I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\"\n", + "\n", + "# This response doesn't complete the email sending task\n", + "result = task_completion(query=query, response=incomplete_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb new file mode 100644 index 00000000..505fee4d --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb @@ -0,0 +1,601 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "28c2bc93", + "metadata": {}, + "source": [ + "# Task Navigation Efficiency Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use task navigation efficiency evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing task responses;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d61cd511", + "metadata": {}, + "source": [ + "The Task Navigation Efficiency evaluator assesses how efficiently an AI agent navigates through task completion by examining:\n", + "- The number of steps taken to complete the task\n", + "- Whether the agent took the most efficient path\n", + "- Comparison against ideal or ground truth navigation steps\n", + "- Unnecessary detours or redundant actions\n", + "\n", + "The evaluator uses a scoring system that measures navigation efficiency:\n", + "\n", + " - Higher scores indicate more efficient navigation\n", + " - Lower scores indicate inefficient paths with unnecessary steps\n", + " - The evaluation considers the optimal path as the baseline\n", + "\n", + "This evaluation focuses on measuring whether the agent takes an efficient route to task completion, minimizing unnecessary steps while still achieving the desired outcome." + ] + }, + { + "cell_type": "markdown", + "id": "e18904e1", + "metadata": {}, + "source": [ + "Task Navigation Efficiency requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response.\n", + "- Ground Truth - (Optional) The ideal or expected sequence of steps to complete the task efficiently.\n", + "- Tool Definitions - (Optional) Tool(s) definition used by Agent to answer the query. Providing tool definitions helps the evaluator understand available actions.\n" + ] + }, + { + "cell_type": "markdown", + "id": "5e4e0a1d", + "metadata": {}, + "source": [ + "### Initialize Task Navigation Efficiency Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc3726cd", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator, _TaskNavigationEfficiencyMatchingMode\n", + "from pprint import pprint\n", + "\n", + "# Initialize with exact match mode\n", + "task_navigation_efficiency = _TaskNavigationEfficiencyEvaluator(\n", + " matching_mode=_TaskNavigationEfficiencyMatchingMode.EXACT_MATCH\n", + ")\n", + "\n", + "# Other examples:\n", + "# For in-order matching (allows extra steps but requires correct order)\n", + "# task_navigation_efficiency = _TaskNavigationEfficiencyEvaluator(matching_mode=TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", + "\n", + "# For any-order matching (most lenient - allows extra steps and different order) \n", + "# task_navigation_efficiency = _TaskNavigationEfficiencyEvaluator(matching_mode=TaskNavigationEfficiencyMatchingMode.ANY_ORDER_MATCH)\n", + "\n", + "# Or use defaults (exact match mode)\n", + "# task_navigation_efficiency = _TaskNavigationEfficiencyEvaluator()" + ] + }, + { + "cell_type": "markdown", + "id": "15a53227", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "1d75457a", + "metadata": {}, + "source": [ + "#### Evaluating Simple Task Navigation" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "0cae15fb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", + " 'precision_score': 0.0,\n", + " 'recall_score': 0.0},\n", + " 'task_navigation_efficiency_label': False,\n", + " 'task_navigation_efficiency_result': 'fail'}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = \"I'll check the weather for you. The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F).\"\n", + "\n", + "# Ground truth represents the ideal efficient path\n", + "ground_truth = [\"fetch_weather(location='Seattle')\", \"return_weather_information\"]\n", + "\n", + "# Basic evaluation without tool definitions\n", + "result = task_navigation_efficiency(query=query, response=response, ground_truth=ground_truth)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "8a9c5e69", + "metadata": {}, + "source": [ + "#### Task Navigation with Tool Context" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "f53dde31", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", + " 'precision_score': 0.0,\n", + " 'recall_score': 0.0},\n", + " 'task_navigation_efficiency_label': False,\n", + " 'task_navigation_efficiency_result': 'fail'}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I'll fetch the weather information for Seattle.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\", \n", + " \"content\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C.\"\n", + " }\n", + "]\n", + "\n", + "ground_truth = [\"fetch_weather\", \"provide_weather_info\"]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = task_navigation_efficiency(\n", + " query=query, \n", + " response=response, \n", + " ground_truth=ground_truth,\n", + " tool_definitions=tool_definitions\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "e4fa5757", + "metadata": {}, + "source": [ + "#### Complex Task with Multiple Steps Navigation" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "c2226715", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", + " 'precision_score': 0.0,\n", + " 'recall_score': 0.0},\n", + " 'task_navigation_efficiency_label': False,\n", + " 'task_navigation_efficiency_result': 'fail'}\n" + ] + } + ], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I'll help you get the weather information for Seattle and send it via email.\",\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"Let me first fetch the current weather for Seattle.\",\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"Now I'll compose and send the email with the weather information.\",\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is partly cloudy with a temperature of 15°C.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "# Ground truth represents the most efficient navigation path\n", + "ground_truth = [\n", + " \"fetch_weather(location='Seattle')\", \n", + " \"compose_email(weather_data)\", \n", + " \"send_email()\"\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = task_navigation_efficiency(\n", + " query=query, \n", + " response=response, \n", + " ground_truth=ground_truth,\n", + " tool_definitions=tool_definitions\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "ae74ac8e", + "metadata": {}, + "source": [ + "#### Query as Conversation History with Navigation Steps\n", + "The evaluator also supports query as a list of messages representing conversation history. This helps evaluate navigation efficiency in the context of a full conversation." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "ccd95640", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", + " 'precision_score': 0.0,\n", + " 'recall_score': 0.0},\n", + " 'task_navigation_efficiency_label': False,\n", + " 'task_navigation_efficiency_result': 'fail'}\n" + ] + } + ], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, I need to plan my day. Can you check the weather in Seattle?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Also, please send me an email summary of the weather so I can reference it later.\"\n", + " }\n", + "]\n", + "\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I'll help you with both tasks. Let me start by checking the weather in Seattle.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"The weather in Seattle is currently 15°C and partly cloudy with light winds.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Now I'll send you an email with this weather information.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Done! I've sent you an email with the weather summary for Seattle. You should receive it shortly.\"\n", + " }\n", + "]\n", + "\n", + "ground_truth = [\n", + " \"fetch_weather(location='Seattle')\",\n", + " \"send_email(weather_summary)\"\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = task_navigation_efficiency(\n", + " query=query_as_conversation, \n", + " response=response, \n", + " ground_truth=ground_truth,\n", + " tool_definitions=tool_definitions\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "2c9111ef", + "metadata": {}, + "source": [ + "#### Example of Inefficient Navigation" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "b4bd7b47", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", + " 'precision_score': 0.0,\n", + " 'recall_score': 0.0},\n", + " 'task_navigation_efficiency_label': False,\n", + " 'task_navigation_efficiency_result': 'fail'}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "inefficient_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Let me think about this request.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"You want to know about the weather. Let me consider what information you need.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\", \n", + " \"content\": \"Seattle is a city in Washington state. Let me check the weather there.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I'll now fetch the weather information.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"The weather in Seattle is currently 15°C and partly cloudy.\"\n", + " }\n", + "]\n", + "\n", + "ground_truth = [\"fetch_weather(location='Seattle')\", \"return_weather_information\"]\n", + "\n", + "# This response shows inefficient navigation with unnecessary steps\n", + "result = task_navigation_efficiency(\n", + " query=query, \n", + " response=inefficient_response, \n", + " ground_truth=ground_truth,\n", + " tool_definitions=tool_definitions\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "4bf2e0ee", + "metadata": {}, + "source": [ + "### Sample 4: Task Navigation with Exact Parameter Matching\n", + "\n", + "This sample demonstrates using `ground_truth` as a Tuple with exact parameter matching. The Tuple format allows you to specify not only which tools should be called, but also what exact parameters should be passed to each tool." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "e4dcf585", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'task_navigation_efficiency_details': {'f1_score': 1.0,\n", + " 'precision_score': 1.0,\n", + " 'recall_score': 1.0},\n", + " 'task_navigation_efficiency_label': True,\n", + " 'task_navigation_efficiency_result': 'pass'}\n" + ] + } + ], + "source": [ + "query = \"Check the weather in Seattle and send me an email about it\"\n", + "\n", + "# Agent response with multiple tool calls\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_1\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"}\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"role\": \"tool\",\n", + " \"tool_call_id\": \"call_weather_1\", \n", + " \"content\": '{\"temperature\": \"15°C\", \"condition\": \"partly cloudy\"}'\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_1\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"user@example.com\",\n", + " \"subject\": \"Seattle Weather Update\",\n", + " \"body\": \"The weather in Seattle is 15°C and partly cloudy.\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"role\": \"tool\",\n", + " \"tool_call_id\": \"call_email_1\",\n", + " \"content\": '{\"status\": \"sent\", \"message_id\": \"msg_123\"}'\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I've checked the weather in Seattle (15°C, partly cloudy) and sent you an email with the details.\"\n", + " }\n", + "]\n", + "\n", + "# Ground truth as Tuple with exact parameter matching\n", + "# This format allows you to specify not just which tools should be called,\n", + "# but also what exact parameters should be passed to each tool\n", + "ground_truth = (\n", + " [\"fetch_weather\", \"send_email\"], # Expected tool sequence\n", + " {\n", + " \"fetch_weather\": {\"location\": \"Seattle\"}, # Exact expected parameters\n", + " \"send_email\": {\n", + " \"recipient\": \"user@example.com\",\n", + " \"subject\": \"Seattle Weather Update\",\n", + " \"body\": \"The weather in Seattle is 15°C and partly cloudy.\"\n", + " }\n", + " }\n", + ")\n", + "\n", + "result = task_navigation_efficiency(response=response, ground_truth=ground_truth)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb index 0aa60d2d..eb161692 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb @@ -70,9 +70,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Class ToolCallAccuracyEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" + ] + } + ], "source": [ "import os\n", "from azure.ai.evaluation import ToolCallAccuracyEvaluator, AzureOpenAIModelConfiguration\n", @@ -105,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -130,9 +138,108 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy_completion_tokens': 292,\n", + " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 1,\n", + " 'excess_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'missing_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", + " 'correct_tool_percentage': 1.0,\n", + " 'tool_call_errors': 0,\n", + " 'tool_name': 'fetch_weather',\n", + " 'tool_success_result': 'pass',\n", + " 'total_calls_required': 1}],\n", + " 'tool_calls_made_by_agent': 1},\n", + " 'tool_call_accuracy_finish_reason': 'stop',\n", + " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_call_accuracy_prompt_tokens': 2124,\n", + " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", + " \"is 'How is the weather in Seattle?'. The \"\n", + " \"available tool is 'fetch_weather', which \"\n", + " 'fetches weather information for a specified '\n", + " 'location. The agent made one tool call: '\n", + " \"'fetch_weather' with the argument 'location': \"\n", + " \"'Seattle'. This is the correct tool and the \"\n", + " 'correct parameter, directly grounded from the '\n", + " \"user's query. Only one call is needed and only \"\n", + " 'one was made, with no excess or missing calls. '\n", + " 'There are no errors mentioned, and the call is '\n", + " 'efficient and relevant. This matches the '\n", + " 'criteria for Tool Call Accuracy Level 5 '\n", + " '(Optimal Solution).',\n", + " 'tool_call_accuracy_result': 'pass',\n", + " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"How is the weather in '\n", + " 'Seattle ?\\\\\", \\\\\"tool_calls\\\\\": '\n", + " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", + " '\\\\\"tool_call_id\\\\\": '\n", + " '\\\\\"call_CUdbkBfvVBla2YP3p24uhElJ\\\\\", '\n", + " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"arguments\\\\\": {\\\\\"location\\\\\": '\n", + " '\\\\\"Seattle\\\\\"}}], \\\\\"tool_definitions\\\\\": '\n", + " '[{\\\\\"id\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", + " 'information for the specified '\n", + " 'location.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The location to '\n", + " 'fetch weather for.\\\\\"}}}}]}\"}]',\n", + " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", + " \"step by step: The user's last query is \"\n", + " \"'How is the weather in Seattle?'. The \"\n", + " \"available tool is 'fetch_weather', which \"\n", + " 'fetches weather information for a '\n", + " 'specified location. The agent made one '\n", + " \"tool call: 'fetch_weather' with the \"\n", + " \"argument 'location': 'Seattle'. This is \"\n", + " 'the correct tool and the correct '\n", + " 'parameter, directly grounded from the '\n", + " \"user's query. Only one call is needed \"\n", + " 'and only one was made, with no excess or '\n", + " 'missing calls. There are no errors '\n", + " 'mentioned, and the call is efficient and '\n", + " 'relevant. This matches the criteria for '\n", + " 'Tool Call Accuracy Level 5 (Optimal '\n", + " 'Solution).\\\\\",\\\\n '\n", + " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"tool_calls_made_by_agent\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", + " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\",\\\\n '\n", + " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n '\n", + " '\\\\\"correct_tool_percentage\\\\\": '\n", + " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", + " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", + " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", + " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n },\\\\n '\n", + " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", + " 'tool_call_accuracy_threshold': 3,\n", + " 'tool_call_accuracy_total_tokens': 2416}\n" + ] + } + ], "source": [ "response = tool_call_accuracy(query=query, tool_calls=tool_call, tool_definitions=tool_definition)\n", "pprint(response)" @@ -147,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -180,9 +287,133 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_tool_call_accuracy': 3.0,\n", + " 'tool_call_accuracy': 3.0,\n", + " 'tool_call_accuracy_completion_tokens': 378,\n", + " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 1,\n", + " 'excess_tool_calls': {'details': [{'excess_count': 1,\n", + " 'tool_name': 'fetch_weather'}],\n", + " 'total': 1},\n", + " 'missing_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", + " 'correct_tool_percentage': 1.0,\n", + " 'tool_call_errors': 0,\n", + " 'tool_name': 'fetch_weather',\n", + " 'tool_success_result': 'pass',\n", + " 'total_calls_required': 1}],\n", + " 'tool_calls_made_by_agent': 2},\n", + " 'tool_call_accuracy_finish_reason': 'stop',\n", + " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_call_accuracy_prompt_tokens': 2172,\n", + " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", + " \"is 'How is the weather in Seattle?'. The only \"\n", + " 'relevant tool in the definitions is '\n", + " \"'fetch_weather', which fetches weather \"\n", + " 'information for a specified location. To answer '\n", + " \"the user's query, the agent should call \"\n", + " \"'fetch_weather' with the parameter 'location' \"\n", + " \"set to 'Seattle'. In the TOOL CALLS TO BE \"\n", + " 'EVALUATED section, the agent made two tool '\n", + " \"calls: one for 'Seattle' (correct and relevant) \"\n", + " \"and one for 'London' (irrelevant, as the user \"\n", + " 'did not ask about London). The parameter for '\n", + " 'the Seattle call is correctly grounded in the '\n", + " 'conversation, but the London call is '\n", + " \"unnecessary and not grounded in the user's \"\n", + " 'query. There are no errors or missing tool '\n", + " 'calls, but there is an unnecessary tool call '\n", + " 'for London, which makes the solution '\n", + " 'inefficient according to the rating '\n", + " 'definitions. Therefore, this matches Level 3: '\n", + " 'Relevant but Inefficient.',\n", + " 'tool_call_accuracy_result': 'pass',\n", + " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"How is the weather in '\n", + " 'Seattle ?\\\\\", \\\\\"tool_calls\\\\\": '\n", + " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", + " '\\\\\"tool_call_id\\\\\": '\n", + " '\\\\\"call_CUdbkBfvVBla2YP3p24uhElJ\\\\\", '\n", + " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"arguments\\\\\": {\\\\\"location\\\\\": '\n", + " '\\\\\"Seattle\\\\\"}}, {\\\\\"type\\\\\": '\n", + " '\\\\\"tool_call\\\\\", \\\\\"tool_call_id\\\\\": '\n", + " '\\\\\"call_CUdbkBfvVBla2YP3p24uhElJ\\\\\", '\n", + " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"arguments\\\\\": {\\\\\"location\\\\\": '\n", + " '\\\\\"London\\\\\"}}], \\\\\"tool_definitions\\\\\": '\n", + " '[{\\\\\"id\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", + " 'information for the specified '\n", + " 'location.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The location to '\n", + " 'fetch weather for.\\\\\"}}}}]}\"}]',\n", + " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", + " \"step by step: The user's last query is \"\n", + " \"'How is the weather in Seattle?'. The \"\n", + " 'only relevant tool in the definitions is '\n", + " \"'fetch_weather', which fetches weather \"\n", + " 'information for a specified location. To '\n", + " \"answer the user's query, the agent \"\n", + " \"should call 'fetch_weather' with the \"\n", + " \"parameter 'location' set to 'Seattle'. \"\n", + " 'In the TOOL CALLS TO BE EVALUATED '\n", + " 'section, the agent made two tool calls: '\n", + " \"one for 'Seattle' (correct and relevant) \"\n", + " \"and one for 'London' (irrelevant, as the \"\n", + " 'user did not ask about London). The '\n", + " 'parameter for the Seattle call is '\n", + " 'correctly grounded in the conversation, '\n", + " 'but the London call is unnecessary and '\n", + " \"not grounded in the user's query. There \"\n", + " 'are no errors or missing tool calls, but '\n", + " 'there is an unnecessary tool call for '\n", + " 'London, which makes the solution '\n", + " 'inefficient according to the rating '\n", + " 'definitions. Therefore, this matches '\n", + " 'Level 3: Relevant but '\n", + " 'Inefficient.\\\\\",\\\\n '\n", + " '\\\\\"tool_calls_success_level\\\\\": 3,\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"tool_calls_made_by_agent\\\\\": 2,\\\\n '\n", + " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", + " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\",\\\\n '\n", + " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n '\n", + " '\\\\\"correct_tool_percentage\\\\\": '\n", + " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", + " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", + " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", + " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 1,\\\\n \\\\\"details\\\\\": '\n", + " '[\\\\n {\\\\n '\n", + " '\\\\\"tool_name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\",\\\\n '\n", + " '\\\\\"excess_count\\\\\": 1\\\\n '\n", + " '}\\\\n ]\\\\n },\\\\n '\n", + " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", + " 'tool_call_accuracy_threshold': 3,\n", + " 'tool_call_accuracy_total_tokens': 2550}\n" + ] + } + ], "source": [ "response = tool_call_accuracy(query=query, tool_calls=tool_calls, tool_definitions=tool_definition)\n", "pprint(response)" @@ -198,7 +429,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -289,9 +520,171 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy_completion_tokens': 426,\n", + " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 2,\n", + " 'excess_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'missing_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", + " 'correct_tool_percentage': 1.0,\n", + " 'tool_call_errors': 0,\n", + " 'tool_name': 'fetch_weather',\n", + " 'tool_success_result': 'pass',\n", + " 'total_calls_required': 1},\n", + " {'correct_calls_made_by_agent': 1,\n", + " 'correct_tool_percentage': 1.0,\n", + " 'tool_call_errors': 0,\n", + " 'tool_name': 'send_email',\n", + " 'tool_success_result': 'pass',\n", + " 'total_calls_required': 1}],\n", + " 'tool_calls_made_by_agent': 2},\n", + " 'tool_call_accuracy_finish_reason': 'stop',\n", + " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_call_accuracy_prompt_tokens': 2332,\n", + " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", + " 'is to send an email with weather information '\n", + " 'for Seattle. The available tools are '\n", + " \"'fetch_weather' (to get weather info for a \"\n", + " \"location) and 'send_email' (to send an email \"\n", + " 'with specified content). The agent first called '\n", + " \"'fetch_weather' with the parameter 'Seattle', \"\n", + " \"which is directly grounded from the user's \"\n", + " 'query. The result returned the weather as '\n", + " \"'Rainy, 14°C'. Next, the agent called \"\n", + " \"'send_email' with the recipient \"\n", + " \"'your_email@example.com', subject 'Weather \"\n", + " \"Information for Seattle', and a body containing \"\n", + " 'the weather information just fetched. The '\n", + " 'recipient email is generic, but since the user '\n", + " 'did not specify an email address, it is '\n", + " 'reasonable to use a placeholder. Both tool '\n", + " 'calls are relevant, correctly parameterized, '\n", + " 'and executed successfully. No unnecessary '\n", + " 'duplicate calls were made, and no errors '\n", + " 'occurred. All necessary tools were used to '\n", + " \"fulfill the user's request. Therefore, this is \"\n", + " 'an optimal solution.',\n", + " 'tool_call_accuracy_result': 'pass',\n", + " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"Can you send me an '\n", + " 'email with weather information for '\n", + " 'Seattle?\\\\\", \\\\\"tool_calls\\\\\": '\n", + " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", + " '\\\\\"tool_call_id\\\\\": '\n", + " '\\\\\"call_CUdbkBfvVBla2YP3p24uhElJ\\\\\", '\n", + " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"arguments\\\\\": {\\\\\"location\\\\\": '\n", + " '\\\\\"Seattle\\\\\"}, \\\\\"tool_result\\\\\": '\n", + " '{\\\\\"weather\\\\\": \\\\\"Rainy, '\n", + " '14\\\\\\\\u00b0C\\\\\"}}, {\\\\\"type\\\\\": '\n", + " '\\\\\"tool_call\\\\\", \\\\\"tool_call_id\\\\\": '\n", + " '\\\\\"call_iq9RuPxqzykebvACgX8pqRW2\\\\\", '\n", + " '\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", + " '\\\\\"arguments\\\\\": {\\\\\"recipient\\\\\": '\n", + " '\\\\\"your_email@example.com\\\\\", '\n", + " '\\\\\"subject\\\\\": \\\\\"Weather Information for '\n", + " 'Seattle\\\\\", \\\\\"body\\\\\": \\\\\"The current '\n", + " 'weather in Seattle is rainy with a '\n", + " 'temperature of 14\\\\\\\\u00b0C.\\\\\"}, '\n", + " '\\\\\"tool_result\\\\\": {\\\\\"message\\\\\": '\n", + " '\\\\\"Email successfully sent to '\n", + " 'your_email@example.com.\\\\\"}}], '\n", + " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Fetches the weather information for '\n", + " 'the specified location.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", + " 'location to fetch weather for.\\\\\"}}}}, '\n", + " '{\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Sends an email with '\n", + " 'the specified subject and body to the '\n", + " 'recipient.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"recipient\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Email address of '\n", + " 'the recipient.\\\\\"}, \\\\\"subject\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Subject of the '\n", + " 'email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", + " 'content of the email.\\\\\"}}}}]}\"}]',\n", + " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", + " \"step by step: The user's last query is \"\n", + " 'to send an email with weather '\n", + " 'information for Seattle. The available '\n", + " \"tools are 'fetch_weather' (to get \"\n", + " 'weather info for a location) and '\n", + " \"'send_email' (to send an email with \"\n", + " 'specified content). The agent first '\n", + " \"called 'fetch_weather' with the \"\n", + " \"parameter 'Seattle', which is directly \"\n", + " \"grounded from the user's query. The \"\n", + " \"result returned the weather as 'Rainy, \"\n", + " \"14\\\\u00b0C'. Next, the agent called \"\n", + " \"'send_email' with the recipient \"\n", + " \"'your_email@example.com', subject \"\n", + " \"'Weather Information for Seattle', and a \"\n", + " 'body containing the weather information '\n", + " 'just fetched. The recipient email is '\n", + " 'generic, but since the user did not '\n", + " 'specify an email address, it is '\n", + " 'reasonable to use a placeholder. Both '\n", + " 'tool calls are relevant, correctly '\n", + " 'parameterized, and executed '\n", + " 'successfully. No unnecessary duplicate '\n", + " 'calls were made, and no errors occurred. '\n", + " 'All necessary tools were used to fulfill '\n", + " \"the user's request. Therefore, this is \"\n", + " 'an optimal solution.\\\\\",\\\\n '\n", + " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"tool_calls_made_by_agent\\\\\": 2,\\\\n '\n", + " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", + " '2,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", + " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\",\\\\n '\n", + " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n '\n", + " '\\\\\"correct_tool_percentage\\\\\": '\n", + " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", + " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", + " '\\\\\"pass\\\\\"\\\\n },\\\\n '\n", + " '{\\\\n \\\\\"tool_name\\\\\": '\n", + " '\\\\\"send_email\\\\\",\\\\n '\n", + " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n '\n", + " '\\\\\"correct_tool_percentage\\\\\": '\n", + " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", + " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", + " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", + " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n },\\\\n '\n", + " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", + " 'tool_call_accuracy_threshold': 3,\n", + " 'tool_call_accuracy_total_tokens': 2758}\n" + ] + } + ], "source": [ "response = tool_call_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", "pprint(response)" @@ -301,39 +694,559 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Batch evaluate and visualize results on Azure AI Foundry\n", - "Batch evaluate to leverage asynchronous evaluation on a dataset. \n", + "#### Response as List[dict] with Tool Definition as Single Dict" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy_completion_tokens': 307,\n", + " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 1,\n", + " 'excess_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'missing_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", + " 'correct_tool_percentage': 1.0,\n", + " 'tool_call_errors': 0,\n", + " 'tool_name': 'book_flight',\n", + " 'tool_success_result': 'pass',\n", + " 'total_calls_required': 1}],\n", + " 'tool_calls_made_by_agent': 1},\n", + " 'tool_call_accuracy_finish_reason': 'stop',\n", + " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_call_accuracy_prompt_tokens': 2137,\n", + " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", + " \"is 'Book a flight to New York for tomorrow.' \"\n", + " \"The available tool is 'book_flight', which \"\n", + " 'books a flight to a specified destination and '\n", + " 'date. The agent made one tool call to '\n", + " \"'book_flight' with 'destination' set to 'New \"\n", + " \"York' and 'date' set to 'tomorrow', both of \"\n", + " \"which are directly grounded in the user's \"\n", + " 'query. Only one call is needed and was made, '\n", + " 'with no errors or unnecessary duplicates. No '\n", + " 'other tools are available or required for this '\n", + " 'task. Therefore, the tool call is fully '\n", + " 'relevant, efficient, and correct, matching the '\n", + " 'criteria for Level 5 (Optimal Solution).',\n", + " 'tool_call_accuracy_result': 'pass',\n", + " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"Book a flight to New '\n", + " 'York for tomorrow\\\\\", \\\\\"tool_calls\\\\\": '\n", + " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", + " '\\\\\"tool_call_id\\\\\": '\n", + " '\\\\\"call_book_flight_456\\\\\", \\\\\"name\\\\\": '\n", + " '\\\\\"book_flight\\\\\", \\\\\"arguments\\\\\": '\n", + " '{\\\\\"destination\\\\\": \\\\\"New York\\\\\", '\n", + " '\\\\\"date\\\\\": \\\\\"tomorrow\\\\\"}}], '\n", + " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", + " '\\\\\"book_flight\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Books a flight to the specified '\n", + " 'destination on the given date.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"destination\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", + " 'destination city.\\\\\"}, \\\\\"date\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The date of the '\n", + " 'flight.\\\\\"}}}}]}\"}]',\n", + " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", + " \"step by step: The user's last query is \"\n", + " \"'Book a flight to New York for \"\n", + " \"tomorrow.' The available tool is \"\n", + " \"'book_flight', which books a flight to a \"\n", + " 'specified destination and date. The '\n", + " 'agent made one tool call to '\n", + " \"'book_flight' with 'destination' set to \"\n", + " \"'New York' and 'date' set to 'tomorrow', \"\n", + " 'both of which are directly grounded in '\n", + " \"the user's query. Only one call is \"\n", + " 'needed and was made, with no errors or '\n", + " 'unnecessary duplicates. No other tools '\n", + " 'are available or required for this task. '\n", + " 'Therefore, the tool call is fully '\n", + " 'relevant, efficient, and correct, '\n", + " 'matching the criteria for Level 5 '\n", + " '(Optimal Solution).\\\\\",\\\\n '\n", + " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"tool_calls_made_by_agent\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", + " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", + " '\\\\\"book_flight\\\\\",\\\\n '\n", + " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n '\n", + " '\\\\\"correct_tool_percentage\\\\\": '\n", + " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", + " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", + " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", + " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n },\\\\n '\n", + " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", + " 'tool_call_accuracy_threshold': 3,\n", + " 'tool_call_accuracy_total_tokens': 2444}\n" + ] + } + ], + "source": [ + "query = \"Book a flight to New York for tomorrow\"\n", + "\n", + "# Response as a simple string instead of a list of messages\n", + "response = \"I've found several flight options to New York for tomorrow. I'll use the booking tool to reserve your seat.\"\n", + "\n", + "tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_book_flight_456\",\n", + " \"name\": \"book_flight\",\n", + " \"arguments\": {\n", + " \"destination\": \"New York\",\n", + " \"date\": \"tomorrow\"\n", + " },\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"book_flight\",\n", + " \"description\": \"Books a flight to the specified destination on the given date.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"destination\": {\"type\": \"string\", \"description\": \"The destination city.\"},\n", + " \"date\": {\"type\": \"string\", \"description\": \"The date of the flight.\"},\n", + " },\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_call_accuracy(query=query, response=response, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy_completion_tokens': 297,\n", + " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 1,\n", + " 'excess_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'missing_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", + " 'correct_tool_percentage': 1.0,\n", + " 'tool_call_errors': 0,\n", + " 'tool_name': 'fetch_weather',\n", + " 'tool_success_result': 'pass',\n", + " 'total_calls_required': 1}],\n", + " 'tool_calls_made_by_agent': 1},\n", + " 'tool_call_accuracy_finish_reason': 'stop',\n", + " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_call_accuracy_prompt_tokens': 2121,\n", + " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", + " \"is 'What's the weather in San Francisco?'. The \"\n", + " \"available tool is 'fetch_weather', which \"\n", + " 'fetches weather information for a specified '\n", + " 'location. The agent made one tool call to '\n", + " \"'fetch_weather' with the parameter 'location' \"\n", + " \"set to 'San Francisco', which is directly \"\n", + " \"grounded from the user's query. The tool call \"\n", + " 'returned a successful result with the weather '\n", + " 'information. Only one call was made, which is '\n", + " 'efficient and sufficient to answer the query. '\n", + " 'No errors occurred, and no excess or missing '\n", + " 'tool calls are present. This matches the '\n", + " 'criteria for Tool Call Accuracy Level 5 '\n", + " '(Optimal Solution).',\n", + " 'tool_call_accuracy_result': 'pass',\n", + " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"What\\'s the weather in '\n", + " 'San Francisco?\\\\\", \\\\\"tool_calls\\\\\": '\n", + " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", + " '\\\\\"tool_call_id\\\\\": '\n", + " '\\\\\"call_sf_weather_789\\\\\", \\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"arguments\\\\\": '\n", + " '{\\\\\"location\\\\\": \\\\\"San Francisco\\\\\"}, '\n", + " '\\\\\"tool_result\\\\\": {\\\\\"weather\\\\\": '\n", + " '\\\\\"Foggy, 18\\\\\\\\u00b0C\\\\\"}}], '\n", + " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Fetches the weather information for '\n", + " 'the specified location.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", + " 'location to fetch weather '\n", + " 'for.\\\\\"}}}}]}\"}]',\n", + " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", + " \"step by step: The user's last query is \"\n", + " \"'What's the weather in San Francisco?'. \"\n", + " \"The available tool is 'fetch_weather', \"\n", + " 'which fetches weather information for a '\n", + " 'specified location. The agent made one '\n", + " \"tool call to 'fetch_weather' with the \"\n", + " \"parameter 'location' set to 'San \"\n", + " \"Francisco', which is directly grounded \"\n", + " \"from the user's query. The tool call \"\n", + " 'returned a successful result with the '\n", + " 'weather information. Only one call was '\n", + " 'made, which is efficient and sufficient '\n", + " 'to answer the query. No errors occurred, '\n", + " 'and no excess or missing tool calls are '\n", + " 'present. This matches the criteria for '\n", + " 'Tool Call Accuracy Level 5 (Optimal '\n", + " 'Solution).\\\\\",\\\\n '\n", + " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"tool_calls_made_by_agent\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", + " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\",\\\\n '\n", + " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n '\n", + " '\\\\\"correct_tool_percentage\\\\\": '\n", + " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", + " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", + " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", + " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n },\\\\n '\n", + " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", + " 'tool_call_accuracy_threshold': 3,\n", + " 'tool_call_accuracy_total_tokens': 2418}\n" + ] + } + ], + "source": [ + "query = \"What's the weather in San Francisco?\"\n", + "\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T18:15:22Z\",\n", + " \"run_id\": \"run_abc123\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_sf_weather_789\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"San Francisco\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T18:15:24Z\",\n", + " \"run_id\": \"run_abc123\",\n", + " \"tool_call_id\": \"call_sf_weather_789\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Foggy, 18°C\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T18:15:25Z\",\n", + " \"run_id\": \"run_abc123\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The weather in San Francisco is currently foggy with a temperature of 18°C.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", "\n", - "Optionally, you can go to AI Foundry URL for rich Azure AI Foundry data visualization. You can inspect the evaluation scores and reasoning to quickly identify bugs and issues of your agent to fix and improve. Make sure to authenticate to Azure using `az login` in your terminal before running this cell.\n" + "result = tool_call_accuracy(query=query, response=response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Query as Conversation History (List of Messages)\n", + "The evaluator also supports query as a list of messages representing conversation history. This helps determine if the Agent used the information in the conversation history to make the right tool calls." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ - "from azure.ai.evaluation import evaluate\n", + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Actually, could you also send me an email with that weather information to john@example.com?\"\n", + " }\n", + "]\n", "\n", - "# This sample files contains the evaluation data in JSONL format. Where each line is a run from agent.\n", - "# This was saved using agent thread and converter.\n", - "file_name = \"evaluation_data.jsonl\"\n", + "tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " },\n", + " {\n", + " \"type\": \"tool_call\", \n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"john@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Here is the weather information you requested.\"\n", + " },\n", + " },\n", + "]\n", "\n", - "response = evaluate(\n", - " data=file_name,\n", - " evaluation_name=\"Tool Call Accuracy Evaluation\",\n", - " evaluators={\n", - " \"tool_call_accuracy\": tool_call_accuracy,\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", " },\n", - " azure_ai_project=os.environ[\"AZURE_AI_PROJECT\"],\n", - ")\n", - "pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')" + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'gpt_tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy': 5.0,\n", + " 'tool_call_accuracy_completion_tokens': 410,\n", + " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 2,\n", + " 'excess_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'missing_tool_calls': {'details': [],\n", + " 'total': 0},\n", + " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", + " 'correct_tool_percentage': 1.0,\n", + " 'tool_call_errors': 0,\n", + " 'tool_name': 'fetch_weather',\n", + " 'tool_success_result': 'pass',\n", + " 'total_calls_required': 1},\n", + " {'correct_calls_made_by_agent': 1,\n", + " 'correct_tool_percentage': 1.0,\n", + " 'tool_call_errors': 0,\n", + " 'tool_name': 'send_email',\n", + " 'tool_success_result': 'pass',\n", + " 'total_calls_required': 1}],\n", + " 'tool_calls_made_by_agent': 2},\n", + " 'tool_call_accuracy_finish_reason': 'stop',\n", + " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_call_accuracy_prompt_tokens': 2331,\n", + " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", + " 'is to check the weather in Seattle and send an '\n", + " 'email with that weather information to '\n", + " 'john@example.com. The relevant available tools '\n", + " \"are 'fetch_weather' (to get the weather for \"\n", + " \"Seattle) and 'send_email' (to send the email \"\n", + " 'with the weather info). The agent made two tool '\n", + " \"calls: one to 'fetch_weather' with 'location' \"\n", + " \"set to 'Seattle', and one to 'send_email' with \"\n", + " 'the correct recipient, subject, and a body '\n", + " 'referencing the weather information. All '\n", + " 'parameters are grounded in the conversation: '\n", + " \"'Seattle' is the location, 'john@example.com' \"\n", + " 'is the recipient, and the subject/body are '\n", + " 'appropriate for the context. No unnecessary '\n", + " 'duplicate tool calls were made, and no errors '\n", + " 'are indicated. Both required tools were called '\n", + " 'exactly once, with correct parameters. '\n", + " 'Therefore, this is an optimal solution '\n", + " 'according to the definitions.',\n", + " 'tool_call_accuracy_result': 'pass',\n", + " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": [{\\\\\"role\\\\\": '\n", + " '\\\\\"system\\\\\", \\\\\"content\\\\\": \\\\\"You are a '\n", + " 'helpful assistant that can fetch weather '\n", + " 'information and send emails.\\\\\"}, '\n", + " '{\\\\\"role\\\\\": \\\\\"user\\\\\", \\\\\"content\\\\\": '\n", + " '\\\\\"Hi, can you check the weather in '\n", + " 'Seattle for me?\\\\\"}, {\\\\\"role\\\\\": '\n", + " '\\\\\"user\\\\\", \\\\\"content\\\\\": \\\\\"Actually, '\n", + " 'could you also send me an email with that '\n", + " 'weather information to '\n", + " 'john@example.com?\\\\\"}], \\\\\"tool_calls\\\\\": '\n", + " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", + " '\\\\\"tool_call_id\\\\\": '\n", + " '\\\\\"call_weather_123\\\\\", \\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"arguments\\\\\": '\n", + " '{\\\\\"location\\\\\": \\\\\"Seattle\\\\\"}}, '\n", + " '{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", + " '\\\\\"tool_call_id\\\\\": \\\\\"call_email_456\\\\\", '\n", + " '\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", + " '\\\\\"arguments\\\\\": {\\\\\"recipient\\\\\": '\n", + " '\\\\\"john@example.com\\\\\", \\\\\"subject\\\\\": '\n", + " '\\\\\"Weather Information for Seattle\\\\\", '\n", + " '\\\\\"body\\\\\": \\\\\"Here is the weather '\n", + " 'information you requested.\\\\\"}}], '\n", + " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Fetches the weather information for '\n", + " 'the specified location.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", + " 'location to fetch weather for.\\\\\"}}}}, '\n", + " '{\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Sends an email with '\n", + " 'the specified subject and body to the '\n", + " 'recipient.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"recipient\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Email address of '\n", + " 'the recipient.\\\\\"}, \\\\\"subject\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Subject of the '\n", + " 'email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", + " 'content of the email.\\\\\"}}}}]}\"}]',\n", + " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", + " \"step by step: The user's last query is \"\n", + " 'to check the weather in Seattle and send '\n", + " 'an email with that weather information '\n", + " 'to john@example.com. The relevant '\n", + " \"available tools are 'fetch_weather' (to \"\n", + " 'get the weather for Seattle) and '\n", + " \"'send_email' (to send the email with the \"\n", + " 'weather info). The agent made two tool '\n", + " \"calls: one to 'fetch_weather' with \"\n", + " \"'location' set to 'Seattle', and one to \"\n", + " \"'send_email' with the correct recipient, \"\n", + " 'subject, and a body referencing the '\n", + " 'weather information. All parameters are '\n", + " \"grounded in the conversation: 'Seattle' \"\n", + " \"is the location, 'john@example.com' is \"\n", + " 'the recipient, and the subject/body are '\n", + " 'appropriate for the context. No '\n", + " 'unnecessary duplicate tool calls were '\n", + " 'made, and no errors are indicated. Both '\n", + " 'required tools were called exactly once, '\n", + " 'with correct parameters. Therefore, this '\n", + " 'is an optimal solution according to the '\n", + " 'definitions.\\\\\",\\\\n '\n", + " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"tool_calls_made_by_agent\\\\\": 2,\\\\n '\n", + " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", + " '2,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", + " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\",\\\\n '\n", + " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n '\n", + " '\\\\\"correct_tool_percentage\\\\\": '\n", + " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", + " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", + " '\\\\\"pass\\\\\"\\\\n },\\\\n '\n", + " '{\\\\n \\\\\"tool_name\\\\\": '\n", + " '\\\\\"send_email\\\\\",\\\\n '\n", + " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", + " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", + " '1,\\\\n '\n", + " '\\\\\"correct_tool_percentage\\\\\": '\n", + " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", + " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", + " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", + " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n },\\\\n '\n", + " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", + " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", + " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", + " 'tool_call_accuracy_threshold': 3,\n", + " 'tool_call_accuracy_total_tokens': 2741}\n" + ] + } + ], + "source": [ + "response = tool_call_accuracy(query=query_as_conversation, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(response)" ] } ], "metadata": { "kernelspec": { - "display_name": "azureai-sample", + "display_name": "samples", "language": "python", "name": "python3" }, @@ -346,7 +1259,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.10.19" } }, "nbformat": 4, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb new file mode 100644 index 00000000..9d76a75c --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb @@ -0,0 +1,995 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cea11cae", + "metadata": {}, + "source": [ + "# Tool Call Success Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use tool call success evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing agent responses;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "49404a44", + "metadata": {}, + "source": [ + "The Tool Call Success evaluator determines whether tool calls done by an AI agent includes failures or not.\n", + "\n", + "This evaluator focuses solely on tool call results and tool definitions, disregarding user's query to the agent, conversation history and agent's final response. Although tool definitions is optional, providing them can help the evaluator better understand the context of the tool calls made by the agent. Please note that this evaluator validates tool calls for potential technical failures like errors, exceptions, timeouts and empty results (only in cases where empty results could indicate a failure). It does not assess the correctness or the tool result itself, like mathematical errors and unrealistic field values like name=\"668656\".\n", + "\n", + "Scoring is binary:\n", + "- TRUE: All tool calls were successful\n", + "- FALSE: At least one tool call failed\n", + "\n", + "This evaluation focuses on measuring the technical success of tool execution, not the semantic correctness of the results." + ] + }, + { + "cell_type": "markdown", + "id": "d91cb876", + "metadata": {}, + "source": [ + "Tool Call Success requires following input:\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator examines tool call results within the response.\n", + "- Tool Definitions - (Optional) Tool(s) definition used by Agent. Providing tool definitions helps the evaluator better understand the context of the tool calls made by the agent.\n" + ] + }, + { + "cell_type": "markdown", + "id": "e07d293d", + "metadata": {}, + "source": [ + "### Initialize Tool Call Success Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "919d1b9a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Class _ToolSuccessEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" + ] + } + ], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._tool_success import _ToolSuccessEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "\n", + "tool_call_success = _ToolSuccessEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "4739878c", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "2352536d", + "metadata": {}, + "source": [ + "#### Evaluating Successful Tool Calls" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "9ddc2e22", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_success': 1.0,\n", + " 'tool_success_completion_tokens': 60,\n", + " 'tool_success_finish_reason': 'stop',\n", + " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_success_prompt_tokens': 2407,\n", + " 'tool_success_reason': 'The fetch_weather tool returned a result containing '\n", + " 'temperature, condition, humidity, and wind, with no '\n", + " 'indication of error or technical failure. '\n", + " \"{'failed_tools': '', 'success': True}\",\n", + " 'tool_success_result': 'pass',\n", + " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '15\\\\\\\\u00b0C', 'condition': \"\n", + " \"'partly cloudy', 'humidity': '68%', 'wind': '8 \"\n", + " 'mph NW\\'}\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", + " 'Fetches the weather information for the '\n", + " 'specified location. (inputs: location)\\\\\"}\"}]',\n", + " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The fetch_weather tool '\n", + " 'returned a result containing temperature, '\n", + " 'condition, humidity, and wind, with no '\n", + " 'indication of error or technical '\n", + " 'failure.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"failed_tools\\\\\": \\\\\"\\\\\",\\\\n '\n", + " '\\\\\"success\\\\\": true\\\\n },\\\\n '\n", + " '\\\\\"success\\\\\": true\\\\n}\"}]',\n", + " 'tool_success_threshold': 1,\n", + " 'tool_success_total_tokens': 2467}\n" + ] + } + ], + "source": [ + "# Successful tool execution\n", + "successful_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_call_success(response=successful_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "b7217ddc", + "metadata": {}, + "source": [ + "#### Tool Definition as Single Dict" + ] + }, + { + "cell_type": "markdown", + "id": "c146de06", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "575d33eb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Agent response could not be parsed, falling back to original response: Tool call executed: fetch_weather(location=\"Paris\")\n", + "Tool result: {\"temperature\": \"18°C\", \"condition\": \"clear sky\", \"humidity\": \"55%\"}\n", + "The weather in Paris is currently clear with a temperature of 18°C.\n", + "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", + "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_success': 1.0,\n", + " 'tool_success_completion_tokens': 67,\n", + " 'tool_success_finish_reason': 'stop',\n", + " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_success_prompt_tokens': 2441,\n", + " 'tool_success_reason': 'The tool fetch_weather returned a result with '\n", + " 'temperature, condition, and humidity fields '\n", + " 'populated. There are no indications of technical '\n", + " 'errors, exceptions, or empty/invalid responses. '\n", + " \"{'failed_tools': '', 'success': True}\",\n", + " 'tool_success_result': 'pass',\n", + " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"tool_calls\\\\\": \\\\\"Tool call executed: '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Paris\\\\\\\\\\\\\")\\\\\\\\nTool '\n", + " 'result: {\\\\\\\\\\\\\"temperature\\\\\\\\\\\\\": '\n", + " '\\\\\\\\\\\\\"18\\\\\\\\u00b0C\\\\\\\\\\\\\", '\n", + " '\\\\\\\\\\\\\"condition\\\\\\\\\\\\\": \\\\\\\\\\\\\"clear '\n", + " 'sky\\\\\\\\\\\\\", \\\\\\\\\\\\\"humidity\\\\\\\\\\\\\": '\n", + " '\\\\\\\\\\\\\"55%\\\\\\\\\\\\\"}\\\\\\\\nThe weather in Paris is '\n", + " 'currently clear with a temperature of '\n", + " '18\\\\\\\\u00b0C.\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " '{\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", + " 'information for the specified location.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", + " 'location to fetch weather for.\\\\\"}}}}}\"}]',\n", + " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The tool fetch_weather '\n", + " 'returned a result with temperature, condition, '\n", + " 'and humidity fields populated. There are no '\n", + " 'indications of technical errors, exceptions, '\n", + " 'or empty/invalid responses.\\\\\",\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n \\\\\"failed_tools\\\\\": '\n", + " '\\\\\"\\\\\",\\\\n \\\\\"success\\\\\": true\\\\n '\n", + " '},\\\\n \\\\\"success\\\\\": true\\\\n}\"}]',\n", + " 'tool_success_threshold': 1,\n", + " 'tool_success_total_tokens': 2508}\n" + ] + } + ], + "source": [ + "# Response as a simple string containing tool call information\n", + "# This format is less common but still valid for the evaluator\n", + "response_str = \"\"\"Tool call executed: fetch_weather(location=\"Paris\")\n", + "Tool result: {\"temperature\": \"18°C\", \"condition\": \"clear sky\", \"humidity\": \"55%\"}\n", + "The weather in Paris is currently clear with a temperature of 18°C.\"\"\"\n", + "\n", + "tool_definition = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = tool_call_success(response=response_str, tool_definitions=tool_definition)\n", + "pprint(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f89ca010", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", + "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n", + "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_success': 0.0,\n", + " 'tool_success_completion_tokens': 66,\n", + " 'tool_success_finish_reason': 'stop',\n", + " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_success_prompt_tokens': 2430,\n", + " 'tool_success_reason': 'The fetch_weather tool returned a result with '\n", + " 'temperature, condition, and humidity fields '\n", + " 'populated. There are no indications of technical '\n", + " 'errors, exceptions, or empty/invalid fields. The tool '\n", + " \"call succeeded. {'failed_tools': '', 'success': True}\",\n", + " 'tool_success_result': 'fail',\n", + " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"New '\n", + " 'York\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '22\\\\\\\\u00b0C', 'condition': \"\n", + " '\\'sunny\\', \\'humidity\\': \\'45%\\'}\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": {\\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Fetches the weather information for the '\n", + " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", + " 'weather for.\\\\\"}}}}}\"}]',\n", + " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The fetch_weather tool '\n", + " 'returned a result with temperature, condition, '\n", + " 'and humidity fields populated. There are no '\n", + " 'indications of technical errors, exceptions, '\n", + " 'or empty/invalid fields. The tool call '\n", + " 'succeeded.\\\\\",\\\\n \\\\\"details\\\\\": '\n", + " '{\\\\n \\\\\"failed_tools\\\\\": \\\\\"\\\\\",\\\\n '\n", + " '\\\\\"success\\\\\": true\\\\n }\\\\n}\"}]',\n", + " 'tool_success_threshold': 1,\n", + " 'tool_success_total_tokens': 2496}\n" + ] + } + ], + "source": [ + "# Successful tool execution with single tool definition dict\n", + "successful_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_nyc_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"New York\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_nyc_weather\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"22°C\", \"condition\": \"sunny\", \"humidity\": \"45%\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The weather in New York is currently sunny with a temperature of 22°C.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = tool_call_success(response=successful_response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "1aeb8c11", + "metadata": {}, + "source": [ + "#### Multiple Successful Tool Calls" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "444ee59c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_success': 1.0,\n", + " 'tool_success_completion_tokens': 70,\n", + " 'tool_success_finish_reason': 'stop',\n", + " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_success_prompt_tokens': 2488,\n", + " 'tool_success_reason': 'Both tool results returned valid, non-empty responses '\n", + " 'with no indication of technical errors, exceptions, '\n", + " 'or failures. The fetch_weather result contains '\n", + " 'weather data, and send_email confirms successful '\n", + " \"delivery. {'failed_tools': '', 'success': True}\",\n", + " 'tool_success_result': 'pass',\n", + " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '14\\\\\\\\u00b0C', 'condition': \"\n", + " \"'rainy', 'precipitation': 'light \"\n", + " \"rain'}\\\\\\\\n[TOOL_CALL] \"\n", + " 'send_email(recipient=\\\\\\\\\\\\\"your_email@example.com\\\\\\\\\\\\\", '\n", + " 'subject=\\\\\\\\\\\\\"Weather Information for '\n", + " 'Seattle\\\\\\\\\\\\\", body=\\\\\\\\\\\\\"Current weather: '\n", + " '14\\\\\\\\u00b0C and rainy with light '\n", + " 'rain.\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] {\\'message\\': '\n", + " \"'Email successfully sent to \"\n", + " \"your_email@example.com.', 'status': \"\n", + " '\\'delivered\\'}\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", + " 'Fetches the weather information for the '\n", + " 'specified location. (inputs: location)\\\\\\\\n- '\n", + " 'send_email: Sends an email with the specified '\n", + " 'subject and body to the recipient. (inputs: '\n", + " 'recipient, subject, body)\\\\\"}\"}]',\n", + " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"Both tool results '\n", + " 'returned valid, non-empty responses with no '\n", + " 'indication of technical errors, exceptions, or '\n", + " 'failures. The fetch_weather result contains '\n", + " 'weather data, and send_email confirms '\n", + " 'successful delivery.\\\\\",\\\\n \\\\\"details\\\\\": '\n", + " '{\\\\n \\\\\"failed_tools\\\\\": \\\\\"\\\\\",\\\\n '\n", + " '\\\\\"success\\\\\": true\\\\n },\\\\n '\n", + " '\\\\\"success\\\\\": true\\\\n}\"}]',\n", + " 'tool_success_threshold': 1,\n", + " 'tool_success_total_tokens': 2558}\n" + ] + } + ], + "source": [ + "# Multiple successful tool executions\n", + "multiple_successful_response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\", \"precipitation\": \"light rain\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"your_email@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Current weather: 14°C and rainy with light rain.\",\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\", \"status\": \"delivered\"}}\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_call_success(response=multiple_successful_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "63e43736", + "metadata": {}, + "source": [ + "#### Tool Call Success without Tool Definitions" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "a0ca062e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to filter tool definitions, returning original list. Error: 'NoneType' object is not iterable\n", + "Tool definitions could not be parsed, falling back to original definitions: None\n", + "Tool definitions could not be parsed, falling back to original definitions: None\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_success': 1.0,\n", + " 'tool_success_completion_tokens': 58,\n", + " 'tool_success_finish_reason': 'stop',\n", + " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_success_prompt_tokens': 2412,\n", + " 'tool_success_reason': 'The tool get_calendar returned a result containing a '\n", + " 'list of events without any indication of error, '\n", + " \"exception, or technical failure. {'failed_tools': '', \"\n", + " \"'success': True}\",\n", + " 'tool_success_result': 'pass',\n", + " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", + " 'get_calendar(start_date=\\\\\\\\\\\\\"2025-01-01\\\\\\\\\\\\\", '\n", + " 'end_date=\\\\\\\\\\\\\"2025-01-31\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'events': [{'date': '2025-01-15', 'title': \"\n", + " \"'Team Meeting'}, {'date': '2025-01-20', \"\n", + " '\\'title\\': \\'Project Review\\'}]}\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": null}\"}]',\n", + " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The tool get_calendar '\n", + " 'returned a result containing a list of events '\n", + " 'without any indication of error, exception, or '\n", + " 'technical failure.\\\\\",\\\\n \\\\\"details\\\\\": '\n", + " '{\\\\n \\\\\"failed_tools\\\\\": \\\\\"\\\\\",\\\\n '\n", + " '\\\\\"success\\\\\": true\\\\n },\\\\n '\n", + " '\\\\\"success\\\\\": true\\\\n}\"}]',\n", + " 'tool_success_threshold': 1,\n", + " 'tool_success_total_tokens': 2470}\n" + ] + } + ], + "source": [ + "# Successful execution without providing tool definitions\n", + "simple_successful_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_calendar_123\",\n", + " \"name\": \"get_calendar\",\n", + " \"arguments\": {\"start_date\": \"2025-01-01\", \"end_date\": \"2025-01-31\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_calendar_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"events\": [{\"date\": \"2025-01-15\", \"title\": \"Team Meeting\"}, {\"date\": \"2025-01-20\", \"title\": \"Project Review\"}]}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I found 2 events in your calendar for January: a Team Meeting on the 15th and a Project Review on the 20th.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "# Evaluation without tool definitions\n", + "result = tool_call_success(response=simple_successful_response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "97c1af0f", + "metadata": {}, + "source": [ + "#### Example of Failed Tool Call" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "cafdee8e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_success': 0.0,\n", + " 'tool_success_completion_tokens': 79,\n", + " 'tool_success_finish_reason': 'stop',\n", + " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_success_prompt_tokens': 2398,\n", + " 'tool_success_reason': \"The fetch_weather tool result contains an 'error' \"\n", + " \"field with 'Location not found', a 'status' of \"\n", + " \"'failed', and a 404 code. This clearly indicates a \"\n", + " \"technical failure in the tool call. {'failed_tools': \"\n", + " \"'fetch_weather', 'success': False}\",\n", + " 'tool_success_result': 'fail',\n", + " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"InvalidCity\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'error': 'Location not found', 'status': \"\n", + " '\\'failed\\', \\'code\\': 404}\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", + " 'Fetches the weather information for the '\n", + " 'specified location. (inputs: location)\\\\\"}\"}]',\n", + " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The fetch_weather tool '\n", + " \"result contains an 'error' field with \"\n", + " \"'Location not found', a 'status' of 'failed', \"\n", + " 'and a 404 code. This clearly indicates a '\n", + " 'technical failure in the tool call.\\\\\",\\\\n '\n", + " '\\\\\"details\\\\\": {\\\\n \\\\\"failed_tools\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\",\\\\n \\\\\"success\\\\\": '\n", + " 'false\\\\n },\\\\n \\\\\"success\\\\\": '\n", + " 'false\\\\n}\"}]',\n", + " 'tool_success_threshold': 1,\n", + " 'tool_success_total_tokens': 2477}\n" + ] + } + ], + "source": [ + "# Failed tool execution with error\n", + "failed_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"InvalidCity\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Location not found\", \"status\": \"failed\", \"code\": 404}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I'm sorry, I couldn't retrieve the weather information due to an error.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score FALSE due to tool failure\n", + "result = tool_call_success(response=failed_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "67694403", + "metadata": {}, + "source": [ + "#### Example of Tool Call Timeout" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "19c7f49d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_success': 0.0,\n", + " 'tool_success_completion_tokens': 80,\n", + " 'tool_success_finish_reason': 'stop',\n", + " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_success_prompt_tokens': 2422,\n", + " 'tool_success_reason': \"The result for send_email contains an 'error' field \"\n", + " \"with 'Request timeout' and a 'status' of 'timeout', \"\n", + " 'which indicates a technical failure due to timeout. '\n", + " 'Therefore, the send_email tool call failed. '\n", + " \"{'failed_tools': 'send_email', 'success': False}\",\n", + " 'tool_success_result': 'fail',\n", + " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", + " 'send_email(recipient=\\\\\\\\\\\\\"test@example.com\\\\\\\\\\\\\", '\n", + " 'subject=\\\\\\\\\\\\\"Test Email\\\\\\\\\\\\\", '\n", + " 'body=\\\\\\\\\\\\\"This is a test '\n", + " 'email.\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] {\\'error\\': '\n", + " \"'Request timeout', 'status': 'timeout', \"\n", + " '\\'timeout_duration\\': \\'30s\\'}\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- send_email: Sends an '\n", + " 'email with the specified subject and body to '\n", + " 'the recipient. (inputs: recipient, subject, '\n", + " 'body)\\\\\"}\"}]',\n", + " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The result for '\n", + " \"send_email contains an 'error' field with \"\n", + " \"'Request timeout' and a 'status' of 'timeout', \"\n", + " 'which indicates a technical failure due to '\n", + " 'timeout. Therefore, the send_email tool call '\n", + " 'failed.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"failed_tools\\\\\": \\\\\"send_email\\\\\",\\\\n '\n", + " '\\\\\"success\\\\\": false\\\\n },\\\\n '\n", + " '\\\\\"success\\\\\": false\\\\n}\"}]',\n", + " 'tool_success_threshold': 1,\n", + " 'tool_success_total_tokens': 2502}\n" + ] + } + ], + "source": [ + "# Tool execution timeout\n", + "timeout_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_789\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"test@example.com\",\n", + " \"subject\": \"Test Email\",\n", + " \"body\": \"This is a test email.\"\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_email_789\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Request timeout\", \"status\": \"timeout\", \"timeout_duration\": \"30s\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I'm unable to send the email due to a timeout error. Please try again later.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score FALSE due to timeout\n", + "result = tool_call_success(response=timeout_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "8a5e147a", + "metadata": {}, + "source": [ + "#### Mixed Results - Some Success, Some Failure" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "c73b8611", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_success': 0.0,\n", + " 'tool_success_completion_tokens': 71,\n", + " 'tool_success_finish_reason': 'stop',\n", + " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_success_prompt_tokens': 2444,\n", + " 'tool_success_reason': 'The fetch_weather tool returned a valid weather '\n", + " 'result, indicating success. The send_email tool '\n", + " \"returned an error message and status 'failed', which \"\n", + " 'is a technical failure. Therefore, the evaluation '\n", + " \"process has failed. {'failed_tools': 'send_email'}\",\n", + " 'tool_success_result': 'fail',\n", + " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '15\\\\\\\\u00b0C', 'condition': \"\n", + " \"'sunny'}\\\\\\\\n[TOOL_CALL] \"\n", + " 'send_email(recipient=\\\\\\\\\\\\\"invalid-email\\\\\\\\\\\\\", '\n", + " 'subject=\\\\\\\\\\\\\"Weather Report\\\\\\\\\\\\\", '\n", + " 'body=\\\\\\\\\\\\\"Weather is sunny in '\n", + " 'Seattle.\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] {\\'error\\': '\n", + " \"'Invalid email format', 'status': \"\n", + " '\\'failed\\'}\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- send_email: Sends an '\n", + " 'email with the specified subject and body to '\n", + " 'the recipient. (inputs: recipient, subject, '\n", + " 'body)\\\\\"}\"}]',\n", + " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The fetch_weather tool '\n", + " 'returned a valid weather result, indicating '\n", + " 'success. The send_email tool returned an error '\n", + " \"message and status 'failed', which is a \"\n", + " 'technical failure. Therefore, the evaluation '\n", + " 'process has failed.\\\\\",\\\\n \\\\\"details\\\\\": '\n", + " '{\\\\n \\\\\"failed_tools\\\\\": '\n", + " '\\\\\"send_email\\\\\"\\\\n },\\\\n \\\\\"success\\\\\": '\n", + " 'false\\\\n}\"}]',\n", + " 'tool_success_threshold': 1,\n", + " 'tool_success_total_tokens': 2515}\n" + ] + } + ], + "source": [ + "# Mixed results - one success, one failure\n", + "mixed_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_success\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_success\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"sunny\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_fail\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"invalid-email\",\n", + " \"subject\": \"Weather Report\",\n", + " \"body\": \"Weather is sunny in Seattle.\"\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_email_fail\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Invalid email format\", \"status\": \"failed\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I got the weather information but failed to send the email due to an invalid email address.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "# This should score FALSE because at least one tool call failed\n", + "result = tool_call_success(response=mixed_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb new file mode 100644 index 00000000..57405431 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb @@ -0,0 +1,581 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8d6b6cf9", + "metadata": {}, + "source": [ + "# Tool Input Accuracy Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use tool input accuracy evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing tool calls;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "74ac1b5c", + "metadata": {}, + "source": [ + "The Tool Input Accuracy evaluator performs a strict binary evaluation (PASS/FAIL) of parameters passed to tool calls. It ensures that ALL parameters meet ALL criteria:\n", + "\n", + "- Parameter grounding: All parameters must be derived from conversation history/query\n", + "- Type compliance: All parameters must match exact types specified in tool definitions\n", + "- Format compliance: All parameters must follow exact format and structure requirements\n", + "- Completeness: All required parameters must be provided\n", + "- No unexpected parameters: Only defined parameters are allowed\n", + "\n", + "The evaluator uses strict binary evaluation:\n", + "\n", + " - 1: Only when ALL criteria are satisfied perfectly for ALL parameters\n", + " - 0: When ANY criterion fails for ANY parameter\n", + "\n", + "This evaluation focuses on ensuring tool call parameters are completely correct without any tolerance for partial correctness." + ] + }, + { + "cell_type": "markdown", + "id": "17346632", + "metadata": {}, + "source": [ + "Tool Input Accuracy requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator extracts tool calls from the response.\n", + "- Tool Definitions - Tool(s) definition used by Agent to answer the query. Required to validate parameter types and structures.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d78194ee", + "metadata": {}, + "source": [ + "### Initialize Tool Input Accuracy Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1663a6b1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._tool_input_accuracy import _ToolInputAccuracyEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "\n", + "tool_input_accuracy = _ToolInputAccuracyEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "f92e8378", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "ba5f6441", + "metadata": {}, + "source": [ + "#### Evaluating Correct Tool Input Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "486057ca", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "e9275f1c", + "metadata": {}, + "source": [ + "#### Tool Definition as Single Dict" + ] + }, + { + "cell_type": "markdown", + "id": "80fc6e82", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "626462d3", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Check the weather in Miami\"\n", + "\n", + "# Response as a simple string containing tool call information\n", + "response_str = \"\"\"I'll check the weather for you. Calling fetch_weather with location=\"Miami\".\"\"\"\n", + "\n", + "tool_definition = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + "}\n", + "\n", + "result = tool_input_accuracy(query=query, response=response_str, tool_definitions=tool_definition)\n", + "pprint(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91e1e4d3", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"What's the temperature in Boston?\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_boston_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Boston\"},\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + "}\n", + "\n", + "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "c351c61a", + "metadata": {}, + "source": [ + "#### Complex Tool Parameters with Multiple Fields" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b1147ab", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you send an email to john@example.com with the subject 'Weather Update' and tell him the weather in Seattle is 15°C and partly cloudy?\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"john@example.com\",\n", + " \"subject\": \"Weather Update\",\n", + " \"body\": \"The weather in Seattle is 15°C and partly cloudy.\"\n", + " },\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " \"required\": [\"recipient\", \"subject\", \"body\"]\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "52468ac9", + "metadata": {}, + "source": [ + "#### Multiple Tool Calls with Different Parameter Requirements" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a341071b", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Rainy, 14°C\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"your_email@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"The current weather in Seattle is rainy with a temperature of 14°C.\",\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\"}}\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " \"required\": [\"recipient\", \"subject\", \"body\"]\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "2b274da2", + "metadata": {}, + "source": [ + "#### Query as Conversation History (List of Messages)\n", + "The evaluator also supports query as a list of messages representing conversation history. This helps validate parameters are grounded in the conversation context." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a90e8827", + "metadata": {}, + "outputs": [], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Please send the results to admin@company.com with the subject 'Daily Weather Report'.\"\n", + " }\n", + "]\n", + "\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\", \n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"admin@company.com\",\n", + " \"subject\": \"Daily Weather Report\",\n", + " \"body\": \"Weather information for Seattle as requested.\"\n", + " },\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " \"required\": [\"recipient\", \"subject\", \"body\"]\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_input_accuracy(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "14c41e90", + "metadata": {}, + "source": [ + "#### Example of Incorrect Tool Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59810a22", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "# Missing required parameter or wrong type\n", + "incorrect_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"city\": \"Seattle\"}, # Wrong parameter name (should be \"location\")\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score 0 due to incorrect parameter name\n", + "result = tool_input_accuracy(query=query, response=incorrect_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "b4a866e0", + "metadata": {}, + "source": [ + "#### Example of Missing Required Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "159cf7e5", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Send an email to john@example.com with weather info\"\n", + "# Missing required parameters\n", + "incomplete_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\"recipient\": \"john@example.com\"}, # Missing subject and body\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " \"required\": [\"recipient\", \"subject\", \"body\"]\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score 0 due to missing required parameters\n", + "result = tool_input_accuracy(query=query, response=incomplete_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb new file mode 100644 index 00000000..43880212 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb @@ -0,0 +1,1082 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d638561f", + "metadata": {}, + "source": [ + "# Tool Output Utilization Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use tool output utilization evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing agent responses;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "41d3647c", + "metadata": {}, + "source": [ + "The Tool Output Utilization Evaluator assesses how effectively an AI agent utilizes the outputs from tools and whether it accurately incorporates this information into its responses.\n", + "\n", + "Scoring is based on two levels:\n", + "1. Pass - The agent effectively utilizes tool outputs and accurately incorporates the information into its response.\n", + "2. Fail - The agent fails to properly utilize tool outputs or incorrectly incorporates the information into its response.\n", + "\n", + "The evaluation includes the score, a brief explanation, and a final pass/fail result.\n", + "\n", + "This evaluation focuses on measuring whether the agent properly uses the information returned by tools to provide accurate and complete responses to users." + ] + }, + { + "cell_type": "markdown", + "id": "246c7bcd", + "metadata": {}, + "source": [ + "Tool Output Utilization requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response.\n", + "- Tool Definitions - Tool(s) definition used by Agent to answer the query. Required to understand the context and expected outputs of the tools.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d386d576", + "metadata": {}, + "source": [ + "### Initialize Tool Output Utilization Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "087dd96d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Class _ToolOutputUtilizationEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" + ] + } + ], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._tool_output_utilization import _ToolOutputUtilizationEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "\n", + "tool_output_utilization = _ToolOutputUtilizationEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "c7b4f98a", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "9ee00e9d", + "metadata": {}, + "source": [ + "#### Evaluating Good Tool Output Utilization" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "017a7afe", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_output_utilization': 1.0,\n", + " 'tool_output_utilization_completion_tokens': 53,\n", + " 'tool_output_utilization_finish_reason': 'stop',\n", + " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_output_utilization_prompt_tokens': 1963,\n", + " 'tool_output_utilization_reason': 'All tool output values (temperature, '\n", + " 'condition, humidity, wind) are correctly '\n", + " 'reflected in the response with accurate '\n", + " 'paraphrasing and no omissions or '\n", + " 'fabrications.',\n", + " 'tool_output_utilization_result': 'pass',\n", + " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"How is the weather '\n", + " 'in Seattle?\\\\\", \\\\\"response\\\\\": '\n", + " '\\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '15\\\\\\\\u00b0C', \"\n", + " \"'condition': 'partly cloudy', \"\n", + " \"'humidity': '68%', 'wind': '8 mph \"\n", + " \"NW'}\\\\\\\\nThe current weather in \"\n", + " 'Seattle is partly cloudy with a '\n", + " 'temperature of 15\\\\\\\\u00b0C. The '\n", + " \"humidity is at 68% and there's a \"\n", + " 'light wind from the northwest at 8 '\n", + " 'mph.\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", + " 'fetch_weather: Fetches the weather '\n", + " 'information for the specified '\n", + " 'location. (inputs: location)\\\\\"}\"}]',\n", + " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", + " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", + " '[],\\\\n \\\\\"reason\\\\\": \\\\\"All tool '\n", + " 'output values (temperature, '\n", + " 'condition, humidity, wind) are '\n", + " 'correctly reflected in the response '\n", + " 'with accurate paraphrasing and no '\n", + " 'omissions or fabrications.\\\\\",\\\\n '\n", + " '\\\\\"label\\\\\": \\\\\"pass\\\\\"\\\\n}\"}]',\n", + " 'tool_output_utilization_threshold': 1,\n", + " 'tool_output_utilization_total_tokens': 2016}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C. The humidity is at 68% and there's a light wind from the northwest at 8 mph.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "78270b9b", + "metadata": {}, + "source": [ + "#### Complex Scenario with Multiple Tool Outputs" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "e8543c52", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", + "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n", + "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_output_utilization': 1.0,\n", + " 'tool_output_utilization_completion_tokens': 51,\n", + " 'tool_output_utilization_finish_reason': 'stop',\n", + " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_output_utilization_prompt_tokens': 2079,\n", + " 'tool_output_utilization_reason': 'All tool outputs (weather details and '\n", + " 'email delivery status) were correctly '\n", + " 'reflected in the response and reused '\n", + " 'accurately in the email body and '\n", + " 'user-facing message.',\n", + " 'tool_output_utilization_result': 'pass',\n", + " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"Can you send me an '\n", + " 'email with weather information for '\n", + " 'Seattle?\\\\\", \\\\\"response\\\\\": '\n", + " '\\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '14\\\\\\\\u00b0C', \"\n", + " \"'condition': 'rainy', \"\n", + " \"'precipitation': 'light rain', \"\n", + " \"'wind': '12 mph SW'}\\\\\\\\n[TOOL_CALL] \"\n", + " 'send_email(recipient=\\\\\\\\\\\\\"your_email@example.com\\\\\\\\\\\\\", '\n", + " 'subject=\\\\\\\\\\\\\"Weather Information '\n", + " 'for Seattle\\\\\\\\\\\\\", '\n", + " 'body=\\\\\\\\\\\\\"Current weather in '\n", + " 'Seattle: 14\\\\\\\\u00b0C and rainy with '\n", + " 'light rain. Wind is from the '\n", + " 'southwest at 12 mph. Stay '\n", + " 'dry!\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'message': 'Email successfully sent \"\n", + " \"to your_email@example.com.', \"\n", + " \"'status': 'delivered'}\\\\\\\\nI have \"\n", + " 'successfully sent you an email with '\n", + " 'the weather information for Seattle. '\n", + " 'The email includes the current '\n", + " 'conditions: 14\\\\\\\\u00b0C and rainy '\n", + " 'with light rain, plus wind details. '\n", + " 'The email was delivered successfully '\n", + " 'to your inbox.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", + " 'fetch_weather: Fetches the weather '\n", + " 'information for the specified '\n", + " 'location. (inputs: location)\\\\\\\\n- '\n", + " 'send_email: Sends an email with the '\n", + " 'specified subject and body to the '\n", + " 'recipient. (inputs: recipient, '\n", + " 'subject, body)\\\\\"}\"}]',\n", + " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", + " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", + " '[],\\\\n \\\\\"reason\\\\\": \\\\\"All tool '\n", + " 'outputs (weather details and email '\n", + " 'delivery status) were correctly '\n", + " 'reflected in the response and '\n", + " 'reused accurately in the email body '\n", + " 'and user-facing message.\\\\\",\\\\n '\n", + " '\\\\\"label\\\\\": \\\\\"pass\\\\\"\\\\n}\"}]',\n", + " 'tool_output_utilization_threshold': 1,\n", + " 'tool_output_utilization_total_tokens': 2130}\n" + ] + } + ], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\", \"precipitation\": \"light rain\", \"wind\": \"12 mph SW\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"your_email@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Current weather in Seattle: 14°C and rainy with light rain. Wind is from the southwest at 12 mph. Stay dry!\",\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\", \"status\": \"delivered\"}}\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The email includes the current conditions: 14°C and rainy with light rain, plus wind details. The email was delivered successfully to your inbox.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "1c52f24c", + "metadata": {}, + "source": [ + "#### Query as Conversation History" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "b3b9bef5", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: [{'role': 'system', 'content': 'You are a helpful assistant that can fetch weather information and send emails.'}, {'role': 'user', 'content': 'Hi, I need to know the weather in Seattle for my trip planning.'}, {'role': 'user', 'content': 'Can you also email me a summary so I can reference it later when packing?'}]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_output_utilization': 1.0,\n", + " 'tool_output_utilization_completion_tokens': 57,\n", + " 'tool_output_utilization_finish_reason': 'stop',\n", + " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_output_utilization_prompt_tokens': 2189,\n", + " 'tool_output_utilization_reason': 'All tool-derived information in the '\n", + " 'response, including weather details and '\n", + " 'email delivery status, matches the prior '\n", + " 'tool outputs and is correctly incorporated '\n", + " 'into both the textual summary and the '\n", + " 'email body.',\n", + " 'tool_output_utilization_result': 'pass',\n", + " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": [{\\\\\"role\\\\\": '\n", + " '\\\\\"system\\\\\", \\\\\"content\\\\\": \\\\\"You '\n", + " 'are a helpful assistant that can '\n", + " 'fetch weather information and send '\n", + " 'emails.\\\\\"}, {\\\\\"role\\\\\": '\n", + " '\\\\\"user\\\\\", \\\\\"content\\\\\": \\\\\"Hi, I '\n", + " 'need to know the weather in Seattle '\n", + " 'for my trip planning.\\\\\"}, '\n", + " '{\\\\\"role\\\\\": \\\\\"user\\\\\", '\n", + " '\\\\\"content\\\\\": \\\\\"Can you also email '\n", + " 'me a summary so I can reference it '\n", + " 'later when packing?\\\\\"}], '\n", + " '\\\\\"response\\\\\": \\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '15\\\\\\\\u00b0C', \"\n", + " \"'condition': 'partly cloudy', \"\n", + " \"'humidity': '68%', 'wind': '8 mph \"\n", + " \"NW', 'forecast': 'no rain \"\n", + " \"expected'}\\\\\\\\n[TOOL_CALL] \"\n", + " 'send_email(recipient=\\\\\\\\\\\\\"user@example.com\\\\\\\\\\\\\", '\n", + " 'subject=\\\\\\\\\\\\\"Seattle Weather '\n", + " 'Summary for Trip Planning\\\\\\\\\\\\\", '\n", + " 'body=\\\\\\\\\\\\\"Weather summary for '\n", + " 'Seattle: Temperature is '\n", + " '15\\\\\\\\u00b0C, partly cloudy '\n", + " 'conditions, humidity at 68%, with '\n", + " 'northwest winds at 8 mph. No rain is '\n", + " 'expected, so light layers should be '\n", + " 'sufficient for your '\n", + " 'trip.\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'message': 'Email sent \"\n", + " \"successfully', 'delivery_status': \"\n", + " \"'delivered'}\\\\\\\\nPerfect! I've \"\n", + " 'checked the weather in Seattle for '\n", + " \"your trip planning. It's currently \"\n", + " '15\\\\\\\\u00b0C and partly cloudy with '\n", + " 'light northwest winds and no rain '\n", + " \"expected. I've also sent you an \"\n", + " 'email summary with all these details '\n", + " 'including packing recommendations '\n", + " 'based on the weather conditions. The '\n", + " 'email was delivered successfully, so '\n", + " 'you can reference it when preparing '\n", + " 'for your trip.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", + " 'fetch_weather: Fetches the weather '\n", + " 'information for the specified '\n", + " 'location. (inputs: location)\\\\\\\\n- '\n", + " 'send_email: Sends an email with the '\n", + " 'specified subject and body to the '\n", + " 'recipient. (inputs: recipient, '\n", + " 'subject, body)\\\\\"}\"}]',\n", + " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", + " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", + " '[],\\\\n \\\\\"reason\\\\\": \\\\\"All '\n", + " 'tool-derived information in the '\n", + " 'response, including weather details '\n", + " 'and email delivery status, matches '\n", + " 'the prior tool outputs and is '\n", + " 'correctly incorporated into both '\n", + " 'the textual summary and the email '\n", + " 'body.\\\\\",\\\\n \\\\\"label\\\\\": '\n", + " '\\\\\"pass\\\\\"\\\\n}\"}]',\n", + " 'tool_output_utilization_threshold': 1,\n", + " 'tool_output_utilization_total_tokens': 2246}\n" + ] + } + ], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, I need to know the weather in Seattle for my trip planning.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you also email me a summary so I can reference it later when packing?\"\n", + " }\n", + "]\n", + "\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\", \"forecast\": \"no rain expected\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"user@example.com\",\n", + " \"subject\": \"Seattle Weather Summary for Trip Planning\",\n", + " \"body\": \"Weather summary for Seattle: Temperature is 15°C, partly cloudy conditions, humidity at 68%, with northwest winds at 8 mph. No rain is expected, so light layers should be sufficient for your trip.\"\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email sent successfully\", \"delivery_status\": \"delivered\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"Perfect! I've checked the weather in Seattle for your trip planning. It's currently 15°C and partly cloudy with light northwest winds and no rain expected. I've also sent you an email summary with all these details including packing recommendations based on the weather conditions. The email was delivered successfully, so you can reference it when preparing for your trip.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_output_utilization(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "199a11bc", + "metadata": {}, + "source": [ + "#### Example of Poor Tool Output Utilization" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "6be39269", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_output_utilization': 0.0,\n", + " 'tool_output_utilization_completion_tokens': 93,\n", + " 'tool_output_utilization_finish_reason': 'stop',\n", + " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_output_utilization_prompt_tokens': 1954,\n", + " 'tool_output_utilization_reason': \"The agent's response omits all specific \"\n", + " 'factual details from the tool output '\n", + " '(temperature, condition, humidity, wind, '\n", + " 'UV index, visibility) and instead provides '\n", + " 'only a vague summary, failing to correctly '\n", + " 'use the tool-derived information. Issues '\n", + " 'found: claim -> OMISSION (expected factual '\n", + " 'details from tool output, saw only vague '\n", + " 'summary) mapped to fetch_weather',\n", + " 'tool_output_utilization_result': 'fail',\n", + " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"How is the weather '\n", + " 'in Seattle?\\\\\", \\\\\"response\\\\\": '\n", + " '\\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '15\\\\\\\\u00b0C', \"\n", + " \"'condition': 'partly cloudy', \"\n", + " \"'humidity': '68%', 'wind': '8 mph \"\n", + " \"NW', 'UV_index': '3', 'visibility': \"\n", + " \"'10 km'}\\\\\\\\nThe weather in Seattle \"\n", + " \"is okay. It's not too bad \"\n", + " 'outside.\\\\\", \\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", + " 'fetch_weather: Fetches the weather '\n", + " 'information for the specified '\n", + " 'location. (inputs: location)\\\\\"}\"}]',\n", + " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", + " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", + " '[\\\\n \\\\\"claim -> OMISSION '\n", + " '(expected factual details from tool '\n", + " 'output, saw only vague summary) '\n", + " 'mapped to fetch_weather\\\\\"\\\\n '\n", + " '],\\\\n \\\\\"reason\\\\\": \\\\\"The '\n", + " \"agent's response omits all specific \"\n", + " 'factual details from the tool '\n", + " 'output (temperature, condition, '\n", + " 'humidity, wind, UV index, '\n", + " 'visibility) and instead provides '\n", + " 'only a vague summary, failing to '\n", + " 'correctly use the tool-derived '\n", + " 'information.\\\\\",\\\\n \\\\\"label\\\\\": '\n", + " '\\\\\"fail\\\\\"\\\\n}\"}]',\n", + " 'tool_output_utilization_threshold': 1,\n", + " 'tool_output_utilization_total_tokens': 2047}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "# Poor utilization - agent doesn't use the detailed tool output effectively\n", + "poor_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\", \"UV_index\": \"3\", \"visibility\": \"10 km\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The weather in Seattle is okay. It's not too bad outside.\", # Poor utilization of detailed weather data\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score poorly due to inadequate use of tool output\n", + "result = tool_output_utilization(query=query, response=poor_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "6b02ba5f", + "metadata": {}, + "source": [ + "#### Example of Incorrect Tool Output Utilization" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "a524ee56", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_output_utilization': 0.0,\n", + " 'tool_output_utilization_completion_tokens': 147,\n", + " 'tool_output_utilization_finish_reason': 'stop',\n", + " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_output_utilization_prompt_tokens': 1944,\n", + " 'tool_output_utilization_reason': \"The agent's response misrepresented both \"\n", + " 'the weather condition and temperature, '\n", + " \"fabricating 'sunny and hot with 25°C' \"\n", + " 'instead of using the tool output of '\n", + " \"'partly cloudy' and '15°C'. Issues found: \"\n", + " 'claim -> MISMATCH (expected partly cloudy '\n", + " 'and 15°C, saw sunny and hot with 25°C) '\n", + " 'mapped to fetch_weather.condition; claim '\n", + " '-> MISMATCH (expected 15°C, saw 25°C) '\n", + " 'mapped to fetch_weather.temperature; claim '\n", + " '-> FABRICATED (no supporting tool field; '\n", + " \"'sunny and hot' not present in tool \"\n", + " 'output)',\n", + " 'tool_output_utilization_result': 'fail',\n", + " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"How is the weather '\n", + " 'in Seattle?\\\\\", \\\\\"response\\\\\": '\n", + " '\\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '15\\\\\\\\u00b0C', \"\n", + " \"'condition': 'partly cloudy', \"\n", + " \"'humidity': '68%'}\\\\\\\\nThe weather \"\n", + " 'in Seattle is currently sunny and '\n", + " 'hot with temperatures reaching '\n", + " \"25\\\\\\\\u00b0C. It's a perfect day for \"\n", + " 'outdoor activities!\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", + " 'fetch_weather: Fetches the weather '\n", + " 'information for the specified '\n", + " 'location. (inputs: location)\\\\\"}\"}]',\n", + " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", + " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", + " '[\\\\n \\\\\"claim -> MISMATCH '\n", + " '(expected partly cloudy and '\n", + " '15\\\\u00b0C, saw sunny and hot with '\n", + " '25\\\\u00b0C) mapped to '\n", + " 'fetch_weather.condition\\\\\",\\\\n '\n", + " '\\\\\"claim -> MISMATCH (expected '\n", + " '15\\\\u00b0C, saw 25\\\\u00b0C) mapped '\n", + " 'to '\n", + " 'fetch_weather.temperature\\\\\",\\\\n '\n", + " '\\\\\"claim -> FABRICATED (no '\n", + " \"supporting tool field; 'sunny and \"\n", + " \"hot' not present in tool \"\n", + " 'output)\\\\\"\\\\n ],\\\\n \\\\\"reason\\\\\": '\n", + " '\\\\\"The agent\\'s response '\n", + " 'misrepresented both the weather '\n", + " 'condition and temperature, '\n", + " \"fabricating 'sunny and hot with \"\n", + " \"25\\\\u00b0C' instead of using the \"\n", + " \"tool output of 'partly cloudy' and \"\n", + " '\\'15\\\\u00b0C\\'.\\\\\",\\\\n '\n", + " '\\\\\"label\\\\\": \\\\\"fail\\\\\"\\\\n}\"}]',\n", + " 'tool_output_utilization_threshold': 1,\n", + " 'tool_output_utilization_total_tokens': 2091}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "# Incorrect utilization - agent misrepresents the tool output\n", + "incorrect_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The weather in Seattle is currently sunny and hot with temperatures reaching 25°C. It's a perfect day for outdoor activities!\", # Incorrect representation of tool output\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "# This should score poorly due to incorrect use of tool output\n", + "result = tool_output_utilization(query=query, response=incorrect_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "bcb8a2cf", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "0845822f", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", + "Conversation history could not be parsed, falling back to original query: What's the weather like in Seattle?\n", + "Agent response could not be parsed, falling back to original response: The weather in Seattle is currently 15°C and partly cloudy with light winds from the northwest.\n", + "Conversation history could not be parsed, falling back to original query: What's the weather like in Seattle?\n", + "Agent response could not be parsed, falling back to original response: The weather in Seattle is currently 15°C and partly cloudy with light winds from the northwest.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_output_utilization': 0.0,\n", + " 'tool_output_utilization_completion_tokens': 66,\n", + " 'tool_output_utilization_finish_reason': 'stop',\n", + " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_output_utilization_prompt_tokens': 1901,\n", + " 'tool_output_utilization_reason': \"The agent's response contains weather \"\n", + " 'details for Seattle, but there is no '\n", + " 'evidence of any tool output being used or '\n", + " 'referenced; all claims are fabricated. '\n", + " 'Issues found: claim -> FABRICATED (no '\n", + " 'supporting tool field)',\n", + " 'tool_output_utilization_result': 'fail',\n", + " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"What\\'s the '\n", + " 'weather like in Seattle?\\\\\", '\n", + " '\\\\\"response\\\\\": \\\\\"The weather in '\n", + " 'Seattle is currently 15\\\\\\\\u00b0C '\n", + " 'and partly cloudy with light winds '\n", + " 'from the northwest.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": '\n", + " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", + " 'fetch_weather: Fetches the weather '\n", + " 'information for the specified '\n", + " 'location. (inputs: location)\\\\\"}\"}]',\n", + " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", + " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", + " '[\\\\n \\\\\"claim -> FABRICATED (no '\n", + " 'supporting tool field)\\\\\"\\\\n '\n", + " '],\\\\n \\\\\"reason\\\\\": \\\\\"The '\n", + " \"agent's response contains weather \"\n", + " 'details for Seattle, but there is '\n", + " 'no evidence of any tool output '\n", + " 'being used or referenced; all '\n", + " 'claims are fabricated.\\\\\",\\\\n '\n", + " '\\\\\"label\\\\\": \\\\\"fail\\\\\"\\\\n}\"}]',\n", + " 'tool_output_utilization_threshold': 1,\n", + " 'tool_output_utilization_total_tokens': 1967}\n" + ] + } + ], + "source": [ + "query = \"What's the weather like in Seattle?\"\n", + "# Response as a simple string (not a list of messages)\n", + "response_str = \"The weather in Seattle is currently 15°C and partly cloudy with light winds from the northwest.\"\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_output_utilization(query=query, response=response_str, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "b4cb9ddd", + "metadata": {}, + "source": [ + "#### Tool Definitions as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "6424b3bb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", + "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n", + "Conversation history could not be parsed, falling back to original query: Check the weather in Seattle\n", + "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n", + "Conversation history could not be parsed, falling back to original query: Check the weather in Seattle\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_output_utilization': 1.0,\n", + " 'tool_output_utilization_completion_tokens': 48,\n", + " 'tool_output_utilization_finish_reason': 'stop',\n", + " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_output_utilization_prompt_tokens': 1957,\n", + " 'tool_output_utilization_reason': \"The agent's response accurately reflects \"\n", + " 'the tool output, correctly reporting both '\n", + " 'the temperature and condition for Seattle '\n", + " 'as provided by the tool result.',\n", + " 'tool_output_utilization_result': 'pass',\n", + " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", + " '\"{\\\\\"query\\\\\": \\\\\"Check the weather '\n", + " 'in Seattle\\\\\", \\\\\"response\\\\\": '\n", + " '\\\\\"[TOOL_CALL] '\n", + " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", + " \"{'temperature': '14\\\\\\\\u00b0C', \"\n", + " \"'condition': 'rainy'}\\\\\\\\nThe \"\n", + " 'current weather in Seattle is rainy '\n", + " 'with a temperature of '\n", + " '14\\\\\\\\u00b0C.\\\\\", '\n", + " '\\\\\"tool_definitions\\\\\": {\\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Fetches the '\n", + " 'weather information for the '\n", + " 'specified location.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"The location to fetch weather '\n", + " 'for.\\\\\"}}}}}\"}]',\n", + " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", + " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", + " '[],\\\\n \\\\\"reason\\\\\": \\\\\"The '\n", + " \"agent's response accurately \"\n", + " 'reflects the tool output, correctly '\n", + " 'reporting both the temperature and '\n", + " 'condition for Seattle as provided '\n", + " 'by the tool result.\\\\\",\\\\n '\n", + " '\\\\\"label\\\\\": \\\\\"pass\\\\\"\\\\n}\"}]',\n", + " 'tool_output_utilization_threshold': 1,\n", + " 'tool_output_utilization_total_tokens': 2005}\n" + ] + } + ], + "source": [ + "query = \"Check the weather in Seattle\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The current weather in Seattle is rainy with a temperature of 14°C.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "# Tool definition as a single dict (not a list)\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb new file mode 100644 index 00000000..04fd5e48 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb @@ -0,0 +1,1278 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "862760b0", + "metadata": {}, + "source": [ + "# Tool Selection Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use tool selection evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing tool calls;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "31a75ee6", + "metadata": {}, + "source": [ + "The Tool Selection evaluator assesses the appropriateness and efficiency of tool choices made by an AI agent by examining:\n", + "- Relevance of selected tools to the conversation\n", + "- Completeness of tool selection according to task requirements\n", + "- Efficiency in avoiding unnecessary or redundant tools\n", + "\n", + "The evaluator uses a binary scoring system:\n", + "\n", + " - Score 0 (Fail): Tools selected are irrelevant, incorrect, or missing essential tools\n", + " - Score 1 (Pass): All needed tools are selected, even if there are redundant tools\n", + "\n", + "This evaluation focuses on measuring whether the right tools were chosen for the task, regardless of how those tools were executed or their parameter correctness." + ] + }, + { + "cell_type": "markdown", + "id": "353f2304", + "metadata": {}, + "source": [ + "Tool Selection requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", + "- Tool Calls - Tool Call(s) made by Agent to answer the query. Optional - if response has tool calls, if not provided evaluator will look for tool calls in response.\n", + "- Response - (Optional) Response from Agent (or any GenAI App). This can be a single text response or a list or messages generated as part of Agent Response. If tool calls are not provided, Tool Selection Evaluator will look at response for tool calls.\n", + "- Tool Definitions - Tool(s) definition used by Agent to answer the query. Required to understand available tools and their purposes.\n" + ] + }, + { + "cell_type": "markdown", + "id": "75125e94", + "metadata": {}, + "source": [ + "### Initialize Tool Selection Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b5f31144", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Class _ToolSelectionEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" + ] + } + ], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._tool_selection import _ToolSelectionEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "\n", + "tool_selection = _ToolSelectionEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "fb435367", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "344fe68e", + "metadata": {}, + "source": [ + "#### Evaluating Single Tool Selection" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "dcb71257", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_selection': 1,\n", + " 'tool_selection_completion_tokens': 236,\n", + " 'tool_selection_details': {'correct_tool_selections': 1,\n", + " 'excessive_tools_list': [],\n", + " 'excessive_tools_used': 0,\n", + " 'missing_tools': 0,\n", + " 'missing_tools_list': [],\n", + " 'tool_selection_accuracy': 100.0,\n", + " 'wrong_tool_selections': 0},\n", + " 'tool_selection_finish_reason': 'stop',\n", + " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_selection_prompt_tokens': 1582,\n", + " 'tool_selection_reason': \"1. The user's query is asking for the current \"\n", + " 'weather in Seattle, which is a request for weather '\n", + " 'information about a specific location. 2. There is '\n", + " 'no prior conversation history or previous tool '\n", + " 'calls, so the agent must select the appropriate '\n", + " 'tool now. 3. The only essential tool required from '\n", + " \"the tool definitions is 'fetch_weather', as it is \"\n", + " 'specifically designed to fetch weather information '\n", + " 'for a given location. 4. The agent selected '\n", + " \"'fetch_weather' in response to the query. 5. This \"\n", + " 'tool is directly relevant and the most appropriate '\n", + " \"for the user's request. 6. No essential tools are \"\n", + " 'missing, and no unnecessary tools (such as '\n", + " \"'send_email') were selected. 7. The tool selection \"\n", + " 'is efficient and complete. 8. Therefore, the tool '\n", + " 'selection passes.',\n", + " 'tool_selection_result': 'pass',\n", + " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"How is the weather in Seattle?\\\\\", '\n", + " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\"], '\n", + " '\\\\\"tool_definitions\\\\\": [{\\\\\"id\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Fetches the weather information for the '\n", + " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", + " 'weather for.\\\\\"}}}}, {\\\\\"id\\\\\": '\n", + " '\\\\\"send_email\\\\\", \\\\\"name\\\\\": '\n", + " '\\\\\"send_email\\\\\", \\\\\"description\\\\\": \\\\\"Sends '\n", + " 'an email with the specified subject and body '\n", + " 'to the recipient.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", + " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", + " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", + " 'content of the email.\\\\\"}}}}]}\"}]',\n", + " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", + " 'is asking for the current weather in '\n", + " 'Seattle, which is a request for weather '\n", + " 'information about a specific location. 2. '\n", + " 'There is no prior conversation history or '\n", + " 'previous tool calls, so the agent must '\n", + " 'select the appropriate tool now. 3. The only '\n", + " 'essential tool required from the tool '\n", + " \"definitions is 'fetch_weather', as it is \"\n", + " 'specifically designed to fetch weather '\n", + " 'information for a given location. 4. The '\n", + " \"agent selected 'fetch_weather' in response \"\n", + " 'to the query. 5. This tool is directly '\n", + " 'relevant and the most appropriate for the '\n", + " \"user's request. 6. No essential tools are \"\n", + " 'missing, and no unnecessary tools (such as '\n", + " \"'send_email') were selected. 7. The tool \"\n", + " 'selection is efficient and complete. 8. '\n", + " 'Therefore, the tool selection '\n", + " 'passes.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"correct_tool_selections\\\\\": 1,\\\\n '\n", + " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", + " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", + " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", + " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", + " 'tool_selection_threshold': 1,\n", + " 'tool_selection_total_tokens': 1818}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "tool_call = {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + "}\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"id\": \"fetch_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"id\": \"send_email\",\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "response = tool_selection(query=query, tool_calls=tool_call, tool_definitions=tool_definitions)\n", + "pprint(response)" + ] + }, + { + "cell_type": "markdown", + "id": "e078b1fa", + "metadata": {}, + "source": [ + "#### Multiple Tool Selections for Complex Task" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "322efe28", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_selection': 1,\n", + " 'tool_selection_completion_tokens': 239,\n", + " 'tool_selection_details': {'correct_tool_selections': 2,\n", + " 'excessive_tools_list': [],\n", + " 'excessive_tools_used': 0,\n", + " 'missing_tools': 0,\n", + " 'missing_tools_list': [],\n", + " 'tool_selection_accuracy': 100.0,\n", + " 'wrong_tool_selections': 0},\n", + " 'tool_selection_finish_reason': 'stop',\n", + " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_selection_prompt_tokens': 1674,\n", + " 'tool_selection_reason': \"1. The user's query is to send an email containing \"\n", + " 'weather information for Seattle. This requires two '\n", + " 'operations: fetching the weather for Seattle and '\n", + " 'sending an email with that information. 2. There is '\n", + " 'no prior conversation history or previous tool '\n", + " 'calls, so all required data must be gathered in '\n", + " 'this step. 3. The essential tools are '\n", + " \"'fetch_weather' (to get the weather data) and \"\n", + " \"'send_email' (to send the email). 4. The agent \"\n", + " \"selected both 'fetch_weather' and 'send_email'. 5. \"\n", + " 'Both tools are directly relevant and appropriate '\n", + " 'for the task. 6. No essential tools are missing, '\n", + " \"and no unnecessary tools (like 'get_calendar') were \"\n", + " 'selected. 7. The tool selection is efficient and '\n", + " 'complete. 8. Therefore, the tool selection passes.',\n", + " 'tool_selection_result': 'pass',\n", + " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"Can you send me an email with weather '\n", + " 'information for Seattle?\\\\\", '\n", + " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"send_email\\\\\"], \\\\\"tool_definitions\\\\\": '\n", + " '[{\\\\\"id\\\\\": \\\\\"fetch_weather\\\\\", \\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Fetches the weather information for the '\n", + " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", + " 'weather for.\\\\\"}}}}, {\\\\\"id\\\\\": '\n", + " '\\\\\"send_email\\\\\", \\\\\"name\\\\\": '\n", + " '\\\\\"send_email\\\\\", \\\\\"description\\\\\": \\\\\"Sends '\n", + " 'an email with the specified subject and body '\n", + " 'to the recipient.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", + " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", + " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", + " 'content of the email.\\\\\"}}}}, {\\\\\"id\\\\\": '\n", + " '\\\\\"get_calendar\\\\\", \\\\\"name\\\\\": '\n", + " '\\\\\"get_calendar\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Retrieves calendar events for a specified '\n", + " 'date range.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"start_date\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Start date for calendar '\n", + " 'events.\\\\\"}, \\\\\"end_date\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"End date '\n", + " 'for calendar events.\\\\\"}}}}]}\"}]',\n", + " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", + " 'is to send an email containing weather '\n", + " 'information for Seattle. This requires two '\n", + " 'operations: fetching the weather for Seattle '\n", + " 'and sending an email with that information. '\n", + " '2. There is no prior conversation history or '\n", + " 'previous tool calls, so all required data '\n", + " 'must be gathered in this step. 3. The '\n", + " \"essential tools are 'fetch_weather' (to get \"\n", + " \"the weather data) and 'send_email' (to send \"\n", + " 'the email). 4. The agent selected both '\n", + " \"'fetch_weather' and 'send_email'. 5. Both \"\n", + " 'tools are directly relevant and appropriate '\n", + " 'for the task. 6. No essential tools are '\n", + " 'missing, and no unnecessary tools (like '\n", + " \"'get_calendar') were selected. 7. The tool \"\n", + " 'selection is efficient and complete. 8. '\n", + " 'Therefore, the tool selection '\n", + " 'passes.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"correct_tool_selections\\\\\": 2,\\\\n '\n", + " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", + " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", + " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", + " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", + " 'tool_selection_threshold': 1,\n", + " 'tool_selection_total_tokens': 1913}\n" + ] + } + ], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " },\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"user@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Weather data will be included here.\",\n", + " },\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"id\": \"fetch_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"id\": \"send_email\",\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + " {\n", + " \"id\": \"get_calendar\",\n", + " \"name\": \"get_calendar\",\n", + " \"description\": \"Retrieves calendar events for a specified date range.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"start_date\": {\"type\": \"string\", \"description\": \"Start date for calendar events.\"},\n", + " \"end_date\": {\"type\": \"string\", \"description\": \"End date for calendar events.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "response = tool_selection(query=query, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(response)" + ] + }, + { + "cell_type": "markdown", + "id": "19b3c8de", + "metadata": {}, + "source": [ + "#### Tool Calls passed as part of `Response` (common for agent case)\n", + "- Tool Selection Evaluator extracts tool calls from response" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "629f3cc7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_selection': 1,\n", + " 'tool_selection_completion_tokens': 239,\n", + " 'tool_selection_details': {'correct_tool_selections': 2,\n", + " 'excessive_tools_list': [],\n", + " 'excessive_tools_used': 0,\n", + " 'missing_tools': 0,\n", + " 'missing_tools_list': [],\n", + " 'tool_selection_accuracy': 100.0,\n", + " 'wrong_tool_selections': 0},\n", + " 'tool_selection_finish_reason': 'stop',\n", + " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_selection_prompt_tokens': 1653,\n", + " 'tool_selection_reason': \"1. The user's query is to send an email containing \"\n", + " 'weather information for Seattle. This requires two '\n", + " 'operations: fetching the weather for Seattle and '\n", + " 'sending an email with that information. 2. There is '\n", + " 'no prior conversation history or previous tool '\n", + " 'calls, so all required data must be gathered in '\n", + " 'this response. 3. The essential tools are '\n", + " \"'fetch_weather' (to get the weather info) and \"\n", + " \"'send_email' (to send the email). 4. The agent \"\n", + " \"selected both 'fetch_weather' and 'send_email'. 5. \"\n", + " 'Both tools are directly relevant and appropriate '\n", + " 'for the task. 6. No essential tools are missing, '\n", + " \"and no unnecessary tools (like 'get_calendar') were \"\n", + " 'selected. 7. The tool selection is efficient and '\n", + " 'complete. 8. Therefore, the tool selection passes.',\n", + " 'tool_selection_result': 'pass',\n", + " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"Can you send me an email with weather '\n", + " 'information for Seattle?\\\\\", '\n", + " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"send_email\\\\\"], \\\\\"tool_definitions\\\\\": '\n", + " '[{\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", + " 'information for the specified location.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"The location to fetch weather for.\\\\\"}}}}, '\n", + " '{\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Sends an email with the '\n", + " 'specified subject and body to the '\n", + " 'recipient.\\\\\", \\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", + " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", + " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", + " 'content of the email.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", + " '\\\\\"get_calendar\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Retrieves calendar events for a specified '\n", + " 'date range.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"start_date\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Start date for calendar '\n", + " 'events.\\\\\"}, \\\\\"end_date\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"End date '\n", + " 'for calendar events.\\\\\"}}}}]}\"}]',\n", + " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", + " 'is to send an email containing weather '\n", + " 'information for Seattle. This requires two '\n", + " 'operations: fetching the weather for Seattle '\n", + " 'and sending an email with that information. '\n", + " '2. There is no prior conversation history or '\n", + " 'previous tool calls, so all required data '\n", + " 'must be gathered in this response. 3. The '\n", + " \"essential tools are 'fetch_weather' (to get \"\n", + " \"the weather info) and 'send_email' (to send \"\n", + " 'the email). 4. The agent selected both '\n", + " \"'fetch_weather' and 'send_email'. 5. Both \"\n", + " 'tools are directly relevant and appropriate '\n", + " 'for the task. 6. No essential tools are '\n", + " 'missing, and no unnecessary tools (like '\n", + " \"'get_calendar') were selected. 7. The tool \"\n", + " 'selection is efficient and complete. 8. '\n", + " 'Therefore, the tool selection '\n", + " 'passes.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"correct_tool_selections\\\\\": 2,\\\\n '\n", + " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", + " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", + " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", + " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", + " 'tool_selection_threshold': 1,\n", + " 'tool_selection_total_tokens': 1892}\n" + ] + } + ], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Rainy, 14°C\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"your_email@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"The current weather in Seattle is rainy with a temperature of 14°C.\",\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\"}}\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is rainy with a temperature of 14°C.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"get_calendar\",\n", + " \"description\": \"Retrieves calendar events for a specified date range.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"start_date\": {\"type\": \"string\", \"description\": \"Start date for calendar events.\"},\n", + " \"end_date\": {\"type\": \"string\", \"description\": \"End date for calendar events.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_selection(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "682839b8", + "metadata": {}, + "source": [ + "#### Query as Conversation History (List of Messages)\n", + "The evaluator also supports query as a list of messages representing conversation history. This helps determine if the Agent selected appropriate tools based on the conversation context." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "218ade7a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: [{'role': 'system', 'content': 'You are a helpful assistant that can fetch weather information and send emails.'}, {'role': 'user', 'content': 'Hi, can you check the weather in Seattle for me?'}, {'role': 'user', 'content': 'Actually, could you also send me an email with that weather information to john@example.com?'}]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_selection': 1,\n", + " 'tool_selection_completion_tokens': 235,\n", + " 'tool_selection_details': {'correct_tool_selections': 2,\n", + " 'excessive_tools_list': [],\n", + " 'excessive_tools_used': 0,\n", + " 'missing_tools': 0,\n", + " 'missing_tools_list': [],\n", + " 'tool_selection_accuracy': 100.0,\n", + " 'wrong_tool_selections': 0},\n", + " 'tool_selection_finish_reason': 'stop',\n", + " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_selection_prompt_tokens': 1719,\n", + " 'tool_selection_reason': \"1. The user's latest query asks for the weather in \"\n", + " 'Seattle and requests that this information be sent '\n", + " 'via email to john@example.com. 2. There is no '\n", + " 'indication in the conversation history that any '\n", + " 'tools have already been called; both requests are '\n", + " 'new. 3. The essential tools required are '\n", + " \"'fetch_weather' (to get the weather information) \"\n", + " \"and 'send_email' (to send the email with that \"\n", + " 'information). 4. The agent selected both '\n", + " \"'fetch_weather' and 'send_email'. 5. Both tools are \"\n", + " \"directly relevant and appropriate for the user's \"\n", + " 'query. 6. No essential tools are missing, and no '\n", + " \"unnecessary tools (like 'get_calendar') were \"\n", + " 'selected. 7. The tool selection is efficient and '\n", + " 'complete. 8. Therefore, the tool selection passes.',\n", + " 'tool_selection_result': 'pass',\n", + " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '[{\\\\\"role\\\\\": \\\\\"system\\\\\", \\\\\"content\\\\\": '\n", + " '\\\\\"You are a helpful assistant that can fetch '\n", + " 'weather information and send emails.\\\\\"}, '\n", + " '{\\\\\"role\\\\\": \\\\\"user\\\\\", \\\\\"content\\\\\": '\n", + " '\\\\\"Hi, can you check the weather in Seattle '\n", + " 'for me?\\\\\"}, {\\\\\"role\\\\\": \\\\\"user\\\\\", '\n", + " '\\\\\"content\\\\\": \\\\\"Actually, could you also '\n", + " 'send me an email with that weather '\n", + " 'information to john@example.com?\\\\\"}], '\n", + " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"send_email\\\\\"], \\\\\"tool_definitions\\\\\": '\n", + " '[{\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", + " 'information for the specified location.\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"The location to fetch weather for.\\\\\"}}}}, '\n", + " '{\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Sends an email with the '\n", + " 'specified subject and body to the '\n", + " 'recipient.\\\\\", \\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", + " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", + " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", + " 'content of the email.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", + " '\\\\\"get_calendar\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Retrieves calendar events for a specified '\n", + " 'date range.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"start_date\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Start date for calendar '\n", + " 'events.\\\\\"}, \\\\\"end_date\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"End date '\n", + " 'for calendar events.\\\\\"}}}}]}\"}]',\n", + " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s latest '\n", + " 'query asks for the weather in Seattle and '\n", + " 'requests that this information be sent via '\n", + " 'email to john@example.com. 2. There is no '\n", + " 'indication in the conversation history that '\n", + " 'any tools have already been called; both '\n", + " 'requests are new. 3. The essential tools '\n", + " \"required are 'fetch_weather' (to get the \"\n", + " \"weather information) and 'send_email' (to \"\n", + " 'send the email with that information). 4. '\n", + " \"The agent selected both 'fetch_weather' and \"\n", + " \"'send_email'. 5. Both tools are directly \"\n", + " \"relevant and appropriate for the user's \"\n", + " 'query. 6. No essential tools are missing, '\n", + " 'and no unnecessary tools (like '\n", + " \"'get_calendar') were selected. 7. The tool \"\n", + " 'selection is efficient and complete. 8. '\n", + " 'Therefore, the tool selection '\n", + " 'passes.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"correct_tool_selections\\\\\": 2,\\\\n '\n", + " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", + " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", + " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", + " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", + " 'tool_selection_threshold': 1,\n", + " 'tool_selection_total_tokens': 1954}\n" + ] + } + ], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Actually, could you also send me an email with that weather information to john@example.com?\"\n", + " }\n", + "]\n", + "\n", + "tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " },\n", + " {\n", + " \"type\": \"tool_call\", \n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"john@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Here is the weather information you requested.\"\n", + " },\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"get_calendar\",\n", + " \"description\": \"Retrieves calendar events for a specified date range.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"start_date\": {\"type\": \"string\", \"description\": \"Start date for calendar events.\"},\n", + " \"end_date\": {\"type\": \"string\", \"description\": \"End date for calendar events.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_selection(query=query_as_conversation, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "406bc90c", + "metadata": {}, + "source": [ + "#### Example of Poor Tool Selection" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2c451e4e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_selection': 0,\n", + " 'tool_selection_completion_tokens': 165,\n", + " 'tool_selection_details': {'correct_tool_selections': 0,\n", + " 'excessive_tools_list': ['get_calendar'],\n", + " 'excessive_tools_used': 1,\n", + " 'missing_tools': 1,\n", + " 'missing_tools_list': ['fetch_weather'],\n", + " 'tool_selection_accuracy': 0.0,\n", + " 'wrong_tool_selections': 1},\n", + " 'tool_selection_finish_reason': 'stop',\n", + " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_selection_prompt_tokens': 1644,\n", + " 'tool_selection_reason': \"The user's query is about the weather in Seattle, \"\n", + " 'which requires fetching weather information. The '\n", + " \"only relevant tool for this is 'fetch_weather'. The \"\n", + " \"agent instead selected 'get_calendar', which is not \"\n", + " \"relevant to the user's request and does not provide \"\n", + " 'weather information. No previous tool calls in the '\n", + " 'conversation provided weather data, so the '\n", + " \"essential tool 'fetch_weather' was missed. The tool \"\n", + " 'selection is therefore inappropriate and '\n", + " 'incomplete.',\n", + " 'tool_selection_result': 'fail',\n", + " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"How is the weather in Seattle?\\\\\", '\n", + " '\\\\\"tool_calls\\\\\": [\\\\\"get_calendar\\\\\"], '\n", + " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Fetches the weather information for the '\n", + " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", + " 'weather for.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", + " '\\\\\"send_email\\\\\", \\\\\"description\\\\\": \\\\\"Sends '\n", + " 'an email with the specified subject and body '\n", + " 'to the recipient.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", + " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", + " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", + " 'content of the email.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", + " '\\\\\"get_calendar\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Retrieves calendar events for a specified '\n", + " 'date range.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"start_date\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Start date for calendar '\n", + " 'events.\\\\\"}, \\\\\"end_date\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"End date '\n", + " 'for calendar events.\\\\\"}}}}]}\"}]',\n", + " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"The user\\'s query is '\n", + " 'about the weather in Seattle, which requires '\n", + " 'fetching weather information. The only '\n", + " \"relevant tool for this is 'fetch_weather'. \"\n", + " \"The agent instead selected 'get_calendar', \"\n", + " \"which is not relevant to the user's request \"\n", + " 'and does not provide weather information. No '\n", + " 'previous tool calls in the conversation '\n", + " 'provided weather data, so the essential tool '\n", + " \"'fetch_weather' was missed. The tool \"\n", + " 'selection is therefore inappropriate and '\n", + " 'incomplete.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"correct_tool_selections\\\\\": 0,\\\\n '\n", + " '\\\\\"wrong_tool_selections\\\\\": 1,\\\\n '\n", + " '\\\\\"excessive_tools_used\\\\\": 1,\\\\n '\n", + " '\\\\\"excessive_tools_list\\\\\": '\n", + " '[\\\\\"get_calendar\\\\\"],\\\\n '\n", + " '\\\\\"missing_tools\\\\\": 1,\\\\n '\n", + " '\\\\\"missing_tools_list\\\\\": '\n", + " '[\\\\\"fetch_weather\\\\\"]\\\\n },\\\\n '\n", + " '\\\\\"score\\\\\": 0\\\\n}\"}]',\n", + " 'tool_selection_threshold': 1,\n", + " 'tool_selection_total_tokens': 1809}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "# Using irrelevant tool for the task\n", + "poor_tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_calendar_123\",\n", + " \"name\": \"get_calendar\",\n", + " \"arguments\": {\"start_date\": \"2025-01-01\", \"end_date\": \"2025-01-31\"},\n", + " },\n", + "]\n", + "\n", + "# This should score poorly as get_calendar is not relevant for weather queries\n", + "result = tool_selection(query=query, tool_calls=poor_tool_calls, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "ab4a46c7", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "e34bec52", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: What's the weather in New York?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_selection': 1,\n", + " 'tool_selection_completion_tokens': 218,\n", + " 'tool_selection_details': {'correct_tool_selections': 1,\n", + " 'excessive_tools_list': [],\n", + " 'excessive_tools_used': 0,\n", + " 'missing_tools': 0,\n", + " 'missing_tools_list': [],\n", + " 'tool_selection_accuracy': 100.0,\n", + " 'wrong_tool_selections': 0},\n", + " 'tool_selection_finish_reason': 'stop',\n", + " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_selection_prompt_tokens': 1568,\n", + " 'tool_selection_reason': \"1. The user's query is asking for the weather in \"\n", + " 'New York, which requires fetching weather '\n", + " 'information for a specific location. 2. There is no '\n", + " 'prior conversation history or previous tool calls, '\n", + " 'so the agent must select the appropriate tool now. '\n", + " '3. The only essential tool for this task, according '\n", + " \"to the tool definitions, is 'fetch_weather'. 4. The \"\n", + " \"agent selected 'fetch_weather' in response to the \"\n", + " 'query. 5. This tool is directly relevant and the '\n", + " \"most appropriate for the user's request. 6. No \"\n", + " 'essential tools are missing, and no unnecessary '\n", + " 'tools were selected. 7. The tool selection is '\n", + " 'efficient and complete. 8. Therefore, the tool '\n", + " 'selection passes.',\n", + " 'tool_selection_result': 'pass',\n", + " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"What\\'s the weather in New York?\\\\\", '\n", + " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\"], '\n", + " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", + " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"Fetches the weather information for the '\n", + " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", + " 'weather for.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", + " '\\\\\"send_email\\\\\", \\\\\"description\\\\\": \\\\\"Sends '\n", + " 'an email with the specified subject and body '\n", + " 'to the recipient.\\\\\", \\\\\"parameters\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", + " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", + " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", + " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", + " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", + " 'content of the email.\\\\\"}}}}]}\"}]',\n", + " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", + " 'is asking for the weather in New York, which '\n", + " 'requires fetching weather information for a '\n", + " 'specific location. 2. There is no prior '\n", + " 'conversation history or previous tool calls, '\n", + " 'so the agent must select the appropriate '\n", + " 'tool now. 3. The only essential tool for '\n", + " 'this task, according to the tool '\n", + " \"definitions, is 'fetch_weather'. 4. The \"\n", + " \"agent selected 'fetch_weather' in response \"\n", + " 'to the query. 5. This tool is directly '\n", + " 'relevant and the most appropriate for the '\n", + " \"user's request. 6. No essential tools are \"\n", + " 'missing, and no unnecessary tools were '\n", + " 'selected. 7. The tool selection is efficient '\n", + " 'and complete. 8. Therefore, the tool '\n", + " 'selection passes.\\\\\",\\\\n \\\\\"details\\\\\": '\n", + " '{\\\\n \\\\\"correct_tool_selections\\\\\": '\n", + " '1,\\\\n \\\\\"wrong_tool_selections\\\\\": '\n", + " '0,\\\\n \\\\\"excessive_tools_used\\\\\": '\n", + " '0,\\\\n \\\\\"excessive_tools_list\\\\\": '\n", + " '[],\\\\n \\\\\"missing_tools\\\\\": 0,\\\\n '\n", + " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", + " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", + " 'tool_selection_threshold': 1,\n", + " 'tool_selection_total_tokens': 1786}\n" + ] + } + ], + "source": [ + "query = \"What's the weather in New York?\"\n", + "\n", + "# Response as a simple string (not commonly used, but supported by the API)\n", + "# When using string response, tool_calls should be provided separately\n", + "response_str = \"The current weather in New York is sunny with a temperature of 22°C.\"\n", + "\n", + "tool_calls = {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_ny_weather_789\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"New York\"},\n", + "}\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_selection(query=query, response=response_str, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "cc857725", + "metadata": {}, + "source": [ + "#### Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "d94822e1", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tool_selection': 1,\n", + " 'tool_selection_completion_tokens': 233,\n", + " 'tool_selection_details': {'correct_tool_selections': 1,\n", + " 'excessive_tools_list': [],\n", + " 'excessive_tools_used': 0,\n", + " 'missing_tools': 0,\n", + " 'missing_tools_list': [],\n", + " 'tool_selection_accuracy': 100.0,\n", + " 'wrong_tool_selections': 0},\n", + " 'tool_selection_finish_reason': 'stop',\n", + " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", + " 'tool_selection_prompt_tokens': 1501,\n", + " 'tool_selection_reason': \"1. The user's query asks for the current weather in \"\n", + " 'Seattle, which requires retrieving weather '\n", + " 'information for a specific location. 2. There is no '\n", + " 'prior conversation history or previous tool calls, '\n", + " 'so the agent must select the appropriate tool now. '\n", + " '3. The only tool defined that is relevant is '\n", + " \"'get_weather', which is designed to fetch current \"\n", + " 'weather for a location. 4. The agent selected '\n", + " \"'get_weather' in response to the query. 5. This \"\n", + " 'tool is directly relevant and the most appropriate '\n", + " \"for the user's request. 6. No essential tools are \"\n", + " \"missing, as 'get_weather' is the only necessary \"\n", + " 'tool and it was selected. 7. No unnecessary or '\n", + " 'redundant tools were selected. 8. Therefore, the '\n", + " 'tool selection is fully appropriate and complete.',\n", + " 'tool_selection_result': 'pass',\n", + " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", + " '\\\\\"How is the weather in Seattle?\\\\\", '\n", + " '\\\\\"tool_calls\\\\\": [\\\\\"get_weather\\\\\"], '\n", + " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", + " '\\\\\"get_weather\\\\\", \\\\\"description\\\\\": \\\\\"Get '\n", + " 'the current weather for a location\\\\\", '\n", + " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", + " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", + " '{\\\\\"type\\\\\": \\\\\"string\\\\\", \\\\\"description\\\\\": '\n", + " '\\\\\"The city and state, e.g. San Francisco, '\n", + " 'CA\\\\\"}, \\\\\"unit\\\\\": {\\\\\"type\\\\\": '\n", + " '\\\\\"string\\\\\", \\\\\"enum\\\\\": [\\\\\"celsius\\\\\", '\n", + " '\\\\\"fahrenheit\\\\\"]}}, \\\\\"required\\\\\": '\n", + " '[\\\\\"location\\\\\"]}}]}\"}]',\n", + " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", + " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", + " 'asks for the current weather in Seattle, '\n", + " 'which requires retrieving weather '\n", + " 'information for a specific location. 2. '\n", + " 'There is no prior conversation history or '\n", + " 'previous tool calls, so the agent must '\n", + " 'select the appropriate tool now. 3. The only '\n", + " 'tool defined that is relevant is '\n", + " \"'get_weather', which is designed to fetch \"\n", + " 'current weather for a location. 4. The agent '\n", + " \"selected 'get_weather' in response to the \"\n", + " 'query. 5. This tool is directly relevant and '\n", + " \"the most appropriate for the user's request. \"\n", + " '6. No essential tools are missing, as '\n", + " \"'get_weather' is the only necessary tool and \"\n", + " 'it was selected. 7. No unnecessary or '\n", + " 'redundant tools were selected. 8. Therefore, '\n", + " 'the tool selection is fully appropriate and '\n", + " 'complete.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", + " '\\\\\"correct_tool_selections\\\\\": 1,\\\\n '\n", + " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", + " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", + " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", + " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", + " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", + " 'tool_selection_threshold': 1,\n", + " 'tool_selection_total_tokens': 1734}\n" + ] + } + ], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"get_weather\",\n", + " \"description\": \"Get the current weather for a location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\"type\": \"string\", \"description\": \"The city and state, e.g. San Francisco, CA\"},\n", + " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]}\n", + " },\n", + " \"required\": [\"location\"]\n", + " }\n", + "}\n", + "\n", + "tool_calls_dict = {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_abc123\",\n", + " \"name\": \"get_weather\",\n", + " \"arguments\": {\"location\": \"Seattle, WA\", \"unit\": \"celsius\"}\n", + "}\n", + "\n", + "result = tool_selection(query=query, tool_definitions=tool_definition_dict, tool_calls=tool_calls_dict)\n", + "\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb index 06142292..77058c92 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -83,7 +83,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -108,9 +108,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'response_completeness': 4,\n", + " 'response_completeness_result': 'pass',\n", + " 'response_completeness_threshold': 3,\n", + " 'response_completeness_reason': 'The response contains most of the relevant information, but misses the explicit purpose of \"city sightseeing\" for Day 1. Otherwise, it is accurate and complete for Day 2.',\n", + " 'response_completeness_prompt_tokens': 1395,\n", + " 'response_completeness_completion_tokens': 176,\n", + " 'response_completeness_total_tokens': 1571,\n", + " 'response_completeness_finish_reason': 'stop',\n", + " 'response_completeness_model': 'gpt-4.1-2025-04-14',\n", + " 'response_completeness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": \\\\\"Itinery: Day 1 check out the downtown district of the city on train; for Day 2, we can rest in hotel.\\\\\", \\\\\"ground_truth\\\\\": \\\\\"Itinery: Day 1 take a train to visit the downtown area for city sightseeing; Day 2 rests in hotel.\\\\\"}\"}]',\n", + " 'response_completeness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think step by step: The ground truth has two statements: (1) Day 1 involves taking a train to visit the downtown area for city sightseeing, and (2) Day 2 involves resting in the hotel. The response covers both days: (1) Day 1 mentions checking out the downtown district on train, which is similar to the ground truth but omits \\\\\"city sightseeing\\\\\" explicitly, and (2) Day 2 mentions resting in the hotel, which matches the ground truth. The main omission is the explicit mention of \\\\\"city sightseeing\\\\\" as the purpose for visiting downtown.\\\\nThe response contains most of the relevant information, but misses the explicit purpose of \\\\\"city sightseeing\\\\\" for Day 1. Otherwise, it is accurate and complete for Day 2.\\\\n4\"}]'}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# agent response is complete\n", "result = response_completeness_evaluator(\n", @@ -122,9 +143,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'response_completeness': 5,\n", + " 'response_completeness_result': 'pass',\n", + " 'response_completeness_threshold': 3,\n", + " 'response_completeness_reason': 'The response fully contains the ground truth information without missing any details.',\n", + " 'response_completeness_prompt_tokens': 1410,\n", + " 'response_completeness_completion_tokens': 120,\n", + " 'response_completeness_total_tokens': 1530,\n", + " 'response_completeness_finish_reason': 'stop',\n", + " 'response_completeness_model': 'gpt-4.1-2025-04-14',\n", + " 'response_completeness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": \\\\\"The order with ID 123 has been shipped and is expected to be delivered on March 15, 2025. However, the order with ID 124 is delayed and should now arrive by March 20, 2025.\\\\\", \\\\\"ground_truth\\\\\": \\\\\"The order with ID 124 is delayed and should now arrive by March 20, 2025.\\\\\"}\"}]',\n", + " 'response_completeness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think step by step: The ground truth contains one statement: \\\\\"The order with ID 124 is delayed and should now arrive by March 20, 2025.\\\\\" The response includes this statement accurately and completely. The response also adds information about another order (ID 123), but this does not detract from the completeness regarding the ground truth statement. All necessary and relevant information from the ground truth is present in the response.\\\\nThe response fully contains the ground truth information without missing any details.\\\\n5\"}]'}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# agent response is incomplete\n", "result = response_completeness_evaluator(\n", @@ -138,79 +180,61 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Prepare ground truth for agent response\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "from pathlib import Path\n", - "\n", - "data = [\n", - " {\n", - " \"response\": \"The temperature of Seattle now is 70 degrees. Based on the temperature, having an outdoor office party is recommended.\",\n", - " \"ground_truth\": \"The temperature of Seattle now is 50 degrees. It will be recommended to bring a jacket in the evening.\",\n", - " },\n", - " {\n", - " \"response\": 'The email draft \"Project Plan\" is attached. Please review and provide feedback.',\n", - " \"ground_truth\": 'The email draft \"Project Plan\" is attached. Please review and provide feedback by EOD.',\n", - " },\n", - " {\n", - " \"response\": \"Based on the retrieved documents, the shareholder meeting discussed the operational efficiency of the company and financing options.\",\n", - " \"ground_truth\": \"The shareholder meeting discussed the compensation package of the company CEO.\",\n", - " },\n", - " {\n", - " \"response\": \"The calendar API returns an error code 500. Please check the server logs for more details.\",\n", - " \"ground_truth\": \"The meeting is scheduled for 2 PM tomorrow. Please confirm your availability by EOD.\",\n", - " },\n", - "]\n", - "\n", - "file_path = \"response_completeness_data.jsonl\"\n", - "\n", - "with Path.open(file_path, \"w\") as file:\n", - " for line in data:\n", - " file.write(json.dumps(line) + \"\\n\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Batch evaluate and visualize results on Azure AI Foundry\n", - "Batch evaluate to leverage asynchronous evaluation on a dataset. \n", - "\n", - "Optionally, you can go to AI Foundry URL for rich Azure AI Foundry data visualization. You can inspect the evaluation scores and reasoning to quickly identify bugs and issues of your agent to fix and improve. Make sure to authenticate to Azure using `az login` in your terminal before running this cell." + "#### Evaluating using Conversation format" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'response_completeness': 1,\n", + " 'response_completeness_result': 'fail',\n", + " 'response_completeness_threshold': 3,\n", + " 'response_completeness_reason': 'There is no information in the ground truth, so the response cannot be evaluated for completeness and is considered fully incomplete.',\n", + " 'response_completeness_prompt_tokens': 1362,\n", + " 'response_completeness_completion_tokens': 109,\n", + " 'response_completeness_total_tokens': 1471,\n", + " 'response_completeness_finish_reason': 'stop',\n", + " 'response_completeness_model': 'gpt-4.1-2025-04-14',\n", + " 'response_completeness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": \\\\\"The weather in Seattle this weekend will be partly cloudy with temperatures around 15\\\\\\\\u00b0C on Saturday.\\\\\", \\\\\"ground_truth\\\\\": \\\\\"\\\\\"}\"}]',\n", + " 'response_completeness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think step by step: The ground truth is completely empty, meaning there is no information provided to compare against the response. According to the definitions, completeness is measured by how well the response reflects the ground truth. Since there are no claims or statements in the ground truth, the response cannot be complete in relation to it.\\\\nThere is no information in the ground truth, so the response cannot be evaluated for completeness and is considered fully incomplete.\\\\n1\"}]'}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "from azure.ai.evaluation import evaluate\n", - "\n", - "azure_ai_project = os.environ[\"AZURE_AI_PROJECT\"]\n", + "# Conversation format - includes user query, expected ground truth, and agent response\n", + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What's the weather forecast for Seattle this weekend?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday.\",\n", + " \"context\": {\n", + " \"ground_truth\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday and 17°C on Sunday, with a 20% chance of rain on Sunday afternoon.\"\n", + " }\n", + " }\n", + " ]\n", + "}\n", "\n", - "response = evaluate(\n", - " data=file_path,\n", - " evaluators={\n", - " \"response_completeness\": response_completeness_evaluator,\n", - " },\n", - " azure_ai_project=azure_ai_project,\n", - ")\n", - "\n", - "pprint(f'AI Foundry URL: {response.get(\"studio_url\")}')" + "result = response_completeness_evaluator(conversation=conversation)\n", + "result" ] } ], "metadata": { "kernelspec": { - "display_name": "azureai-sample", + "display_name": "samples", "language": "python", "name": "python3" }, @@ -223,7 +247,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.10.19" } }, "nbformat": 4, From cf1e56421ec75d3c741f861b7dc58f60a29e2d35 Mon Sep 17 00:00:00 2001 From: v-mradi Date: Tue, 4 Nov 2025 01:09:03 +0200 Subject: [PATCH 2/7] Updating evaluators and adding new evaluator --- .../AI_Judge_Evaluator_Coherence.ipynb | 396 +++++++ .../AI_Judge_Evaluator_Fluency.ipynb | 530 +--------- .../AI_Judge_Evaluator_Groundedness.ipynb | 583 +--------- ...AI_Judge_Evaluator_Intent_Resolution.ipynb | 266 +---- ...udge_Evaluator_Response_Completeness.ipynb | 192 ++++ .../AI_Judge_Evaluator_Task_Adherence.ipynb | 306 +----- .../AI_Judge_Evaluator_Task_Completion.ipynb | 469 +-------- ...Evaluator_Task_Navigation_Efficiency.ipynb | 107 +- ...I_Judge_Evaluator_Tool_Call_Accuracy.ipynb | 814 +------------- ...AI_Judge_Evaluator_Tool_Call_Success.ipynb | 995 ------------------ ..._Judge_Evaluator_Tool_Input_Accuracy.ipynb | 27 +- ...ge_Evaluator_Tool_Output_Utilization.ipynb | 532 +--------- .../AI_Judge_Evaluator_Tool_Selection.ipynb | 736 +------------ .../AI_Judge_Evaluator_Tool_Success.ipynb | 619 +++++++++++ ...dge_Evaluators_Response_Completeness.ipynb | 256 ----- 15 files changed, 1391 insertions(+), 5437 deletions(-) create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb new file mode 100644 index 00000000..fb5acbc7 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb @@ -0,0 +1,396 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "da22a6e7", + "metadata": {}, + "source": [ + "# Coherence Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use the Coherence evaluator to assess the quality and readability of AI-generated responses. The evaluator measures the ability of the language model to generate text that reads naturally, flows smoothly, and resembles human-like language.\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "id": "48f8cb64", + "metadata": {}, + "source": [ + "The Coherence evaluator assesses the ability of the language model to generate text that reads naturally, flows smoothly, and resembles human-like language in its responses. It measures the readability and user-friendliness of the model's generated responses.\n", + "\n", + "Coherence scores range from 1 to 5:\n", + "\n", + "
\n",
+    "Score 1: Very Poor - The response is completely incoherent with no logical flow or connection between ideas.\n",
+    "Score 2: Poor - The response has significant coherence issues with disjointed ideas and unclear transitions.\n",
+    "Score 3: Fair - The response is somewhat coherent but has noticeable gaps in logical flow or organization.\n",
+    "Score 4: Good - The response is mostly coherent with good flow and logical connections between ideas.\n",
+    "Score 5: Excellent - The response is perfectly coherent with smooth, natural flow and clear logical progression.\n",
+    "
\n", + "\n", + "The evaluation requires one of the following input patterns:\n", + "\n", + "**Pattern 1: Query-Response Evaluation**\n", + "- Query: The user's question or prompt. (string)\n", + "- Response: The AI-generated response to be evaluated for coherence. (string)\n", + "\n", + "**Pattern 2: Conversation Evaluation**\n", + "- Conversation: A multi-turn conversation where each turn is evaluated for coherence. (dict with \"messages\" key)" + ] + }, + { + "cell_type": "markdown", + "id": "542c1340", + "metadata": {}, + "source": [ + "### Initialize Coherence Evaluator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35799119", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import CoherenceEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "coherence_evaluator = CoherenceEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "b10aff00", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "724adfc8", + "metadata": {}, + "source": [ + "#### Query and Response as Strings (str) - Excellent Coherence" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82a1be93", + "metadata": {}, + "outputs": [], + "source": [ + "# Excellent coherence - smooth flow, clear logical progression\n", + "query = \"Can you explain how machine learning works?\"\n", + "response = \"Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed. It works by analyzing data patterns to make predictions or decisions. First, the system is trained on a dataset, which allows it to identify patterns and relationships. Then, it uses these learned patterns to make predictions on new, unseen data. The accuracy improves over time as the model processes more data and refines its understanding.\"\n", + "\n", + "result = coherence_evaluator(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "34fbd16a", + "metadata": {}, + "source": [ + "#### Query and Response as Strings (str) - Good Coherence" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4046db9", + "metadata": {}, + "outputs": [], + "source": [ + "# Good coherence - mostly logical with minor flow issues\n", + "query = \"What are the benefits of cloud computing?\"\n", + "response = \"Cloud computing offers several advantages. Cost savings is one major benefit because companies don't need to invest in physical servers. Scalability is another advantage. You can also access your data from anywhere with an internet connection. Additionally, cloud providers handle maintenance and updates, which reduces the IT burden on organizations.\"\n", + "\n", + "result = coherence_evaluator(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "5336a182", + "metadata": {}, + "source": [ + "#### Query and Response as Strings (str) - Fair Coherence" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24ff7903", + "metadata": {}, + "outputs": [], + "source": [ + "# Fair coherence - noticeable gaps in logical flow\n", + "query = \"How does photosynthesis work?\"\n", + "response = \"Plants use sunlight to make food. Chlorophyll is green. The process happens in leaves. Carbon dioxide and water are important. Oxygen is released as a byproduct. This is why plants are important for the environment. The energy from sunlight is converted into chemical energy.\"\n", + "\n", + "result = coherence_evaluator(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "bc250127", + "metadata": {}, + "source": [ + "#### Query and Response as Strings (str) - Poor Coherence" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62ad5478", + "metadata": {}, + "outputs": [], + "source": [ + "# Poor coherence - disjointed ideas and unclear transitions\n", + "query = \"What is blockchain technology?\"\n", + "response = \"Blockchain is decentralized. Bitcoin uses it. Blocks contain data. Cryptography is involved. Banks might use blockchain. It's secure because of the chain. Mining is how new blocks are added. Digital currencies exist. The ledger is distributed.\"\n", + "\n", + "result = coherence_evaluator(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "0a559cdd", + "metadata": {}, + "source": [ + "#### Query and Response as Strings (str) - Technical Question" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a769e940", + "metadata": {}, + "outputs": [], + "source": [ + "# Technical response with excellent coherence\n", + "query = \"How do I optimize database query performance?\"\n", + "response = \"To optimize database query performance, start by analyzing your query execution plans to identify bottlenecks. Next, ensure you have appropriate indexes on columns used in WHERE clauses and JOIN conditions. Consider denormalizing data in read-heavy scenarios to reduce the number of joins required. Additionally, implement query result caching for frequently accessed data. Finally, regularly update database statistics to help the query optimizer make better decisions about execution plans.\"\n", + "\n", + "result = coherence_evaluator(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "1d22e3d6", + "metadata": {}, + "source": [ + "#### Conversation as Dict with Messages - Multi-turn Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "556c197b", + "metadata": {}, + "outputs": [], + "source": [ + "# Multi-turn conversation - evaluates coherence across turns\n", + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"I'm planning a trip to Japan. What should I know?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Japan is a fascinating destination with a rich cultural heritage. First, you should know that the best times to visit are spring (March-May) for cherry blossoms and fall (September-November) for autumn colors. The country has an excellent public transportation system, particularly the Shinkansen bullet train network. Additionally, learning a few basic Japanese phrases will be appreciated by locals, though many signs in major cities have English translations.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What about the food?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Japanese cuisine is incredibly diverse and delicious. Beyond the well-known sushi and ramen, you should try regional specialties like okonomiyaki in Osaka or motsunabe in Fukuoka. Most restaurants display plastic food models in their windows, which makes ordering easier even with a language barrier. It's also important to note that tipping is not customary in Japan and can sometimes be considered rude.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Are there any cultural customs I should be aware of?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Yes, there are several important customs to observe. Always remove your shoes when entering homes, temples, and some traditional restaurants. Bowing is a common greeting, though a slight nod is acceptable for foreigners. When using chopsticks, never stick them upright in rice as this resembles a funeral ritual. Finally, be mindful of noise levels on public transportation, as talking on phones or having loud conversations is considered impolite.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = coherence_evaluator(conversation=conversation)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "174a8ad4", + "metadata": {}, + "source": [ + "#### Conversation as Dict with Messages - Agent with Tool Calls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1977ad35", + "metadata": {}, + "outputs": [], + "source": [ + "# Conversation with tool calls - evaluates coherence of responses\n", + "conversation_with_tools = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you check the weather in Seattle and send me an email with the forecast?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I'll help you with that. First, let me check the current weather conditions in Seattle, and then I'll send you an email with the detailed forecast.\"\n", + " },\n", + " {\n", + " \"role\": \"tool\",\n", + " \"content\": \"Current weather in Seattle: Temperature: 15°C, Condition: Partly cloudy, Humidity: 65%, Wind: 12 mph NW, Forecast: Light rain expected in the evening\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I've retrieved the weather information for Seattle. Currently, it's 15°C with partly cloudy skies. The humidity is at 65% with winds from the northwest at 12 mph. The forecast indicates light rain is expected later this evening. I've sent you an email containing this complete weather report along with recommendations to bring an umbrella if you're planning to go out later today.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = coherence_evaluator(conversation=conversation_with_tools)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "f30e316a", + "metadata": {}, + "source": [ + "#### Conversation as Dict with Messages - Mixed Coherence Quality" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35562030", + "metadata": {}, + "outputs": [], + "source": [ + "# Conversation with varying coherence levels\n", + "conversation_mixed = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Tell me about renewable energy sources.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Renewable energy comes from natural sources. Solar panels convert sunlight. Wind turbines exist. Hydroelectric power uses water. These are cleaner than fossil fuels. The environment benefits. Many countries are investing in renewables.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Which one is the most efficient?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"The efficiency of renewable energy sources varies depending on several factors. Hydroelectric power is generally the most efficient, with conversion rates of up to 90% under optimal conditions. Solar panels typically achieve 15-22% efficiency, while wind turbines can reach 35-45% efficiency. However, efficiency alone doesn't determine the best choice, as factors like geographic location, initial investment costs, and environmental impact all play crucial roles in selecting the most appropriate renewable energy source for a specific situation.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = coherence_evaluator(conversation=conversation_mixed)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "b9b730ac", + "metadata": {}, + "source": [ + "#### Conversation as Dict with Messages - Short Conversation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54dfbd45", + "metadata": {}, + "outputs": [], + "source": [ + "# Short, coherent conversation\n", + "conversation_short = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What's the capital of France?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"The capital of France is Paris. It's located in the north-central part of the country along the Seine River and is known for its iconic landmarks like the Eiffel Tower, the Louvre Museum, and Notre-Dame Cathedral.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = coherence_evaluator(conversation=conversation_short)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb index 37edf785..f2dd32bd 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb @@ -19,14 +19,13 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "pip install azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." ] }, { @@ -65,7 +64,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "bfd097f8", "metadata": {}, "outputs": [], @@ -102,51 +101,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "3c2df39a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'fluency': 4.0,\n", - " 'fluency_completion_tokens': 140,\n", - " 'fluency_finish_reason': 'stop',\n", - " 'fluency_model': 'gpt-4.1-2025-04-14',\n", - " 'fluency_prompt_tokens': 948,\n", - " 'fluency_reason': 'The response is well-articulated, coherent, and uses '\n", - " 'varied sentence structures and vocabulary appropriate for '\n", - " 'the context, with no significant errors.',\n", - " 'fluency_result': 'pass',\n", - " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", - " '\\\\\"The weather in Seattle is currently partly cloudy '\n", - " 'with a temperature of 15\\\\\\\\u00b0C. The forecast '\n", - " 'indicates that conditions will remain stable '\n", - " 'throughout the day, with a gentle breeze from the '\n", - " 'northwest.\\\\\"}\"}]',\n", - " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", - " 'step by step: The response is grammatically correct '\n", - " 'and uses appropriate vocabulary for a weather '\n", - " 'report. Sentences are well-structured and coherent, '\n", - " 'with logical flow between the current conditions '\n", - " 'and the forecast. There are no noticeable errors, '\n", - " 'and the text is easy to read and understand. While '\n", - " 'the vocabulary is not highly sophisticated, it is '\n", - " 'varied and precise for the context. The sentence '\n", - " 'structure is somewhat complex, especially in the '\n", - " 'second sentence, which uses a subordinate '\n", - " 'clause.\\\\nThe response is '\n", - " 'well-articulated, coherent, and uses varied '\n", - " 'sentence structures and vocabulary appropriate for '\n", - " 'the context, with no significant '\n", - " 'errors.\\\\n4\"}]',\n", - " 'fluency_threshold': 3,\n", - " 'fluency_total_tokens': 1088,\n", - " 'gpt_fluency': 4.0}\n" - ] - } - ], + "outputs": [], "source": [ "# Excellent fluency - proper grammar, syntax, and vocabulary\n", "response_excellent = \"The weather in Seattle is currently partly cloudy with a temperature of 15°C. The forecast indicates that conditions will remain stable throughout the day, with a gentle breeze from the northwest.\"\n", @@ -165,47 +123,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "8dcf34b8", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'fluency': 3.0,\n", - " 'fluency_completion_tokens': 127,\n", - " 'fluency_finish_reason': 'stop',\n", - " 'fluency_model': 'gpt-4.1-2025-04-14',\n", - " 'fluency_prompt_tokens': 937,\n", - " 'fluency_reason': 'The response is clear and coherent but contains some '\n", - " 'grammatical mistakes and lacks sentence complexity, '\n", - " 'fitting the definition of Competent Fluency.',\n", - " 'fluency_result': 'pass',\n", - " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", - " '\\\\\"The meeting has been schedule for tomorrow at 2 '\n", - " 'PM. Please make sure to reviewing the attached '\n", - " 'documents before attending the session.\\\\\"}\"}]',\n", - " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", - " 'step by step: The response communicates the main '\n", - " 'idea clearly: a meeting is scheduled and documents '\n", - " 'should be reviewed beforehand. However, there are '\n", - " 'noticeable grammatical errors (\\\\\"has been '\n", - " 'schedule\\\\\" should be \\\\\"has been scheduled\\\\\" and '\n", - " '\\\\\"to reviewing\\\\\" should be \\\\\"to review\\\\\"). '\n", - " 'Vocabulary is adequate but not varied, and sentence '\n", - " 'structure is simple. Despite the errors, the '\n", - " 'message is coherent and understandable with minimal '\n", - " 'effort.\\\\nThe response is clear and '\n", - " 'coherent but contains some grammatical mistakes and '\n", - " 'lacks sentence complexity, fitting the definition '\n", - " 'of Competent Fluency.\\\\n3\"}]',\n", - " 'fluency_threshold': 3,\n", - " 'fluency_total_tokens': 1064,\n", - " 'gpt_fluency': 3.0}\n" - ] - } - ], + "outputs": [], "source": [ "# Good fluency - mostly fluent with minor grammatical issues\n", "response_good = \"The meeting has been schedule for tomorrow at 2 PM. Please make sure to reviewing the attached documents before attending the session.\"\n", @@ -224,49 +145,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "c30fc6bd", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'fluency': 2.0,\n", - " 'fluency_completion_tokens': 146,\n", - " 'fluency_finish_reason': 'stop',\n", - " 'fluency_model': 'gpt-4.1-2025-04-14',\n", - " 'fluency_prompt_tokens': 934,\n", - " 'fluency_reason': 'The response has frequent grammatical errors and awkward '\n", - " 'phrasing, but the main idea is still partially clear.',\n", - " 'fluency_result': 'fail',\n", - " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", - " '\\\\\"The order you placing yesterday is being process '\n", - " 'and will shipped soon. Tracking information send to '\n", - " 'your email when available.\\\\\"}\"}]',\n", - " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", - " 'step by step: The response contains several '\n", - " 'grammatical errors (\\\\\"The order you placing '\n", - " 'yesterday,\\\\\" \\\\\"is being process,\\\\\" \\\\\"will '\n", - " 'shipped soon,\\\\\" \\\\\"Tracking information send to '\n", - " 'your email\\\\\"), which make the sentences awkward '\n", - " 'and difficult to read. Vocabulary is very basic and '\n", - " 'there is little sentence complexity. Despite the '\n", - " 'errors, the main idea can be understood with some '\n", - " 'effort, but the phrasing is repetitive and awkward. '\n", - " 'This fits the description of Basic Fluency, where '\n", - " 'frequent errors and limited vocabulary are present, '\n", - " 'but the message is partially '\n", - " 'understandable.\\\\nThe response has '\n", - " 'frequent grammatical errors and awkward phrasing, '\n", - " 'but the main idea is still partially '\n", - " 'clear.\\\\n2\"}]',\n", - " 'fluency_threshold': 3,\n", - " 'fluency_total_tokens': 1080,\n", - " 'gpt_fluency': 2.0}\n" - ] - } - ], + "outputs": [], "source": [ "# Fair fluency - understandable but contains noticeable errors\n", "response_fair = \"The order you placing yesterday is being process and will shipped soon. Tracking information send to your email when available.\"\n", @@ -285,46 +167,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "d9c624ab", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'fluency': 1.0,\n", - " 'fluency_completion_tokens': 108,\n", - " 'fluency_finish_reason': 'stop',\n", - " 'fluency_model': 'gpt-4.1-2025-04-14',\n", - " 'fluency_prompt_tokens': 932,\n", - " 'fluency_reason': 'The response is mostly incoherent and contains pervasive '\n", - " 'grammatical errors, fitting the definition of Emergent '\n", - " 'Fluency.',\n", - " 'fluency_result': 'fail',\n", - " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", - " '\\\\\"Email draft attach is. You review and giving '\n", - " 'feedback must. Important very for project success it '\n", - " 'being.\\\\\"}\"}]',\n", - " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", - " 'step by step: The response contains fragmented '\n", - " 'sentences and awkward phrasing (\\\\\"Email draft '\n", - " 'attach is. You review and giving feedback must. '\n", - " 'Important very for project success it being.\\\\\"). '\n", - " 'There are pervasive grammatical errors, and the '\n", - " 'vocabulary is extremely limited. The message is '\n", - " 'difficult to understand and lacks coherence, making '\n", - " 'it largely incomprehensible.\\\\nThe '\n", - " 'response is mostly incoherent and contains '\n", - " 'pervasive grammatical errors, fitting the '\n", - " 'definition of Emergent '\n", - " 'Fluency.\\\\n1\"}]',\n", - " 'fluency_threshold': 3,\n", - " 'fluency_total_tokens': 1040,\n", - " 'gpt_fluency': 1.0}\n" - ] - } - ], + "outputs": [], "source": [ "# Poor fluency - frequent grammatical errors and awkward phrasing\n", "response_poor = \"Email draft attach is. You review and giving feedback must. Important very for project success it being.\"\n", @@ -343,53 +189,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "36e92f0b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'fluency': 4.0,\n", - " 'fluency_completion_tokens': 150,\n", - " 'fluency_finish_reason': 'stop',\n", - " 'fluency_model': 'gpt-4.1-2025-04-14',\n", - " 'fluency_prompt_tokens': 949,\n", - " 'fluency_reason': 'The response is well-articulated, clear, and uses varied '\n", - " 'technical vocabulary with good grammar and logical flow, '\n", - " 'fitting the definition of proficient fluency.',\n", - " 'fluency_result': 'pass',\n", - " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": \\\\\"To '\n", - " 'resolve the authentication error, please verify that '\n", - " 'your API credentials are correctly configured in the '\n", - " 'environment variables. Additionally, ensure that the '\n", - " 'service principal has the necessary permissions '\n", - " 'assigned at the subscription level.\\\\\"}\"}]',\n", - " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", - " 'step by step: The response is grammatically correct '\n", - " 'and uses appropriate technical vocabulary (\\\\\"API '\n", - " 'credentials,\\\\\" \\\\\"environment variables,\\\\\" '\n", - " '\\\\\"service principal,\\\\\" \\\\\"permissions,\\\\\" '\n", - " '\\\\\"subscription level\\\\\"). Sentences are '\n", - " 'well-structured and connected logically. There are '\n", - " 'no noticeable errors, and the message is clear and '\n", - " 'coherent. The vocabulary is varied within the '\n", - " 'technical context, and the text flows smoothly. '\n", - " 'However, the sentence structure and vocabulary, '\n", - " 'while proficient, do not reach the level of '\n", - " 'exceptional fluency or stylistic '\n", - " 'sophistication.\\\\nThe response is '\n", - " 'well-articulated, clear, and uses varied technical '\n", - " 'vocabulary with good grammar and logical flow, '\n", - " 'fitting the definition of proficient '\n", - " 'fluency.\\\\n4\"}]',\n", - " 'fluency_threshold': 3,\n", - " 'fluency_total_tokens': 1099,\n", - " 'gpt_fluency': 4.0}\n" - ] - } - ], + "outputs": [], "source": [ "# Technical response with excellent fluency\n", "response_technical = \"To resolve the authentication error, please verify that your API credentials are correctly configured in the environment variables. Additionally, ensure that the service principal has the necessary permissions assigned at the subscription level.\"\n", @@ -408,157 +211,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "9ccaff56", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'evaluation_per_turn': {'fluency': [3.0, 3.0, 4.0],\n", - " 'fluency_completion_tokens': [119, 121, 136],\n", - " 'fluency_finish_reason': ['stop', 'stop', 'stop'],\n", - " 'fluency_model': ['gpt-4.1-2025-04-14',\n", - " 'gpt-4.1-2025-04-14',\n", - " 'gpt-4.1-2025-04-14'],\n", - " 'fluency_prompt_tokens': [938, 938, 937],\n", - " 'fluency_reason': ['The response is clear, correct, '\n", - " 'and coherent, but it does not '\n", - " 'demonstrate complex sentence '\n", - " 'structure or advanced vocabulary.',\n", - " 'The response is clear and '\n", - " 'coherent with only a minor '\n", - " 'grammatical error, fitting the '\n", - " 'definition of Competent Fluency.',\n", - " 'The response is well-articulated, '\n", - " 'coherent, and uses varied '\n", - " 'vocabulary and sentence '\n", - " 'structure, fitting the definition '\n", - " 'of Proficient Fluency.'],\n", - " 'fluency_result': ['pass', 'pass', 'pass'],\n", - " 'fluency_sample_input': ['[{\"role\": \"user\", '\n", - " '\"content\": '\n", - " '\"{\\\\\"response\\\\\": \\\\\"Of '\n", - " \"course! I'd be happy to \"\n", - " 'help you understand this '\n", - " \"week's weather forecast. \"\n", - " 'The outlook shows a mix of '\n", - " 'sunny and cloudy '\n", - " 'conditions.\\\\\"}\"}]',\n", - " '[{\"role\": \"user\", '\n", - " '\"content\": '\n", - " '\"{\\\\\"response\\\\\": '\n", - " '\\\\\"Temperatures will '\n", - " 'ranging from 12\\\\\\\\u00b0C '\n", - " 'to 18\\\\\\\\u00b0C throughout '\n", - " 'the week, with the warmest '\n", - " 'days being Wednesday and '\n", - " 'Thursday.\\\\\"}\"}]',\n", - " '[{\"role\": \"user\", '\n", - " '\"content\": '\n", - " '\"{\\\\\"response\\\\\": \\\\\"Yes, '\n", - " 'it would be advisable to '\n", - " 'carry an umbrella, '\n", - " 'particularly on Friday when '\n", - " 'there is a 60% chance of '\n", - " 'precipitation.\\\\\"}\"}]'],\n", - " 'fluency_sample_output': ['[{\"role\": \"assistant\", '\n", - " '\"content\": \"Let\\'s '\n", - " 'think step by step: The '\n", - " 'response is grammatically '\n", - " 'correct and uses a polite, '\n", - " 'clear tone. Vocabulary is '\n", - " 'appropriate, though not '\n", - " 'particularly advanced or '\n", - " 'varied. Sentences are '\n", - " 'well-structured but '\n", - " 'relatively simple. The '\n", - " 'message is coherent and '\n", - " 'easily understood, with no '\n", - " 'awkward phrasing or '\n", - " 'errors. However, it lacks '\n", - " 'complexity and '\n", - " 'sophistication in sentence '\n", - " 'structure and vocabulary, '\n", - " 'which are required for '\n", - " 'higher '\n", - " 'scores.\\\\nThe '\n", - " 'response is clear, '\n", - " 'correct, and coherent, but '\n", - " 'it does not demonstrate '\n", - " 'complex sentence structure '\n", - " 'or advanced '\n", - " 'vocabulary.\\\\n3\"}]',\n", - " '[{\"role\": \"assistant\", '\n", - " '\"content\": \"Let\\'s '\n", - " 'think step by step: The '\n", - " 'response is mostly clear '\n", - " 'and communicates the '\n", - " 'temperature range and the '\n", - " 'warmest days. There is a '\n", - " 'minor grammatical error '\n", - " '(\\\\\"will ranging\\\\\" should '\n", - " 'be \\\\\"will range\\\\\" or '\n", - " '\\\\\"will be ranging\\\\\"), '\n", - " 'but the rest of the '\n", - " 'sentence is correct and '\n", - " 'easily understood. '\n", - " 'Vocabulary is adequate, '\n", - " 'and sentence structure is '\n", - " 'simple but coherent. The '\n", - " 'message is clear with '\n", - " 'minimal effort required to '\n", - " 'understand.\\\\nThe '\n", - " 'response is clear and '\n", - " 'coherent with only a minor '\n", - " 'grammatical error, fitting '\n", - " 'the definition of '\n", - " 'Competent '\n", - " 'Fluency.\\\\n3\"}]',\n", - " '[{\"role\": \"assistant\", '\n", - " '\"content\": \"Let\\'s '\n", - " 'think step by step: The '\n", - " 'response is grammatically '\n", - " 'correct and uses '\n", - " 'appropriate vocabulary '\n", - " '(\\\\\"advisable,\\\\\" '\n", - " '\\\\\"particularly,\\\\\" '\n", - " '\\\\\"precipitation\\\\\"). The '\n", - " 'sentence is '\n", - " 'well-structured and flows '\n", - " 'smoothly. There is a clear '\n", - " 'connection between the '\n", - " 'advice and the reason '\n", - " 'provided. There are no '\n", - " 'noticeable errors, and the '\n", - " 'message is coherent and '\n", - " 'easy to understand. The '\n", - " 'vocabulary is varied but '\n", - " 'not highly sophisticated, '\n", - " 'and the sentence structure '\n", - " 'is complex but not '\n", - " 'exceptionally '\n", - " 'so.\\\\nThe '\n", - " 'response is '\n", - " 'well-articulated, '\n", - " 'coherent, and uses varied '\n", - " 'vocabulary and sentence '\n", - " 'structure, fitting the '\n", - " 'definition of Proficient '\n", - " 'Fluency.\\\\n4\"}]'],\n", - " 'fluency_threshold': [3, 3, 3],\n", - " 'fluency_total_tokens': [1057, 1059, 1073],\n", - " 'gpt_fluency': [3.0, 3.0, 4.0]},\n", - " 'fluency': 3.3333333333333335,\n", - " 'fluency_completion_tokens': 125.33333333333333,\n", - " 'fluency_prompt_tokens': 937.6666666666666,\n", - " 'fluency_threshold': 3.0,\n", - " 'fluency_total_tokens': 1063.0,\n", - " 'gpt_fluency': 3.3333333333333335}\n" - ] - } - ], + "outputs": [], "source": [ "# Multi-turn conversation - evaluates fluency of each assistant turn\n", "conversation = {\n", @@ -604,48 +260,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "2658f610", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'fluency': 4.0,\n", - " 'fluency_completion_tokens': 123,\n", - " 'fluency_finish_reason': 'stop',\n", - " 'fluency_model': 'gpt-4.1-2025-04-14',\n", - " 'fluency_prompt_tokens': 929,\n", - " 'fluency_reason': 'The response demonstrates proficient fluency with good '\n", - " 'grammar, clear structure, and logical flow, but does not '\n", - " 'reach the level of exceptional fluency due to its '\n", - " 'simplicity.',\n", - " 'fluency_result': 'pass',\n", - " 'fluency_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", - " '\\\\\"I\\'ll fetch the current weather information for '\n", - " 'Seattle and send you an email with the '\n", - " 'details.\\\\\"}\"}]',\n", - " 'fluency_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think '\n", - " 'step by step: The response is grammatically correct '\n", - " 'and uses appropriate vocabulary. The sentence '\n", - " 'structure is clear and coherent, with no errors. '\n", - " 'The message is easily understood and flows '\n", - " 'smoothly, but it is relatively simple and lacks '\n", - " 'complex or varied sentence structures. There is no '\n", - " 'sophisticated vocabulary or nuanced expression, but '\n", - " 'the response is well-articulated and '\n", - " 'logical.\\\\nThe response demonstrates '\n", - " 'proficient fluency with good grammar, clear '\n", - " 'structure, and logical flow, but does not reach the '\n", - " 'level of exceptional fluency due to its '\n", - " 'simplicity.\\\\n4\"}]',\n", - " 'fluency_threshold': 3,\n", - " 'fluency_total_tokens': 1052,\n", - " 'gpt_fluency': 4.0}\n" - ] - } - ], + "outputs": [], "source": [ "# Conversation with tool calls - evaluates fluency of assistant responses\n", "conversation_with_tools = {\n", @@ -683,114 +301,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "c0d8bf46", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'evaluation_per_turn': {'fluency': [2.0, 4.0],\n", - " 'fluency_completion_tokens': [124, 127],\n", - " 'fluency_finish_reason': ['stop', 'stop'],\n", - " 'fluency_model': ['gpt-4.1-2025-04-14',\n", - " 'gpt-4.1-2025-04-14'],\n", - " 'fluency_prompt_tokens': [927, 942],\n", - " 'fluency_reason': ['The response has frequent '\n", - " 'grammatical errors and limited '\n", - " 'vocabulary, but the main idea is '\n", - " 'partially understandable.',\n", - " 'The response is well-written, '\n", - " 'clear, and coherent, with good '\n", - " 'grammar and appropriate '\n", - " 'vocabulary, but lacks the '\n", - " 'complexity and nuance required '\n", - " 'for the highest score.'],\n", - " 'fluency_result': ['fail', 'pass'],\n", - " 'fluency_sample_input': ['[{\"role\": \"user\", '\n", - " '\"content\": '\n", - " '\"{\\\\\"response\\\\\": \\\\\"Your '\n", - " 'order is being process '\n", - " 'currently and will be ship '\n", - " 'very soon to you.\\\\\"}\"}]',\n", - " '[{\"role\": \"user\", '\n", - " '\"content\": '\n", - " '\"{\\\\\"response\\\\\": \\\\\"The '\n", - " 'estimated delivery date for '\n", - " 'your order is March 15, '\n", - " '2025. You will receive '\n", - " 'tracking information via '\n", - " 'email once the package has '\n", - " 'been dispatched.\\\\\"}\"}]'],\n", - " 'fluency_sample_output': ['[{\"role\": \"assistant\", '\n", - " '\"content\": \"Let\\'s '\n", - " 'think step by step: The '\n", - " 'response communicates a '\n", - " 'simple idea about an order '\n", - " 'being processed and '\n", - " 'shipped. However, there '\n", - " 'are noticeable grammatical '\n", - " 'errors (\\\\\"being '\n", - " 'process\\\\\" should be '\n", - " '\\\\\"being processed\\\\\", '\n", - " '\\\\\"will be ship\\\\\" should '\n", - " 'be \\\\\"will be shipped\\\\\"), '\n", - " 'and the sentence structure '\n", - " 'is awkward. Vocabulary is '\n", - " 'very basic, and the '\n", - " 'sentences are short and '\n", - " 'not well-constructed. '\n", - " 'Despite these issues, the '\n", - " 'main message can be '\n", - " 'understood with some '\n", - " 'effort.\\\\nThe '\n", - " 'response has frequent '\n", - " 'grammatical errors and '\n", - " 'limited vocabulary, but '\n", - " 'the main idea is partially '\n", - " 'understandable.\\\\n2\"}]',\n", - " '[{\"role\": \"assistant\", '\n", - " '\"content\": \"Let\\'s '\n", - " 'think step by step: The '\n", - " 'response is grammatically '\n", - " 'correct and uses clear, '\n", - " 'precise language. '\n", - " 'Vocabulary is appropriate '\n", - " 'for the context, and '\n", - " 'sentences are '\n", - " 'well-structured and '\n", - " 'coherent. There are no '\n", - " 'errors, and the message is '\n", - " 'easy to understand. '\n", - " 'However, the vocabulary '\n", - " 'and sentence structure are '\n", - " 'straightforward and not '\n", - " 'particularly sophisticated '\n", - " 'or varied, which suggests '\n", - " 'it is not at the highest '\n", - " 'level of '\n", - " 'fluency.\\\\nThe '\n", - " 'response is well-written, '\n", - " 'clear, and coherent, with '\n", - " 'good grammar and '\n", - " 'appropriate vocabulary, '\n", - " 'but lacks the complexity '\n", - " 'and nuance required for '\n", - " 'the highest '\n", - " 'score.\\\\n4\"}]'],\n", - " 'fluency_threshold': [3, 3],\n", - " 'fluency_total_tokens': [1051, 1069],\n", - " 'gpt_fluency': [2.0, 4.0]},\n", - " 'fluency': 3.0,\n", - " 'fluency_completion_tokens': 125.5,\n", - " 'fluency_prompt_tokens': 934.5,\n", - " 'fluency_threshold': 3.0,\n", - " 'fluency_total_tokens': 1060.0,\n", - " 'gpt_fluency': 3.0}\n" - ] - } - ], + "outputs": [], "source": [ "# Conversation with varying fluency levels across turns\n", "conversation_mixed = {\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb index 4830ad40..9dd74c95 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "298d067a", "metadata": {}, "outputs": [], @@ -129,61 +129,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "4e5fa898", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== Well-Grounded Weather Response ===\n", - "{'gpt_groundedness': 4.0,\n", - " 'groundedness': 4.0,\n", - " 'groundedness_completion_tokens': 156,\n", - " 'groundedness_finish_reason': 'stop',\n", - " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", - " 'groundedness_prompt_tokens': 1205,\n", - " 'groundedness_reason': 'The response is correct and covers most key points '\n", - " 'but leaves out some specific details (humidity, wind, '\n", - " 'visibility) present in the context.',\n", - " 'groundedness_result': 'pass',\n", - " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", - " '\\\\\"The current weather in Seattle is rainy with '\n", - " \"a temperature of 14\\\\\\\\u00b0C. It's typical \"\n", - " 'Pacific Northwest weather for this time of year '\n", - " 'with overcast skies and light '\n", - " 'precipitation.\\\\\", \\\\\"context\\\\\": \\\\\"Current '\n", - " 'weather data for Seattle shows rainy conditions '\n", - " 'with a temperature of 14\\\\\\\\u00b0C. \\\\\\\\nThe '\n", - " 'forecast indicates overcast skies with light '\n", - " 'precipitation typical for Pacific Northwest '\n", - " 'weather. \\\\\\\\nHumidity is at 85% with winds '\n", - " 'from the southwest at 12 mph. Visibility is '\n", - " 'reduced to 8 miles due to rain.\\\\\"}\"}]',\n", - " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", - " 'think step by step: The context provides '\n", - " 'several details: rainy conditions, temperature '\n", - " 'of 14\\\\u00b0C, typical Pacific Northwest '\n", - " 'weather, overcast skies, light precipitation, '\n", - " 'humidity at 85%, southwest winds at 12 mph, '\n", - " 'and reduced visibility to 8 miles. The '\n", - " 'response accurately mentions rainy conditions, '\n", - " 'the temperature, typical Pacific Northwest '\n", - " 'weather, overcast skies, and light '\n", - " 'precipitation. However, it omits details about '\n", - " 'humidity, wind speed/direction, and visibility '\n", - " 'reduction. The response is accurate but does '\n", - " 'not include all relevant details from the '\n", - " 'context.\\\\nThe response is correct '\n", - " 'and covers most key points but leaves out some '\n", - " 'specific details (humidity, wind, visibility) '\n", - " 'present in the context.\\\\n4\"}]',\n", - " 'groundedness_threshold': 3,\n", - " 'groundedness_total_tokens': 1361}\n" - ] - } - ], + "outputs": [], "source": [ "# Example of a well-grounded response using weather context\n", "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", @@ -207,59 +156,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "6de6a51e", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== Partially Grounded Response ===\n", - "{'gpt_groundedness': 2.0,\n", - " 'groundedness': 2.0,\n", - " 'groundedness_completion_tokens': 139,\n", - " 'groundedness_finish_reason': 'stop',\n", - " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", - " 'groundedness_prompt_tokens': 1185,\n", - " 'groundedness_reason': 'The response includes accurate information from the '\n", - " 'context but also introduces incorrect and unsupported '\n", - " 'details, making it unreliable.',\n", - " 'groundedness_result': 'fail',\n", - " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": '\n", - " '\\\\\"The current weather in Seattle is rainy with '\n", - " 'a temperature of 14\\\\\\\\u00b0C. The city is '\n", - " 'experiencing its wettest month in 50 years, and '\n", - " 'the mayor has declared a weather emergency due '\n", - " 'to flooding concerns.\\\\\", \\\\\"context\\\\\": '\n", - " '\\\\\"Current weather data for Seattle shows rainy '\n", - " 'conditions with a temperature of 14\\\\\\\\u00b0C. '\n", - " '\\\\\\\\nThe forecast indicates overcast skies with '\n", - " 'light precipitation typical for Pacific '\n", - " 'Northwest weather.\\\\\"}\"}]',\n", - " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", - " 'think step by step: The context states that '\n", - " 'Seattle is experiencing rainy conditions with '\n", - " 'a temperature of 14\\\\u00b0C, and the forecast '\n", - " 'is for overcast skies with light precipitation '\n", - " 'typical for the region. The response '\n", - " 'accurately repeats the current weather and '\n", - " 'temperature, which is grounded. However, it '\n", - " 'adds that Seattle is experiencing its wettest '\n", - " 'month in 50 years and that the mayor has '\n", - " 'declared a weather emergency due to flooding '\n", - " 'concerns. These claims are not supported or '\n", - " 'mentioned in the context, making them '\n", - " 'incorrect additions.\\\\nThe response '\n", - " 'includes accurate information from the context '\n", - " 'but also introduces incorrect and unsupported '\n", - " 'details, making it '\n", - " 'unreliable.\\\\n2\"}]',\n", - " 'groundedness_threshold': 3,\n", - " 'groundedness_total_tokens': 1324}\n" - ] - } - ], + "outputs": [], "source": [ "# Example of partially grounded response with unsupported claims\n", "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", @@ -282,62 +182,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "c77316c4", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== Enhanced Evaluation with Weather Query ===\n", - "{'gpt_groundedness': 4.0,\n", - " 'groundedness': 4.0,\n", - " 'groundedness_completion_tokens': 157,\n", - " 'groundedness_finish_reason': 'stop',\n", - " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", - " 'groundedness_prompt_tokens': 1412,\n", - " 'groundedness_reason': 'The response is accurate and relevant, but it is '\n", - " 'missing the wind speed and direction mentioned in the '\n", - " 'context, making it slightly incomplete.',\n", - " 'groundedness_result': 'pass',\n", - " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"How is the weather in Seattle?\\\\\", '\n", - " '\\\\\"response\\\\\": \\\\\"The weather in Seattle is '\n", - " 'rainy with a temperature of 14\\\\\\\\u00b0C. These '\n", - " 'are typical Pacific Northwest conditions with '\n", - " 'overcast skies.\\\\\", \\\\\"context\\\\\": \\\\\"Weather '\n", - " 'report for Seattle, Washington: Currently '\n", - " 'experiencing rainy weather with temperature at '\n", - " '14\\\\\\\\u00b0C. \\\\\\\\nOvercast conditions with '\n", - " 'light rain are expected to continue. The '\n", - " 'current conditions are typical for the Pacific '\n", - " 'Northwest region \\\\\\\\nduring this season. Wind '\n", - " 'speed is moderate at 12 mph from southwest '\n", - " 'direction.\\\\\"}\"}]',\n", - " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", - " 'think step by step: The query asks about the '\n", - " 'current weather in Seattle. The context '\n", - " 'provides a detailed weather report: rainy, '\n", - " '14\\\\u00b0C, overcast, light rain, typical for '\n", - " 'the region, and moderate wind from the '\n", - " \"southwest at 12 mph. The response states it's \"\n", - " 'rainy, 14\\\\u00b0C, typical Pacific Northwest '\n", - " 'conditions, and overcast skies. The response '\n", - " 'accurately reflects the rain, temperature, '\n", - " 'typicality, and overcast conditions, but omits '\n", - " 'the wind speed and direction. No incorrect '\n", - " 'information is present, but not all details '\n", - " 'are included.\\\\nThe response is '\n", - " 'accurate and relevant, but it is missing the '\n", - " 'wind speed and direction mentioned in the '\n", - " 'context, making it slightly '\n", - " 'incomplete.\\\\n4\"}]',\n", - " 'groundedness_threshold': 3,\n", - " 'groundedness_total_tokens': 1569}\n" - ] - } - ], + "outputs": [], "source": [ "# Example with query for enhanced evaluation - consistent with ToolCallAccuracyEvaluator\n", "query = \"How is the weather in Seattle?\"\n", @@ -364,70 +212,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "72b922ee", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== Agent Response with File Search ===\n", - "{'gpt_groundedness': 5.0,\n", - " 'groundedness': 5.0,\n", - " 'groundedness_completion_tokens': 183,\n", - " 'groundedness_finish_reason': 'stop',\n", - " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", - " 'groundedness_prompt_tokens': 1435,\n", - " 'groundedness_reason': 'The response is fully relevant, complete, and '\n", - " 'directly answers the query with specific weather '\n", - " 'information for Seattle, as would be expected from a '\n", - " 'tool call.',\n", - " 'groundedness_result': 'pass',\n", - " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"Can you get me the current weather '\n", - " 'information for Seattle?\\\\\", \\\\\"response\\\\\": '\n", - " '[{\\\\\"role\\\\\": \\\\\"assistant\\\\\", \\\\\"content\\\\\": '\n", - " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", - " '\\\\\"tool_call_id\\\\\": '\n", - " '\\\\\"call_filesearch_weather_123\\\\\", \\\\\"name\\\\\": '\n", - " '\\\\\"file_search\\\\\", \\\\\"arguments\\\\\": '\n", - " '{\\\\\"query\\\\\": \\\\\"current weather Seattle '\n", - " 'temperature conditions\\\\\"}}]}, {\\\\\"role\\\\\": '\n", - " '\\\\\"assistant\\\\\", \\\\\"content\\\\\": \\\\\"Based on the '\n", - " 'weather data, Seattle is currently experiencing '\n", - " 'rainy weather with a temperature of '\n", - " '14\\\\\\\\u00b0C. The conditions include overcast '\n", - " 'skies and light precipitation, which is typical '\n", - " 'for the Pacific Northwest.\\\\\"}], \\\\\"context\\\\\": '\n", - " '\\\\\"<>\\\\\"}\"}]',\n", - " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", - " 'think step by step: The context is empty, so I '\n", - " 'must rely on the tool results and the query. '\n", - " 'The query asks for the current weather '\n", - " 'information for Seattle. The response first '\n", - " 'shows a tool call to search for current '\n", - " 'weather in Seattle, then provides a weather '\n", - " 'report: rainy, 14\\\\u00b0C, overcast skies, '\n", - " 'light precipitation, typical for the Pacific '\n", - " 'Northwest. Since there is no explicit tool '\n", - " 'result shown, I must judge groundedness based '\n", - " 'on whether the response is plausible and '\n", - " 'relevant to the query. The answer is directly '\n", - " 'relevant, provides specific details, and does '\n", - " 'not introduce unrelated or incorrect '\n", - " 'information. There is no evidence of error or '\n", - " 'missing key details for the '\n", - " 'query.\\\\nThe response is fully '\n", - " 'relevant, complete, and directly answers the '\n", - " 'query with specific weather information for '\n", - " 'Seattle, as would be expected from a tool '\n", - " 'call.\\\\n5\"}]',\n", - " 'groundedness_threshold': 3,\n", - " 'groundedness_total_tokens': 1618}\n" - ] - } - ], + "outputs": [], "source": [ "# Example of agent evaluation with file_search tool for weather information\n", "query = \"Can you get me the current weather information for Seattle?\"\n", @@ -496,261 +284,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "0f37ae51", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== Conversation Evaluation ===\n", - "{'evaluation_per_turn': {'gpt_groundedness': [5.0, 5.0, 5.0],\n", - " 'groundedness': [5.0, 5.0, 5.0],\n", - " 'groundedness_completion_tokens': [107, 161, 130],\n", - " 'groundedness_finish_reason': ['stop', 'stop', 'stop'],\n", - " 'groundedness_model': ['gpt-4.1-2025-04-14',\n", - " 'gpt-4.1-2025-04-14',\n", - " 'gpt-4.1-2025-04-14'],\n", - " 'groundedness_prompt_tokens': [1402, 1409, 1427],\n", - " 'groundedness_reason': ['The response is fully '\n", - " 'correct and complete, '\n", - " 'directly reflecting all the '\n", - " 'information provided in the '\n", - " \"context about Seattle's \"\n", - " 'weather.',\n", - " 'The response is fully '\n", - " 'accurate, complete, and '\n", - " 'directly grounded in the '\n", - " 'provided context, addressing '\n", - " 'the query with all relevant '\n", - " 'details.',\n", - " 'The response is fully '\n", - " 'grounded in the context, '\n", - " 'accurately summarizing the '\n", - " 'weather for both cities as '\n", - " 'described, and directly '\n", - " 'addresses the query.'],\n", - " 'groundedness_result': ['pass', 'pass', 'pass'],\n", - " 'groundedness_sample_input': ['[{\"role\": \"user\", '\n", - " '\"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"Can '\n", - " 'you check the weather '\n", - " 'in Seattle for me?\\\\\", '\n", - " '\\\\\"response\\\\\": '\n", - " '\\\\\"According to the '\n", - " 'current weather data, '\n", - " 'Seattle is '\n", - " 'experiencing rainy '\n", - " 'conditions with a '\n", - " 'temperature of '\n", - " '14\\\\\\\\u00b0C and '\n", - " 'overcast skies.\\\\\", '\n", - " '\\\\\"context\\\\\": '\n", - " '\\\\\"{\\'global_context\\': '\n", - " \"'Weather data shows \"\n", - " 'Seattle currently has '\n", - " 'rainy conditions at '\n", - " '14\\\\\\\\u00b0C with '\n", - " 'overcast skies. London '\n", - " 'shows cloudy weather '\n", - " 'at 8\\\\\\\\u00b0C with '\n", - " 'partly cloudy '\n", - " 'conditions. Both '\n", - " 'cities are '\n", - " 'experiencing typical '\n", - " 'seasonal weather '\n", - " 'patterns.\\'}\\\\\"}\"}]',\n", - " '[{\"role\": \"user\", '\n", - " '\"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"How '\n", - " 'does that compare to '\n", - " 'London?\\\\\", '\n", - " '\\\\\"response\\\\\": '\n", - " '\\\\\"London is currently '\n", - " 'cloudier but drier '\n", - " 'than Seattle, with a '\n", - " 'temperature of '\n", - " '8\\\\\\\\u00b0C and partly '\n", - " 'cloudy conditions. '\n", - " 'Seattle is warmer but '\n", - " 'rainier at '\n", - " '14\\\\\\\\u00b0C.\\\\\", '\n", - " '\\\\\"context\\\\\": '\n", - " '\\\\\"{\\'global_context\\': '\n", - " \"'Weather data shows \"\n", - " 'Seattle currently has '\n", - " 'rainy conditions at '\n", - " '14\\\\\\\\u00b0C with '\n", - " 'overcast skies. London '\n", - " 'shows cloudy weather '\n", - " 'at 8\\\\\\\\u00b0C with '\n", - " 'partly cloudy '\n", - " 'conditions. Both '\n", - " 'cities are '\n", - " 'experiencing typical '\n", - " 'seasonal weather '\n", - " 'patterns.\\'}\\\\\"}\"}]',\n", - " '[{\"role\": \"user\", '\n", - " '\"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"Can '\n", - " 'you email me a summary '\n", - " \"of both cities' \"\n", - " 'weather?\\\\\", '\n", - " '\\\\\"response\\\\\": \\\\\"I '\n", - " 'can provide you with a '\n", - " 'weather summary: '\n", - " 'Seattle has rainy '\n", - " 'weather at '\n", - " '14\\\\\\\\u00b0C with '\n", - " 'overcast skies, while '\n", - " 'London has partly '\n", - " 'cloudy conditions at '\n", - " '8\\\\\\\\u00b0C. However, '\n", - " 'I would need email '\n", - " 'access to send this '\n", - " 'information to '\n", - " 'you.\\\\\", '\n", - " '\\\\\"context\\\\\": '\n", - " '\\\\\"{\\'global_context\\': '\n", - " \"'Weather data shows \"\n", - " 'Seattle currently has '\n", - " 'rainy conditions at '\n", - " '14\\\\\\\\u00b0C with '\n", - " 'overcast skies. London '\n", - " 'shows cloudy weather '\n", - " 'at 8\\\\\\\\u00b0C with '\n", - " 'partly cloudy '\n", - " 'conditions. Both '\n", - " 'cities are '\n", - " 'experiencing typical '\n", - " 'seasonal weather '\n", - " 'patterns.\\'}\\\\\"}\"}]'],\n", - " 'groundedness_sample_output': ['[{\"role\": '\n", - " '\"assistant\", '\n", - " '\"content\": '\n", - " '\"Let\\'s think '\n", - " 'step by step: The '\n", - " 'query asks for the '\n", - " 'current weather in '\n", - " 'Seattle. The context '\n", - " 'provides specific '\n", - " 'details: Seattle has '\n", - " 'rainy conditions, '\n", - " '14\\\\u00b0C '\n", - " 'temperature, and '\n", - " 'overcast skies. The '\n", - " 'response repeats '\n", - " 'these details '\n", - " 'accurately and '\n", - " 'completely, without '\n", - " 'adding unrelated or '\n", - " 'incorrect '\n", - " 'information. All '\n", - " 'relevant details from '\n", - " 'the context are '\n", - " 'included in the '\n", - " 'response.\\\\nThe '\n", - " 'response is fully '\n", - " 'correct and complete, '\n", - " 'directly reflecting '\n", - " 'all the information '\n", - " 'provided in the '\n", - " 'context about '\n", - " \"Seattle's \"\n", - " 'weather.\\\\n5\"}]',\n", - " '[{\"role\": '\n", - " '\"assistant\", '\n", - " '\"content\": '\n", - " '\"Let\\'s think '\n", - " 'step by step: The '\n", - " 'context provides '\n", - " 'current weather data '\n", - " 'for Seattle and '\n", - " 'London, specifying '\n", - " 'that Seattle is rainy '\n", - " 'and 14\\\\u00b0C with '\n", - " 'overcast skies, while '\n", - " 'London is 8\\\\u00b0C '\n", - " 'and partly cloudy. '\n", - " 'The query asks for a '\n", - " 'comparison to London. '\n", - " 'The response '\n", - " 'accurately compares '\n", - " 'the two cities: it '\n", - " 'states London is '\n", - " 'cloudier but drier '\n", - " '(correct, as London '\n", - " 'is partly cloudy and '\n", - " 'not rainy), and '\n", - " 'Seattle is warmer but '\n", - " 'rainier (correct, as '\n", - " 'Seattle is 14\\\\u00b0C '\n", - " 'and rainy). All '\n", - " 'relevant details from '\n", - " 'the context are '\n", - " 'included, and no '\n", - " 'extraneous or '\n", - " 'incorrect information '\n", - " 'is '\n", - " 'added.\\\\nThe '\n", - " 'response is fully '\n", - " 'accurate, complete, '\n", - " 'and directly grounded '\n", - " 'in the provided '\n", - " 'context, addressing '\n", - " 'the query with all '\n", - " 'relevant '\n", - " 'details.\\\\n5\"}]',\n", - " '[{\"role\": '\n", - " '\"assistant\", '\n", - " '\"content\": '\n", - " '\"Let\\'s think '\n", - " 'step by step: The '\n", - " 'query asks for an '\n", - " 'emailed summary of '\n", - " 'the weather in both '\n", - " 'Seattle and London. '\n", - " 'The context provides '\n", - " 'specific weather '\n", - " 'details for both '\n", - " 'cities. The response '\n", - " 'summarizes the '\n", - " 'weather for both '\n", - " 'cities accurately and '\n", - " 'completely, matching '\n", - " 'the context. It also '\n", - " 'notes the need for '\n", - " 'email access, which '\n", - " 'is a relevant '\n", - " 'clarification but '\n", - " 'does not detract from '\n", - " 'the completeness or '\n", - " 'accuracy of the '\n", - " 'summary. No incorrect '\n", - " 'or unrelated '\n", - " 'information is '\n", - " 'introduced.\\\\nThe '\n", - " 'response is fully '\n", - " 'grounded in the '\n", - " 'context, accurately '\n", - " 'summarizing the '\n", - " 'weather for both '\n", - " 'cities as described, '\n", - " 'and directly '\n", - " 'addresses the '\n", - " 'query.\\\\n5\"}]'],\n", - " 'groundedness_threshold': [3, 3, 3],\n", - " 'groundedness_total_tokens': [1509, 1570, 1557]},\n", - " 'gpt_groundedness': 5.0,\n", - " 'groundedness': 5.0,\n", - " 'groundedness_completion_tokens': 132.66666666666666,\n", - " 'groundedness_prompt_tokens': 1412.6666666666667,\n", - " 'groundedness_threshold': 3.0,\n", - " 'groundedness_total_tokens': 1545.3333333333333}\n" - ] - } - ], + "outputs": [], "source": [ "# Example of conversation evaluation - using weather and email theme for consistency\n", "conversation = {\n", @@ -799,63 +336,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "e46ac0ec", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "=== Email Context Evaluation ===\n", - "{'gpt_groundedness': 5.0,\n", - " 'groundedness': 5.0,\n", - " 'groundedness_completion_tokens': 142,\n", - " 'groundedness_finish_reason': 'stop',\n", - " 'groundedness_model': 'gpt-4.1-2025-04-14',\n", - " 'groundedness_prompt_tokens': 1421,\n", - " 'groundedness_reason': 'The response is fully correct and complete, directly '\n", - " 'addressing the query and including all relevant '\n", - " 'details from the context.',\n", - " 'groundedness_result': 'pass',\n", - " 'groundedness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"Can you send me an email with weather '\n", - " 'information for Seattle?\\\\\", \\\\\"response\\\\\": '\n", - " '\\\\\"I can send you an email with the Seattle '\n", - " 'weather information. The current weather shows '\n", - " 'rainy conditions at 14\\\\\\\\u00b0C with overcast '\n", - " \"skies. I'll prepare this information for your \"\n", - " 'email.\\\\\", \\\\\"context\\\\\": \\\\\"Email service is '\n", - " 'available. Current weather data for Seattle: '\n", - " 'rainy conditions, 14\\\\\\\\u00b0C temperature, '\n", - " '\\\\\\\\novercast skies with light precipitation. '\n", - " 'User email preferences show john@example.com as '\n", - " 'primary contact. \\\\\\\\nEmail system can send '\n", - " 'weather reports with current conditions and '\n", - " 'forecasts.\\\\\"}\"}]',\n", - " 'groundedness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s '\n", - " 'think step by step: The query asks for an '\n", - " 'email with Seattle weather information. The '\n", - " 'context confirms that email service is '\n", - " \"available, the user's email is known, and \"\n", - " 'provides current weather details (rainy, '\n", - " '14\\\\u00b0C, overcast, light precipitation). '\n", - " 'The response states it can send an email, '\n", - " 'summarizes the current weather accurately, and '\n", - " 'says it will prepare the information for '\n", - " 'email. All relevant details from the context '\n", - " 'are included, and the response is directly '\n", - " 'tied to the query and context without adding '\n", - " 'unrelated information.\\\\nThe response '\n", - " 'is fully correct and complete, directly '\n", - " 'addressing the query and including all '\n", - " 'relevant details from the '\n", - " 'context.\\\\n5\"}]',\n", - " 'groundedness_threshold': 3,\n", - " 'groundedness_total_tokens': 1563}\n" - ] - } - ], + "outputs": [], "source": [ "# Example with email context - consistent with ToolCallAccuracyEvaluator email scenarios\n", "query = \"Can you send me an email with weather information for Seattle?\"\n", @@ -870,47 +354,6 @@ "print(\"=== Email Context Evaluation ===\")\n", "pprint(result)" ] - }, - { - "cell_type": "markdown", - "id": "a617af7b", - "metadata": {}, - "source": [ - "## Understanding the Results\n", - "\n", - "The Groundedness evaluator returns a dictionary with key metrics:\n", - "\n", - "- **`groundedness`**: Main score (1-5) indicating how well-grounded the response is\n", - "- **`groundedness_result`**: Binary result (\"pass\" or \"fail\") based on threshold\n", - "- **`groundedness_threshold`**: Threshold for pass/fail determination (default: 3)\n", - "- **`groundedness_reason`**: Detailed explanation of the grounding assessment\n", - "- **Token usage information**: For monitoring costs and performance\n", - "\n", - "### Interpreting Scores:\n", - "- **Score 5**: Fully grounded - all claims supported by context\n", - "- **Score 4**: Mostly grounded - most claims supported, minor unsupported details\n", - "- **Score 3**: Partially grounded - some claims supported, some not\n", - "- **Score 2**: Mostly ungrounded - few claims supported by context\n", - "- **Score 1**: Completely ungrounded - no claims supported by context\n", - "\n", - "### Best Practices:\n", - "1. **Provide Comprehensive Context**: Include all relevant source material\n", - "2. **Use Clear Queries**: When provided, queries help focus the evaluation\n", - "3. **Monitor Agent Tools**: For agent responses, ensure tool results are properly grounded\n", - "4. **Regular Evaluation**: Test groundedness across different domains and use cases\n", - "\n", - "### Use Cases:\n", - "- **RAG Applications**: Ensure retrieved information supports responses\n", - "- **Customer Support**: Verify responses are based on knowledge base\n", - "- **Agent Systems**: Validate tool-based responses against retrieved data\n", - "- **Content Generation**: Ensure factual accuracy and source attribution\n", - "- **Multi-turn Conversations**: Maintain context grounding across dialogue\n", - "\n", - "### Comparison with Other Evaluators:\n", - "- **vs Relevance**: Groundedness checks factual support, Relevance checks topic alignment\n", - "- **vs Tool Call Accuracy**: Groundedness validates response content, Tool Call Accuracy validates tool usage\n", - "- **Combined Use**: Use all three evaluators for comprehensive agent evaluation" - ] } ], "metadata": { diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb index e68dde18..0b73807b 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb @@ -20,14 +20,13 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "pip install azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." ] }, { @@ -61,17 +60,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Class IntentResolutionEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", @@ -104,53 +95,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: What are the opening hours of the Eiffel Tower?\n", - "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Opening hours of the Eiffel Tower are 9:00 AM to 11:00 PM.\n", - "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Opening hours of the Eiffel Tower are 9:00 AM to 11:00 PM.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_intent_resolution': 5.0,\n", - " 'intent_resolution': 5.0,\n", - " 'intent_resolution_completion_tokens': 52,\n", - " 'intent_resolution_finish_reason': 'stop',\n", - " 'intent_resolution_model': 'gpt-4.1-2025-04-14',\n", - " 'intent_resolution_prompt_tokens': 1907,\n", - " 'intent_resolution_reason': 'The user asked for the opening hours of the '\n", - " 'Eiffel Tower. The agent provided a clear and '\n", - " 'direct answer with specific times, fully '\n", - " \"satisfying the user's intent without any notable \"\n", - " 'gaps or errors.',\n", - " 'intent_resolution_result': 'pass',\n", - " 'intent_resolution_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"What are the opening '\n", - " 'hours of the Eiffel Tower?\\\\\", '\n", - " '\\\\\"response\\\\\": \\\\\"Opening hours of the '\n", - " 'Eiffel Tower are 9:00 AM to 11:00 PM.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": null}\"}]',\n", - " 'intent_resolution_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The user asked for '\n", - " 'the opening hours of the Eiffel Tower. '\n", - " 'The agent provided a clear and direct '\n", - " 'answer with specific times, fully '\n", - " \"satisfying the user's intent without any \"\n", - " 'notable gaps or errors.\\\\\",\\\\n '\n", - " '\\\\\"score\\\\\": 5\\\\n}\"}]',\n", - " 'intent_resolution_threshold': 3,\n", - " 'intent_resolution_total_tokens': 1959}\n" - ] - } - ], + "outputs": [], "source": [ "# Success example. Intent is identified and understood and the response correctly resolves user intent\n", "result = intent_resolution_evaluator(\n", @@ -162,55 +109,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: What is the opening hours of the Eiffel Tower?\n", - "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Please check the official website for the up-to-date information on Eiffel Tower opening hours.\n", - "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Please check the official website for the up-to-date information on Eiffel Tower opening hours.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_intent_resolution': 2.0,\n", - " 'intent_resolution': 2.0,\n", - " 'intent_resolution_completion_tokens': 57,\n", - " 'intent_resolution_finish_reason': 'stop',\n", - " 'intent_resolution_model': 'gpt-4.1-2025-04-14',\n", - " 'intent_resolution_prompt_tokens': 1905,\n", - " 'intent_resolution_reason': 'The user asked for the opening hours of the '\n", - " 'Eiffel Tower. The agent did not provide any '\n", - " 'actual information, only redirecting the user to '\n", - " 'the official website, leaving the intent '\n", - " 'unresolved and offering no direct assistance.',\n", - " 'intent_resolution_result': 'fail',\n", - " 'intent_resolution_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"What is the opening '\n", - " 'hours of the Eiffel Tower?\\\\\", '\n", - " '\\\\\"response\\\\\": \\\\\"Please check the '\n", - " 'official website for the up-to-date '\n", - " 'information on Eiffel Tower opening '\n", - " 'hours.\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " 'null}\"}]',\n", - " 'intent_resolution_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The user asked for '\n", - " 'the opening hours of the Eiffel Tower. '\n", - " 'The agent did not provide any actual '\n", - " 'information, only redirecting the user to '\n", - " 'the official website, leaving the intent '\n", - " 'unresolved and offering no direct '\n", - " 'assistance.\\\\\",\\\\n \\\\\"score\\\\\": 2\\\\n}\"}]',\n", - " 'intent_resolution_threshold': 3,\n", - " 'intent_resolution_total_tokens': 1962}\n" - ] - } - ], + "outputs": [], "source": [ "# Failure example. Even though intent is correctly identified, the response does not resolve the user intent\n", "result = intent_resolution_evaluator(\n", @@ -229,63 +130,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: What's the status of order #789?\n", - "Empty agent response extracted, likely due to input schema change. Falling back to using the original response: Order #789 has been shipped and will arrive on March 18, 2025.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_intent_resolution': 5.0,\n", - " 'intent_resolution': 5.0,\n", - " 'intent_resolution_completion_tokens': 59,\n", - " 'intent_resolution_finish_reason': 'stop',\n", - " 'intent_resolution_model': 'gpt-4.1-2025-04-14',\n", - " 'intent_resolution_prompt_tokens': 1904,\n", - " 'intent_resolution_reason': 'The user asked for the status of order #789. The '\n", - " 'agent provided a clear and complete update, '\n", - " 'stating the order has shipped and specifying the '\n", - " 'expected arrival date, fully resolving the '\n", - " \"user's intent with accurate and relevant \"\n", - " 'information.',\n", - " 'intent_resolution_result': 'pass',\n", - " 'intent_resolution_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"What\\'s the status of '\n", - " 'order #789?\\\\\", \\\\\"response\\\\\": \\\\\"Order '\n", - " '#789 has been shipped and will arrive on '\n", - " 'March 18, 2025.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": {\\\\\"name\\\\\": '\n", - " '\\\\\"get_order\\\\\", \\\\\"description\\\\\": \\\\\"Get '\n", - " 'the details of a specific order.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"order_id\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", - " 'order ID to get the details '\n", - " 'for.\\\\\"}}}}}\"}]',\n", - " 'intent_resolution_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The user asked for '\n", - " 'the status of order #789. The agent '\n", - " 'provided a clear and complete update, '\n", - " 'stating the order has shipped and '\n", - " 'specifying the expected arrival date, '\n", - " \"fully resolving the user's intent with \"\n", - " 'accurate and relevant '\n", - " 'information.\\\\\",\\\\n \\\\\"score\\\\\": '\n", - " '5\\\\n}\"}]',\n", - " 'intent_resolution_threshold': 3,\n", - " 'intent_resolution_total_tokens': 1963}\n" - ] - } - ], + "outputs": [], "source": [ "# Example with a single tool definition dict instead of a list\n", "query = \"What's the status of order #789?\"\n", @@ -317,92 +164,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_intent_resolution': 5.0,\n", - " 'intent_resolution': 5.0,\n", - " 'intent_resolution_completion_tokens': 59,\n", - " 'intent_resolution_finish_reason': 'stop',\n", - " 'intent_resolution_model': 'gpt-4.1-2025-04-14',\n", - " 'intent_resolution_prompt_tokens': 1999,\n", - " 'intent_resolution_reason': 'The user requested status updates for their last '\n", - " 'two orders on account #888. The agent provided '\n", - " 'clear, specific information for both orders, '\n", - " 'including shipping and expected delivery dates, '\n", - " \"fully resolving the user's intent with accuracy \"\n", - " 'and completeness.',\n", - " 'intent_resolution_result': 'pass',\n", - " 'intent_resolution_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"User turn 1:\\\\\\\\n Hi, I '\n", - " 'need help with the last 2 orders on my '\n", - " 'account #888. Could you please update me '\n", - " 'on their status?\\\\\\\\n\\\\\\\\n\\\\\", '\n", - " '\\\\\"response\\\\\": \\\\\"Hello! Let me quickly '\n", - " 'look up your account details.\\\\\\\\nThanks '\n", - " 'for your patience. I see two orders on '\n", - " 'your account. Let me fetch the details for '\n", - " 'both.\\\\\\\\nThe order with ID 123 has been '\n", - " 'shipped and is expected to be delivered on '\n", - " 'March 15, 2025. However, the order with ID '\n", - " '124 is delayed and should now arrive by '\n", - " 'March 20, 2025. Is there anything else I '\n", - " 'can help you with?\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", - " '\\\\\"get_orders\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Get the list of orders for a given '\n", - " 'account number.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"account_number\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The account number '\n", - " 'to get the orders for.\\\\\"}}}}, '\n", - " '{\\\\\"name\\\\\": \\\\\"get_order\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Get the details of a '\n", - " 'specific order.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"order_id\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The order ID to get '\n", - " 'the details for.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", - " '\\\\\"initiate_return\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Initiate the return process for an '\n", - " 'order.\\\\\", \\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"order_id\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", - " 'order ID for the return process.\\\\\"}}}}, '\n", - " '{\\\\\"name\\\\\": '\n", - " '\\\\\"update_shipping_address\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Update the shipping '\n", - " 'address for a given account.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"account_number\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", - " 'account number to update.\\\\\"}, '\n", - " '\\\\\"new_address\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", - " 'new shipping address.\\\\\"}}}}]}\"}]',\n", - " 'intent_resolution_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The user requested '\n", - " 'status updates for their last two orders '\n", - " 'on account #888. The agent provided '\n", - " 'clear, specific information for both '\n", - " 'orders, including shipping and expected '\n", - " 'delivery dates, fully resolving the '\n", - " \"user's intent with accuracy and \"\n", - " 'completeness.\\\\\",\\\\n \\\\\"score\\\\\": '\n", - " '5\\\\n}\"}]',\n", - " 'intent_resolution_threshold': 3,\n", - " 'intent_resolution_total_tokens': 2058}\n" - ] - } - ], + "outputs": [], "source": [ "query = [\n", " {\"role\": \"system\", \"content\": \"You are a friendly and helpful customer service agent.\"},\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb new file mode 100644 index 00000000..1e94bbe1 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb @@ -0,0 +1,192 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Response Completeness Evaluator" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting Started\n", + "\n", + "This sample demonstrates how to use Response Completeness Evaluator on agent's response when ground truth is provided. This evaluator is helpful when you have ground truth to assess the quality of the agent's final response. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Response Completeness evaluator assesses the quality of an agent response by examining how well it aligns with the provided ground truth. The evaluation is based on the following scoring system:\n", + "\n", + "
\n",
+    "Score 1: Fully incomplete: The response misses all necessary and relevant information compared to the ground truth.\n",
+    "Score 2: Barely complete: The response contains only a small percentage of the necessary information.\n",
+    "Score 3: Moderately complete: The response includes about half of the necessary information.\n",
+    "Score 4: Mostly complete: The response contains most of the necessary information, with only minor omissions.\n",
+    "Score 5: Fully complete: The response perfectly matches all necessary and relevant information from the ground truth.\n",
+    "
\n", + "\n", + "The evaluation requires the following inputs:\n", + "\n", + "- Response: The response to be evaluated. (string)\n", + "- Ground Truth: The correct and complete information against which the response is compared. (string)\n", + "\n", + "The evaluator uses these inputs to determine the completeness score, ensuring that the response meaningfully addresses the query while adhering to the provided definitions and data." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize Completeness Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.evaluation import ResponseCompletenessEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "import os\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "\n", + "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluating for a ground_truth and response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# agent response is complete\n", + "result = response_completeness_evaluator(\n", + " response=\"Itinery: Day 1 check out the downtown district of the city on train; for Day 2, we can rest in hotel.\",\n", + " ground_truth=\"Itinery: Day 1 take a train to visit the downtown area for city sightseeing; Day 2 rests in hotel.\",\n", + ")\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# agent response is incomplete\n", + "result = response_completeness_evaluator(\n", + " response=\"The order with ID 123 has been shipped and is expected to be delivered on March 15, 2025. However, the order with ID 124 is delayed and should now arrive by March 20, 2025.\",\n", + " ground_truth=\"The order with ID 124 is delayed and should now arrive by March 20, 2025.\",\n", + ")\n", + "result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluating using Conversation format" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Conversation format - includes user query, expected ground truth, and agent response\n", + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What's the weather forecast for Seattle this weekend?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday.\",\n", + " \"context\": {\n", + " \"ground_truth\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday and 17°C on Sunday, with a 20% chance of rain on Sunday afternoon.\"\n", + " }\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = response_completeness_evaluator(conversation=conversation)\n", + "result" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb index 2126e759..1a260070 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb @@ -6,8 +6,6 @@ "source": [ "# Task Adherence Evaluator\n", "\n", - "# Intent Resolution Evaluator\n", - "\n", "## Objective\n", "This sample demonstrates to how to use task adherence evaluator on agent data. The supported input formats include:\n", "- simple data such as strings;\n", @@ -22,33 +20,13 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project url, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Getting Started\n", - "\n", - "This sample demonstrates how to use Task Adherence Evaluator\n", - "Before running the sample:\n", - "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "pip install azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project url, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the AI model, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." ] }, { @@ -83,17 +61,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Class TaskAdherenceEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "from azure.ai.evaluation import TaskAdherenceEvaluator, AzureOpenAIModelConfiguration\n", @@ -124,63 +94,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: What are the best practices for maintaining a healthy rose garden during the summer?\n", - "Agent response could not be parsed, falling back to original response: Make sure to water your roses regularly and trim them occasionally.\n", - "Agent response could not be parsed, falling back to original response: Make sure to water your roses regularly and trim them occasionally.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_task_adherence': 3.0,\n", - " 'task_adherence': 3.0,\n", - " 'task_adherence_completion_tokens': 117,\n", - " 'task_adherence_finish_reason': 'stop',\n", - " 'task_adherence_model': 'gpt-4.1-2025-04-14',\n", - " 'task_adherence_prompt_tokens': 2831,\n", - " 'task_adherence_reason': 'The intended task was to provide best practices for '\n", - " 'maintaining a healthy rose garden in summer. The '\n", - " 'assistant gave only basic advice (watering and '\n", - " 'trimming), omitting key practices like pest '\n", - " 'control, mulching, and fertilization. No system '\n", - " 'constraints or tool requirements were present.',\n", - " 'task_adherence_result': 'pass',\n", - " 'task_adherence_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"What are the best practices for '\n", - " 'maintaining a healthy rose garden during the '\n", - " 'summer?\\\\\", \\\\\"response\\\\\": \\\\\"Make sure to '\n", - " 'water your roses regularly and trim them '\n", - " 'occasionally.\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " 'null}\"}]',\n", - " 'task_adherence_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The intended task was '\n", - " 'to provide best practices for maintaining a '\n", - " 'healthy rose garden in summer. The assistant '\n", - " 'gave only basic advice (watering and '\n", - " 'trimming), omitting key practices like pest '\n", - " 'control, mulching, and fertilization. No '\n", - " 'system constraints or tool requirements were '\n", - " 'present.\\\\\",\\\\n \\\\\"agent_perceived_task\\\\\": '\n", - " '\\\\\"Give basic tips for summer rose garden '\n", - " 'care.\\\\\",\\\\n \\\\\"actual_task\\\\\": \\\\\"Provide '\n", - " 'comprehensive best practices for maintaining '\n", - " 'a healthy rose garden during summer.\\\\\",\\\\n '\n", - " '\\\\\"tools_used\\\\\": [],\\\\n '\n", - " '\\\\\"expected_tool_calls\\\\\": [],\\\\n '\n", - " '\\\\\"score\\\\\": 3\\\\n}\"}]',\n", - " 'task_adherence_threshold': 3,\n", - " 'task_adherence_total_tokens': 2948}\n" - ] - } - ], + "outputs": [], "source": [ "# Failure example, there's only a vague adherence to the task\n", "result = task_adherence_evaluator(\n", @@ -192,77 +108,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: What are the best practices for maintaining a healthy rose garden during the summer?\n", - "Agent response could not be parsed, falling back to original response: For optimal summer care of your rose garden, start by watering deeply early in the morning to ensure the roots are well-hydrated without encouraging fungal growth. Apply a 2-3 inch layer of organic mulch around the base of the plants to conserve moisture and regulate soil temperature. Fertilize with a balanced rose fertilizer every 4 to 6 weeks to support healthy growth. Prune away any dead or diseased wood to promote good air circulation, and inspect regularly for pests such as aphids or spider mites, treating them promptly with an appropriate organic insecticidal soap. Finally, ensure that your roses receive at least 6 hours of direct sunlight daily for robust flowering.\n", - "Agent response could not be parsed, falling back to original response: For optimal summer care of your rose garden, start by watering deeply early in the morning to ensure the roots are well-hydrated without encouraging fungal growth. Apply a 2-3 inch layer of organic mulch around the base of the plants to conserve moisture and regulate soil temperature. Fertilize with a balanced rose fertilizer every 4 to 6 weeks to support healthy growth. Prune away any dead or diseased wood to promote good air circulation, and inspect regularly for pests such as aphids or spider mites, treating them promptly with an appropriate organic insecticidal soap. Finally, ensure that your roses receive at least 6 hours of direct sunlight daily for robust flowering.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_task_adherence': 5.0,\n", - " 'task_adherence': 5.0,\n", - " 'task_adherence_completion_tokens': 118,\n", - " 'task_adherence_finish_reason': 'stop',\n", - " 'task_adherence_model': 'gpt-4.1-2025-04-14',\n", - " 'task_adherence_prompt_tokens': 2955,\n", - " 'task_adherence_reason': 'The assistant provided a thorough, well-structured '\n", - " 'list of best practices for summer rose garden care, '\n", - " \"directly addressing the user's request. No system \"\n", - " 'constraints or tool requirements were present, and '\n", - " 'the response was complete and relevant.',\n", - " 'task_adherence_result': 'pass',\n", - " 'task_adherence_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"What are the best practices for '\n", - " 'maintaining a healthy rose garden during the '\n", - " 'summer?\\\\\", \\\\\"response\\\\\": \\\\\"For optimal '\n", - " 'summer care of your rose garden, start by '\n", - " 'watering deeply early in the morning to '\n", - " 'ensure the roots are well-hydrated without '\n", - " 'encouraging fungal growth. Apply a 2-3 inch '\n", - " 'layer of organic mulch around the base of the '\n", - " 'plants to conserve moisture and regulate soil '\n", - " 'temperature. Fertilize with a balanced rose '\n", - " 'fertilizer every 4 to 6 weeks to support '\n", - " 'healthy growth. Prune away any dead or '\n", - " 'diseased wood to promote good air '\n", - " 'circulation, and inspect regularly for pests '\n", - " 'such as aphids or spider mites, treating them '\n", - " 'promptly with an appropriate organic '\n", - " 'insecticidal soap. Finally, ensure that your '\n", - " 'roses receive at least 6 hours of direct '\n", - " 'sunlight daily for robust flowering.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": null}\"}]',\n", - " 'task_adherence_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The assistant provided '\n", - " 'a thorough, well-structured list of best '\n", - " 'practices for summer rose garden care, '\n", - " \"directly addressing the user's request. No \"\n", - " 'system constraints or tool requirements were '\n", - " 'present, and the response was complete and '\n", - " 'relevant.\\\\\",\\\\n '\n", - " '\\\\\"agent_perceived_task\\\\\": \\\\\"List best '\n", - " 'practices for maintaining a healthy rose '\n", - " 'garden during summer.\\\\\",\\\\n '\n", - " '\\\\\"actual_task\\\\\": \\\\\"Provide best practices '\n", - " 'for maintaining a healthy rose garden during '\n", - " 'the summer, with no system constraints or '\n", - " 'tool requirements.\\\\\",\\\\n \\\\\"tools_used\\\\\": '\n", - " '[],\\\\n \\\\\"expected_tool_calls\\\\\": [],\\\\n '\n", - " '\\\\\"score\\\\\": 5\\\\n}\"}]',\n", - " 'task_adherence_threshold': 3,\n", - " 'task_adherence_total_tokens': 3073}\n" - ] - } - ], + "outputs": [], "source": [ "# Success example, full adherence to the task\n", "result = task_adherence_evaluator(\n", @@ -281,67 +129,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: Can you recommend a science fiction book for me?\n", - "Agent response could not be parsed, falling back to original response: I found 'Dune' by Frank Herbert for you. It's a classic science fiction novel set in a distant future.\n", - "Tool definitions could not be parsed, falling back to original definitions: {'name': 'get_book', 'description': 'Retrieve a book recommendation for a specified genre.', 'parameters': {'type': 'object', 'properties': {'genre': {'type': 'string', 'description': 'The genre for which a book recommendation is requested.'}}}}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_task_adherence': 5.0,\n", - " 'task_adherence': 5.0,\n", - " 'task_adherence_completion_tokens': 99,\n", - " 'task_adherence_finish_reason': 'stop',\n", - " 'task_adherence_model': 'gpt-4.1-2025-04-14',\n", - " 'task_adherence_prompt_tokens': 2893,\n", - " 'task_adherence_reason': 'The assistant provided a relevant science fiction '\n", - " \"book recommendation, directly fulfilling the user's \"\n", - " 'request. There were no system constraints or tool '\n", - " 'requirements, so the lack of tool use does not '\n", - " 'impact the score.',\n", - " 'task_adherence_result': 'pass',\n", - " 'task_adherence_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"Can you recommend a science fiction book '\n", - " 'for me?\\\\\", \\\\\"response\\\\\": \\\\\"I found '\n", - " \"'Dune' by Frank Herbert for you. It's a \"\n", - " 'classic science fiction novel set in a '\n", - " 'distant future.\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " '{\\\\\"name\\\\\": \\\\\"get_book\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Retrieve a book '\n", - " 'recommendation for a specified genre.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"genre\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The genre '\n", - " 'for which a book recommendation is '\n", - " 'requested.\\\\\"}}}}}\"}]',\n", - " 'task_adherence_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The assistant provided '\n", - " 'a relevant science fiction book '\n", - " 'recommendation, directly fulfilling the '\n", - " \"user's request. There were no system \"\n", - " 'constraints or tool requirements, so the '\n", - " 'lack of tool use does not impact the '\n", - " 'score.\\\\\",\\\\n \\\\\"agent_perceived_task\\\\\": '\n", - " '\\\\\"recommend a science fiction book\\\\\",\\\\n '\n", - " '\\\\\"actual_task\\\\\": \\\\\"Recommend a science '\n", - " 'fiction book (no system constraints or tool '\n", - " 'requirements)\\\\\",\\\\n \\\\\"tools_used\\\\\": '\n", - " '[],\\\\n \\\\\"expected_tool_calls\\\\\": [],\\\\n '\n", - " '\\\\\"score\\\\\": 5\\\\n}\"}]',\n", - " 'task_adherence_threshold': 3,\n", - " 'task_adherence_total_tokens': 2992}\n" - ] - } - ], + "outputs": [], "source": [ "# Example with a single tool definition dict\n", "query = \"Can you recommend a science fiction book for me?\"\n", @@ -375,75 +165,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_task_adherence': 5.0,\n", - " 'task_adherence': 5.0,\n", - " 'task_adherence_completion_tokens': 114,\n", - " 'task_adherence_finish_reason': 'stop',\n", - " 'task_adherence_model': 'gpt-4.1-2025-04-14',\n", - " 'task_adherence_prompt_tokens': 2986,\n", - " 'task_adherence_reason': \"The assistant correctly identified the user's \"\n", - " 'request for a historical fiction recommendation, '\n", - " 'used the appropriate tool, and provided a relevant '\n", - " 'book with a brief summary. All system and user '\n", - " 'requirements were met without error.',\n", - " 'task_adherence_result': 'pass',\n", - " 'task_adherence_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"SYSTEM_PROMPT:\\\\\\\\n You are an expert in '\n", - " 'literature and at provid can provide book '\n", - " 'recommendations.\\\\\\\\n\\\\\\\\nUser turn 1:\\\\\\\\n '\n", - " 'I love historical fiction. Can you recommend '\n", - " 'a good book from that genre?\\\\\\\\n\\\\\\\\n\\\\\", '\n", - " '\\\\\"response\\\\\": \\\\\"Let me fetch a '\n", - " 'recommendation for historical '\n", - " 'fiction.\\\\\\\\n[TOOL_CALL] '\n", - " 'get_book(genre=\\\\\\\\\\\\\"historical '\n", - " 'fiction\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] { '\n", - " '\\\\\\\\\\\\\"book\\\\\\\\\\\\\": { \\\\\\\\\\\\\"title\\\\\\\\\\\\\": '\n", - " '\\\\\\\\\\\\\"The Pillars of the Earth\\\\\\\\\\\\\", '\n", - " '\\\\\\\\\\\\\"author\\\\\\\\\\\\\": \\\\\\\\\\\\\"Ken '\n", - " 'Follett\\\\\\\\\\\\\", \\\\\\\\\\\\\"summary\\\\\\\\\\\\\": '\n", - " '\\\\\\\\\\\\\"A captivating tale set in medieval '\n", - " 'England that weaves historical events with '\n", - " 'personal drama.\\\\\\\\\\\\\" } }\\\\\\\\nBased on our '\n", - " \"records, I recommend 'The Pillars of the \"\n", - " \"Earth' by Ken Follett. This novel is an \"\n", - " 'excellent example of historical fiction with '\n", - " 'a rich narrative and well-developed '\n", - " 'characters. Would you like more details or '\n", - " 'another suggestion?\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- get_book: Retrieve '\n", - " 'a book recommendation for a specified genre. '\n", - " '(inputs: genre)\\\\\"}\"}]',\n", - " 'task_adherence_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The assistant '\n", - " \"correctly identified the user's request for \"\n", - " 'a historical fiction recommendation, used '\n", - " 'the appropriate tool, and provided a '\n", - " 'relevant book with a brief summary. All '\n", - " 'system and user requirements were met '\n", - " 'without error.\\\\\",\\\\n '\n", - " '\\\\\"agent_perceived_task\\\\\": \\\\\"recommend a '\n", - " 'good historical fiction book using available '\n", - " 'records and tools\\\\\",\\\\n \\\\\"actual_task\\\\\": '\n", - " '\\\\\"Recommend a good historical fiction book, '\n", - " 'using expertise and available tools as '\n", - " 'needed\\\\\",\\\\n \\\\\"tools_used\\\\\": '\n", - " '[\\\\\"get_book\\\\\"],\\\\n '\n", - " '\\\\\"expected_tool_calls\\\\\": '\n", - " '[\\\\\"get_book\\\\\"],\\\\n \\\\\"score\\\\\": 5\\\\n}\"}]',\n", - " 'task_adherence_threshold': 3,\n", - " 'task_adherence_total_tokens': 3100}\n" - ] - } - ], + "outputs": [], "source": [ "query = [\n", " {\"role\": \"system\", \"content\": \"You are an expert in literature and at provid can provide book recommendations.\"},\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb index 6da7483d..05958d1e 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb @@ -21,14 +21,13 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "pip install azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." ] }, { @@ -70,18 +69,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "0a5fae33", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Class _TaskCompletionEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "from azure.ai.evaluation._evaluators._task_completion import _TaskCompletionEvaluator\n", @@ -117,72 +108,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "57d4592d", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", - "Agent response could not be parsed, falling back to original response: The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\n", - "Agent response could not be parsed, falling back to original response: The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_completion': True,\n", - " 'task_completion_completion_tokens': 107,\n", - " 'task_completion_details': {'completion_gaps': 'None; all relevant weather '\n", - " 'details were provided.',\n", - " 'delivered_outcome': 'Agent delivered a detailed '\n", - " 'weather update with '\n", - " 'temperature, cloud cover, '\n", - " 'wind, humidity, and '\n", - " 'precipitation forecast.',\n", - " 'task_requirements': 'User requested the current '\n", - " 'weather conditions in '\n", - " 'Seattle.'},\n", - " 'task_completion_finish_reason': 'stop',\n", - " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", - " 'task_completion_prompt_tokens': 1771,\n", - " 'task_completion_reason': 'The agent provided a complete and current weather '\n", - " 'report for Seattle, including temperature, '\n", - " 'conditions, wind, humidity, and precipitation '\n", - " 'outlook.',\n", - " 'task_completion_result': 'pass',\n", - " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"How is the weather in Seattle?\\\\\", '\n", - " '\\\\\"response\\\\\": \\\\\"The current weather in '\n", - " 'Seattle is partly cloudy with a temperature '\n", - " \"of 15\\\\\\\\u00b0C (59\\\\\\\\u00b0F). There's a \"\n", - " 'light breeze from the northwest at 8 mph, '\n", - " 'and the humidity is at 68%. No precipitation '\n", - " 'is expected for the rest of the day.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": null}\"}]',\n", - " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The agent provided a '\n", - " 'complete and current weather report for '\n", - " 'Seattle, including temperature, conditions, '\n", - " 'wind, humidity, and precipitation '\n", - " 'outlook.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"task_requirements\\\\\": \\\\\"User requested '\n", - " 'the current weather conditions in '\n", - " 'Seattle.\\\\\",\\\\n \\\\\"delivered_outcome\\\\\": '\n", - " '\\\\\"Agent delivered a detailed weather '\n", - " 'update with temperature, cloud cover, wind, '\n", - " 'humidity, and precipitation '\n", - " 'forecast.\\\\\",\\\\n \\\\\"completion_gaps\\\\\": '\n", - " '\\\\\"None; all relevant weather details were '\n", - " 'provided.\\\\\"\\\\n },\\\\n \\\\\"success\\\\\": '\n", - " 'true\\\\n}\"}]',\n", - " 'task_completion_total_tokens': 1878}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "response = \"The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\"\n", @@ -202,82 +131,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "71ea8725", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", - "Agent response could not be parsed, falling back to original response: I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\n", - "Agent response could not be parsed, falling back to original response: I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_completion': True,\n", - " 'task_completion_completion_tokens': 131,\n", - " 'task_completion_details': {'completion_gaps': 'No gaps; all relevant weather '\n", - " 'details were provided as '\n", - " 'requested.',\n", - " 'delivered_outcome': 'The agent delivered a '\n", - " 'concise weather summary: '\n", - " 'partly cloudy, 15°C, light '\n", - " 'breeze, and no rain '\n", - " 'expected today.',\n", - " 'task_requirements': 'The user asked for the '\n", - " 'current weather conditions '\n", - " 'in Seattle, expecting a '\n", - " 'usable summary of '\n", - " 'temperature, sky condition, '\n", - " 'and precipitation.'},\n", - " 'task_completion_finish_reason': 'stop',\n", - " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", - " 'task_completion_prompt_tokens': 1752,\n", - " 'task_completion_reason': 'The agent provided a current weather report for '\n", - " 'Seattle, including cloud cover, temperature, wind, '\n", - " \"and precipitation, fully addressing the user's \"\n", - " 'request.',\n", - " 'task_completion_result': 'pass',\n", - " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"How is the weather in Seattle?\\\\\", '\n", - " '\\\\\"response\\\\\": \\\\\"I\\'ve checked the weather '\n", - " \"for Seattle and found that it's currently \"\n", - " 'partly cloudy with a temperature of '\n", - " \"15\\\\\\\\u00b0C. There's a light breeze and no \"\n", - " 'rain expected today.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", - " 'Fetches the weather information for the '\n", - " 'specified location. (inputs: '\n", - " 'location)\\\\\"}\"}]',\n", - " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The agent provided a '\n", - " 'current weather report for Seattle, '\n", - " 'including cloud cover, temperature, wind, '\n", - " 'and precipitation, fully addressing the '\n", - " 'user\\'s request.\\\\\",\\\\n \\\\\"details\\\\\": '\n", - " '{\\\\n \\\\\"task_requirements\\\\\": \\\\\"The '\n", - " 'user asked for the current weather '\n", - " 'conditions in Seattle, expecting a usable '\n", - " 'summary of temperature, sky condition, and '\n", - " 'precipitation.\\\\\",\\\\n '\n", - " '\\\\\"delivered_outcome\\\\\": \\\\\"The agent '\n", - " 'delivered a concise weather summary: partly '\n", - " 'cloudy, 15\\\\u00b0C, light breeze, and no '\n", - " 'rain expected today.\\\\\",\\\\n '\n", - " '\\\\\"completion_gaps\\\\\": \\\\\"No gaps; all '\n", - " 'relevant weather details were provided as '\n", - " 'requested.\\\\\"\\\\n },\\\\n \\\\\"success\\\\\": '\n", - " 'true\\\\n}\"}]',\n", - " 'task_completion_total_tokens': 1883}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "response = \"I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\"\n", @@ -307,74 +164,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "9a9cd459", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: What's the current temperature in Boston?\n", - "Agent response could not be parsed, falling back to original response: The current temperature in Boston is 22°C (72°F) with clear skies. It's a beautiful day with low humidity and a gentle breeze.\n", - "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_completion': True,\n", - " 'task_completion_completion_tokens': 98,\n", - " 'task_completion_details': {'completion_gaps': 'None; all requested '\n", - " 'information was provided.',\n", - " 'delivered_outcome': 'Agent delivered the current '\n", - " 'temperature in both Celsius '\n", - " 'and Fahrenheit, plus brief '\n", - " 'weather conditions.',\n", - " 'task_requirements': 'User requested the current '\n", - " 'temperature in Boston.'},\n", - " 'task_completion_finish_reason': 'stop',\n", - " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", - " 'task_completion_prompt_tokens': 1750,\n", - " 'task_completion_reason': 'The agent provided the current temperature in '\n", - " 'Boston, including both Celsius and Fahrenheit, '\n", - " \"which fully answers the user's request.\",\n", - " 'task_completion_result': 'pass',\n", - " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"What\\'s the current temperature in '\n", - " 'Boston?\\\\\", \\\\\"response\\\\\": \\\\\"The current '\n", - " 'temperature in Boston is 22\\\\\\\\u00b0C '\n", - " \"(72\\\\\\\\u00b0F) with clear skies. It's a \"\n", - " 'beautiful day with low humidity and a gentle '\n", - " 'breeze.\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " '{\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", - " 'information for the specified location.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", - " 'weather for.\\\\\"}}}}}\"}]',\n", - " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The agent provided '\n", - " 'the current temperature in Boston, '\n", - " 'including both Celsius and Fahrenheit, '\n", - " \"which fully answers the user's \"\n", - " 'request.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"task_requirements\\\\\": \\\\\"User requested '\n", - " 'the current temperature in '\n", - " 'Boston.\\\\\",\\\\n \\\\\"delivered_outcome\\\\\": '\n", - " '\\\\\"Agent delivered the current temperature '\n", - " 'in both Celsius and Fahrenheit, plus brief '\n", - " 'weather conditions.\\\\\",\\\\n '\n", - " '\\\\\"completion_gaps\\\\\": \\\\\"None; all '\n", - " 'requested information was provided.\\\\\"\\\\n '\n", - " '},\\\\n \\\\\"success\\\\\": true\\\\n}\"}]',\n", - " 'task_completion_total_tokens': 1848}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"What's the current temperature in Boston?\"\n", "response = \"The current temperature in Boston is 22°C (72°F) with clear skies. It's a beautiful day with low humidity and a gentle breeze.\"\n", @@ -402,82 +195,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "915768db", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_completion': True,\n", - " 'task_completion_completion_tokens': 133,\n", - " 'task_completion_details': {'completion_gaps': 'No gaps if the email was '\n", - " 'truly sent; otherwise, the '\n", - " 'task would be incomplete.',\n", - " 'delivered_outcome': 'Agent states an email was '\n", - " \"sent with Seattle's weather \"\n", - " 'details (partly cloudy, '\n", - " \"15°C) to the user's \"\n", - " 'registered address.',\n", - " 'task_requirements': 'Send an email to the user '\n", - " 'containing current weather '\n", - " 'information for Seattle.'},\n", - " 'task_completion_finish_reason': 'stop',\n", - " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", - " 'task_completion_prompt_tokens': 1784,\n", - " 'task_completion_reason': 'The agent claims to have sent an email with '\n", - " 'Seattle weather information, including specific '\n", - " 'details. If the email was actually sent as stated, '\n", - " \"the user's request is fully met.\",\n", - " 'task_completion_result': 'pass',\n", - " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"Can you send me an email with weather '\n", - " 'information for Seattle?\\\\\", \\\\\"response\\\\\": '\n", - " '\\\\\"I\\'ll get the current weather information '\n", - " 'for Seattle and then send you an email with '\n", - " 'the details.\\\\\\\\nI have successfully sent '\n", - " 'you an email with the weather information '\n", - " 'for Seattle. The current weather is partly '\n", - " 'cloudy with a temperature of 15\\\\\\\\u00b0C. '\n", - " 'You should receive the email shortly at your '\n", - " 'registered email address.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", - " 'Fetches the weather information for the '\n", - " 'specified location. (inputs: location)\\\\\\\\n- '\n", - " 'send_email: Sends an email with the '\n", - " 'specified subject and body to the recipient. '\n", - " '(inputs: recipient, subject, body)\\\\\"}\"}]',\n", - " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The agent claims to '\n", - " 'have sent an email with Seattle weather '\n", - " 'information, including specific details. If '\n", - " 'the email was actually sent as stated, the '\n", - " 'user\\'s request is fully met.\\\\\",\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"task_requirements\\\\\": \\\\\"Send an email '\n", - " 'to the user containing current weather '\n", - " 'information for Seattle.\\\\\",\\\\n '\n", - " '\\\\\"delivered_outcome\\\\\": \\\\\"Agent states an '\n", - " \"email was sent with Seattle's weather \"\n", - " 'details (partly cloudy, 15\\\\u00b0C) to the '\n", - " 'user\\'s registered address.\\\\\",\\\\n '\n", - " '\\\\\"completion_gaps\\\\\": \\\\\"No gaps if the '\n", - " 'email was truly sent; otherwise, the task '\n", - " 'would be incomplete.\\\\\"\\\\n },\\\\n '\n", - " '\\\\\"success\\\\\": true\\\\n}\"}]',\n", - " 'task_completion_total_tokens': 1917}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"Can you send me an email with weather information for Seattle?\"\n", "response = [\n", @@ -543,89 +264,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "2c4772be", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: [{'role': 'system', 'content': 'You are a helpful assistant that can fetch weather information and send emails.'}, {'role': 'user', 'content': 'Hi, I need to plan my day. Can you check the weather in Seattle for me?'}, {'role': 'user', 'content': 'Also, please send me an email summary of the weather so I can reference it later.'}]\n", - "Agent response could not be parsed, falling back to original response: I've checked the weather in Seattle for you. It's currently 15°C and partly cloudy with light winds. I've also sent you an email summary with these details so you can reference them throughout your day. The task has been completed successfully.\n", - "Agent response could not be parsed, falling back to original response: I've checked the weather in Seattle for you. It's currently 15°C and partly cloudy with light winds. I've also sent you an email summary with these details so you can reference them throughout your day. The task has been completed successfully.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_completion': True,\n", - " 'task_completion_completion_tokens': 116,\n", - " 'task_completion_details': {'completion_gaps': 'None; both weather '\n", - " 'information and email summary '\n", - " 'were delivered as requested.',\n", - " 'delivered_outcome': 'Current weather details for '\n", - " 'Seattle and confirmation '\n", - " 'that an email summary was '\n", - " 'sent to the user.',\n", - " 'task_requirements': 'Check Seattle weather and '\n", - " 'send an email summary of '\n", - " 'the weather for later '\n", - " 'reference.'},\n", - " 'task_completion_finish_reason': 'stop',\n", - " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", - " 'task_completion_prompt_tokens': 1845,\n", - " 'task_completion_reason': 'The agent provided the current Seattle weather and '\n", - " 'confirmed sending an email summary, fulfilling '\n", - " 'both user requests. The deliverables are complete '\n", - " 'and actionable.',\n", - " 'task_completion_result': 'pass',\n", - " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '[{\\\\\"role\\\\\": \\\\\"system\\\\\", \\\\\"content\\\\\": '\n", - " '\\\\\"You are a helpful assistant that can '\n", - " 'fetch weather information and send '\n", - " 'emails.\\\\\"}, {\\\\\"role\\\\\": \\\\\"user\\\\\", '\n", - " '\\\\\"content\\\\\": \\\\\"Hi, I need to plan my day. '\n", - " 'Can you check the weather in Seattle for '\n", - " 'me?\\\\\"}, {\\\\\"role\\\\\": \\\\\"user\\\\\", '\n", - " '\\\\\"content\\\\\": \\\\\"Also, please send me an '\n", - " 'email summary of the weather so I can '\n", - " 'reference it later.\\\\\"}], \\\\\"response\\\\\": '\n", - " '\\\\\"I\\'ve checked the weather in Seattle for '\n", - " \"you. It's currently 15\\\\\\\\u00b0C and partly \"\n", - " \"cloudy with light winds. I've also sent you \"\n", - " 'an email summary with these details so you '\n", - " 'can reference them throughout your day. The '\n", - " 'task has been completed successfully.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", - " 'Fetches the weather information for the '\n", - " 'specified location. (inputs: location)\\\\\\\\n- '\n", - " 'send_email: Sends an email with the '\n", - " 'specified subject and body to the recipient. '\n", - " '(inputs: recipient, subject, body)\\\\\"}\"}]',\n", - " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The agent provided '\n", - " 'the current Seattle weather and confirmed '\n", - " 'sending an email summary, fulfilling both '\n", - " 'user requests. The deliverables are '\n", - " 'complete and actionable.\\\\\",\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"task_requirements\\\\\": \\\\\"Check Seattle '\n", - " 'weather and send an email summary of the '\n", - " 'weather for later reference.\\\\\",\\\\n '\n", - " '\\\\\"delivered_outcome\\\\\": \\\\\"Current weather '\n", - " 'details for Seattle and confirmation that '\n", - " 'an email summary was sent to the '\n", - " 'user.\\\\\",\\\\n \\\\\"completion_gaps\\\\\": '\n", - " '\\\\\"None; both weather information and email '\n", - " 'summary were delivered as requested.\\\\\"\\\\n '\n", - " '},\\\\n \\\\\"success\\\\\": true\\\\n}\"}]',\n", - " 'task_completion_total_tokens': 1961}\n" - ] - } - ], + "outputs": [], "source": [ "# Query as conversation history instead of a single string\n", "query_as_conversation = [\n", @@ -682,83 +324,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "9986fa70", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n", - "Agent response could not be parsed, falling back to original response: I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\n", - "Agent response could not be parsed, falling back to original response: I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_completion': False,\n", - " 'task_completion_completion_tokens': 128,\n", - " 'task_completion_details': {'completion_gaps': 'No email was sent, no '\n", - " 'specific weather information '\n", - " 'included, and the deliverable '\n", - " 'is not actionable or usable '\n", - " 'as requested.',\n", - " 'delivered_outcome': 'General comment about '\n", - " \"Seattle's usual weather, no \"\n", - " 'email sent, no specific '\n", - " 'weather details provided.',\n", - " 'task_requirements': 'Send an email containing '\n", - " 'weather information for '\n", - " 'Seattle.'},\n", - " 'task_completion_finish_reason': 'stop',\n", - " 'task_completion_model': 'gpt-4.1-2025-04-14',\n", - " 'task_completion_prompt_tokens': 1761,\n", - " 'task_completion_reason': 'The agent did not send an email or provide '\n", - " 'specific weather information for Seattle. The '\n", - " 'response only gave a general statement about '\n", - " 'typical weather, failing to deliver the requested '\n", - " 'actionable outcome.',\n", - " 'task_completion_result': 'fail',\n", - " 'task_completion_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"Can you send me an email with weather '\n", - " 'information for Seattle?\\\\\", \\\\\"response\\\\\": '\n", - " '\\\\\"I can see that you want weather '\n", - " 'information for Seattle. The weather there '\n", - " 'is usually quite nice this time of year, '\n", - " 'with temperatures ranging from mild to warm '\n", - " 'depending on the season.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", - " 'Fetches the weather information for the '\n", - " 'specified location. (inputs: location)\\\\\\\\n- '\n", - " 'send_email: Sends an email with the '\n", - " 'specified subject and body to the recipient. '\n", - " '(inputs: recipient, subject, body)\\\\\"}\"}]',\n", - " 'task_completion_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The agent did not '\n", - " 'send an email or provide specific weather '\n", - " 'information for Seattle. The response only '\n", - " 'gave a general statement about typical '\n", - " 'weather, failing to deliver the requested '\n", - " 'actionable outcome.\\\\\",\\\\n \\\\\"details\\\\\": '\n", - " '{\\\\n \\\\\"task_requirements\\\\\": \\\\\"Send an '\n", - " 'email containing weather information for '\n", - " 'Seattle.\\\\\",\\\\n \\\\\"delivered_outcome\\\\\": '\n", - " '\\\\\"General comment about Seattle\\'s usual '\n", - " 'weather, no email sent, no specific weather '\n", - " 'details provided.\\\\\",\\\\n '\n", - " '\\\\\"completion_gaps\\\\\": \\\\\"No email was '\n", - " 'sent, no specific weather information '\n", - " 'included, and the deliverable is not '\n", - " 'actionable or usable as requested.\\\\\"\\\\n '\n", - " '},\\\\n \\\\\"success\\\\\": false\\\\n}\"}]',\n", - " 'task_completion_total_tokens': 1889}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"Can you send me an email with weather information for Seattle?\"\n", "incomplete_response = \"I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\"\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb index 505fee4d..a6ea032d 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb @@ -17,18 +17,13 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "This is a deterministic evaluator that compares navigation paths.\n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "pip install azure-ai-evaluation\n", "```\n", - "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + "No environment variables are required for this evaluator." ] }, { @@ -116,22 +111,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "0cae15fb", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", - " 'precision_score': 0.0,\n", - " 'recall_score': 0.0},\n", - " 'task_navigation_efficiency_label': False,\n", - " 'task_navigation_efficiency_result': 'fail'}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "response = \"I'll check the weather for you. The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F).\"\n", @@ -154,22 +137,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "f53dde31", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", - " 'precision_score': 0.0,\n", - " 'recall_score': 0.0},\n", - " 'task_navigation_efficiency_label': False,\n", - " 'task_navigation_efficiency_result': 'fail'}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "response = [\n", @@ -215,22 +186,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "c2226715", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", - " 'precision_score': 0.0,\n", - " 'recall_score': 0.0},\n", - " 'task_navigation_efficiency_label': False,\n", - " 'task_navigation_efficiency_result': 'fail'}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"Can you send me an email with weather information for Seattle?\"\n", "response = [\n", @@ -330,22 +289,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "ccd95640", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", - " 'precision_score': 0.0,\n", - " 'recall_score': 0.0},\n", - " 'task_navigation_efficiency_label': False,\n", - " 'task_navigation_efficiency_result': 'fail'}\n" - ] - } - ], + "outputs": [], "source": [ "# Query as conversation history instead of a single string\n", "query_as_conversation = [\n", @@ -429,22 +376,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "b4bd7b47", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_navigation_efficiency_details': {'f1_score': 0.0,\n", - " 'precision_score': 0.0,\n", - " 'recall_score': 0.0},\n", - " 'task_navigation_efficiency_label': False,\n", - " 'task_navigation_efficiency_result': 'fail'}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "inefficient_response = [\n", @@ -494,22 +429,10 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "e4dcf585", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'task_navigation_efficiency_details': {'f1_score': 1.0,\n", - " 'precision_score': 1.0,\n", - " 'recall_score': 1.0},\n", - " 'task_navigation_efficiency_label': True,\n", - " 'task_navigation_efficiency_result': 'pass'}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"Check the weather in Seattle and send me an email about it\"\n", "\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb index eb161692..f55620fc 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb @@ -20,14 +20,13 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "pip install azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." ] }, { @@ -70,17 +69,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Class ToolCallAccuracyEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "from azure.ai.evaluation import ToolCallAccuracyEvaluator, AzureOpenAIModelConfiguration\n", @@ -113,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -138,108 +129,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy_completion_tokens': 292,\n", - " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 1,\n", - " 'excess_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'missing_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", - " 'correct_tool_percentage': 1.0,\n", - " 'tool_call_errors': 0,\n", - " 'tool_name': 'fetch_weather',\n", - " 'tool_success_result': 'pass',\n", - " 'total_calls_required': 1}],\n", - " 'tool_calls_made_by_agent': 1},\n", - " 'tool_call_accuracy_finish_reason': 'stop',\n", - " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_call_accuracy_prompt_tokens': 2124,\n", - " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", - " \"is 'How is the weather in Seattle?'. The \"\n", - " \"available tool is 'fetch_weather', which \"\n", - " 'fetches weather information for a specified '\n", - " 'location. The agent made one tool call: '\n", - " \"'fetch_weather' with the argument 'location': \"\n", - " \"'Seattle'. This is the correct tool and the \"\n", - " 'correct parameter, directly grounded from the '\n", - " \"user's query. Only one call is needed and only \"\n", - " 'one was made, with no excess or missing calls. '\n", - " 'There are no errors mentioned, and the call is '\n", - " 'efficient and relevant. This matches the '\n", - " 'criteria for Tool Call Accuracy Level 5 '\n", - " '(Optimal Solution).',\n", - " 'tool_call_accuracy_result': 'pass',\n", - " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"How is the weather in '\n", - " 'Seattle ?\\\\\", \\\\\"tool_calls\\\\\": '\n", - " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", - " '\\\\\"tool_call_id\\\\\": '\n", - " '\\\\\"call_CUdbkBfvVBla2YP3p24uhElJ\\\\\", '\n", - " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"arguments\\\\\": {\\\\\"location\\\\\": '\n", - " '\\\\\"Seattle\\\\\"}}], \\\\\"tool_definitions\\\\\": '\n", - " '[{\\\\\"id\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", - " 'information for the specified '\n", - " 'location.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The location to '\n", - " 'fetch weather for.\\\\\"}}}}]}\"}]',\n", - " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", - " \"step by step: The user's last query is \"\n", - " \"'How is the weather in Seattle?'. The \"\n", - " \"available tool is 'fetch_weather', which \"\n", - " 'fetches weather information for a '\n", - " 'specified location. The agent made one '\n", - " \"tool call: 'fetch_weather' with the \"\n", - " \"argument 'location': 'Seattle'. This is \"\n", - " 'the correct tool and the correct '\n", - " 'parameter, directly grounded from the '\n", - " \"user's query. Only one call is needed \"\n", - " 'and only one was made, with no excess or '\n", - " 'missing calls. There are no errors '\n", - " 'mentioned, and the call is efficient and '\n", - " 'relevant. This matches the criteria for '\n", - " 'Tool Call Accuracy Level 5 (Optimal '\n", - " 'Solution).\\\\\",\\\\n '\n", - " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"tool_calls_made_by_agent\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", - " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\",\\\\n '\n", - " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n '\n", - " '\\\\\"correct_tool_percentage\\\\\": '\n", - " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", - " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", - " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", - " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n },\\\\n '\n", - " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", - " 'tool_call_accuracy_threshold': 3,\n", - " 'tool_call_accuracy_total_tokens': 2416}\n" - ] - } - ], + "outputs": [], "source": [ "response = tool_call_accuracy(query=query, tool_calls=tool_call, tool_definitions=tool_definition)\n", "pprint(response)" @@ -254,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -287,133 +179,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_tool_call_accuracy': 3.0,\n", - " 'tool_call_accuracy': 3.0,\n", - " 'tool_call_accuracy_completion_tokens': 378,\n", - " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 1,\n", - " 'excess_tool_calls': {'details': [{'excess_count': 1,\n", - " 'tool_name': 'fetch_weather'}],\n", - " 'total': 1},\n", - " 'missing_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", - " 'correct_tool_percentage': 1.0,\n", - " 'tool_call_errors': 0,\n", - " 'tool_name': 'fetch_weather',\n", - " 'tool_success_result': 'pass',\n", - " 'total_calls_required': 1}],\n", - " 'tool_calls_made_by_agent': 2},\n", - " 'tool_call_accuracy_finish_reason': 'stop',\n", - " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_call_accuracy_prompt_tokens': 2172,\n", - " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", - " \"is 'How is the weather in Seattle?'. The only \"\n", - " 'relevant tool in the definitions is '\n", - " \"'fetch_weather', which fetches weather \"\n", - " 'information for a specified location. To answer '\n", - " \"the user's query, the agent should call \"\n", - " \"'fetch_weather' with the parameter 'location' \"\n", - " \"set to 'Seattle'. In the TOOL CALLS TO BE \"\n", - " 'EVALUATED section, the agent made two tool '\n", - " \"calls: one for 'Seattle' (correct and relevant) \"\n", - " \"and one for 'London' (irrelevant, as the user \"\n", - " 'did not ask about London). The parameter for '\n", - " 'the Seattle call is correctly grounded in the '\n", - " 'conversation, but the London call is '\n", - " \"unnecessary and not grounded in the user's \"\n", - " 'query. There are no errors or missing tool '\n", - " 'calls, but there is an unnecessary tool call '\n", - " 'for London, which makes the solution '\n", - " 'inefficient according to the rating '\n", - " 'definitions. Therefore, this matches Level 3: '\n", - " 'Relevant but Inefficient.',\n", - " 'tool_call_accuracy_result': 'pass',\n", - " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"How is the weather in '\n", - " 'Seattle ?\\\\\", \\\\\"tool_calls\\\\\": '\n", - " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", - " '\\\\\"tool_call_id\\\\\": '\n", - " '\\\\\"call_CUdbkBfvVBla2YP3p24uhElJ\\\\\", '\n", - " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"arguments\\\\\": {\\\\\"location\\\\\": '\n", - " '\\\\\"Seattle\\\\\"}}, {\\\\\"type\\\\\": '\n", - " '\\\\\"tool_call\\\\\", \\\\\"tool_call_id\\\\\": '\n", - " '\\\\\"call_CUdbkBfvVBla2YP3p24uhElJ\\\\\", '\n", - " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"arguments\\\\\": {\\\\\"location\\\\\": '\n", - " '\\\\\"London\\\\\"}}], \\\\\"tool_definitions\\\\\": '\n", - " '[{\\\\\"id\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", - " 'information for the specified '\n", - " 'location.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The location to '\n", - " 'fetch weather for.\\\\\"}}}}]}\"}]',\n", - " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", - " \"step by step: The user's last query is \"\n", - " \"'How is the weather in Seattle?'. The \"\n", - " 'only relevant tool in the definitions is '\n", - " \"'fetch_weather', which fetches weather \"\n", - " 'information for a specified location. To '\n", - " \"answer the user's query, the agent \"\n", - " \"should call 'fetch_weather' with the \"\n", - " \"parameter 'location' set to 'Seattle'. \"\n", - " 'In the TOOL CALLS TO BE EVALUATED '\n", - " 'section, the agent made two tool calls: '\n", - " \"one for 'Seattle' (correct and relevant) \"\n", - " \"and one for 'London' (irrelevant, as the \"\n", - " 'user did not ask about London). The '\n", - " 'parameter for the Seattle call is '\n", - " 'correctly grounded in the conversation, '\n", - " 'but the London call is unnecessary and '\n", - " \"not grounded in the user's query. There \"\n", - " 'are no errors or missing tool calls, but '\n", - " 'there is an unnecessary tool call for '\n", - " 'London, which makes the solution '\n", - " 'inefficient according to the rating '\n", - " 'definitions. Therefore, this matches '\n", - " 'Level 3: Relevant but '\n", - " 'Inefficient.\\\\\",\\\\n '\n", - " '\\\\\"tool_calls_success_level\\\\\": 3,\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"tool_calls_made_by_agent\\\\\": 2,\\\\n '\n", - " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", - " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\",\\\\n '\n", - " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n '\n", - " '\\\\\"correct_tool_percentage\\\\\": '\n", - " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", - " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", - " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", - " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 1,\\\\n \\\\\"details\\\\\": '\n", - " '[\\\\n {\\\\n '\n", - " '\\\\\"tool_name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\",\\\\n '\n", - " '\\\\\"excess_count\\\\\": 1\\\\n '\n", - " '}\\\\n ]\\\\n },\\\\n '\n", - " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", - " 'tool_call_accuracy_threshold': 3,\n", - " 'tool_call_accuracy_total_tokens': 2550}\n" - ] - } - ], + "outputs": [], "source": [ "response = tool_call_accuracy(query=query, tool_calls=tool_calls, tool_definitions=tool_definition)\n", "pprint(response)" @@ -429,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -520,183 +288,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy_completion_tokens': 426,\n", - " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 2,\n", - " 'excess_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'missing_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", - " 'correct_tool_percentage': 1.0,\n", - " 'tool_call_errors': 0,\n", - " 'tool_name': 'fetch_weather',\n", - " 'tool_success_result': 'pass',\n", - " 'total_calls_required': 1},\n", - " {'correct_calls_made_by_agent': 1,\n", - " 'correct_tool_percentage': 1.0,\n", - " 'tool_call_errors': 0,\n", - " 'tool_name': 'send_email',\n", - " 'tool_success_result': 'pass',\n", - " 'total_calls_required': 1}],\n", - " 'tool_calls_made_by_agent': 2},\n", - " 'tool_call_accuracy_finish_reason': 'stop',\n", - " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_call_accuracy_prompt_tokens': 2332,\n", - " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", - " 'is to send an email with weather information '\n", - " 'for Seattle. The available tools are '\n", - " \"'fetch_weather' (to get weather info for a \"\n", - " \"location) and 'send_email' (to send an email \"\n", - " 'with specified content). The agent first called '\n", - " \"'fetch_weather' with the parameter 'Seattle', \"\n", - " \"which is directly grounded from the user's \"\n", - " 'query. The result returned the weather as '\n", - " \"'Rainy, 14°C'. Next, the agent called \"\n", - " \"'send_email' with the recipient \"\n", - " \"'your_email@example.com', subject 'Weather \"\n", - " \"Information for Seattle', and a body containing \"\n", - " 'the weather information just fetched. The '\n", - " 'recipient email is generic, but since the user '\n", - " 'did not specify an email address, it is '\n", - " 'reasonable to use a placeholder. Both tool '\n", - " 'calls are relevant, correctly parameterized, '\n", - " 'and executed successfully. No unnecessary '\n", - " 'duplicate calls were made, and no errors '\n", - " 'occurred. All necessary tools were used to '\n", - " \"fulfill the user's request. Therefore, this is \"\n", - " 'an optimal solution.',\n", - " 'tool_call_accuracy_result': 'pass',\n", - " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"Can you send me an '\n", - " 'email with weather information for '\n", - " 'Seattle?\\\\\", \\\\\"tool_calls\\\\\": '\n", - " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", - " '\\\\\"tool_call_id\\\\\": '\n", - " '\\\\\"call_CUdbkBfvVBla2YP3p24uhElJ\\\\\", '\n", - " '\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"arguments\\\\\": {\\\\\"location\\\\\": '\n", - " '\\\\\"Seattle\\\\\"}, \\\\\"tool_result\\\\\": '\n", - " '{\\\\\"weather\\\\\": \\\\\"Rainy, '\n", - " '14\\\\\\\\u00b0C\\\\\"}}, {\\\\\"type\\\\\": '\n", - " '\\\\\"tool_call\\\\\", \\\\\"tool_call_id\\\\\": '\n", - " '\\\\\"call_iq9RuPxqzykebvACgX8pqRW2\\\\\", '\n", - " '\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", - " '\\\\\"arguments\\\\\": {\\\\\"recipient\\\\\": '\n", - " '\\\\\"your_email@example.com\\\\\", '\n", - " '\\\\\"subject\\\\\": \\\\\"Weather Information for '\n", - " 'Seattle\\\\\", \\\\\"body\\\\\": \\\\\"The current '\n", - " 'weather in Seattle is rainy with a '\n", - " 'temperature of 14\\\\\\\\u00b0C.\\\\\"}, '\n", - " '\\\\\"tool_result\\\\\": {\\\\\"message\\\\\": '\n", - " '\\\\\"Email successfully sent to '\n", - " 'your_email@example.com.\\\\\"}}], '\n", - " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Fetches the weather information for '\n", - " 'the specified location.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", - " 'location to fetch weather for.\\\\\"}}}}, '\n", - " '{\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Sends an email with '\n", - " 'the specified subject and body to the '\n", - " 'recipient.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"recipient\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Email address of '\n", - " 'the recipient.\\\\\"}, \\\\\"subject\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Subject of the '\n", - " 'email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", - " 'content of the email.\\\\\"}}}}]}\"}]',\n", - " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", - " \"step by step: The user's last query is \"\n", - " 'to send an email with weather '\n", - " 'information for Seattle. The available '\n", - " \"tools are 'fetch_weather' (to get \"\n", - " 'weather info for a location) and '\n", - " \"'send_email' (to send an email with \"\n", - " 'specified content). The agent first '\n", - " \"called 'fetch_weather' with the \"\n", - " \"parameter 'Seattle', which is directly \"\n", - " \"grounded from the user's query. The \"\n", - " \"result returned the weather as 'Rainy, \"\n", - " \"14\\\\u00b0C'. Next, the agent called \"\n", - " \"'send_email' with the recipient \"\n", - " \"'your_email@example.com', subject \"\n", - " \"'Weather Information for Seattle', and a \"\n", - " 'body containing the weather information '\n", - " 'just fetched. The recipient email is '\n", - " 'generic, but since the user did not '\n", - " 'specify an email address, it is '\n", - " 'reasonable to use a placeholder. Both '\n", - " 'tool calls are relevant, correctly '\n", - " 'parameterized, and executed '\n", - " 'successfully. No unnecessary duplicate '\n", - " 'calls were made, and no errors occurred. '\n", - " 'All necessary tools were used to fulfill '\n", - " \"the user's request. Therefore, this is \"\n", - " 'an optimal solution.\\\\\",\\\\n '\n", - " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"tool_calls_made_by_agent\\\\\": 2,\\\\n '\n", - " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", - " '2,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", - " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\",\\\\n '\n", - " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n '\n", - " '\\\\\"correct_tool_percentage\\\\\": '\n", - " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", - " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", - " '\\\\\"pass\\\\\"\\\\n },\\\\n '\n", - " '{\\\\n \\\\\"tool_name\\\\\": '\n", - " '\\\\\"send_email\\\\\",\\\\n '\n", - " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n '\n", - " '\\\\\"correct_tool_percentage\\\\\": '\n", - " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", - " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", - " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", - " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n },\\\\n '\n", - " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", - " 'tool_call_accuracy_threshold': 3,\n", - " 'tool_call_accuracy_total_tokens': 2758}\n" - ] - } - ], + "outputs": [], "source": [ "response = tool_call_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", "pprint(response)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Response as List[dict] with Tool Definition as Single Dict" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -706,111 +305,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy_completion_tokens': 307,\n", - " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 1,\n", - " 'excess_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'missing_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", - " 'correct_tool_percentage': 1.0,\n", - " 'tool_call_errors': 0,\n", - " 'tool_name': 'book_flight',\n", - " 'tool_success_result': 'pass',\n", - " 'total_calls_required': 1}],\n", - " 'tool_calls_made_by_agent': 1},\n", - " 'tool_call_accuracy_finish_reason': 'stop',\n", - " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_call_accuracy_prompt_tokens': 2137,\n", - " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", - " \"is 'Book a flight to New York for tomorrow.' \"\n", - " \"The available tool is 'book_flight', which \"\n", - " 'books a flight to a specified destination and '\n", - " 'date. The agent made one tool call to '\n", - " \"'book_flight' with 'destination' set to 'New \"\n", - " \"York' and 'date' set to 'tomorrow', both of \"\n", - " \"which are directly grounded in the user's \"\n", - " 'query. Only one call is needed and was made, '\n", - " 'with no errors or unnecessary duplicates. No '\n", - " 'other tools are available or required for this '\n", - " 'task. Therefore, the tool call is fully '\n", - " 'relevant, efficient, and correct, matching the '\n", - " 'criteria for Level 5 (Optimal Solution).',\n", - " 'tool_call_accuracy_result': 'pass',\n", - " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"Book a flight to New '\n", - " 'York for tomorrow\\\\\", \\\\\"tool_calls\\\\\": '\n", - " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", - " '\\\\\"tool_call_id\\\\\": '\n", - " '\\\\\"call_book_flight_456\\\\\", \\\\\"name\\\\\": '\n", - " '\\\\\"book_flight\\\\\", \\\\\"arguments\\\\\": '\n", - " '{\\\\\"destination\\\\\": \\\\\"New York\\\\\", '\n", - " '\\\\\"date\\\\\": \\\\\"tomorrow\\\\\"}}], '\n", - " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", - " '\\\\\"book_flight\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Books a flight to the specified '\n", - " 'destination on the given date.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"destination\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", - " 'destination city.\\\\\"}, \\\\\"date\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The date of the '\n", - " 'flight.\\\\\"}}}}]}\"}]',\n", - " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", - " \"step by step: The user's last query is \"\n", - " \"'Book a flight to New York for \"\n", - " \"tomorrow.' The available tool is \"\n", - " \"'book_flight', which books a flight to a \"\n", - " 'specified destination and date. The '\n", - " 'agent made one tool call to '\n", - " \"'book_flight' with 'destination' set to \"\n", - " \"'New York' and 'date' set to 'tomorrow', \"\n", - " 'both of which are directly grounded in '\n", - " \"the user's query. Only one call is \"\n", - " 'needed and was made, with no errors or '\n", - " 'unnecessary duplicates. No other tools '\n", - " 'are available or required for this task. '\n", - " 'Therefore, the tool call is fully '\n", - " 'relevant, efficient, and correct, '\n", - " 'matching the criteria for Level 5 '\n", - " '(Optimal Solution).\\\\\",\\\\n '\n", - " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"tool_calls_made_by_agent\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", - " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", - " '\\\\\"book_flight\\\\\",\\\\n '\n", - " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n '\n", - " '\\\\\"correct_tool_percentage\\\\\": '\n", - " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", - " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", - " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", - " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n },\\\\n '\n", - " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", - " 'tool_call_accuracy_threshold': 3,\n", - " 'tool_call_accuracy_total_tokens': 2444}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"Book a flight to New York for tomorrow\"\n", "\n", @@ -847,113 +344,18 @@ "pprint(result)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Response as List[dict] with Tool Definition as Single Dict" + ] + }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy_completion_tokens': 297,\n", - " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 1,\n", - " 'excess_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'missing_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", - " 'correct_tool_percentage': 1.0,\n", - " 'tool_call_errors': 0,\n", - " 'tool_name': 'fetch_weather',\n", - " 'tool_success_result': 'pass',\n", - " 'total_calls_required': 1}],\n", - " 'tool_calls_made_by_agent': 1},\n", - " 'tool_call_accuracy_finish_reason': 'stop',\n", - " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_call_accuracy_prompt_tokens': 2121,\n", - " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", - " \"is 'What's the weather in San Francisco?'. The \"\n", - " \"available tool is 'fetch_weather', which \"\n", - " 'fetches weather information for a specified '\n", - " 'location. The agent made one tool call to '\n", - " \"'fetch_weather' with the parameter 'location' \"\n", - " \"set to 'San Francisco', which is directly \"\n", - " \"grounded from the user's query. The tool call \"\n", - " 'returned a successful result with the weather '\n", - " 'information. Only one call was made, which is '\n", - " 'efficient and sufficient to answer the query. '\n", - " 'No errors occurred, and no excess or missing '\n", - " 'tool calls are present. This matches the '\n", - " 'criteria for Tool Call Accuracy Level 5 '\n", - " '(Optimal Solution).',\n", - " 'tool_call_accuracy_result': 'pass',\n", - " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"What\\'s the weather in '\n", - " 'San Francisco?\\\\\", \\\\\"tool_calls\\\\\": '\n", - " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", - " '\\\\\"tool_call_id\\\\\": '\n", - " '\\\\\"call_sf_weather_789\\\\\", \\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"arguments\\\\\": '\n", - " '{\\\\\"location\\\\\": \\\\\"San Francisco\\\\\"}, '\n", - " '\\\\\"tool_result\\\\\": {\\\\\"weather\\\\\": '\n", - " '\\\\\"Foggy, 18\\\\\\\\u00b0C\\\\\"}}], '\n", - " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Fetches the weather information for '\n", - " 'the specified location.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", - " 'location to fetch weather '\n", - " 'for.\\\\\"}}}}]}\"}]',\n", - " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", - " \"step by step: The user's last query is \"\n", - " \"'What's the weather in San Francisco?'. \"\n", - " \"The available tool is 'fetch_weather', \"\n", - " 'which fetches weather information for a '\n", - " 'specified location. The agent made one '\n", - " \"tool call to 'fetch_weather' with the \"\n", - " \"parameter 'location' set to 'San \"\n", - " \"Francisco', which is directly grounded \"\n", - " \"from the user's query. The tool call \"\n", - " 'returned a successful result with the '\n", - " 'weather information. Only one call was '\n", - " 'made, which is efficient and sufficient '\n", - " 'to answer the query. No errors occurred, '\n", - " 'and no excess or missing tool calls are '\n", - " 'present. This matches the criteria for '\n", - " 'Tool Call Accuracy Level 5 (Optimal '\n", - " 'Solution).\\\\\",\\\\n '\n", - " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"tool_calls_made_by_agent\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", - " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\",\\\\n '\n", - " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n '\n", - " '\\\\\"correct_tool_percentage\\\\\": '\n", - " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", - " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", - " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", - " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n },\\\\n '\n", - " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", - " 'tool_call_accuracy_threshold': 3,\n", - " 'tool_call_accuracy_total_tokens': 2418}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"What's the weather in San Francisco?\"\n", "\n", @@ -1014,7 +416,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1079,165 +481,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'gpt_tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy': 5.0,\n", - " 'tool_call_accuracy_completion_tokens': 410,\n", - " 'tool_call_accuracy_details': {'correct_tool_calls_made_by_agent': 2,\n", - " 'excess_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'missing_tool_calls': {'details': [],\n", - " 'total': 0},\n", - " 'per_tool_call_details': [{'correct_calls_made_by_agent': 1,\n", - " 'correct_tool_percentage': 1.0,\n", - " 'tool_call_errors': 0,\n", - " 'tool_name': 'fetch_weather',\n", - " 'tool_success_result': 'pass',\n", - " 'total_calls_required': 1},\n", - " {'correct_calls_made_by_agent': 1,\n", - " 'correct_tool_percentage': 1.0,\n", - " 'tool_call_errors': 0,\n", - " 'tool_name': 'send_email',\n", - " 'tool_success_result': 'pass',\n", - " 'total_calls_required': 1}],\n", - " 'tool_calls_made_by_agent': 2},\n", - " 'tool_call_accuracy_finish_reason': 'stop',\n", - " 'tool_call_accuracy_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_call_accuracy_prompt_tokens': 2331,\n", - " 'tool_call_accuracy_reason': \"Let's think step by step: The user's last query \"\n", - " 'is to check the weather in Seattle and send an '\n", - " 'email with that weather information to '\n", - " 'john@example.com. The relevant available tools '\n", - " \"are 'fetch_weather' (to get the weather for \"\n", - " \"Seattle) and 'send_email' (to send the email \"\n", - " 'with the weather info). The agent made two tool '\n", - " \"calls: one to 'fetch_weather' with 'location' \"\n", - " \"set to 'Seattle', and one to 'send_email' with \"\n", - " 'the correct recipient, subject, and a body '\n", - " 'referencing the weather information. All '\n", - " 'parameters are grounded in the conversation: '\n", - " \"'Seattle' is the location, 'john@example.com' \"\n", - " 'is the recipient, and the subject/body are '\n", - " 'appropriate for the context. No unnecessary '\n", - " 'duplicate tool calls were made, and no errors '\n", - " 'are indicated. Both required tools were called '\n", - " 'exactly once, with correct parameters. '\n", - " 'Therefore, this is an optimal solution '\n", - " 'according to the definitions.',\n", - " 'tool_call_accuracy_result': 'pass',\n", - " 'tool_call_accuracy_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": [{\\\\\"role\\\\\": '\n", - " '\\\\\"system\\\\\", \\\\\"content\\\\\": \\\\\"You are a '\n", - " 'helpful assistant that can fetch weather '\n", - " 'information and send emails.\\\\\"}, '\n", - " '{\\\\\"role\\\\\": \\\\\"user\\\\\", \\\\\"content\\\\\": '\n", - " '\\\\\"Hi, can you check the weather in '\n", - " 'Seattle for me?\\\\\"}, {\\\\\"role\\\\\": '\n", - " '\\\\\"user\\\\\", \\\\\"content\\\\\": \\\\\"Actually, '\n", - " 'could you also send me an email with that '\n", - " 'weather information to '\n", - " 'john@example.com?\\\\\"}], \\\\\"tool_calls\\\\\": '\n", - " '[{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", - " '\\\\\"tool_call_id\\\\\": '\n", - " '\\\\\"call_weather_123\\\\\", \\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"arguments\\\\\": '\n", - " '{\\\\\"location\\\\\": \\\\\"Seattle\\\\\"}}, '\n", - " '{\\\\\"type\\\\\": \\\\\"tool_call\\\\\", '\n", - " '\\\\\"tool_call_id\\\\\": \\\\\"call_email_456\\\\\", '\n", - " '\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", - " '\\\\\"arguments\\\\\": {\\\\\"recipient\\\\\": '\n", - " '\\\\\"john@example.com\\\\\", \\\\\"subject\\\\\": '\n", - " '\\\\\"Weather Information for Seattle\\\\\", '\n", - " '\\\\\"body\\\\\": \\\\\"Here is the weather '\n", - " 'information you requested.\\\\\"}}], '\n", - " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Fetches the weather information for '\n", - " 'the specified location.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", - " 'location to fetch weather for.\\\\\"}}}}, '\n", - " '{\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Sends an email with '\n", - " 'the specified subject and body to the '\n", - " 'recipient.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"recipient\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Email address of '\n", - " 'the recipient.\\\\\"}, \\\\\"subject\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Subject of the '\n", - " 'email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", - " 'content of the email.\\\\\"}}}}]}\"}]',\n", - " 'tool_call_accuracy_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"chain_of_thought\\\\\": \\\\\"Let\\'s think '\n", - " \"step by step: The user's last query is \"\n", - " 'to check the weather in Seattle and send '\n", - " 'an email with that weather information '\n", - " 'to john@example.com. The relevant '\n", - " \"available tools are 'fetch_weather' (to \"\n", - " 'get the weather for Seattle) and '\n", - " \"'send_email' (to send the email with the \"\n", - " 'weather info). The agent made two tool '\n", - " \"calls: one to 'fetch_weather' with \"\n", - " \"'location' set to 'Seattle', and one to \"\n", - " \"'send_email' with the correct recipient, \"\n", - " 'subject, and a body referencing the '\n", - " 'weather information. All parameters are '\n", - " \"grounded in the conversation: 'Seattle' \"\n", - " \"is the location, 'john@example.com' is \"\n", - " 'the recipient, and the subject/body are '\n", - " 'appropriate for the context. No '\n", - " 'unnecessary duplicate tool calls were '\n", - " 'made, and no errors are indicated. Both '\n", - " 'required tools were called exactly once, '\n", - " 'with correct parameters. Therefore, this '\n", - " 'is an optimal solution according to the '\n", - " 'definitions.\\\\\",\\\\n '\n", - " '\\\\\"tool_calls_success_level\\\\\": 5,\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"tool_calls_made_by_agent\\\\\": 2,\\\\n '\n", - " '\\\\\"correct_tool_calls_made_by_agent\\\\\": '\n", - " '2,\\\\n \\\\\"per_tool_call_details\\\\\": '\n", - " '[\\\\n {\\\\n \\\\\"tool_name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\",\\\\n '\n", - " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n '\n", - " '\\\\\"correct_tool_percentage\\\\\": '\n", - " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", - " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", - " '\\\\\"pass\\\\\"\\\\n },\\\\n '\n", - " '{\\\\n \\\\\"tool_name\\\\\": '\n", - " '\\\\\"send_email\\\\\",\\\\n '\n", - " '\\\\\"total_calls_required\\\\\": 1,\\\\n '\n", - " '\\\\\"correct_calls_made_by_agent\\\\\": '\n", - " '1,\\\\n '\n", - " '\\\\\"correct_tool_percentage\\\\\": '\n", - " '1.0,\\\\n \\\\\"tool_call_errors\\\\\": '\n", - " '0,\\\\n \\\\\"tool_success_result\\\\\": '\n", - " '\\\\\"pass\\\\\"\\\\n }\\\\n ],\\\\n '\n", - " '\\\\\"excess_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n },\\\\n '\n", - " '\\\\\"missing_tool_calls\\\\\": {\\\\n '\n", - " '\\\\\"total\\\\\": 0,\\\\n \\\\\"details\\\\\": '\n", - " '[]\\\\n }\\\\n }\\\\n}\"}]',\n", - " 'tool_call_accuracy_threshold': 3,\n", - " 'tool_call_accuracy_total_tokens': 2741}\n" - ] - } - ], + "outputs": [], "source": [ "response = tool_call_accuracy(query=query_as_conversation, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", "pprint(response)" diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb deleted file mode 100644 index 9d76a75c..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb +++ /dev/null @@ -1,995 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "cea11cae", - "metadata": {}, - "source": [ - "# Tool Call Success Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use tool call success evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings and `dict` describing agent responses;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" - ] - }, - { - "cell_type": "markdown", - "id": "49404a44", - "metadata": {}, - "source": [ - "The Tool Call Success evaluator determines whether tool calls done by an AI agent includes failures or not.\n", - "\n", - "This evaluator focuses solely on tool call results and tool definitions, disregarding user's query to the agent, conversation history and agent's final response. Although tool definitions is optional, providing them can help the evaluator better understand the context of the tool calls made by the agent. Please note that this evaluator validates tool calls for potential technical failures like errors, exceptions, timeouts and empty results (only in cases where empty results could indicate a failure). It does not assess the correctness or the tool result itself, like mathematical errors and unrealistic field values like name=\"668656\".\n", - "\n", - "Scoring is binary:\n", - "- TRUE: All tool calls were successful\n", - "- FALSE: At least one tool call failed\n", - "\n", - "This evaluation focuses on measuring the technical success of tool execution, not the semantic correctness of the results." - ] - }, - { - "cell_type": "markdown", - "id": "d91cb876", - "metadata": {}, - "source": [ - "Tool Call Success requires following input:\n", - "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator examines tool call results within the response.\n", - "- Tool Definitions - (Optional) Tool(s) definition used by Agent. Providing tool definitions helps the evaluator better understand the context of the tool calls made by the agent.\n" - ] - }, - { - "cell_type": "markdown", - "id": "e07d293d", - "metadata": {}, - "source": [ - "### Initialize Tool Call Success Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "919d1b9a", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Class _ToolSuccessEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" - ] - } - ], - "source": [ - "import os\n", - "from azure.ai.evaluation._evaluators._tool_success import _ToolSuccessEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "\n", - "tool_call_success = _ToolSuccessEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "4739878c", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "id": "2352536d", - "metadata": {}, - "source": [ - "#### Evaluating Successful Tool Calls" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "9ddc2e22", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_success': 1.0,\n", - " 'tool_success_completion_tokens': 60,\n", - " 'tool_success_finish_reason': 'stop',\n", - " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_success_prompt_tokens': 2407,\n", - " 'tool_success_reason': 'The fetch_weather tool returned a result containing '\n", - " 'temperature, condition, humidity, and wind, with no '\n", - " 'indication of error or technical failure. '\n", - " \"{'failed_tools': '', 'success': True}\",\n", - " 'tool_success_result': 'pass',\n", - " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '15\\\\\\\\u00b0C', 'condition': \"\n", - " \"'partly cloudy', 'humidity': '68%', 'wind': '8 \"\n", - " 'mph NW\\'}\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", - " 'Fetches the weather information for the '\n", - " 'specified location. (inputs: location)\\\\\"}\"}]',\n", - " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The fetch_weather tool '\n", - " 'returned a result containing temperature, '\n", - " 'condition, humidity, and wind, with no '\n", - " 'indication of error or technical '\n", - " 'failure.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"failed_tools\\\\\": \\\\\"\\\\\",\\\\n '\n", - " '\\\\\"success\\\\\": true\\\\n },\\\\n '\n", - " '\\\\\"success\\\\\": true\\\\n}\"}]',\n", - " 'tool_success_threshold': 1,\n", - " 'tool_success_total_tokens': 2467}\n" - ] - } - ], - "source": [ - "# Successful tool execution\n", - "successful_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " }\n", - "]\n", - "\n", - "result = tool_call_success(response=successful_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "b7217ddc", - "metadata": {}, - "source": [ - "#### Tool Definition as Single Dict" - ] - }, - { - "cell_type": "markdown", - "id": "c146de06", - "metadata": {}, - "source": [ - "#### Response as String (str)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "575d33eb", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Agent response could not be parsed, falling back to original response: Tool call executed: fetch_weather(location=\"Paris\")\n", - "Tool result: {\"temperature\": \"18°C\", \"condition\": \"clear sky\", \"humidity\": \"55%\"}\n", - "The weather in Paris is currently clear with a temperature of 18°C.\n", - "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", - "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_success': 1.0,\n", - " 'tool_success_completion_tokens': 67,\n", - " 'tool_success_finish_reason': 'stop',\n", - " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_success_prompt_tokens': 2441,\n", - " 'tool_success_reason': 'The tool fetch_weather returned a result with '\n", - " 'temperature, condition, and humidity fields '\n", - " 'populated. There are no indications of technical '\n", - " 'errors, exceptions, or empty/invalid responses. '\n", - " \"{'failed_tools': '', 'success': True}\",\n", - " 'tool_success_result': 'pass',\n", - " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"tool_calls\\\\\": \\\\\"Tool call executed: '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Paris\\\\\\\\\\\\\")\\\\\\\\nTool '\n", - " 'result: {\\\\\\\\\\\\\"temperature\\\\\\\\\\\\\": '\n", - " '\\\\\\\\\\\\\"18\\\\\\\\u00b0C\\\\\\\\\\\\\", '\n", - " '\\\\\\\\\\\\\"condition\\\\\\\\\\\\\": \\\\\\\\\\\\\"clear '\n", - " 'sky\\\\\\\\\\\\\", \\\\\\\\\\\\\"humidity\\\\\\\\\\\\\": '\n", - " '\\\\\\\\\\\\\"55%\\\\\\\\\\\\\"}\\\\\\\\nThe weather in Paris is '\n", - " 'currently clear with a temperature of '\n", - " '18\\\\\\\\u00b0C.\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " '{\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", - " 'information for the specified location.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"The '\n", - " 'location to fetch weather for.\\\\\"}}}}}\"}]',\n", - " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The tool fetch_weather '\n", - " 'returned a result with temperature, condition, '\n", - " 'and humidity fields populated. There are no '\n", - " 'indications of technical errors, exceptions, '\n", - " 'or empty/invalid responses.\\\\\",\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n \\\\\"failed_tools\\\\\": '\n", - " '\\\\\"\\\\\",\\\\n \\\\\"success\\\\\": true\\\\n '\n", - " '},\\\\n \\\\\"success\\\\\": true\\\\n}\"}]',\n", - " 'tool_success_threshold': 1,\n", - " 'tool_success_total_tokens': 2508}\n" - ] - } - ], - "source": [ - "# Response as a simple string containing tool call information\n", - "# This format is less common but still valid for the evaluator\n", - "response_str = \"\"\"Tool call executed: fetch_weather(location=\"Paris\")\n", - "Tool result: {\"temperature\": \"18°C\", \"condition\": \"clear sky\", \"humidity\": \"55%\"}\n", - "The weather in Paris is currently clear with a temperature of 18°C.\"\"\"\n", - "\n", - "tool_definition = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", - "\n", - "result = tool_call_success(response=response_str, tool_definitions=tool_definition)\n", - "pprint(result)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "f89ca010", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", - "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n", - "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_success': 0.0,\n", - " 'tool_success_completion_tokens': 66,\n", - " 'tool_success_finish_reason': 'stop',\n", - " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_success_prompt_tokens': 2430,\n", - " 'tool_success_reason': 'The fetch_weather tool returned a result with '\n", - " 'temperature, condition, and humidity fields '\n", - " 'populated. There are no indications of technical '\n", - " 'errors, exceptions, or empty/invalid fields. The tool '\n", - " \"call succeeded. {'failed_tools': '', 'success': True}\",\n", - " 'tool_success_result': 'fail',\n", - " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"New '\n", - " 'York\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '22\\\\\\\\u00b0C', 'condition': \"\n", - " '\\'sunny\\', \\'humidity\\': \\'45%\\'}\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": {\\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Fetches the weather information for the '\n", - " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", - " 'weather for.\\\\\"}}}}}\"}]',\n", - " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The fetch_weather tool '\n", - " 'returned a result with temperature, condition, '\n", - " 'and humidity fields populated. There are no '\n", - " 'indications of technical errors, exceptions, '\n", - " 'or empty/invalid fields. The tool call '\n", - " 'succeeded.\\\\\",\\\\n \\\\\"details\\\\\": '\n", - " '{\\\\n \\\\\"failed_tools\\\\\": \\\\\"\\\\\",\\\\n '\n", - " '\\\\\"success\\\\\": true\\\\n }\\\\n}\"}]',\n", - " 'tool_success_threshold': 1,\n", - " 'tool_success_total_tokens': 2496}\n" - ] - } - ], - "source": [ - "# Successful tool execution with single tool definition dict\n", - "successful_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_nyc_weather\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"New York\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_nyc_weather\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"22°C\", \"condition\": \"sunny\", \"humidity\": \"45%\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The weather in New York is currently sunny with a temperature of 22°C.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definition_dict = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", - "\n", - "result = tool_call_success(response=successful_response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "1aeb8c11", - "metadata": {}, - "source": [ - "#### Multiple Successful Tool Calls" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "444ee59c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_success': 1.0,\n", - " 'tool_success_completion_tokens': 70,\n", - " 'tool_success_finish_reason': 'stop',\n", - " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_success_prompt_tokens': 2488,\n", - " 'tool_success_reason': 'Both tool results returned valid, non-empty responses '\n", - " 'with no indication of technical errors, exceptions, '\n", - " 'or failures. The fetch_weather result contains '\n", - " 'weather data, and send_email confirms successful '\n", - " \"delivery. {'failed_tools': '', 'success': True}\",\n", - " 'tool_success_result': 'pass',\n", - " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '14\\\\\\\\u00b0C', 'condition': \"\n", - " \"'rainy', 'precipitation': 'light \"\n", - " \"rain'}\\\\\\\\n[TOOL_CALL] \"\n", - " 'send_email(recipient=\\\\\\\\\\\\\"your_email@example.com\\\\\\\\\\\\\", '\n", - " 'subject=\\\\\\\\\\\\\"Weather Information for '\n", - " 'Seattle\\\\\\\\\\\\\", body=\\\\\\\\\\\\\"Current weather: '\n", - " '14\\\\\\\\u00b0C and rainy with light '\n", - " 'rain.\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] {\\'message\\': '\n", - " \"'Email successfully sent to \"\n", - " \"your_email@example.com.', 'status': \"\n", - " '\\'delivered\\'}\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", - " 'Fetches the weather information for the '\n", - " 'specified location. (inputs: location)\\\\\\\\n- '\n", - " 'send_email: Sends an email with the specified '\n", - " 'subject and body to the recipient. (inputs: '\n", - " 'recipient, subject, body)\\\\\"}\"}]',\n", - " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"Both tool results '\n", - " 'returned valid, non-empty responses with no '\n", - " 'indication of technical errors, exceptions, or '\n", - " 'failures. The fetch_weather result contains '\n", - " 'weather data, and send_email confirms '\n", - " 'successful delivery.\\\\\",\\\\n \\\\\"details\\\\\": '\n", - " '{\\\\n \\\\\"failed_tools\\\\\": \\\\\"\\\\\",\\\\n '\n", - " '\\\\\"success\\\\\": true\\\\n },\\\\n '\n", - " '\\\\\"success\\\\\": true\\\\n}\"}]',\n", - " 'tool_success_threshold': 1,\n", - " 'tool_success_total_tokens': 2558}\n" - ] - } - ], - "source": [ - "# Multiple successful tool executions\n", - "multiple_successful_response = [\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\", \"precipitation\": \"light rain\"}}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"your_email@example.com\",\n", - " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"Current weather: 14°C and rainy with light rain.\",\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"role\": \"tool\",\n", - " \"content\": [\n", - " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\", \"status\": \"delivered\"}}\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I have successfully sent you an email with the weather information for Seattle.\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = tool_call_success(response=multiple_successful_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "63e43736", - "metadata": {}, - "source": [ - "#### Tool Call Success without Tool Definitions" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "a0ca062e", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Failed to filter tool definitions, returning original list. Error: 'NoneType' object is not iterable\n", - "Tool definitions could not be parsed, falling back to original definitions: None\n", - "Tool definitions could not be parsed, falling back to original definitions: None\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_success': 1.0,\n", - " 'tool_success_completion_tokens': 58,\n", - " 'tool_success_finish_reason': 'stop',\n", - " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_success_prompt_tokens': 2412,\n", - " 'tool_success_reason': 'The tool get_calendar returned a result containing a '\n", - " 'list of events without any indication of error, '\n", - " \"exception, or technical failure. {'failed_tools': '', \"\n", - " \"'success': True}\",\n", - " 'tool_success_result': 'pass',\n", - " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", - " 'get_calendar(start_date=\\\\\\\\\\\\\"2025-01-01\\\\\\\\\\\\\", '\n", - " 'end_date=\\\\\\\\\\\\\"2025-01-31\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'events': [{'date': '2025-01-15', 'title': \"\n", - " \"'Team Meeting'}, {'date': '2025-01-20', \"\n", - " '\\'title\\': \\'Project Review\\'}]}\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": null}\"}]',\n", - " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The tool get_calendar '\n", - " 'returned a result containing a list of events '\n", - " 'without any indication of error, exception, or '\n", - " 'technical failure.\\\\\",\\\\n \\\\\"details\\\\\": '\n", - " '{\\\\n \\\\\"failed_tools\\\\\": \\\\\"\\\\\",\\\\n '\n", - " '\\\\\"success\\\\\": true\\\\n },\\\\n '\n", - " '\\\\\"success\\\\\": true\\\\n}\"}]',\n", - " 'tool_success_threshold': 1,\n", - " 'tool_success_total_tokens': 2470}\n" - ] - } - ], - "source": [ - "# Successful execution without providing tool definitions\n", - "simple_successful_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_calendar_123\",\n", - " \"name\": \"get_calendar\",\n", - " \"arguments\": {\"start_date\": \"2025-01-01\", \"end_date\": \"2025-01-31\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_calendar_123\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"events\": [{\"date\": \"2025-01-15\", \"title\": \"Team Meeting\"}, {\"date\": \"2025-01-20\", \"title\": \"Project Review\"}]}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I found 2 events in your calendar for January: a Team Meeting on the 15th and a Project Review on the 20th.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "# Evaluation without tool definitions\n", - "result = tool_call_success(response=simple_successful_response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "97c1af0f", - "metadata": {}, - "source": [ - "#### Example of Failed Tool Call" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "cafdee8e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_success': 0.0,\n", - " 'tool_success_completion_tokens': 79,\n", - " 'tool_success_finish_reason': 'stop',\n", - " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_success_prompt_tokens': 2398,\n", - " 'tool_success_reason': \"The fetch_weather tool result contains an 'error' \"\n", - " \"field with 'Location not found', a 'status' of \"\n", - " \"'failed', and a 404 code. This clearly indicates a \"\n", - " \"technical failure in the tool call. {'failed_tools': \"\n", - " \"'fetch_weather', 'success': False}\",\n", - " 'tool_success_result': 'fail',\n", - " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"InvalidCity\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'error': 'Location not found', 'status': \"\n", - " '\\'failed\\', \\'code\\': 404}\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- fetch_weather: '\n", - " 'Fetches the weather information for the '\n", - " 'specified location. (inputs: location)\\\\\"}\"}]',\n", - " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The fetch_weather tool '\n", - " \"result contains an 'error' field with \"\n", - " \"'Location not found', a 'status' of 'failed', \"\n", - " 'and a 404 code. This clearly indicates a '\n", - " 'technical failure in the tool call.\\\\\",\\\\n '\n", - " '\\\\\"details\\\\\": {\\\\n \\\\\"failed_tools\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\",\\\\n \\\\\"success\\\\\": '\n", - " 'false\\\\n },\\\\n \\\\\"success\\\\\": '\n", - " 'false\\\\n}\"}]',\n", - " 'tool_success_threshold': 1,\n", - " 'tool_success_total_tokens': 2477}\n" - ] - } - ], - "source": [ - "# Failed tool execution with error\n", - "failed_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_456\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"InvalidCity\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_456\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Location not found\", \"status\": \"failed\", \"code\": 404}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I'm sorry, I couldn't retrieve the weather information due to an error.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " }\n", - "]\n", - "\n", - "# This should score FALSE due to tool failure\n", - "result = tool_call_success(response=failed_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "67694403", - "metadata": {}, - "source": [ - "#### Example of Tool Call Timeout" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "19c7f49d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_success': 0.0,\n", - " 'tool_success_completion_tokens': 80,\n", - " 'tool_success_finish_reason': 'stop',\n", - " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_success_prompt_tokens': 2422,\n", - " 'tool_success_reason': \"The result for send_email contains an 'error' field \"\n", - " \"with 'Request timeout' and a 'status' of 'timeout', \"\n", - " 'which indicates a technical failure due to timeout. '\n", - " 'Therefore, the send_email tool call failed. '\n", - " \"{'failed_tools': 'send_email', 'success': False}\",\n", - " 'tool_success_result': 'fail',\n", - " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", - " 'send_email(recipient=\\\\\\\\\\\\\"test@example.com\\\\\\\\\\\\\", '\n", - " 'subject=\\\\\\\\\\\\\"Test Email\\\\\\\\\\\\\", '\n", - " 'body=\\\\\\\\\\\\\"This is a test '\n", - " 'email.\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] {\\'error\\': '\n", - " \"'Request timeout', 'status': 'timeout', \"\n", - " '\\'timeout_duration\\': \\'30s\\'}\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- send_email: Sends an '\n", - " 'email with the specified subject and body to '\n", - " 'the recipient. (inputs: recipient, subject, '\n", - " 'body)\\\\\"}\"}]',\n", - " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The result for '\n", - " \"send_email contains an 'error' field with \"\n", - " \"'Request timeout' and a 'status' of 'timeout', \"\n", - " 'which indicates a technical failure due to '\n", - " 'timeout. Therefore, the send_email tool call '\n", - " 'failed.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"failed_tools\\\\\": \\\\\"send_email\\\\\",\\\\n '\n", - " '\\\\\"success\\\\\": false\\\\n },\\\\n '\n", - " '\\\\\"success\\\\\": false\\\\n}\"}]',\n", - " 'tool_success_threshold': 1,\n", - " 'tool_success_total_tokens': 2502}\n" - ] - } - ], - "source": [ - "# Tool execution timeout\n", - "timeout_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_email_789\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"test@example.com\",\n", - " \"subject\": \"Test Email\",\n", - " \"body\": \"This is a test email.\"\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_email_789\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Request timeout\", \"status\": \"timeout\", \"timeout_duration\": \"30s\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I'm unable to send the email due to a timeout error. Please try again later.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "# This should score FALSE due to timeout\n", - "result = tool_call_success(response=timeout_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "8a5e147a", - "metadata": {}, - "source": [ - "#### Mixed Results - Some Success, Some Failure" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "c73b8611", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_success': 0.0,\n", - " 'tool_success_completion_tokens': 71,\n", - " 'tool_success_finish_reason': 'stop',\n", - " 'tool_success_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_success_prompt_tokens': 2444,\n", - " 'tool_success_reason': 'The fetch_weather tool returned a valid weather '\n", - " 'result, indicating success. The send_email tool '\n", - " \"returned an error message and status 'failed', which \"\n", - " 'is a technical failure. Therefore, the evaluation '\n", - " \"process has failed. {'failed_tools': 'send_email'}\",\n", - " 'tool_success_result': 'fail',\n", - " 'tool_success_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"tool_calls\\\\\": \\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '15\\\\\\\\u00b0C', 'condition': \"\n", - " \"'sunny'}\\\\\\\\n[TOOL_CALL] \"\n", - " 'send_email(recipient=\\\\\\\\\\\\\"invalid-email\\\\\\\\\\\\\", '\n", - " 'subject=\\\\\\\\\\\\\"Weather Report\\\\\\\\\\\\\", '\n", - " 'body=\\\\\\\\\\\\\"Weather is sunny in '\n", - " 'Seattle.\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] {\\'error\\': '\n", - " \"'Invalid email format', 'status': \"\n", - " '\\'failed\\'}\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- send_email: Sends an '\n", - " 'email with the specified subject and body to '\n", - " 'the recipient. (inputs: recipient, subject, '\n", - " 'body)\\\\\"}\"}]',\n", - " 'tool_success_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The fetch_weather tool '\n", - " 'returned a valid weather result, indicating '\n", - " 'success. The send_email tool returned an error '\n", - " \"message and status 'failed', which is a \"\n", - " 'technical failure. Therefore, the evaluation '\n", - " 'process has failed.\\\\\",\\\\n \\\\\"details\\\\\": '\n", - " '{\\\\n \\\\\"failed_tools\\\\\": '\n", - " '\\\\\"send_email\\\\\"\\\\n },\\\\n \\\\\"success\\\\\": '\n", - " 'false\\\\n}\"}]',\n", - " 'tool_success_threshold': 1,\n", - " 'tool_success_total_tokens': 2515}\n" - ] - } - ], - "source": [ - "# Mixed results - one success, one failure\n", - "mixed_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_success\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_success\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"sunny\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_email_fail\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"invalid-email\",\n", - " \"subject\": \"Weather Report\",\n", - " \"body\": \"Weather is sunny in Seattle.\"\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_email_fail\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Invalid email format\", \"status\": \"failed\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I got the weather information but failed to send the email due to an invalid email address.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "# This should score FALSE because at least one tool call failed\n", - "result = tool_call_success(response=mixed_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb index 57405431..711472fb 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb @@ -21,14 +21,13 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "pip install azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." ] }, { @@ -148,14 +147,6 @@ "pprint(result)" ] }, - { - "cell_type": "markdown", - "id": "e9275f1c", - "metadata": {}, - "source": [ - "#### Tool Definition as Single Dict" - ] - }, { "cell_type": "markdown", "id": "80fc6e82", @@ -190,6 +181,14 @@ "pprint(result)" ] }, + { + "cell_type": "markdown", + "id": "e9275f1c", + "metadata": {}, + "source": [ + "#### Response as a List and Tool Definition as Single Dict" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb index 43880212..317f7c5a 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb @@ -21,14 +21,13 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "pip install azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" ] }, { @@ -68,18 +67,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "087dd96d", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Class _ToolOutputUtilizationEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "from azure.ai.evaluation._evaluators._tool_output_utilization import _ToolOutputUtilizationEvaluator\n", @@ -115,66 +106,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "017a7afe", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_output_utilization': 1.0,\n", - " 'tool_output_utilization_completion_tokens': 53,\n", - " 'tool_output_utilization_finish_reason': 'stop',\n", - " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_output_utilization_prompt_tokens': 1963,\n", - " 'tool_output_utilization_reason': 'All tool output values (temperature, '\n", - " 'condition, humidity, wind) are correctly '\n", - " 'reflected in the response with accurate '\n", - " 'paraphrasing and no omissions or '\n", - " 'fabrications.',\n", - " 'tool_output_utilization_result': 'pass',\n", - " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"How is the weather '\n", - " 'in Seattle?\\\\\", \\\\\"response\\\\\": '\n", - " '\\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '15\\\\\\\\u00b0C', \"\n", - " \"'condition': 'partly cloudy', \"\n", - " \"'humidity': '68%', 'wind': '8 mph \"\n", - " \"NW'}\\\\\\\\nThe current weather in \"\n", - " 'Seattle is partly cloudy with a '\n", - " 'temperature of 15\\\\\\\\u00b0C. The '\n", - " \"humidity is at 68% and there's a \"\n", - " 'light wind from the northwest at 8 '\n", - " 'mph.\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", - " 'fetch_weather: Fetches the weather '\n", - " 'information for the specified '\n", - " 'location. (inputs: location)\\\\\"}\"}]',\n", - " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", - " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", - " '[],\\\\n \\\\\"reason\\\\\": \\\\\"All tool '\n", - " 'output values (temperature, '\n", - " 'condition, humidity, wind) are '\n", - " 'correctly reflected in the response '\n", - " 'with accurate paraphrasing and no '\n", - " 'omissions or fabrications.\\\\\",\\\\n '\n", - " '\\\\\"label\\\\\": \\\\\"pass\\\\\"\\\\n}\"}]',\n", - " 'tool_output_utilization_threshold': 1,\n", - " 'tool_output_utilization_total_tokens': 2016}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "response = [\n", @@ -230,85 +165,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "e8543c52", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", - "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n", - "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_output_utilization': 1.0,\n", - " 'tool_output_utilization_completion_tokens': 51,\n", - " 'tool_output_utilization_finish_reason': 'stop',\n", - " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_output_utilization_prompt_tokens': 2079,\n", - " 'tool_output_utilization_reason': 'All tool outputs (weather details and '\n", - " 'email delivery status) were correctly '\n", - " 'reflected in the response and reused '\n", - " 'accurately in the email body and '\n", - " 'user-facing message.',\n", - " 'tool_output_utilization_result': 'pass',\n", - " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"Can you send me an '\n", - " 'email with weather information for '\n", - " 'Seattle?\\\\\", \\\\\"response\\\\\": '\n", - " '\\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '14\\\\\\\\u00b0C', \"\n", - " \"'condition': 'rainy', \"\n", - " \"'precipitation': 'light rain', \"\n", - " \"'wind': '12 mph SW'}\\\\\\\\n[TOOL_CALL] \"\n", - " 'send_email(recipient=\\\\\\\\\\\\\"your_email@example.com\\\\\\\\\\\\\", '\n", - " 'subject=\\\\\\\\\\\\\"Weather Information '\n", - " 'for Seattle\\\\\\\\\\\\\", '\n", - " 'body=\\\\\\\\\\\\\"Current weather in '\n", - " 'Seattle: 14\\\\\\\\u00b0C and rainy with '\n", - " 'light rain. Wind is from the '\n", - " 'southwest at 12 mph. Stay '\n", - " 'dry!\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'message': 'Email successfully sent \"\n", - " \"to your_email@example.com.', \"\n", - " \"'status': 'delivered'}\\\\\\\\nI have \"\n", - " 'successfully sent you an email with '\n", - " 'the weather information for Seattle. '\n", - " 'The email includes the current '\n", - " 'conditions: 14\\\\\\\\u00b0C and rainy '\n", - " 'with light rain, plus wind details. '\n", - " 'The email was delivered successfully '\n", - " 'to your inbox.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", - " 'fetch_weather: Fetches the weather '\n", - " 'information for the specified '\n", - " 'location. (inputs: location)\\\\\\\\n- '\n", - " 'send_email: Sends an email with the '\n", - " 'specified subject and body to the '\n", - " 'recipient. (inputs: recipient, '\n", - " 'subject, body)\\\\\"}\"}]',\n", - " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", - " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", - " '[],\\\\n \\\\\"reason\\\\\": \\\\\"All tool '\n", - " 'outputs (weather details and email '\n", - " 'delivery status) were correctly '\n", - " 'reflected in the response and '\n", - " 'reused accurately in the email body '\n", - " 'and user-facing message.\\\\\",\\\\n '\n", - " '\\\\\"label\\\\\": \\\\\"pass\\\\\"\\\\n}\"}]',\n", - " 'tool_output_utilization_threshold': 1,\n", - " 'tool_output_utilization_total_tokens': 2130}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"Can you send me an email with weather information for Seattle?\"\n", "response = [\n", @@ -408,103 +268,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "b3b9bef5", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: [{'role': 'system', 'content': 'You are a helpful assistant that can fetch weather information and send emails.'}, {'role': 'user', 'content': 'Hi, I need to know the weather in Seattle for my trip planning.'}, {'role': 'user', 'content': 'Can you also email me a summary so I can reference it later when packing?'}]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_output_utilization': 1.0,\n", - " 'tool_output_utilization_completion_tokens': 57,\n", - " 'tool_output_utilization_finish_reason': 'stop',\n", - " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_output_utilization_prompt_tokens': 2189,\n", - " 'tool_output_utilization_reason': 'All tool-derived information in the '\n", - " 'response, including weather details and '\n", - " 'email delivery status, matches the prior '\n", - " 'tool outputs and is correctly incorporated '\n", - " 'into both the textual summary and the '\n", - " 'email body.',\n", - " 'tool_output_utilization_result': 'pass',\n", - " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": [{\\\\\"role\\\\\": '\n", - " '\\\\\"system\\\\\", \\\\\"content\\\\\": \\\\\"You '\n", - " 'are a helpful assistant that can '\n", - " 'fetch weather information and send '\n", - " 'emails.\\\\\"}, {\\\\\"role\\\\\": '\n", - " '\\\\\"user\\\\\", \\\\\"content\\\\\": \\\\\"Hi, I '\n", - " 'need to know the weather in Seattle '\n", - " 'for my trip planning.\\\\\"}, '\n", - " '{\\\\\"role\\\\\": \\\\\"user\\\\\", '\n", - " '\\\\\"content\\\\\": \\\\\"Can you also email '\n", - " 'me a summary so I can reference it '\n", - " 'later when packing?\\\\\"}], '\n", - " '\\\\\"response\\\\\": \\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '15\\\\\\\\u00b0C', \"\n", - " \"'condition': 'partly cloudy', \"\n", - " \"'humidity': '68%', 'wind': '8 mph \"\n", - " \"NW', 'forecast': 'no rain \"\n", - " \"expected'}\\\\\\\\n[TOOL_CALL] \"\n", - " 'send_email(recipient=\\\\\\\\\\\\\"user@example.com\\\\\\\\\\\\\", '\n", - " 'subject=\\\\\\\\\\\\\"Seattle Weather '\n", - " 'Summary for Trip Planning\\\\\\\\\\\\\", '\n", - " 'body=\\\\\\\\\\\\\"Weather summary for '\n", - " 'Seattle: Temperature is '\n", - " '15\\\\\\\\u00b0C, partly cloudy '\n", - " 'conditions, humidity at 68%, with '\n", - " 'northwest winds at 8 mph. No rain is '\n", - " 'expected, so light layers should be '\n", - " 'sufficient for your '\n", - " 'trip.\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'message': 'Email sent \"\n", - " \"successfully', 'delivery_status': \"\n", - " \"'delivered'}\\\\\\\\nPerfect! I've \"\n", - " 'checked the weather in Seattle for '\n", - " \"your trip planning. It's currently \"\n", - " '15\\\\\\\\u00b0C and partly cloudy with '\n", - " 'light northwest winds and no rain '\n", - " \"expected. I've also sent you an \"\n", - " 'email summary with all these details '\n", - " 'including packing recommendations '\n", - " 'based on the weather conditions. The '\n", - " 'email was delivered successfully, so '\n", - " 'you can reference it when preparing '\n", - " 'for your trip.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", - " 'fetch_weather: Fetches the weather '\n", - " 'information for the specified '\n", - " 'location. (inputs: location)\\\\\\\\n- '\n", - " 'send_email: Sends an email with the '\n", - " 'specified subject and body to the '\n", - " 'recipient. (inputs: recipient, '\n", - " 'subject, body)\\\\\"}\"}]',\n", - " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", - " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", - " '[],\\\\n \\\\\"reason\\\\\": \\\\\"All '\n", - " 'tool-derived information in the '\n", - " 'response, including weather details '\n", - " 'and email delivery status, matches '\n", - " 'the prior tool outputs and is '\n", - " 'correctly incorporated into both '\n", - " 'the textual summary and the email '\n", - " 'body.\\\\\",\\\\n \\\\\"label\\\\\": '\n", - " '\\\\\"pass\\\\\"\\\\n}\"}]',\n", - " 'tool_output_utilization_threshold': 1,\n", - " 'tool_output_utilization_total_tokens': 2246}\n" - ] - } - ], + "outputs": [], "source": [ "# Query as conversation history instead of a single string\n", "query_as_conversation = [\n", @@ -607,75 +374,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "6be39269", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_output_utilization': 0.0,\n", - " 'tool_output_utilization_completion_tokens': 93,\n", - " 'tool_output_utilization_finish_reason': 'stop',\n", - " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_output_utilization_prompt_tokens': 1954,\n", - " 'tool_output_utilization_reason': \"The agent's response omits all specific \"\n", - " 'factual details from the tool output '\n", - " '(temperature, condition, humidity, wind, '\n", - " 'UV index, visibility) and instead provides '\n", - " 'only a vague summary, failing to correctly '\n", - " 'use the tool-derived information. Issues '\n", - " 'found: claim -> OMISSION (expected factual '\n", - " 'details from tool output, saw only vague '\n", - " 'summary) mapped to fetch_weather',\n", - " 'tool_output_utilization_result': 'fail',\n", - " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"How is the weather '\n", - " 'in Seattle?\\\\\", \\\\\"response\\\\\": '\n", - " '\\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '15\\\\\\\\u00b0C', \"\n", - " \"'condition': 'partly cloudy', \"\n", - " \"'humidity': '68%', 'wind': '8 mph \"\n", - " \"NW', 'UV_index': '3', 'visibility': \"\n", - " \"'10 km'}\\\\\\\\nThe weather in Seattle \"\n", - " \"is okay. It's not too bad \"\n", - " 'outside.\\\\\", \\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", - " 'fetch_weather: Fetches the weather '\n", - " 'information for the specified '\n", - " 'location. (inputs: location)\\\\\"}\"}]',\n", - " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", - " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", - " '[\\\\n \\\\\"claim -> OMISSION '\n", - " '(expected factual details from tool '\n", - " 'output, saw only vague summary) '\n", - " 'mapped to fetch_weather\\\\\"\\\\n '\n", - " '],\\\\n \\\\\"reason\\\\\": \\\\\"The '\n", - " \"agent's response omits all specific \"\n", - " 'factual details from the tool '\n", - " 'output (temperature, condition, '\n", - " 'humidity, wind, UV index, '\n", - " 'visibility) and instead provides '\n", - " 'only a vague summary, failing to '\n", - " 'correctly use the tool-derived '\n", - " 'information.\\\\\",\\\\n \\\\\"label\\\\\": '\n", - " '\\\\\"fail\\\\\"\\\\n}\"}]',\n", - " 'tool_output_utilization_threshold': 1,\n", - " 'tool_output_utilization_total_tokens': 2047}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "# Poor utilization - agent doesn't use the detailed tool output effectively\n", @@ -733,87 +435,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "a524ee56", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n", - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_output_utilization': 0.0,\n", - " 'tool_output_utilization_completion_tokens': 147,\n", - " 'tool_output_utilization_finish_reason': 'stop',\n", - " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_output_utilization_prompt_tokens': 1944,\n", - " 'tool_output_utilization_reason': \"The agent's response misrepresented both \"\n", - " 'the weather condition and temperature, '\n", - " \"fabricating 'sunny and hot with 25°C' \"\n", - " 'instead of using the tool output of '\n", - " \"'partly cloudy' and '15°C'. Issues found: \"\n", - " 'claim -> MISMATCH (expected partly cloudy '\n", - " 'and 15°C, saw sunny and hot with 25°C) '\n", - " 'mapped to fetch_weather.condition; claim '\n", - " '-> MISMATCH (expected 15°C, saw 25°C) '\n", - " 'mapped to fetch_weather.temperature; claim '\n", - " '-> FABRICATED (no supporting tool field; '\n", - " \"'sunny and hot' not present in tool \"\n", - " 'output)',\n", - " 'tool_output_utilization_result': 'fail',\n", - " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"How is the weather '\n", - " 'in Seattle?\\\\\", \\\\\"response\\\\\": '\n", - " '\\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '15\\\\\\\\u00b0C', \"\n", - " \"'condition': 'partly cloudy', \"\n", - " \"'humidity': '68%'}\\\\\\\\nThe weather \"\n", - " 'in Seattle is currently sunny and '\n", - " 'hot with temperatures reaching '\n", - " \"25\\\\\\\\u00b0C. It's a perfect day for \"\n", - " 'outdoor activities!\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", - " 'fetch_weather: Fetches the weather '\n", - " 'information for the specified '\n", - " 'location. (inputs: location)\\\\\"}\"}]',\n", - " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", - " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", - " '[\\\\n \\\\\"claim -> MISMATCH '\n", - " '(expected partly cloudy and '\n", - " '15\\\\u00b0C, saw sunny and hot with '\n", - " '25\\\\u00b0C) mapped to '\n", - " 'fetch_weather.condition\\\\\",\\\\n '\n", - " '\\\\\"claim -> MISMATCH (expected '\n", - " '15\\\\u00b0C, saw 25\\\\u00b0C) mapped '\n", - " 'to '\n", - " 'fetch_weather.temperature\\\\\",\\\\n '\n", - " '\\\\\"claim -> FABRICATED (no '\n", - " \"supporting tool field; 'sunny and \"\n", - " \"hot' not present in tool \"\n", - " 'output)\\\\\"\\\\n ],\\\\n \\\\\"reason\\\\\": '\n", - " '\\\\\"The agent\\'s response '\n", - " 'misrepresented both the weather '\n", - " 'condition and temperature, '\n", - " \"fabricating 'sunny and hot with \"\n", - " \"25\\\\u00b0C' instead of using the \"\n", - " \"tool output of 'partly cloudy' and \"\n", - " '\\'15\\\\u00b0C\\'.\\\\\",\\\\n '\n", - " '\\\\\"label\\\\\": \\\\\"fail\\\\\"\\\\n}\"}]',\n", - " 'tool_output_utilization_threshold': 1,\n", - " 'tool_output_utilization_total_tokens': 2091}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "# Incorrect utilization - agent misrepresents the tool output\n", @@ -860,65 +485,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "0845822f", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", - "Conversation history could not be parsed, falling back to original query: What's the weather like in Seattle?\n", - "Agent response could not be parsed, falling back to original response: The weather in Seattle is currently 15°C and partly cloudy with light winds from the northwest.\n", - "Conversation history could not be parsed, falling back to original query: What's the weather like in Seattle?\n", - "Agent response could not be parsed, falling back to original response: The weather in Seattle is currently 15°C and partly cloudy with light winds from the northwest.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_output_utilization': 0.0,\n", - " 'tool_output_utilization_completion_tokens': 66,\n", - " 'tool_output_utilization_finish_reason': 'stop',\n", - " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_output_utilization_prompt_tokens': 1901,\n", - " 'tool_output_utilization_reason': \"The agent's response contains weather \"\n", - " 'details for Seattle, but there is no '\n", - " 'evidence of any tool output being used or '\n", - " 'referenced; all claims are fabricated. '\n", - " 'Issues found: claim -> FABRICATED (no '\n", - " 'supporting tool field)',\n", - " 'tool_output_utilization_result': 'fail',\n", - " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"What\\'s the '\n", - " 'weather like in Seattle?\\\\\", '\n", - " '\\\\\"response\\\\\": \\\\\"The weather in '\n", - " 'Seattle is currently 15\\\\\\\\u00b0C '\n", - " 'and partly cloudy with light winds '\n", - " 'from the northwest.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": '\n", - " '\\\\\"TOOL_DEFINITIONS:\\\\\\\\n- '\n", - " 'fetch_weather: Fetches the weather '\n", - " 'information for the specified '\n", - " 'location. (inputs: location)\\\\\"}\"}]',\n", - " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", - " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", - " '[\\\\n \\\\\"claim -> FABRICATED (no '\n", - " 'supporting tool field)\\\\\"\\\\n '\n", - " '],\\\\n \\\\\"reason\\\\\": \\\\\"The '\n", - " \"agent's response contains weather \"\n", - " 'details for Seattle, but there is '\n", - " 'no evidence of any tool output '\n", - " 'being used or referenced; all '\n", - " 'claims are fabricated.\\\\\",\\\\n '\n", - " '\\\\\"label\\\\\": \\\\\"fail\\\\\"\\\\n}\"}]',\n", - " 'tool_output_utilization_threshold': 1,\n", - " 'tool_output_utilization_total_tokens': 1967}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"What's the weather like in Seattle?\"\n", "# Response as a simple string (not a list of messages)\n", @@ -949,70 +519,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "6424b3bb", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Failed to filter tool definitions, returning original list. Error: 'str' object has no attribute 'get'\n", - "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n", - "Conversation history could not be parsed, falling back to original query: Check the weather in Seattle\n", - "Tool definitions could not be parsed, falling back to original definitions: {'name': 'fetch_weather', 'description': 'Fetches the weather information for the specified location.', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The location to fetch weather for.'}}}}\n", - "Conversation history could not be parsed, falling back to original query: Check the weather in Seattle\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_output_utilization': 1.0,\n", - " 'tool_output_utilization_completion_tokens': 48,\n", - " 'tool_output_utilization_finish_reason': 'stop',\n", - " 'tool_output_utilization_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_output_utilization_prompt_tokens': 1957,\n", - " 'tool_output_utilization_reason': \"The agent's response accurately reflects \"\n", - " 'the tool output, correctly reporting both '\n", - " 'the temperature and condition for Seattle '\n", - " 'as provided by the tool result.',\n", - " 'tool_output_utilization_result': 'pass',\n", - " 'tool_output_utilization_sample_input': '[{\"role\": \"user\", \"content\": '\n", - " '\"{\\\\\"query\\\\\": \\\\\"Check the weather '\n", - " 'in Seattle\\\\\", \\\\\"response\\\\\": '\n", - " '\\\\\"[TOOL_CALL] '\n", - " 'fetch_weather(location=\\\\\\\\\\\\\"Seattle\\\\\\\\\\\\\")\\\\\\\\n[TOOL_RESULT] '\n", - " \"{'temperature': '14\\\\\\\\u00b0C', \"\n", - " \"'condition': 'rainy'}\\\\\\\\nThe \"\n", - " 'current weather in Seattle is rainy '\n", - " 'with a temperature of '\n", - " '14\\\\\\\\u00b0C.\\\\\", '\n", - " '\\\\\"tool_definitions\\\\\": {\\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Fetches the '\n", - " 'weather information for the '\n", - " 'specified location.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"The location to fetch weather '\n", - " 'for.\\\\\"}}}}}\"}]',\n", - " 'tool_output_utilization_sample_output': '[{\"role\": \"assistant\", \"content\": '\n", - " '\"{\\\\n \\\\\"faulty_details\\\\\": '\n", - " '[],\\\\n \\\\\"reason\\\\\": \\\\\"The '\n", - " \"agent's response accurately \"\n", - " 'reflects the tool output, correctly '\n", - " 'reporting both the temperature and '\n", - " 'condition for Seattle as provided '\n", - " 'by the tool result.\\\\\",\\\\n '\n", - " '\\\\\"label\\\\\": \\\\\"pass\\\\\"\\\\n}\"}]',\n", - " 'tool_output_utilization_threshold': 1,\n", - " 'tool_output_utilization_total_tokens': 2005}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"Check the weather in Seattle\"\n", "response = [\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb index 04fd5e48..18ebcbf1 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb @@ -21,14 +21,13 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", + "pip install azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" ] }, { @@ -71,18 +70,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "b5f31144", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Class _ToolSelectionEvaluator: This is an experimental class, and may change at any time. Please see https://aka.ms/azuremlexperimental for more information.\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "from azure.ai.evaluation._evaluators._tool_selection import _ToolSelectionEvaluator\n", @@ -118,107 +109,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "dcb71257", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_selection': 1,\n", - " 'tool_selection_completion_tokens': 236,\n", - " 'tool_selection_details': {'correct_tool_selections': 1,\n", - " 'excessive_tools_list': [],\n", - " 'excessive_tools_used': 0,\n", - " 'missing_tools': 0,\n", - " 'missing_tools_list': [],\n", - " 'tool_selection_accuracy': 100.0,\n", - " 'wrong_tool_selections': 0},\n", - " 'tool_selection_finish_reason': 'stop',\n", - " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_selection_prompt_tokens': 1582,\n", - " 'tool_selection_reason': \"1. The user's query is asking for the current \"\n", - " 'weather in Seattle, which is a request for weather '\n", - " 'information about a specific location. 2. There is '\n", - " 'no prior conversation history or previous tool '\n", - " 'calls, so the agent must select the appropriate '\n", - " 'tool now. 3. The only essential tool required from '\n", - " \"the tool definitions is 'fetch_weather', as it is \"\n", - " 'specifically designed to fetch weather information '\n", - " 'for a given location. 4. The agent selected '\n", - " \"'fetch_weather' in response to the query. 5. This \"\n", - " 'tool is directly relevant and the most appropriate '\n", - " \"for the user's request. 6. No essential tools are \"\n", - " 'missing, and no unnecessary tools (such as '\n", - " \"'send_email') were selected. 7. The tool selection \"\n", - " 'is efficient and complete. 8. Therefore, the tool '\n", - " 'selection passes.',\n", - " 'tool_selection_result': 'pass',\n", - " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"How is the weather in Seattle?\\\\\", '\n", - " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\"], '\n", - " '\\\\\"tool_definitions\\\\\": [{\\\\\"id\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Fetches the weather information for the '\n", - " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", - " 'weather for.\\\\\"}}}}, {\\\\\"id\\\\\": '\n", - " '\\\\\"send_email\\\\\", \\\\\"name\\\\\": '\n", - " '\\\\\"send_email\\\\\", \\\\\"description\\\\\": \\\\\"Sends '\n", - " 'an email with the specified subject and body '\n", - " 'to the recipient.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", - " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", - " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", - " 'content of the email.\\\\\"}}}}]}\"}]',\n", - " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", - " 'is asking for the current weather in '\n", - " 'Seattle, which is a request for weather '\n", - " 'information about a specific location. 2. '\n", - " 'There is no prior conversation history or '\n", - " 'previous tool calls, so the agent must '\n", - " 'select the appropriate tool now. 3. The only '\n", - " 'essential tool required from the tool '\n", - " \"definitions is 'fetch_weather', as it is \"\n", - " 'specifically designed to fetch weather '\n", - " 'information for a given location. 4. The '\n", - " \"agent selected 'fetch_weather' in response \"\n", - " 'to the query. 5. This tool is directly '\n", - " 'relevant and the most appropriate for the '\n", - " \"user's request. 6. No essential tools are \"\n", - " 'missing, and no unnecessary tools (such as '\n", - " \"'send_email') were selected. 7. The tool \"\n", - " 'selection is efficient and complete. 8. '\n", - " 'Therefore, the tool selection '\n", - " 'passes.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"correct_tool_selections\\\\\": 1,\\\\n '\n", - " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", - " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", - " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", - " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", - " 'tool_selection_threshold': 1,\n", - " 'tool_selection_total_tokens': 1818}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "tool_call = {\n", @@ -267,116 +161,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "322efe28", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_selection': 1,\n", - " 'tool_selection_completion_tokens': 239,\n", - " 'tool_selection_details': {'correct_tool_selections': 2,\n", - " 'excessive_tools_list': [],\n", - " 'excessive_tools_used': 0,\n", - " 'missing_tools': 0,\n", - " 'missing_tools_list': [],\n", - " 'tool_selection_accuracy': 100.0,\n", - " 'wrong_tool_selections': 0},\n", - " 'tool_selection_finish_reason': 'stop',\n", - " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_selection_prompt_tokens': 1674,\n", - " 'tool_selection_reason': \"1. The user's query is to send an email containing \"\n", - " 'weather information for Seattle. This requires two '\n", - " 'operations: fetching the weather for Seattle and '\n", - " 'sending an email with that information. 2. There is '\n", - " 'no prior conversation history or previous tool '\n", - " 'calls, so all required data must be gathered in '\n", - " 'this step. 3. The essential tools are '\n", - " \"'fetch_weather' (to get the weather data) and \"\n", - " \"'send_email' (to send the email). 4. The agent \"\n", - " \"selected both 'fetch_weather' and 'send_email'. 5. \"\n", - " 'Both tools are directly relevant and appropriate '\n", - " 'for the task. 6. No essential tools are missing, '\n", - " \"and no unnecessary tools (like 'get_calendar') were \"\n", - " 'selected. 7. The tool selection is efficient and '\n", - " 'complete. 8. Therefore, the tool selection passes.',\n", - " 'tool_selection_result': 'pass',\n", - " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"Can you send me an email with weather '\n", - " 'information for Seattle?\\\\\", '\n", - " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"send_email\\\\\"], \\\\\"tool_definitions\\\\\": '\n", - " '[{\\\\\"id\\\\\": \\\\\"fetch_weather\\\\\", \\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Fetches the weather information for the '\n", - " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", - " 'weather for.\\\\\"}}}}, {\\\\\"id\\\\\": '\n", - " '\\\\\"send_email\\\\\", \\\\\"name\\\\\": '\n", - " '\\\\\"send_email\\\\\", \\\\\"description\\\\\": \\\\\"Sends '\n", - " 'an email with the specified subject and body '\n", - " 'to the recipient.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", - " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", - " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", - " 'content of the email.\\\\\"}}}}, {\\\\\"id\\\\\": '\n", - " '\\\\\"get_calendar\\\\\", \\\\\"name\\\\\": '\n", - " '\\\\\"get_calendar\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Retrieves calendar events for a specified '\n", - " 'date range.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"start_date\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Start date for calendar '\n", - " 'events.\\\\\"}, \\\\\"end_date\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"End date '\n", - " 'for calendar events.\\\\\"}}}}]}\"}]',\n", - " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", - " 'is to send an email containing weather '\n", - " 'information for Seattle. This requires two '\n", - " 'operations: fetching the weather for Seattle '\n", - " 'and sending an email with that information. '\n", - " '2. There is no prior conversation history or '\n", - " 'previous tool calls, so all required data '\n", - " 'must be gathered in this step. 3. The '\n", - " \"essential tools are 'fetch_weather' (to get \"\n", - " \"the weather data) and 'send_email' (to send \"\n", - " 'the email). 4. The agent selected both '\n", - " \"'fetch_weather' and 'send_email'. 5. Both \"\n", - " 'tools are directly relevant and appropriate '\n", - " 'for the task. 6. No essential tools are '\n", - " 'missing, and no unnecessary tools (like '\n", - " \"'get_calendar') were selected. 7. The tool \"\n", - " 'selection is efficient and complete. 8. '\n", - " 'Therefore, the tool selection '\n", - " 'passes.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"correct_tool_selections\\\\\": 2,\\\\n '\n", - " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", - " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", - " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", - " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", - " 'tool_selection_threshold': 1,\n", - " 'tool_selection_total_tokens': 1913}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"Can you send me an email with weather information for Seattle?\"\n", "tool_calls = [\n", @@ -450,114 +238,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "629f3cc7", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: Can you send me an email with weather information for Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_selection': 1,\n", - " 'tool_selection_completion_tokens': 239,\n", - " 'tool_selection_details': {'correct_tool_selections': 2,\n", - " 'excessive_tools_list': [],\n", - " 'excessive_tools_used': 0,\n", - " 'missing_tools': 0,\n", - " 'missing_tools_list': [],\n", - " 'tool_selection_accuracy': 100.0,\n", - " 'wrong_tool_selections': 0},\n", - " 'tool_selection_finish_reason': 'stop',\n", - " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_selection_prompt_tokens': 1653,\n", - " 'tool_selection_reason': \"1. The user's query is to send an email containing \"\n", - " 'weather information for Seattle. This requires two '\n", - " 'operations: fetching the weather for Seattle and '\n", - " 'sending an email with that information. 2. There is '\n", - " 'no prior conversation history or previous tool '\n", - " 'calls, so all required data must be gathered in '\n", - " 'this response. 3. The essential tools are '\n", - " \"'fetch_weather' (to get the weather info) and \"\n", - " \"'send_email' (to send the email). 4. The agent \"\n", - " \"selected both 'fetch_weather' and 'send_email'. 5. \"\n", - " 'Both tools are directly relevant and appropriate '\n", - " 'for the task. 6. No essential tools are missing, '\n", - " \"and no unnecessary tools (like 'get_calendar') were \"\n", - " 'selected. 7. The tool selection is efficient and '\n", - " 'complete. 8. Therefore, the tool selection passes.',\n", - " 'tool_selection_result': 'pass',\n", - " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"Can you send me an email with weather '\n", - " 'information for Seattle?\\\\\", '\n", - " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"send_email\\\\\"], \\\\\"tool_definitions\\\\\": '\n", - " '[{\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", - " 'information for the specified location.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"The location to fetch weather for.\\\\\"}}}}, '\n", - " '{\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Sends an email with the '\n", - " 'specified subject and body to the '\n", - " 'recipient.\\\\\", \\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", - " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", - " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", - " 'content of the email.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", - " '\\\\\"get_calendar\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Retrieves calendar events for a specified '\n", - " 'date range.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"start_date\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Start date for calendar '\n", - " 'events.\\\\\"}, \\\\\"end_date\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"End date '\n", - " 'for calendar events.\\\\\"}}}}]}\"}]',\n", - " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", - " 'is to send an email containing weather '\n", - " 'information for Seattle. This requires two '\n", - " 'operations: fetching the weather for Seattle '\n", - " 'and sending an email with that information. '\n", - " '2. There is no prior conversation history or '\n", - " 'previous tool calls, so all required data '\n", - " 'must be gathered in this response. 3. The '\n", - " \"essential tools are 'fetch_weather' (to get \"\n", - " \"the weather info) and 'send_email' (to send \"\n", - " 'the email). 4. The agent selected both '\n", - " \"'fetch_weather' and 'send_email'. 5. Both \"\n", - " 'tools are directly relevant and appropriate '\n", - " 'for the task. 6. No essential tools are '\n", - " 'missing, and no unnecessary tools (like '\n", - " \"'get_calendar') were selected. 7. The tool \"\n", - " 'selection is efficient and complete. 8. '\n", - " 'Therefore, the tool selection '\n", - " 'passes.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"correct_tool_selections\\\\\": 2,\\\\n '\n", - " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", - " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", - " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", - " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", - " 'tool_selection_threshold': 1,\n", - " 'tool_selection_total_tokens': 1892}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"Can you send me an email with weather information for Seattle?\"\n", "response = [\n", @@ -669,121 +353,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "218ade7a", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: [{'role': 'system', 'content': 'You are a helpful assistant that can fetch weather information and send emails.'}, {'role': 'user', 'content': 'Hi, can you check the weather in Seattle for me?'}, {'role': 'user', 'content': 'Actually, could you also send me an email with that weather information to john@example.com?'}]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_selection': 1,\n", - " 'tool_selection_completion_tokens': 235,\n", - " 'tool_selection_details': {'correct_tool_selections': 2,\n", - " 'excessive_tools_list': [],\n", - " 'excessive_tools_used': 0,\n", - " 'missing_tools': 0,\n", - " 'missing_tools_list': [],\n", - " 'tool_selection_accuracy': 100.0,\n", - " 'wrong_tool_selections': 0},\n", - " 'tool_selection_finish_reason': 'stop',\n", - " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_selection_prompt_tokens': 1719,\n", - " 'tool_selection_reason': \"1. The user's latest query asks for the weather in \"\n", - " 'Seattle and requests that this information be sent '\n", - " 'via email to john@example.com. 2. There is no '\n", - " 'indication in the conversation history that any '\n", - " 'tools have already been called; both requests are '\n", - " 'new. 3. The essential tools required are '\n", - " \"'fetch_weather' (to get the weather information) \"\n", - " \"and 'send_email' (to send the email with that \"\n", - " 'information). 4. The agent selected both '\n", - " \"'fetch_weather' and 'send_email'. 5. Both tools are \"\n", - " \"directly relevant and appropriate for the user's \"\n", - " 'query. 6. No essential tools are missing, and no '\n", - " \"unnecessary tools (like 'get_calendar') were \"\n", - " 'selected. 7. The tool selection is efficient and '\n", - " 'complete. 8. Therefore, the tool selection passes.',\n", - " 'tool_selection_result': 'pass',\n", - " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '[{\\\\\"role\\\\\": \\\\\"system\\\\\", \\\\\"content\\\\\": '\n", - " '\\\\\"You are a helpful assistant that can fetch '\n", - " 'weather information and send emails.\\\\\"}, '\n", - " '{\\\\\"role\\\\\": \\\\\"user\\\\\", \\\\\"content\\\\\": '\n", - " '\\\\\"Hi, can you check the weather in Seattle '\n", - " 'for me?\\\\\"}, {\\\\\"role\\\\\": \\\\\"user\\\\\", '\n", - " '\\\\\"content\\\\\": \\\\\"Actually, could you also '\n", - " 'send me an email with that weather '\n", - " 'information to john@example.com?\\\\\"}], '\n", - " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"send_email\\\\\"], \\\\\"tool_definitions\\\\\": '\n", - " '[{\\\\\"name\\\\\": \\\\\"fetch_weather\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Fetches the weather '\n", - " 'information for the specified location.\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"The location to fetch weather for.\\\\\"}}}}, '\n", - " '{\\\\\"name\\\\\": \\\\\"send_email\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Sends an email with the '\n", - " 'specified subject and body to the '\n", - " 'recipient.\\\\\", \\\\\"parameters\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", - " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", - " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", - " 'content of the email.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", - " '\\\\\"get_calendar\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Retrieves calendar events for a specified '\n", - " 'date range.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"start_date\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Start date for calendar '\n", - " 'events.\\\\\"}, \\\\\"end_date\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"End date '\n", - " 'for calendar events.\\\\\"}}}}]}\"}]',\n", - " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s latest '\n", - " 'query asks for the weather in Seattle and '\n", - " 'requests that this information be sent via '\n", - " 'email to john@example.com. 2. There is no '\n", - " 'indication in the conversation history that '\n", - " 'any tools have already been called; both '\n", - " 'requests are new. 3. The essential tools '\n", - " \"required are 'fetch_weather' (to get the \"\n", - " \"weather information) and 'send_email' (to \"\n", - " 'send the email with that information). 4. '\n", - " \"The agent selected both 'fetch_weather' and \"\n", - " \"'send_email'. 5. Both tools are directly \"\n", - " \"relevant and appropriate for the user's \"\n", - " 'query. 6. No essential tools are missing, '\n", - " 'and no unnecessary tools (like '\n", - " \"'get_calendar') were selected. 7. The tool \"\n", - " 'selection is efficient and complete. 8. '\n", - " 'Therefore, the tool selection '\n", - " 'passes.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"correct_tool_selections\\\\\": 2,\\\\n '\n", - " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", - " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", - " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", - " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", - " 'tool_selection_threshold': 1,\n", - " 'tool_selection_total_tokens': 1954}\n" - ] - } - ], + "outputs": [], "source": [ "# Query as conversation history instead of a single string\n", "query_as_conversation = [\n", @@ -868,102 +441,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "2c451e4e", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_selection': 0,\n", - " 'tool_selection_completion_tokens': 165,\n", - " 'tool_selection_details': {'correct_tool_selections': 0,\n", - " 'excessive_tools_list': ['get_calendar'],\n", - " 'excessive_tools_used': 1,\n", - " 'missing_tools': 1,\n", - " 'missing_tools_list': ['fetch_weather'],\n", - " 'tool_selection_accuracy': 0.0,\n", - " 'wrong_tool_selections': 1},\n", - " 'tool_selection_finish_reason': 'stop',\n", - " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_selection_prompt_tokens': 1644,\n", - " 'tool_selection_reason': \"The user's query is about the weather in Seattle, \"\n", - " 'which requires fetching weather information. The '\n", - " \"only relevant tool for this is 'fetch_weather'. The \"\n", - " \"agent instead selected 'get_calendar', which is not \"\n", - " \"relevant to the user's request and does not provide \"\n", - " 'weather information. No previous tool calls in the '\n", - " 'conversation provided weather data, so the '\n", - " \"essential tool 'fetch_weather' was missed. The tool \"\n", - " 'selection is therefore inappropriate and '\n", - " 'incomplete.',\n", - " 'tool_selection_result': 'fail',\n", - " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"How is the weather in Seattle?\\\\\", '\n", - " '\\\\\"tool_calls\\\\\": [\\\\\"get_calendar\\\\\"], '\n", - " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Fetches the weather information for the '\n", - " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", - " 'weather for.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", - " '\\\\\"send_email\\\\\", \\\\\"description\\\\\": \\\\\"Sends '\n", - " 'an email with the specified subject and body '\n", - " 'to the recipient.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", - " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", - " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", - " 'content of the email.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", - " '\\\\\"get_calendar\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Retrieves calendar events for a specified '\n", - " 'date range.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"start_date\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Start date for calendar '\n", - " 'events.\\\\\"}, \\\\\"end_date\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"End date '\n", - " 'for calendar events.\\\\\"}}}}]}\"}]',\n", - " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"The user\\'s query is '\n", - " 'about the weather in Seattle, which requires '\n", - " 'fetching weather information. The only '\n", - " \"relevant tool for this is 'fetch_weather'. \"\n", - " \"The agent instead selected 'get_calendar', \"\n", - " \"which is not relevant to the user's request \"\n", - " 'and does not provide weather information. No '\n", - " 'previous tool calls in the conversation '\n", - " 'provided weather data, so the essential tool '\n", - " \"'fetch_weather' was missed. The tool \"\n", - " 'selection is therefore inappropriate and '\n", - " 'incomplete.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"correct_tool_selections\\\\\": 0,\\\\n '\n", - " '\\\\\"wrong_tool_selections\\\\\": 1,\\\\n '\n", - " '\\\\\"excessive_tools_used\\\\\": 1,\\\\n '\n", - " '\\\\\"excessive_tools_list\\\\\": '\n", - " '[\\\\\"get_calendar\\\\\"],\\\\n '\n", - " '\\\\\"missing_tools\\\\\": 1,\\\\n '\n", - " '\\\\\"missing_tools_list\\\\\": '\n", - " '[\\\\\"fetch_weather\\\\\"]\\\\n },\\\\n '\n", - " '\\\\\"score\\\\\": 0\\\\n}\"}]',\n", - " 'tool_selection_threshold': 1,\n", - " 'tool_selection_total_tokens': 1809}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "# Using irrelevant tool for the task\n", @@ -991,100 +472,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "e34bec52", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: What's the weather in New York?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_selection': 1,\n", - " 'tool_selection_completion_tokens': 218,\n", - " 'tool_selection_details': {'correct_tool_selections': 1,\n", - " 'excessive_tools_list': [],\n", - " 'excessive_tools_used': 0,\n", - " 'missing_tools': 0,\n", - " 'missing_tools_list': [],\n", - " 'tool_selection_accuracy': 100.0,\n", - " 'wrong_tool_selections': 0},\n", - " 'tool_selection_finish_reason': 'stop',\n", - " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_selection_prompt_tokens': 1568,\n", - " 'tool_selection_reason': \"1. The user's query is asking for the weather in \"\n", - " 'New York, which requires fetching weather '\n", - " 'information for a specific location. 2. There is no '\n", - " 'prior conversation history or previous tool calls, '\n", - " 'so the agent must select the appropriate tool now. '\n", - " '3. The only essential tool for this task, according '\n", - " \"to the tool definitions, is 'fetch_weather'. 4. The \"\n", - " \"agent selected 'fetch_weather' in response to the \"\n", - " 'query. 5. This tool is directly relevant and the '\n", - " \"most appropriate for the user's request. 6. No \"\n", - " 'essential tools are missing, and no unnecessary '\n", - " 'tools were selected. 7. The tool selection is '\n", - " 'efficient and complete. 8. Therefore, the tool '\n", - " 'selection passes.',\n", - " 'tool_selection_result': 'pass',\n", - " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"What\\'s the weather in New York?\\\\\", '\n", - " '\\\\\"tool_calls\\\\\": [\\\\\"fetch_weather\\\\\"], '\n", - " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", - " '\\\\\"fetch_weather\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"Fetches the weather information for the '\n", - " 'specified location.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"location\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"The location to fetch '\n", - " 'weather for.\\\\\"}}}}, {\\\\\"name\\\\\": '\n", - " '\\\\\"send_email\\\\\", \\\\\"description\\\\\": \\\\\"Sends '\n", - " 'an email with the specified subject and body '\n", - " 'to the recipient.\\\\\", \\\\\"parameters\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"object\\\\\", \\\\\"properties\\\\\": '\n", - " '{\\\\\"recipient\\\\\": {\\\\\"type\\\\\": \\\\\"string\\\\\", '\n", - " '\\\\\"description\\\\\": \\\\\"Email address of the '\n", - " 'recipient.\\\\\"}, \\\\\"subject\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Subject '\n", - " 'of the email.\\\\\"}, \\\\\"body\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"description\\\\\": \\\\\"Body '\n", - " 'content of the email.\\\\\"}}}}]}\"}]',\n", - " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", - " 'is asking for the weather in New York, which '\n", - " 'requires fetching weather information for a '\n", - " 'specific location. 2. There is no prior '\n", - " 'conversation history or previous tool calls, '\n", - " 'so the agent must select the appropriate '\n", - " 'tool now. 3. The only essential tool for '\n", - " 'this task, according to the tool '\n", - " \"definitions, is 'fetch_weather'. 4. The \"\n", - " \"agent selected 'fetch_weather' in response \"\n", - " 'to the query. 5. This tool is directly '\n", - " 'relevant and the most appropriate for the '\n", - " \"user's request. 6. No essential tools are \"\n", - " 'missing, and no unnecessary tools were '\n", - " 'selected. 7. The tool selection is efficient '\n", - " 'and complete. 8. Therefore, the tool '\n", - " 'selection passes.\\\\\",\\\\n \\\\\"details\\\\\": '\n", - " '{\\\\n \\\\\"correct_tool_selections\\\\\": '\n", - " '1,\\\\n \\\\\"wrong_tool_selections\\\\\": '\n", - " '0,\\\\n \\\\\"excessive_tools_used\\\\\": '\n", - " '0,\\\\n \\\\\"excessive_tools_list\\\\\": '\n", - " '[],\\\\n \\\\\"missing_tools\\\\\": 0,\\\\n '\n", - " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", - " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", - " 'tool_selection_threshold': 1,\n", - " 'tool_selection_total_tokens': 1786}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"What's the weather in New York?\"\n", "\n", @@ -1136,95 +527,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "d94822e1", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Conversation history could not be parsed, falling back to original query: How is the weather in Seattle?\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'tool_selection': 1,\n", - " 'tool_selection_completion_tokens': 233,\n", - " 'tool_selection_details': {'correct_tool_selections': 1,\n", - " 'excessive_tools_list': [],\n", - " 'excessive_tools_used': 0,\n", - " 'missing_tools': 0,\n", - " 'missing_tools_list': [],\n", - " 'tool_selection_accuracy': 100.0,\n", - " 'wrong_tool_selections': 0},\n", - " 'tool_selection_finish_reason': 'stop',\n", - " 'tool_selection_model': 'gpt-4.1-2025-04-14',\n", - " 'tool_selection_prompt_tokens': 1501,\n", - " 'tool_selection_reason': \"1. The user's query asks for the current weather in \"\n", - " 'Seattle, which requires retrieving weather '\n", - " 'information for a specific location. 2. There is no '\n", - " 'prior conversation history or previous tool calls, '\n", - " 'so the agent must select the appropriate tool now. '\n", - " '3. The only tool defined that is relevant is '\n", - " \"'get_weather', which is designed to fetch current \"\n", - " 'weather for a location. 4. The agent selected '\n", - " \"'get_weather' in response to the query. 5. This \"\n", - " 'tool is directly relevant and the most appropriate '\n", - " \"for the user's request. 6. No essential tools are \"\n", - " \"missing, as 'get_weather' is the only necessary \"\n", - " 'tool and it was selected. 7. No unnecessary or '\n", - " 'redundant tools were selected. 8. Therefore, the '\n", - " 'tool selection is fully appropriate and complete.',\n", - " 'tool_selection_result': 'pass',\n", - " 'tool_selection_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"query\\\\\": '\n", - " '\\\\\"How is the weather in Seattle?\\\\\", '\n", - " '\\\\\"tool_calls\\\\\": [\\\\\"get_weather\\\\\"], '\n", - " '\\\\\"tool_definitions\\\\\": [{\\\\\"name\\\\\": '\n", - " '\\\\\"get_weather\\\\\", \\\\\"description\\\\\": \\\\\"Get '\n", - " 'the current weather for a location\\\\\", '\n", - " '\\\\\"parameters\\\\\": {\\\\\"type\\\\\": \\\\\"object\\\\\", '\n", - " '\\\\\"properties\\\\\": {\\\\\"location\\\\\": '\n", - " '{\\\\\"type\\\\\": \\\\\"string\\\\\", \\\\\"description\\\\\": '\n", - " '\\\\\"The city and state, e.g. San Francisco, '\n", - " 'CA\\\\\"}, \\\\\"unit\\\\\": {\\\\\"type\\\\\": '\n", - " '\\\\\"string\\\\\", \\\\\"enum\\\\\": [\\\\\"celsius\\\\\", '\n", - " '\\\\\"fahrenheit\\\\\"]}}, \\\\\"required\\\\\": '\n", - " '[\\\\\"location\\\\\"]}}]}\"}]',\n", - " 'tool_selection_sample_output': '[{\"role\": \"assistant\", \"content\": \"{\\\\n '\n", - " '\\\\\"explanation\\\\\": \\\\\"1. The user\\'s query '\n", - " 'asks for the current weather in Seattle, '\n", - " 'which requires retrieving weather '\n", - " 'information for a specific location. 2. '\n", - " 'There is no prior conversation history or '\n", - " 'previous tool calls, so the agent must '\n", - " 'select the appropriate tool now. 3. The only '\n", - " 'tool defined that is relevant is '\n", - " \"'get_weather', which is designed to fetch \"\n", - " 'current weather for a location. 4. The agent '\n", - " \"selected 'get_weather' in response to the \"\n", - " 'query. 5. This tool is directly relevant and '\n", - " \"the most appropriate for the user's request. \"\n", - " '6. No essential tools are missing, as '\n", - " \"'get_weather' is the only necessary tool and \"\n", - " 'it was selected. 7. No unnecessary or '\n", - " 'redundant tools were selected. 8. Therefore, '\n", - " 'the tool selection is fully appropriate and '\n", - " 'complete.\\\\\",\\\\n \\\\\"details\\\\\": {\\\\n '\n", - " '\\\\\"correct_tool_selections\\\\\": 1,\\\\n '\n", - " '\\\\\"wrong_tool_selections\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_used\\\\\": 0,\\\\n '\n", - " '\\\\\"excessive_tools_list\\\\\": [],\\\\n '\n", - " '\\\\\"missing_tools\\\\\": 0,\\\\n '\n", - " '\\\\\"missing_tools_list\\\\\": []\\\\n },\\\\n '\n", - " '\\\\\"score\\\\\": 1\\\\n}\"}]',\n", - " 'tool_selection_threshold': 1,\n", - " 'tool_selection_total_tokens': 1734}\n" - ] - } - ], + "outputs": [], "source": [ "query = \"How is the weather in Seattle?\"\n", "\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb new file mode 100644 index 00000000..a277a4e5 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb @@ -0,0 +1,619 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cea11cae", + "metadata": {}, + "source": [ + "# Tool Call Success Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use tool call success evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing agent responses;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "id": "49404a44", + "metadata": {}, + "source": [ + "The Tool Call Success evaluator determines whether tool calls done by an AI agent includes failures or not.\n", + "\n", + "This evaluator focuses solely on tool call results and tool definitions, disregarding user's query to the agent, conversation history and agent's final response. Although tool definitions is optional, providing them can help the evaluator better understand the context of the tool calls made by the agent. Please note that this evaluator validates tool calls for potential technical failures like errors, exceptions, timeouts and empty results (only in cases where empty results could indicate a failure). It does not assess the correctness or the tool result itself, like mathematical errors and unrealistic field values like name=\"668656\".\n", + "\n", + "Scoring is binary:\n", + "- TRUE: All tool calls were successful\n", + "- FALSE: At least one tool call failed\n", + "\n", + "This evaluation focuses on measuring the technical success of tool execution, not the semantic correctness of the results." + ] + }, + { + "cell_type": "markdown", + "id": "d91cb876", + "metadata": {}, + "source": [ + "Tool Call Success requires following input:\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator examines tool call results within the response.\n", + "- Tool Definitions - (Optional) Tool(s) definition used by Agent. Providing tool definitions helps the evaluator better understand the context of the tool calls made by the agent.\n" + ] + }, + { + "cell_type": "markdown", + "id": "e07d293d", + "metadata": {}, + "source": [ + "### Initialize Tool Call Success Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "919d1b9a", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._tool_success import _ToolSuccessEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "\n", + "tool_call_success = _ToolSuccessEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "4739878c", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "2352536d", + "metadata": {}, + "source": [ + "#### Evaluating Successful Tool Calls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ddc2e22", + "metadata": {}, + "outputs": [], + "source": [ + "# Successful tool execution\n", + "successful_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_call_success(response=successful_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "c146de06", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "575d33eb", + "metadata": {}, + "outputs": [], + "source": [ + "# Response as a simple string containing tool call information\n", + "# This format is less common but still valid for the evaluator\n", + "response_str = \"\"\"Tool call executed: fetch_weather(location=\"Paris\")\n", + "Tool result: {\"temperature\": \"18°C\", \"condition\": \"clear sky\", \"humidity\": \"55%\"}\n", + "The weather in Paris is currently clear with a temperature of 18°C.\"\"\"\n", + "\n", + "tool_definition = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = tool_call_success(response=response_str, tool_definitions=tool_definition)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "b7217ddc", + "metadata": {}, + "source": [ + "#### Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f89ca010", + "metadata": {}, + "outputs": [], + "source": [ + "# Successful tool execution with single tool definition dict\n", + "successful_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_nyc_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"New York\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_nyc_weather\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"22°C\", \"condition\": \"sunny\", \"humidity\": \"45%\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The weather in New York is currently sunny with a temperature of 22°C.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = tool_call_success(response=successful_response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "1aeb8c11", + "metadata": {}, + "source": [ + "#### Multiple Successful Tool Calls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "444ee59c", + "metadata": {}, + "outputs": [], + "source": [ + "# Multiple successful tool executions\n", + "multiple_successful_response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\", \"precipitation\": \"light rain\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"your_email@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Current weather: 14°C and rainy with light rain.\",\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\", \"status\": \"delivered\"}}\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_call_success(response=multiple_successful_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "63e43736", + "metadata": {}, + "source": [ + "#### Tool Call Success without Tool Definitions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0ca062e", + "metadata": {}, + "outputs": [], + "source": [ + "# Successful execution without providing tool definitions\n", + "simple_successful_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_calendar_123\",\n", + " \"name\": \"get_calendar\",\n", + " \"arguments\": {\"start_date\": \"2025-01-01\", \"end_date\": \"2025-01-31\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_calendar_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"events\": [{\"date\": \"2025-01-15\", \"title\": \"Team Meeting\"}, {\"date\": \"2025-01-20\", \"title\": \"Project Review\"}]}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I found 2 events in your calendar for January: a Team Meeting on the 15th and a Project Review on the 20th.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "# Evaluation without tool definitions\n", + "result = tool_call_success(response=simple_successful_response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "97c1af0f", + "metadata": {}, + "source": [ + "#### Example of Failed Tool Call" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cafdee8e", + "metadata": {}, + "outputs": [], + "source": [ + "# Failed tool execution with error\n", + "failed_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"InvalidCity\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Location not found\", \"status\": \"failed\", \"code\": 404}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I'm sorry, I couldn't retrieve the weather information due to an error.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score FALSE due to tool failure\n", + "result = tool_call_success(response=failed_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "67694403", + "metadata": {}, + "source": [ + "#### Example of Tool Call Timeout" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19c7f49d", + "metadata": {}, + "outputs": [], + "source": [ + "# Tool execution timeout\n", + "timeout_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_789\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"test@example.com\",\n", + " \"subject\": \"Test Email\",\n", + " \"body\": \"This is a test email.\"\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_email_789\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Request timeout\", \"status\": \"timeout\", \"timeout_duration\": \"30s\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I'm unable to send the email due to a timeout error. Please try again later.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score FALSE due to timeout\n", + "result = tool_call_success(response=timeout_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "8a5e147a", + "metadata": {}, + "source": [ + "#### Mixed Results - Some Success, Some Failure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c73b8611", + "metadata": {}, + "outputs": [], + "source": [ + "# Mixed results - one success, one failure\n", + "mixed_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_success\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_success\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"sunny\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_fail\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"invalid-email\",\n", + " \"subject\": \"Weather Report\",\n", + " \"body\": \"Weather is sunny in Seattle.\"\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_email_fail\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Invalid email format\", \"status\": \"failed\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I got the weather information but failed to send the email due to an invalid email address.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "# This should score FALSE because at least one tool call failed\n", + "result = tool_call_success(response=mixed_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb deleted file mode 100644 index 77058c92..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb +++ /dev/null @@ -1,256 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Response Completeness Evaluator" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Getting Started\n", - "\n", - "This sample demonstrates how to use Response Completeness Evaluator on agent's response when ground truth is provided. This evaluator is helpful when you have ground truth to assess the quality of the agent's final response. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", - "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The Response Completeness evaluator assesses the quality of an agent response by examining how well it aligns with the provided ground truth. The evaluation is based on the following scoring system:\n", - "\n", - "
\n",
-    "Score 1: Fully incomplete: The response misses all necessary and relevant information compared to the ground truth.\n",
-    "Score 2: Barely complete: The response contains only a small percentage of the necessary information.\n",
-    "Score 3: Moderately complete: The response includes about half of the necessary information.\n",
-    "Score 4: Mostly complete: The response contains most of the necessary information, with only minor omissions.\n",
-    "Score 5: Fully complete: The response perfectly matches all necessary and relevant information from the ground truth.\n",
-    "
\n", - "\n", - "The evaluation requires the following inputs:\n", - "\n", - "- Response: The response to be evaluated. (string)\n", - "- Ground Truth: The correct and complete information against which the response is compared. (string)\n", - "\n", - "The evaluator uses these inputs to determine the completeness score, ensuring that the response meaningfully addresses the query while adhering to the provided definitions and data." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Initialize Completeness Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.evaluation import ResponseCompletenessEvaluator, AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "import os\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", - "\n", - "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Evaluating for a ground_truth and response" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'response_completeness': 4,\n", - " 'response_completeness_result': 'pass',\n", - " 'response_completeness_threshold': 3,\n", - " 'response_completeness_reason': 'The response contains most of the relevant information, but misses the explicit purpose of \"city sightseeing\" for Day 1. Otherwise, it is accurate and complete for Day 2.',\n", - " 'response_completeness_prompt_tokens': 1395,\n", - " 'response_completeness_completion_tokens': 176,\n", - " 'response_completeness_total_tokens': 1571,\n", - " 'response_completeness_finish_reason': 'stop',\n", - " 'response_completeness_model': 'gpt-4.1-2025-04-14',\n", - " 'response_completeness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": \\\\\"Itinery: Day 1 check out the downtown district of the city on train; for Day 2, we can rest in hotel.\\\\\", \\\\\"ground_truth\\\\\": \\\\\"Itinery: Day 1 take a train to visit the downtown area for city sightseeing; Day 2 rests in hotel.\\\\\"}\"}]',\n", - " 'response_completeness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think step by step: The ground truth has two statements: (1) Day 1 involves taking a train to visit the downtown area for city sightseeing, and (2) Day 2 involves resting in the hotel. The response covers both days: (1) Day 1 mentions checking out the downtown district on train, which is similar to the ground truth but omits \\\\\"city sightseeing\\\\\" explicitly, and (2) Day 2 mentions resting in the hotel, which matches the ground truth. The main omission is the explicit mention of \\\\\"city sightseeing\\\\\" as the purpose for visiting downtown.\\\\nThe response contains most of the relevant information, but misses the explicit purpose of \\\\\"city sightseeing\\\\\" for Day 1. Otherwise, it is accurate and complete for Day 2.\\\\n4\"}]'}" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# agent response is complete\n", - "result = response_completeness_evaluator(\n", - " response=\"Itinery: Day 1 check out the downtown district of the city on train; for Day 2, we can rest in hotel.\",\n", - " ground_truth=\"Itinery: Day 1 take a train to visit the downtown area for city sightseeing; Day 2 rests in hotel.\",\n", - ")\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'response_completeness': 5,\n", - " 'response_completeness_result': 'pass',\n", - " 'response_completeness_threshold': 3,\n", - " 'response_completeness_reason': 'The response fully contains the ground truth information without missing any details.',\n", - " 'response_completeness_prompt_tokens': 1410,\n", - " 'response_completeness_completion_tokens': 120,\n", - " 'response_completeness_total_tokens': 1530,\n", - " 'response_completeness_finish_reason': 'stop',\n", - " 'response_completeness_model': 'gpt-4.1-2025-04-14',\n", - " 'response_completeness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": \\\\\"The order with ID 123 has been shipped and is expected to be delivered on March 15, 2025. However, the order with ID 124 is delayed and should now arrive by March 20, 2025.\\\\\", \\\\\"ground_truth\\\\\": \\\\\"The order with ID 124 is delayed and should now arrive by March 20, 2025.\\\\\"}\"}]',\n", - " 'response_completeness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think step by step: The ground truth contains one statement: \\\\\"The order with ID 124 is delayed and should now arrive by March 20, 2025.\\\\\" The response includes this statement accurately and completely. The response also adds information about another order (ID 123), but this does not detract from the completeness regarding the ground truth statement. All necessary and relevant information from the ground truth is present in the response.\\\\nThe response fully contains the ground truth information without missing any details.\\\\n5\"}]'}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# agent response is incomplete\n", - "result = response_completeness_evaluator(\n", - " response=\"The order with ID 123 has been shipped and is expected to be delivered on March 15, 2025. However, the order with ID 124 is delayed and should now arrive by March 20, 2025.\",\n", - " ground_truth=\"The order with ID 124 is delayed and should now arrive by March 20, 2025.\",\n", - ")\n", - "result" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Evaluating using Conversation format" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'response_completeness': 1,\n", - " 'response_completeness_result': 'fail',\n", - " 'response_completeness_threshold': 3,\n", - " 'response_completeness_reason': 'There is no information in the ground truth, so the response cannot be evaluated for completeness and is considered fully incomplete.',\n", - " 'response_completeness_prompt_tokens': 1362,\n", - " 'response_completeness_completion_tokens': 109,\n", - " 'response_completeness_total_tokens': 1471,\n", - " 'response_completeness_finish_reason': 'stop',\n", - " 'response_completeness_model': 'gpt-4.1-2025-04-14',\n", - " 'response_completeness_sample_input': '[{\"role\": \"user\", \"content\": \"{\\\\\"response\\\\\": \\\\\"The weather in Seattle this weekend will be partly cloudy with temperatures around 15\\\\\\\\u00b0C on Saturday.\\\\\", \\\\\"ground_truth\\\\\": \\\\\"\\\\\"}\"}]',\n", - " 'response_completeness_sample_output': '[{\"role\": \"assistant\", \"content\": \"Let\\'s think step by step: The ground truth is completely empty, meaning there is no information provided to compare against the response. According to the definitions, completeness is measured by how well the response reflects the ground truth. Since there are no claims or statements in the ground truth, the response cannot be complete in relation to it.\\\\nThere is no information in the ground truth, so the response cannot be evaluated for completeness and is considered fully incomplete.\\\\n1\"}]'}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Conversation format - includes user query, expected ground truth, and agent response\n", - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What's the weather forecast for Seattle this weekend?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday.\",\n", - " \"context\": {\n", - " \"ground_truth\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday and 17°C on Sunday, with a 20% chance of rain on Sunday afternoon.\"\n", - " }\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = response_completeness_evaluator(conversation=conversation)\n", - "result" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From dedfd9e55f9597268aa53566967b26da2ca2132f Mon Sep 17 00:00:00 2001 From: v-mradi Date: Tue, 4 Nov 2025 16:06:50 +0200 Subject: [PATCH 3/7] Removing useless samples --- .../AI_Judge_Evaluator_Coherence.ipynb | 195 +----------------- .../AI_Judge_Evaluator_Fluency.ipynb | 85 +------- .../AI_Judge_Evaluator_Groundedness.ipynb | 55 +---- .../AI_Judge_Evaluator_Relevance.ipynb | 121 +---------- ...udge_Evaluator_Response_Completeness.ipynb | 21 +- ..._Judge_Evaluator_Tool_Input_Accuracy.ipynb | 146 ------------- ...ge_Evaluator_Tool_Output_Utilization.ipynb | 50 ----- .../AI_Judge_Evaluator_Tool_Selection.ipynb | 3 +- .../AI_Judge_Evaluator_Tool_Success.ipynb | 147 +------------ 9 files changed, 34 insertions(+), 789 deletions(-) diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb index fb5acbc7..48194df1 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb @@ -97,7 +97,7 @@ "id": "724adfc8", "metadata": {}, "source": [ - "#### Query and Response as Strings (str) - Excellent Coherence" + "#### Query and Response as Strings (str)" ] }, { @@ -107,7 +107,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Excellent coherence - smooth flow, clear logical progression\n", "query = \"Can you explain how machine learning works?\"\n", "response = \"Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed. It works by analyzing data patterns to make predictions or decisions. First, the system is trained on a dataset, which allows it to identify patterns and relationships. Then, it uses these learned patterns to make predictions on new, unseen data. The accuracy improves over time as the model processes more data and refines its understanding.\"\n", "\n", @@ -115,58 +114,12 @@ "pprint(result)" ] }, - { - "cell_type": "markdown", - "id": "34fbd16a", - "metadata": {}, - "source": [ - "#### Query and Response as Strings (str) - Good Coherence" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c4046db9", - "metadata": {}, - "outputs": [], - "source": [ - "# Good coherence - mostly logical with minor flow issues\n", - "query = \"What are the benefits of cloud computing?\"\n", - "response = \"Cloud computing offers several advantages. Cost savings is one major benefit because companies don't need to invest in physical servers. Scalability is another advantage. You can also access your data from anywhere with an internet connection. Additionally, cloud providers handle maintenance and updates, which reduces the IT burden on organizations.\"\n", - "\n", - "result = coherence_evaluator(query=query, response=response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "5336a182", - "metadata": {}, - "source": [ - "#### Query and Response as Strings (str) - Fair Coherence" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24ff7903", - "metadata": {}, - "outputs": [], - "source": [ - "# Fair coherence - noticeable gaps in logical flow\n", - "query = \"How does photosynthesis work?\"\n", - "response = \"Plants use sunlight to make food. Chlorophyll is green. The process happens in leaves. Carbon dioxide and water are important. Oxygen is released as a byproduct. This is why plants are important for the environment. The energy from sunlight is converted into chemical energy.\"\n", - "\n", - "result = coherence_evaluator(query=query, response=response)\n", - "pprint(result)" - ] - }, { "cell_type": "markdown", "id": "bc250127", "metadata": {}, "source": [ - "#### Query and Response as Strings (str) - Poor Coherence" + "#### Example of Poor Coherence" ] }, { @@ -176,7 +129,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Poor coherence - disjointed ideas and unclear transitions\n", + "# Poor coherence example\n", "query = \"What is blockchain technology?\"\n", "response = \"Blockchain is decentralized. Bitcoin uses it. Blocks contain data. Cryptography is involved. Banks might use blockchain. It's secure because of the chain. Mining is how new blocks are added. Digital currencies exist. The ledger is distributed.\"\n", "\n", @@ -184,35 +137,12 @@ "pprint(result)" ] }, - { - "cell_type": "markdown", - "id": "0a559cdd", - "metadata": {}, - "source": [ - "#### Query and Response as Strings (str) - Technical Question" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a769e940", - "metadata": {}, - "outputs": [], - "source": [ - "# Technical response with excellent coherence\n", - "query = \"How do I optimize database query performance?\"\n", - "response = \"To optimize database query performance, start by analyzing your query execution plans to identify bottlenecks. Next, ensure you have appropriate indexes on columns used in WHERE clauses and JOIN conditions. Consider denormalizing data in read-heavy scenarios to reduce the number of joins required. Additionally, implement query result caching for frequently accessed data. Finally, regularly update database statistics to help the query optimizer make better decisions about execution plans.\"\n", - "\n", - "result = coherence_evaluator(query=query, response=response)\n", - "pprint(result)" - ] - }, { "cell_type": "markdown", "id": "1d22e3d6", "metadata": {}, "source": [ - "#### Conversation as Dict with Messages - Multi-turn Evaluation" + "#### Conversation as Dict with Messages" ] }, { @@ -222,7 +152,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Multi-turn conversation - evaluates coherence across turns\n", + "# Multi-turn conversation\n", "conversation = {\n", " \"messages\": [\n", " {\n", @@ -255,121 +185,6 @@ "result = coherence_evaluator(conversation=conversation)\n", "pprint(result)" ] - }, - { - "cell_type": "markdown", - "id": "174a8ad4", - "metadata": {}, - "source": [ - "#### Conversation as Dict with Messages - Agent with Tool Calls" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1977ad35", - "metadata": {}, - "outputs": [], - "source": [ - "# Conversation with tool calls - evaluates coherence of responses\n", - "conversation_with_tools = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Can you check the weather in Seattle and send me an email with the forecast?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'll help you with that. First, let me check the current weather conditions in Seattle, and then I'll send you an email with the detailed forecast.\"\n", - " },\n", - " {\n", - " \"role\": \"tool\",\n", - " \"content\": \"Current weather in Seattle: Temperature: 15°C, Condition: Partly cloudy, Humidity: 65%, Wind: 12 mph NW, Forecast: Light rain expected in the evening\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I've retrieved the weather information for Seattle. Currently, it's 15°C with partly cloudy skies. The humidity is at 65% with winds from the northwest at 12 mph. The forecast indicates light rain is expected later this evening. I've sent you an email containing this complete weather report along with recommendations to bring an umbrella if you're planning to go out later today.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = coherence_evaluator(conversation=conversation_with_tools)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "f30e316a", - "metadata": {}, - "source": [ - "#### Conversation as Dict with Messages - Mixed Coherence Quality" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35562030", - "metadata": {}, - "outputs": [], - "source": [ - "# Conversation with varying coherence levels\n", - "conversation_mixed = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Tell me about renewable energy sources.\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Renewable energy comes from natural sources. Solar panels convert sunlight. Wind turbines exist. Hydroelectric power uses water. These are cleaner than fossil fuels. The environment benefits. Many countries are investing in renewables.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Which one is the most efficient?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The efficiency of renewable energy sources varies depending on several factors. Hydroelectric power is generally the most efficient, with conversion rates of up to 90% under optimal conditions. Solar panels typically achieve 15-22% efficiency, while wind turbines can reach 35-45% efficiency. However, efficiency alone doesn't determine the best choice, as factors like geographic location, initial investment costs, and environmental impact all play crucial roles in selecting the most appropriate renewable energy source for a specific situation.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = coherence_evaluator(conversation=conversation_mixed)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "b9b730ac", - "metadata": {}, - "source": [ - "#### Conversation as Dict with Messages - Short Conversation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "54dfbd45", - "metadata": {}, - "outputs": [], - "source": [ - "# Short, coherent conversation\n", - "conversation_short = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What's the capital of France?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The capital of France is Paris. It's located in the north-central part of the country along the Seine River and is known for its iconic landmarks like the Eiffel Tower, the Louvre Museum, and Notre-Dame Cathedral.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = coherence_evaluator(conversation=conversation_short)\n", - "pprint(result)" - ] } ], "metadata": { diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb index f2dd32bd..4c3f6809 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb @@ -96,7 +96,7 @@ "id": "46dc8414", "metadata": {}, "source": [ - "#### Response as String (str) - Excellent Fluency" + "#### Response as String (str)" ] }, { @@ -106,54 +106,9 @@ "metadata": {}, "outputs": [], "source": [ - "# Excellent fluency - proper grammar, syntax, and vocabulary\n", - "response_excellent = \"The weather in Seattle is currently partly cloudy with a temperature of 15°C. The forecast indicates that conditions will remain stable throughout the day, with a gentle breeze from the northwest.\"\n", + "response = \"The weather in Seattle is currently partly cloudy with a temperature of 15°C. The forecast indicates that conditions will remain stable throughout the day, with a gentle breeze from the northwest.\"\n", "\n", - "result = fluency_evaluator(response=response_excellent)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "5ca4dfac", - "metadata": {}, - "source": [ - "#### Response as String (str) - Good Fluency with Minor Issues" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8dcf34b8", - "metadata": {}, - "outputs": [], - "source": [ - "# Good fluency - mostly fluent with minor grammatical issues\n", - "response_good = \"The meeting has been schedule for tomorrow at 2 PM. Please make sure to reviewing the attached documents before attending the session.\"\n", - "\n", - "result = fluency_evaluator(response=response_good)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "a89fc139", - "metadata": {}, - "source": [ - "#### Response as String (str) - Fair Fluency" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c30fc6bd", - "metadata": {}, - "outputs": [], - "source": [ - "# Fair fluency - understandable but contains noticeable errors\n", - "response_fair = \"The order you placing yesterday is being process and will shipped soon. Tracking information send to your email when available.\"\n", - "\n", - "result = fluency_evaluator(response=response_fair)\n", + "result = fluency_evaluator(response=response)\n", "pprint(result)" ] }, @@ -162,7 +117,7 @@ "id": "59709c65", "metadata": {}, "source": [ - "#### Response as String (str) - Poor Fluency" + "#### Example of Poor Fluency" ] }, { @@ -172,32 +127,10 @@ "metadata": {}, "outputs": [], "source": [ - "# Poor fluency - frequent grammatical errors and awkward phrasing\n", - "response_poor = \"Email draft attach is. You review and giving feedback must. Important very for project success it being.\"\n", - "\n", - "result = fluency_evaluator(response=response_poor)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "e8934a3c", - "metadata": {}, - "source": [ - "#### Response as String (str) - Technical Content" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36e92f0b", - "metadata": {}, - "outputs": [], - "source": [ - "# Technical response with excellent fluency\n", - "response_technical = \"To resolve the authentication error, please verify that your API credentials are correctly configured in the environment variables. Additionally, ensure that the service principal has the necessary permissions assigned at the subscription level.\"\n", + "# Poor fluency example\n", + "response = \"Email draft attach is. You review and giving feedback must. Important very for project success it being.\"\n", "\n", - "result = fluency_evaluator(response=response_technical)\n", + "result = fluency_evaluator(response=response)\n", "pprint(result)" ] }, @@ -206,7 +139,7 @@ "id": "b26dd4bb", "metadata": {}, "source": [ - "#### Conversation as List[dict] - Multi-turn Evaluation" + "#### Conversation as Dict with Messages" ] }, { @@ -216,7 +149,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Multi-turn conversation - evaluates fluency of each assistant turn\n", + "# Multi-turn conversation\n", "conversation = {\n", " \"messages\": [\n", " {\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb index 9dd74c95..6c78ac1d 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb @@ -123,8 +123,7 @@ "id": "b73b1ccd", "metadata": {}, "source": [ - "### 1. Well-Grounded Weather Response\n", - "Using weather information consistent with ToolCallAccuracyEvaluator examples." + "### Response and Context as Strings (str)" ] }, { @@ -142,7 +141,6 @@ "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", "\n", "result = groundedness_evaluator(response=response, context=context)\n", - "print(\"=== Well-Grounded Weather Response ===\")\n", "pprint(result)" ] }, @@ -151,7 +149,7 @@ "id": "0c03f745", "metadata": {}, "source": [ - "### 2. Partially Grounded Response (with hallucination)" + "### Example of Ungrounded Response" ] }, { @@ -161,14 +159,13 @@ "metadata": {}, "outputs": [], "source": [ - "# Example of partially grounded response with unsupported claims\n", + "# Example of ungrounded response with unsupported claims\n", "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", "The forecast indicates overcast skies with light precipitation typical for Pacific Northwest weather.\"\"\"\n", "\n", "response = \"The current weather in Seattle is rainy with a temperature of 14°C. The city is experiencing its wettest month in 50 years, and the mayor has declared a weather emergency due to flooding concerns.\"\n", "\n", "result = groundedness_evaluator(response=response, context=context)\n", - "print(\"=== Partially Grounded Response ===\")\n", "pprint(result)" ] }, @@ -177,7 +174,7 @@ "id": "9052eb13", "metadata": {}, "source": [ - "### 3. Enhanced Evaluation with Weather Query" + "### Query, Response, and Context as Strings (str)" ] }, { @@ -187,7 +184,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Example with query for enhanced evaluation - consistent with ToolCallAccuracyEvaluator\n", "query = \"How is the weather in Seattle?\"\n", "\n", "context = \"\"\"Weather report for Seattle, Washington: Currently experiencing rainy weather with temperature at 14°C. \n", @@ -197,7 +193,6 @@ "response = \"The weather in Seattle is rainy with a temperature of 14°C. These are typical Pacific Northwest conditions with overcast skies.\"\n", "\n", "result = groundedness_evaluator(response=response, context=context, query=query)\n", - "print(\"=== Enhanced Evaluation with Weather Query ===\")\n", "pprint(result)" ] }, @@ -206,8 +201,7 @@ "id": "fd0a420f", "metadata": {}, "source": [ - "### 4. Agent Response with File Search Tool\n", - "This evaluates agent responses that use file_search tools to retrieve weather information - consistent with email/weather theme." + "### Query as String (str), Response as List[dict], Tool Definitions as List[dict]" ] }, { @@ -217,10 +211,9 @@ "metadata": {}, "outputs": [], "source": [ - "# Example of agent evaluation with file_search tool for weather information\n", "query = \"Can you get me the current weather information for Seattle?\"\n", "\n", - "# Simulated agent response with file_search tool call - similar to ToolCallAccuracyEvaluator pattern\n", + "# Agent response with file_search tool call\n", "agent_response = [\n", " {\n", " \"role\": \"assistant\",\n", @@ -269,7 +262,6 @@ " response=agent_response, \n", " tool_definitions=tool_definitions\n", ")\n", - "print(\"=== Agent Response with File Search ===\")\n", "pprint(result)" ] }, @@ -278,8 +270,7 @@ "id": "6dc31ae0", "metadata": {}, "source": [ - "### 5. Conversation Evaluation\n", - "Evaluating groundedness in multi-turn conversations about weather and email requests." + "### Conversation as Dict with Context and Messages" ] }, { @@ -289,7 +280,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Example of conversation evaluation - using weather and email theme for consistency\n", "conversation = {\n", " \"context\": \"Weather data shows Seattle currently has rainy conditions at 14°C with overcast skies. London shows cloudy weather at 8°C with partly cloudy conditions. Both cities are experiencing typical seasonal weather patterns.\",\n", " \"messages\": [\n", @@ -321,37 +311,6 @@ "}\n", "\n", "result = groundedness_evaluator(conversation=conversation)\n", - "print(\"=== Conversation Evaluation ===\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "dbe6e815", - "metadata": {}, - "source": [ - "### 6. Email Context Evaluation\n", - "Evaluating groundedness when dealing with email-related responses, consistent with ToolCallAccuracyEvaluator." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e46ac0ec", - "metadata": {}, - "outputs": [], - "source": [ - "# Example with email context - consistent with ToolCallAccuracyEvaluator email scenarios\n", - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "\n", - "context = \"\"\"Email service is available. Current weather data for Seattle: rainy conditions, 14°C temperature, \n", - "overcast skies with light precipitation. User email preferences show john@example.com as primary contact. \n", - "Email system can send weather reports with current conditions and forecasts.\"\"\"\n", - "\n", - "response = \"I can send you an email with the Seattle weather information. The current weather shows rainy conditions at 14°C with overcast skies. I'll prepare this information for your email.\"\n", - "\n", - "result = groundedness_evaluator(response=response, context=context, query=query)\n", - "print(\"=== Email Context Evaluation ===\")\n", "pprint(result)" ] } diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb index f3da5ca5..efe311f3 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb @@ -110,7 +110,7 @@ "id": "83d8d0ec", "metadata": {}, "source": [ - "### 1. Highly Relevant Response" + "### Query and Response as Strings (str)" ] }, { @@ -120,12 +120,10 @@ "metadata": {}, "outputs": [], "source": [ - "# Example of a highly relevant response - using weather query similar to ToolCallAccuracyEvaluator\n", "query = \"How is the weather in Seattle?\"\n", "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", "\n", "result = relevance_evaluator(query=query, response=response)\n", - "print(\"=== Highly Relevant Response ===\")\n", "pprint(result)" ] }, @@ -134,7 +132,7 @@ "id": "89266048", "metadata": {}, "source": [ - "### 2. Partially Relevant Response" + "### Example of Irrelevant Response" ] }, { @@ -144,36 +142,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Example of a partially relevant response\n", - "query = \"How do I bake a chocolate cake?\"\n", - "response = \"Chocolate is made from cocoa beans and was first discovered by the Mayans. There are many types of chocolate including dark, milk, and white chocolate.\"\n", - "\n", - "result = relevance_evaluator(query=query, response=response)\n", - "print(\"=== Partially Relevant Response ===\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "aeb6961e", - "metadata": {}, - "source": [ - "### 3. Irrelevant Response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1cb62f5e", - "metadata": {}, - "outputs": [], - "source": [ - "# Example of an irrelevant response\n", + "# Irrelevant response example\n", "query = \"What are the symptoms of flu?\"\n", "response = \"The stock market had significant fluctuations today, with technology stocks leading the decline.\"\n", "\n", "result = relevance_evaluator(query=query, response=response)\n", - "print(\"=== Irrelevant Response ===\")\n", "pprint(result)" ] }, @@ -182,8 +155,7 @@ "id": "bd158e43", "metadata": {}, "source": [ - "### 4. Conversation Evaluation\n", - "The evaluator also supports evaluating multi-turn conversations using the `conversation` parameter." + "### Conversation as Dict with Messages" ] }, { @@ -193,7 +165,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Example of conversation evaluation\n", "conversation = {\n", " \"messages\": [\n", " {\n", @@ -216,90 +187,6 @@ "}\n", "\n", "result = relevance_evaluator(conversation=conversation)\n", - "print(\"=== Conversation Evaluation ===\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "105c8839", - "metadata": {}, - "source": [ - "### 5. Technical Query Evaluation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3b73bf46", - "metadata": {}, - "outputs": [], - "source": [ - "# Example of technical query evaluation\n", - "query = \"How do I implement a binary search algorithm in Python?\"\n", - "response = \"\"\"Here's a Python implementation of binary search:\n", - "\n", - "def binary_search(arr, target):\n", - " left, right = 0, len(arr) - 1\n", - " \n", - " while left <= right:\n", - " mid = (left + right) // 2\n", - " \n", - " if arr[mid] == target:\n", - " return mid\n", - " elif arr[mid] < target:\n", - " left = mid + 1\n", - " else:\n", - " right = mid - 1\n", - " \n", - " return -1\n", - "\n", - "This algorithm works on sorted arrays and has O(log n) time complexity.\"\"\"\n", - "\n", - "result = relevance_evaluator(query=query, response=response)\n", - "print(\"=== Technical Query Evaluation ===\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "c98f8d61", - "metadata": {}, - "source": [ - "### 6. Customer Service Scenario" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b1604f3d", - "metadata": {}, - "outputs": [], - "source": [ - "# Example of customer service conversation\n", - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"I received a damaged product in my order #12345. What can you do to help?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'm sorry to hear about the damaged product in order #12345. I can help you with this issue. Let me check your order details and arrange either a replacement or refund. Could you please describe the damage and let me know your preference for resolution?\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"The screen of the tablet has a crack. I'd prefer a replacement.\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I understand you received a tablet with a cracked screen. I'll immediately process a replacement for you. You'll receive a new tablet within 2-3 business days, and we'll include a prepaid return label for the damaged unit. Is your shipping address still the same as in the original order?\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = relevance_evaluator(conversation=conversation)\n", - "print(\"=== Customer Service Scenario ===\")\n", "pprint(result)" ] } diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb index 1e94bbe1..95e6d57a 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb @@ -102,7 +102,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Evaluating for a ground_truth and response" + "#### Response and Ground Truth as Strings (str)" ] }, { @@ -111,7 +111,6 @@ "metadata": {}, "outputs": [], "source": [ - "# agent response is complete\n", "result = response_completeness_evaluator(\n", " response=\"Itinery: Day 1 check out the downtown district of the city on train; for Day 2, we can rest in hotel.\",\n", " ground_truth=\"Itinery: Day 1 take a train to visit the downtown area for city sightseeing; Day 2 rests in hotel.\",\n", @@ -119,25 +118,11 @@ "result" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# agent response is incomplete\n", - "result = response_completeness_evaluator(\n", - " response=\"The order with ID 123 has been shipped and is expected to be delivered on March 15, 2025. However, the order with ID 124 is delayed and should now arrive by March 20, 2025.\",\n", - " ground_truth=\"The order with ID 124 is delayed and should now arrive by March 20, 2025.\",\n", - ")\n", - "result" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### Evaluating using Conversation format" + "#### Conversation as Dict with Ground Truth in Context" ] }, { @@ -146,7 +131,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Conversation format - includes user query, expected ground truth, and agent response\n", + "# Conversation format with ground truth in context\n", "conversation = {\n", " \"messages\": [\n", " {\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb index 711472fb..a1cdb299 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb @@ -279,100 +279,6 @@ "pprint(result)" ] }, - { - "cell_type": "markdown", - "id": "52468ac9", - "metadata": {}, - "source": [ - "#### Multiple Tool Calls with Different Parameter Requirements" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a341071b", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "response = [\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Rainy, 14°C\"}}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"your_email@example.com\",\n", - " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"The current weather in Seattle is rainy with a temperature of 14°C.\",\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"role\": \"tool\",\n", - " \"content\": [\n", - " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\"}}\n", - " ],\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " \"required\": [\"location\"]\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " \"required\": [\"recipient\", \"subject\", \"body\"]\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, { "cell_type": "markdown", "id": "2b274da2", @@ -510,58 +416,6 @@ "result = tool_input_accuracy(query=query, response=incorrect_response, tool_definitions=tool_definitions)\n", "pprint(result)" ] - }, - { - "cell_type": "markdown", - "id": "b4a866e0", - "metadata": {}, - "source": [ - "#### Example of Missing Required Parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "159cf7e5", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Send an email to john@example.com with weather info\"\n", - "# Missing required parameters\n", - "incomplete_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_email_456\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\"recipient\": \"john@example.com\"}, # Missing subject and body\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " \"required\": [\"recipient\", \"subject\", \"body\"]\n", - " },\n", - " }\n", - "]\n", - "\n", - "# This should score 0 due to missing required parameters\n", - "result = tool_input_accuracy(query=query, response=incomplete_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] } ], "metadata": { diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb index 317f7c5a..c0c9dfb8 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb @@ -425,56 +425,6 @@ "pprint(result)" ] }, - { - "cell_type": "markdown", - "id": "6b02ba5f", - "metadata": {}, - "source": [ - "#### Example of Incorrect Tool Output Utilization" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a524ee56", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "# Incorrect utilization - agent misrepresents the tool output\n", - "incorrect_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The weather in Seattle is currently sunny and hot with temperatures reaching 25°C. It's a perfect day for outdoor activities!\", # Incorrect representation of tool output\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "# This should score poorly due to incorrect use of tool output\n", - "result = tool_output_utilization(query=query, response=incorrect_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, { "cell_type": "markdown", "id": "bcb8a2cf", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb index 18ebcbf1..aadb165b 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb @@ -232,8 +232,7 @@ "id": "19b3c8de", "metadata": {}, "source": [ - "#### Tool Calls passed as part of `Response` (common for agent case)\n", - "- Tool Selection Evaluator extracts tool calls from response" + "#### Tool Calls passed as part of `Response` (common for agent case)" ] }, { diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb index a277a4e5..ca8d8e3b 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb @@ -5,10 +5,10 @@ "id": "cea11cae", "metadata": {}, "source": [ - "# Tool Call Success Evaluator\n", + "# Tool Success Evaluator\n", "\n", "## Objective\n", - "This sample demonstrates how to use tool call success evaluator on agent data. The supported input formats include:\n", + "This sample demonstrates how to use tool success evaluator on agent data. The supported input formats include:\n", "- simple data such as strings and `dict` describing agent responses;\n", "- user-agent conversations in the form of list of agent messages. \n", "\n", @@ -35,7 +35,7 @@ "id": "49404a44", "metadata": {}, "source": [ - "The Tool Call Success evaluator determines whether tool calls done by an AI agent includes failures or not.\n", + "The Tool Success evaluator determines whether tool calls done by an AI agent includes failures or not.\n", "\n", "This evaluator focuses solely on tool call results and tool definitions, disregarding user's query to the agent, conversation history and agent's final response. Although tool definitions is optional, providing them can help the evaluator better understand the context of the tool calls made by the agent. Please note that this evaluator validates tool calls for potential technical failures like errors, exceptions, timeouts and empty results (only in cases where empty results could indicate a failure). It does not assess the correctness or the tool result itself, like mathematical errors and unrealistic field values like name=\"668656\".\n", "\n", @@ -51,7 +51,7 @@ "id": "d91cb876", "metadata": {}, "source": [ - "Tool Call Success requires following input:\n", + "Tool Success requires following input:\n", "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator examines tool call results within the response.\n", "- Tool Definitions - (Optional) Tool(s) definition used by Agent. Providing tool definitions helps the evaluator better understand the context of the tool calls made by the agent.\n" ] @@ -61,7 +61,7 @@ "id": "e07d293d", "metadata": {}, "source": [ - "### Initialize Tool Call Success Evaluator\n" + "### Initialize Tool Success Evaluator\n" ] }, { @@ -456,143 +456,6 @@ "result = tool_call_success(response=failed_response, tool_definitions=tool_definitions)\n", "pprint(result)" ] - }, - { - "cell_type": "markdown", - "id": "67694403", - "metadata": {}, - "source": [ - "#### Example of Tool Call Timeout" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19c7f49d", - "metadata": {}, - "outputs": [], - "source": [ - "# Tool execution timeout\n", - "timeout_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_email_789\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"test@example.com\",\n", - " \"subject\": \"Test Email\",\n", - " \"body\": \"This is a test email.\"\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_email_789\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Request timeout\", \"status\": \"timeout\", \"timeout_duration\": \"30s\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I'm unable to send the email due to a timeout error. Please try again later.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "# This should score FALSE due to timeout\n", - "result = tool_call_success(response=timeout_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "8a5e147a", - "metadata": {}, - "source": [ - "#### Mixed Results - Some Success, Some Failure" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c73b8611", - "metadata": {}, - "outputs": [], - "source": [ - "# Mixed results - one success, one failure\n", - "mixed_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_success\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_success\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"sunny\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_email_fail\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"invalid-email\",\n", - " \"subject\": \"Weather Report\",\n", - " \"body\": \"Weather is sunny in Seattle.\"\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_email_fail\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Invalid email format\", \"status\": \"failed\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I got the weather information but failed to send the email due to an invalid email address.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "# This should score FALSE because at least one tool call failed\n", - "result = tool_call_success(response=mixed_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] } ], "metadata": { From cda9038b6bf48db6fc87245dede6bcdc14ffbfaf Mon Sep 17 00:00:00 2001 From: v-mradi Date: Wed, 5 Nov 2025 17:35:37 +0200 Subject: [PATCH 4/7] Updating response completeness and task navigation efficiency evaluators --- ...udge_Evaluator_Response_Completeness.ipynb | 51 ++- ...Evaluator_Task_Navigation_Efficiency.ipynb | 412 +++++++----------- 2 files changed, 199 insertions(+), 264 deletions(-) diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb index 95e6d57a..00cd4eaa 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb @@ -112,8 +112,48 @@ "outputs": [], "source": [ "result = response_completeness_evaluator(\n", - " response=\"Itinery: Day 1 check out the downtown district of the city on train; for Day 2, we can rest in hotel.\",\n", - " ground_truth=\"Itinery: Day 1 take a train to visit the downtown area for city sightseeing; Day 2 rests in hotel.\",\n", + " response=\"The capital of Japan\",\n", + " ground_truth=\"The capital of Japan is Tokyo.\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = response_completeness_evaluator(\n", + " response=\"The capital of Japan is Tokyo.\",\n", + " ground_truth=\"The capital of Japan is Tokyo.\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluate with a reasoning model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.evaluation import ResponseCompletenessEvaluator , AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "# set is_reasoning_model to True in case the model is a reasoning model (ex: o3-mini, o1-preview)\n", + "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config,\n", + " is_reasoning_model=False)\n", + "\n", + "result = response_completeness_evaluator(\n", + " response=\"The capital of Japan is Tokyo.\",\n", + " ground_truth=\"The capital of Japan is Tokyo.\"\n", ")\n", "result" ] @@ -141,11 +181,10 @@ " {\n", " \"role\": \"assistant\",\n", " \"content\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday.\",\n", - " \"context\": {\n", - " \"ground_truth\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday and 17°C on Sunday, with a 20% chance of rain on Sunday afternoon.\"\n", - " }\n", + " \"ground_truth\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday and 17°C on Sunday, with a 20% chance of rain on Sunday afternoon.\"\n", " }\n", - " ]\n", + " ],\n", + " \n", "}\n", "\n", "result = response_completeness_evaluator(conversation=conversation)\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb index a6ea032d..1d8615a0 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb @@ -31,31 +31,26 @@ "id": "d61cd511", "metadata": {}, "source": [ - "The Task Navigation Efficiency evaluator assesses how efficiently an AI agent navigates through task completion by examining:\n", - "- The number of steps taken to complete the task\n", - "- Whether the agent took the most efficient path\n", - "- Comparison against ideal or ground truth navigation steps\n", - "- Unnecessary detours or redundant actions\n", + "The Task Navigation Efficiency Evaluator measures how efficiently an agent navigates through a sequence of actions compared to an optimal task completion path.\n", "\n", - "The evaluator uses a scoring system that measures navigation efficiency:\n", + "The evaluator provides comprehensive evaluation with both binary matching results and additional detailed P\\R\\F1 results:\n", "\n", - " - Higher scores indicate more efficient navigation\n", - " - Lower scores indicate inefficient paths with unnecessary steps\n", - " - The evaluation considers the optimal path as the baseline\n", + "**Primary Result:**\n", + "- **Binary Match Result**: Pass/Fail based on the selected matching mode\n", "\n", - "This evaluation focuses on measuring whether the agent takes an efficient route to task completion, minimizing unnecessary steps while still achieving the desired outcome." - ] - }, - { - "cell_type": "markdown", - "id": "e18904e1", - "metadata": {}, - "source": [ - "Task Navigation Efficiency requires following input:\n", - "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", - "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response.\n", - "- Ground Truth - (Optional) The ideal or expected sequence of steps to complete the task efficiently.\n", - "- Tool Definitions - (Optional) Tool(s) definition used by Agent to answer the query. Providing tool definitions helps the evaluator understand available actions.\n" + "**Available Matching Modes:**\n", + "- **Exact Match**: Agent's tool calls must exactly match the ground truth (default)\n", + "- **In-Order Match**: All ground truth steps must appear in correct order (allows extra steps)\n", + "- **Any-Order Match**: All ground truth steps must appear with sufficient frequency (most lenient)\n", + "\n", + "**Properties Bag Additional Metrics (0.0 - 1.0):**\n", + "- **Precision**: How many of the agent's steps were necessary (relevant to ground truth)\n", + "- **Recall**: How many of the required steps were executed by the agent \n", + "- **F1 Score**: Harmonic mean of precision and recall\n", + "\n", + "The evaluation requires the following inputs:\n", + "- **Response**: The agent's response containing tool calls as a list of messages or string\n", + "- **Ground Truth**: List of expected tool/action steps as strings, or tuple with parameters for matching" ] }, { @@ -73,24 +68,23 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", "from azure.ai.evaluation._evaluators._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator, _TaskNavigationEfficiencyMatchingMode\n", "from pprint import pprint\n", "\n", "# Initialize with exact match mode\n", - "task_navigation_efficiency = _TaskNavigationEfficiencyEvaluator(\n", + "task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(\n", " matching_mode=_TaskNavigationEfficiencyMatchingMode.EXACT_MATCH\n", ")\n", "\n", "# Other examples:\n", "# For in-order matching (allows extra steps but requires correct order)\n", - "# task_navigation_efficiency = _TaskNavigationEfficiencyEvaluator(matching_mode=TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", + "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", "\n", "# For any-order matching (most lenient - allows extra steps and different order) \n", - "# task_navigation_efficiency = _TaskNavigationEfficiencyEvaluator(matching_mode=TaskNavigationEfficiencyMatchingMode.ANY_ORDER_MATCH)\n", + "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.ANY_ORDER_MATCH)\n", "\n", "# Or use defaults (exact match mode)\n", - "# task_navigation_efficiency = _TaskNavigationEfficiencyEvaluator()" + "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator()" ] }, { @@ -106,7 +100,7 @@ "id": "1d75457a", "metadata": {}, "source": [ - "#### Evaluating Simple Task Navigation" + "#### Sample 1: Perfect Path (Exact Match)" ] }, { @@ -116,14 +110,26 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", - "response = \"I'll check the weather for you. The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F).\"\n", + "# Agent follows the exact optimal path\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\", \n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"analyze\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"report\", \"arguments\": {}}],\n", + " },\n", + "]\n", "\n", - "# Ground truth represents the ideal efficient path\n", - "ground_truth = [\"fetch_weather(location='Seattle')\", \"return_weather_information\"]\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", "\n", - "# Basic evaluation without tool definitions\n", - "result = task_navigation_efficiency(query=query, response=response, ground_truth=ground_truth)\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"Perfect Path Results:\")\n", "pprint(result)" ] }, @@ -132,7 +138,7 @@ "id": "8a9c5e69", "metadata": {}, "source": [ - "#### Task Navigation with Tool Context" + "#### Sample 2: Efficient Path with Extra Steps" ] }, { @@ -142,37 +148,30 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", + "# Agent performs all required steps but with extra unnecessary step\n", "response = [\n", " {\n", " \"role\": \"assistant\",\n", - " \"content\": \"I'll fetch the weather information for Seattle.\"\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"validate\", \"arguments\": {}}],\n", " },\n", " {\n", " \"role\": \"assistant\", \n", - " \"content\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C.\"\n", - " }\n", - "]\n", - "\n", - "ground_truth = [\"fetch_weather\", \"provide_weather_info\"]\n", - "\n", - "tool_definitions = [\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"analyze\", \"arguments\": {}}],\n", + " },\n", " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " }\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"report\", \"arguments\": {}}],\n", + " },\n", "]\n", "\n", - "result = task_navigation_efficiency(\n", - " query=query, \n", - " response=response, \n", - " ground_truth=ground_truth,\n", - " tool_definitions=tool_definitions\n", - ")\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nPath with Extra Steps Results:\")\n", "pprint(result)" ] }, @@ -181,7 +180,7 @@ "id": "e4fa5757", "metadata": {}, "source": [ - "#### Complex Task with Multiple Steps Navigation" + "#### Sample 3: Inefficient Path (Wrong Order)" ] }, { @@ -191,90 +190,29 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", + "# Agent performs all required steps but in wrong order\n", "response = [\n", " {\n", - " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I'll help you get the weather information for Seattle and send it via email.\",\n", - " }\n", - " ],\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"report\", \"arguments\": {}}],\n", " },\n", " {\n", - " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"Let me first fetch the current weather for Seattle.\",\n", - " }\n", - " ],\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"search\", \"arguments\": {}}],\n", " },\n", " {\n", - " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"Now I'll compose and send the email with the weather information.\",\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is partly cloudy with a temperature of 15°C.\",\n", - " }\n", - " ],\n", + " \"role\": \"assistant\", \n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"analyze\", \"arguments\": {}}],\n", " },\n", "]\n", "\n", - "# Ground truth represents the most efficient navigation path\n", - "ground_truth = [\n", - " \"fetch_weather(location='Seattle')\", \n", - " \"compose_email(weather_data)\", \n", - " \"send_email()\"\n", - "]\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", + "# Using in-order matching mode to demonstrate the difference\n", + "in_order_task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", "\n", - "result = task_navigation_efficiency(\n", - " query=query, \n", - " response=response, \n", - " ground_truth=ground_truth,\n", - " tool_definitions=tool_definitions\n", - ")\n", + "result = in_order_task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nWrong Order Results:\")\n", "pprint(result)" ] }, @@ -283,8 +221,7 @@ "id": "ae74ac8e", "metadata": {}, "source": [ - "#### Query as Conversation History with Navigation Steps\n", - "The evaluator also supports query as a list of messages representing conversation history. This helps evaluate navigation efficiency in the context of a full conversation." + "#### Sample 4: Incomplete Path (Missing Steps)" ] }, { @@ -294,75 +231,22 @@ "metadata": {}, "outputs": [], "source": [ - "# Query as conversation history instead of a single string\n", - "query_as_conversation = [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"Hi, I need to plan my day. Can you check the weather in Seattle?\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Also, please send me an email summary of the weather so I can reference it later.\"\n", - " }\n", - "]\n", - "\n", + "# Agent performs only some of the required steps (incomplete)\n", "response = [\n", " {\n", " \"role\": \"assistant\",\n", - " \"content\": \"I'll help you with both tasks. Let me start by checking the weather in Seattle.\"\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", " },\n", " {\n", " \"role\": \"assistant\",\n", - " \"content\": \"The weather in Seattle is currently 15°C and partly cloudy with light winds.\"\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"analyze\", \"arguments\": {}}],\n", " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Now I'll send you an email with this weather information.\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Done! I've sent you an email with the weather summary for Seattle. You should receive it shortly.\"\n", - " }\n", - "]\n", - "\n", - "ground_truth = [\n", - " \"fetch_weather(location='Seattle')\",\n", - " \"send_email(weather_summary)\"\n", "]\n", "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", "\n", - "result = task_navigation_efficiency(\n", - " query=query_as_conversation, \n", - " response=response, \n", - " ground_truth=ground_truth,\n", - " tool_definitions=tool_definitions\n", - ")\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nMissing Steps Results:\")\n", "pprint(result)" ] }, @@ -371,7 +255,7 @@ "id": "2c9111ef", "metadata": {}, "source": [ - "#### Example of Inefficient Navigation" + "#### Sample 5: Real-World Customer Service Scenario" ] }, { @@ -381,39 +265,30 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", - "inefficient_response = [\n", + "# Real-world example: Customer service agent handling a refund request\n", + "response = [\n", " {\n", " \"role\": \"assistant\",\n", - " \"content\": \"Let me think about this request.\"\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"lookup_order\", \"arguments\": {\"order_id\": \"12345\"}}],\n", " },\n", " {\n", " \"role\": \"assistant\",\n", - " \"content\": \"You want to know about the weather. Let me consider what information you need.\"\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"check_inventory\", \"arguments\": {\"product_id\": \"ABC123\"}}],\n", " },\n", " {\n", " \"role\": \"assistant\", \n", - " \"content\": \"Seattle is a city in Washington state. Let me check the weather there.\"\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"calculate_refund\", \"arguments\": {\"order_id\": \"12345\"}}],\n", " },\n", " {\n", " \"role\": \"assistant\",\n", - " \"content\": \"I'll now fetch the weather information.\"\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"process_refund\", \"arguments\": {\"order_id\": \"12345\", \"amount\": \"29.99\"}}],\n", " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The weather in Seattle is currently 15°C and partly cloudy.\"\n", - " }\n", "]\n", "\n", - "ground_truth = [\"fetch_weather(location='Seattle')\", \"return_weather_information\"]\n", + "ground_truth = [\"lookup_order\", \"calculate_refund\", \"process_refund\"]\n", "\n", - "# This response shows inefficient navigation with unnecessary steps\n", - "result = task_navigation_efficiency(\n", - " query=query, \n", - " response=inefficient_response, \n", - " ground_truth=ground_truth,\n", - " tool_definitions=tool_definitions\n", - ")\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nCustomer Service Results:\")\n", "pprint(result)" ] }, @@ -422,9 +297,7 @@ "id": "4bf2e0ee", "metadata": {}, "source": [ - "### Sample 4: Task Navigation with Exact Parameter Matching\n", - "\n", - "This sample demonstrates using `ground_truth` as a Tuple with exact parameter matching. The Tuple format allows you to specify not only which tools should be called, but also what exact parameters should be passed to each tool." + "#### Sample 6: Complex Path with Duplicates" ] }, { @@ -434,68 +307,91 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Check the weather in Seattle and send me an email about it\"\n", - "\n", - "# Agent response with multiple tool calls\n", + "# Agent repeats some steps and includes extra ones\n", "response = [\n", " {\n", " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_1\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"}\n", - " }\n", - " ]\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", " },\n", " {\n", - " \"role\": \"tool\",\n", - " \"tool_call_id\": \"call_weather_1\", \n", - " \"content\": '{\"temperature\": \"15°C\", \"condition\": \"partly cloudy\"}'\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"search\", \"arguments\": {}}], # duplicate\n", " },\n", " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_email_1\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"user@example.com\",\n", - " \"subject\": \"Seattle Weather Update\",\n", - " \"body\": \"The weather in Seattle is 15°C and partly cloudy.\"\n", - " }\n", - " }\n", - " ]\n", + " \"role\": \"assistant\", \n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"validate\", \"arguments\": {}}], # extra step\n", " },\n", " {\n", - " \"role\": \"tool\",\n", - " \"tool_call_id\": \"call_email_1\",\n", - " \"content\": '{\"status\": \"sent\", \"message_id\": \"msg_123\"}'\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"analyze\", \"arguments\": {}}],\n", " },\n", " {\n", " \"role\": \"assistant\",\n", - " \"content\": \"I've checked the weather in Seattle (15°C, partly cloudy) and sent you an email with the details.\"\n", - " }\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_5\", \"name\": \"report\", \"arguments\": {}}],\n", + " },\n", "]\n", "\n", - "# Ground truth as Tuple with exact parameter matching\n", - "# This format allows you to specify not just which tools should be called,\n", - "# but also what exact parameters should be passed to each tool\n", - "ground_truth = (\n", - " [\"fetch_weather\", \"send_email\"], # Expected tool sequence\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nComplex Path with Duplicates Results:\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "478c00fb", + "metadata": {}, + "source": [ + "#### Sample 7: Tuple Format with Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7404f830", + "metadata": {}, + "outputs": [], + "source": [ + "# TaskNavigationEfficiencyEvaluator also supports tuple format with parameters for exact parameter matching\n", + "response_with_params = [\n", " {\n", - " \"fetch_weather\": {\"location\": \"Seattle\"}, # Exact expected parameters\n", - " \"send_email\": {\n", - " \"recipient\": \"user@example.com\",\n", - " \"subject\": \"Seattle Weather Update\",\n", - " \"body\": \"The weather in Seattle is 15°C and partly cloudy.\"\n", - " }\n", - " }\n", - ")\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {\"query\": \"test\"}}],\n", + " },\n", + "]\n", + "\n", + "# Ground truth using tuple format: (tool_names, parameters_dict)\n", + "# Parameters must match exactly for tools to be considered matching\n", + "ground_truth_with_params = ([\"search\"], {\"search\": {\"query\": \"test\"}})\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=response_with_params, ground_truth=ground_truth_with_params)\n", + "print(\"\\nTuple Format with Parameters Results:\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "38d2de9b", + "metadata": {}, + "source": [ + "#### Sample 8: String Response Input Type" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a0a853b", + "metadata": {}, + "outputs": [], + "source": [ + "# Demonstrate string response input type\n", + "# The string response should contain structured tool call information that can be parsed\n", + "string_response = \"I'll help you with that. Let me search for information, then analyze the results, and finally provide a report.\"\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", "\n", - "result = task_navigation_efficiency(response=response, ground_truth=ground_truth)\n", + "result = task_navigation_efficiency_evaluator(response=string_response, ground_truth=ground_truth)\n", + "print(\"\\nString Response Results:\")\n", "pprint(result)" ] } From 7522b1972dc09f20e8e291f253e3106f8f1178fc Mon Sep 17 00:00:00 2001 From: v-mradi Date: Wed, 5 Nov 2025 20:27:21 +0200 Subject: [PATCH 5/7] Adding data source configs and data mapping for all evaluators --- .../AI_Judge_Evaluator_Coherence.ipynb | 235 +++++++ .../AI_Judge_Evaluator_Fluency.ipynb | 312 +++++++++ .../AI_Judge_Evaluator_Groundedness.ipynb | 394 +++++++++++ ...AI_Judge_Evaluator_Intent_Resolution.ipynb | 401 +++++++++++ .../AI_Judge_Evaluator_Relevance.ipynb | 237 +++++++ ...udge_Evaluator_Response_Completeness.ipynb | 229 +++++++ .../AI_Judge_Evaluator_Task_Adherence.ipynb | 318 +++++++++ .../AI_Judge_Evaluator_Task_Completion.ipynb | 411 +++++++++++ ...Evaluator_Task_Navigation_Efficiency.ipynb | 442 ++++++++++++ ...I_Judge_Evaluator_Tool_Call_Accuracy.ipynb | 575 ++++++++++++++++ ..._Judge_Evaluator_Tool_Input_Accuracy.ipynb | 483 +++++++++++++ ...ge_Evaluator_Tool_Output_Utilization.ipynb | 591 ++++++++++++++++ .../AI_Judge_Evaluator_Tool_Selection.ipynb | 644 ++++++++++++++++++ .../AI_Judge_Evaluator_Tool_Success.ipynb | 519 ++++++++++++++ 14 files changed, 5791 insertions(+) create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb new file mode 100644 index 00000000..2380a14c --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb @@ -0,0 +1,235 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "da22a6e7", + "metadata": {}, + "source": [ + "# Coherence Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use the Coherence evaluator to assess the quality and readability of AI-generated responses. The evaluator measures the ability of the language model to generate text that reads naturally, flows smoothly, and resembles human-like language.\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "id": "48f8cb64", + "metadata": {}, + "source": [ + "The Coherence evaluator assesses the ability of the language model to generate text that reads naturally, flows smoothly, and resembles human-like language in its responses. It measures the readability and user-friendliness of the model's generated responses.\n", + "\n", + "Coherence scores range from 1 to 5:\n", + "\n", + "
\n",
+    "Score 1: Very Poor - The response is completely incoherent with no logical flow or connection between ideas.\n",
+    "Score 2: Poor - The response has significant coherence issues with disjointed ideas and unclear transitions.\n",
+    "Score 3: Fair - The response is somewhat coherent but has noticeable gaps in logical flow or organization.\n",
+    "Score 4: Good - The response is mostly coherent with good flow and logical connections between ideas.\n",
+    "Score 5: Excellent - The response is perfectly coherent with smooth, natural flow and clear logical progression.\n",
+    "
\n", + "\n", + "The evaluation requires one of the following input patterns:\n", + "\n", + "**Pattern 1: Query-Response Evaluation**\n", + "- Query: The user's question or prompt. (string)\n", + "- Response: The AI-generated response to be evaluated for coherence. (string)\n", + "\n", + "**Pattern 2: Conversation Evaluation**\n", + "- Conversation: A multi-turn conversation where each turn is evaluated for coherence. (dict with \"messages\" key)" + ] + }, + { + "cell_type": "markdown", + "id": "542c1340", + "metadata": {}, + "source": [ + "### Initialize Coherence Evaluator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35799119", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import CoherenceEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"type\": \"string\"\n", + " },\n", + " \"response\": {\n", + " \"type\": \"string\"\n", + " }\n", + " },\n", + " \"required\": []\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", + "\n", + "coherence_evaluator = CoherenceEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "b10aff00", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "724adfc8", + "metadata": {}, + "source": [ + "#### Query and Response as Strings (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82a1be93", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you explain how machine learning works?\"\n", + "response = \"Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed. It works by analyzing data patterns to make predictions or decisions. First, the system is trained on a dataset, which allows it to identify patterns and relationships. Then, it uses these learned patterns to make predictions on new, unseen data. The accuracy improves over time as the model processes more data and refines its understanding.\"\n", + "\n", + "result = coherence_evaluator(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "bc250127", + "metadata": {}, + "source": [ + "#### Example of Poor Coherence" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62ad5478", + "metadata": {}, + "outputs": [], + "source": [ + "# Poor coherence example\n", + "query = \"What is blockchain technology?\"\n", + "response = \"Blockchain is decentralized. Bitcoin uses it. Blocks contain data. Cryptography is involved. Banks might use blockchain. It's secure because of the chain. Mining is how new blocks are added. Digital currencies exist. The ledger is distributed.\"\n", + "\n", + "result = coherence_evaluator(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "1d22e3d6", + "metadata": {}, + "source": [ + "#### Conversation as Dict with Messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "556c197b", + "metadata": {}, + "outputs": [], + "source": [ + "# Multi-turn conversation\n", + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"I'm planning a trip to Japan. What should I know?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Japan is a fascinating destination with a rich cultural heritage. First, you should know that the best times to visit are spring (March-May) for cherry blossoms and fall (September-November) for autumn colors. The country has an excellent public transportation system, particularly the Shinkansen bullet train network. Additionally, learning a few basic Japanese phrases will be appreciated by locals, though many signs in major cities have English translations.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What about the food?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Japanese cuisine is incredibly diverse and delicious. Beyond the well-known sushi and ramen, you should try regional specialties like okonomiyaki in Osaka or motsunabe in Fukuoka. Most restaurants display plastic food models in their windows, which makes ordering easier even with a language barrier. It's also important to note that tipping is not customary in Japan and can sometimes be considered rude.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Are there any cultural customs I should be aware of?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Yes, there are several important customs to observe. Always remove your shoes when entering homes, temples, and some traditional restaurants. Bowing is a common greeting, though a slight nod is acceptable for foreigners. When using chopsticks, never stick them upright in rice as this resembles a funeral ritual. Finally, be mindful of noise levels on public transportation, as talking on phones or having loud conversations is considered impolite.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = coherence_evaluator(conversation=conversation)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb new file mode 100644 index 00000000..7d0ce118 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb @@ -0,0 +1,312 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "21599c07", + "metadata": {}, + "source": [ + "# Fluency Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use the Fluency evaluator to assess the linguistic quality of AI-generated responses. The evaluator measures how well generated text conforms to grammatical rules, syntactic structures, and appropriate vocabulary usage.\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "id": "7add5ec1", + "metadata": {}, + "source": [ + "The Fluency evaluator assesses the extent to which generated text conforms to grammatical rules, syntactic structures, and appropriate vocabulary usage, resulting in linguistically correct responses.\n", + "\n", + "Fluency scores range from 1 to 5:\n", + "\n", + "
\n",
+    "Score 1: Very Poor - The response is incomprehensible with severe grammatical errors and improper vocabulary.\n",
+    "Score 2: Poor - The response has frequent grammatical errors and awkward phrasing that hinder understanding.\n",
+    "Score 3: Fair - The response is understandable but contains noticeable grammatical errors or awkward expressions.\n",
+    "Score 4: Good - The response is mostly fluent with minor grammatical issues that don't significantly impact readability.\n",
+    "Score 5: Excellent - The response is perfectly fluent with proper grammar, syntax, and vocabulary usage.\n",
+    "
\n", + "\n", + "The evaluation requires one of the following input patterns:\n", + "\n", + "**Pattern 1: Single Response Evaluation**\n", + "- Response: The text response to be evaluated for fluency. (string)\n", + "\n", + "**Pattern 2: Conversation Evaluation**\n", + "- Conversation: A multi-turn conversation where each turn is evaluated for fluency. (List[dict])" + ] + }, + { + "cell_type": "markdown", + "id": "2abd4d05", + "metadata": {}, + "source": [ + "### Initialize Fluency Evaluator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfd097f8", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import FluencyEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"type\": \"string\"\n", + " },\n", + " \"response\": {\n", + " \"type\": \"string\"\n", + " }\n", + " },\n", + " \"required\": []\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", + "\n", + "fluency_evaluator = FluencyEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "4793759a", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "46dc8414", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c2df39a", + "metadata": {}, + "outputs": [], + "source": [ + "response = \"The weather in Seattle is currently partly cloudy with a temperature of 15°C. The forecast indicates that conditions will remain stable throughout the day, with a gentle breeze from the northwest.\"\n", + "\n", + "result = fluency_evaluator(response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "59709c65", + "metadata": {}, + "source": [ + "#### Example of Poor Fluency" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9c624ab", + "metadata": {}, + "outputs": [], + "source": [ + "# Poor fluency example\n", + "response = \"Email draft attach is. You review and giving feedback must. Important very for project success it being.\"\n", + "\n", + "result = fluency_evaluator(response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "b26dd4bb", + "metadata": {}, + "source": [ + "#### Conversation as Dict with Messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ccaff56", + "metadata": {}, + "outputs": [], + "source": [ + "# Multi-turn conversation\n", + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you help me understand the weather forecast for this week?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Of course! I'd be happy to help you understand this week's weather forecast. The outlook shows a mix of sunny and cloudy conditions.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What about the temperature?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Temperatures will ranging from 12°C to 18°C throughout the week, with the warmest days being Wednesday and Thursday.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Should I bring an umbrella?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Yes, it would be advisable to carry an umbrella, particularly on Friday when there is a 60% chance of precipitation.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = fluency_evaluator(conversation=conversation)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "0ba89e73", + "metadata": {}, + "source": [ + "#### Conversation as List[dict] - Agent with Tool Calls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2658f610", + "metadata": {}, + "outputs": [], + "source": [ + "# Conversation with tool calls - evaluates fluency of assistant responses\n", + "conversation_with_tools = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you send me an email with the weather report for Seattle?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I'll fetch the current weather information for Seattle and send you an email with the details.\"\n", + " },\n", + " {\n", + " \"role\": \"tool\",\n", + " \"content\": \"Temperature: 15°C, Condition: Partly cloudy, Wind: 8 mph NW\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I have successfully sent you an email containing the weather report for Seattle. The current conditions are partly cloudy with a temperature of 15°C and light winds from the northwest.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = fluency_evaluator(conversation=conversation_with_tools)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "ff71d193", + "metadata": {}, + "source": [ + "#### Conversation as List[dict] - Mixed Fluency Quality" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0d8bf46", + "metadata": {}, + "outputs": [], + "source": [ + "# Conversation with varying fluency levels across turns\n", + "conversation_mixed = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What's the status of my order?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Your order is being process currently and will be ship very soon to you.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"When will it arrive?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"The estimated delivery date for your order is March 15, 2025. You will receive tracking information via email once the package has been dispatched.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = fluency_evaluator(conversation=conversation_mixed)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb new file mode 100644 index 00000000..060c7ee3 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb @@ -0,0 +1,394 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6a3f10c0", + "metadata": {}, + "source": [ + "# Groundedness Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use the Groundedness evaluator to assess whether AI-generated responses are grounded in the provided context. The evaluator supports multiple input formats including:\n", + "- Simple response and context evaluation\n", + "- Query, response, and context evaluation\n", + "- Agent responses with tool calls (file_search)\n", + "- Multi-turn conversations\n", + "\n", + "This notebook uses consistent examples with the ToolCallAccuracyEvaluator for better comparison across evaluators.\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook.\n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities.\n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure OpenAI Endpoint to be used for evaluation\n", + "3) **AZURE_OPENAI_API_KEY** - Azure OpenAI Key to be used for evaluation\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure OpenAI API version to be used for evaluation" + ] + }, + { + "cell_type": "markdown", + "id": "fe3af2a1", + "metadata": {}, + "source": [ + "## What is Groundedness?\n", + "\n", + "The Groundedness evaluator assesses the correspondence between claims in an AI-generated response and the source context. It ensures that responses are substantiated by the provided context, preventing hallucinations and unsupported claims.\n", + "\n", + "**Key Points:**\n", + "- Even factually correct responses are considered ungrounded if they can't be verified against the provided context\n", + "- Essential for RAG (Retrieval-Augmented Generation) applications\n", + "- Helps ensure AI responses are trustworthy and verifiable\n", + "\n", + "**Scoring:** Groundedness scores range from 1 to 5, with:\n", + "- **1**: Completely ungrounded - no claims supported by context\n", + "- **2**: Mostly ungrounded - few claims supported\n", + "- **3**: Partially grounded - some claims supported\n", + "- **4**: Mostly grounded - most claims supported\n", + "- **5**: Fully grounded - all claims supported by context" + ] + }, + { + "cell_type": "markdown", + "id": "bd8a00fb", + "metadata": {}, + "source": [ + "## Groundedness Evaluator Input Requirements\n", + "\n", + "The Groundedness evaluator supports multiple input formats:\n", + "\n", + "1. **Basic Context Evaluation:**\n", + " - `response`: The AI response to evaluate (str)\n", + " - `context`: The source context/documents (str)\n", + " - `query`: Optional query for enhanced evaluation (str)\n", + "\n", + "2. **Agent Tool Evaluation:**\n", + " - `query`: The user query (str)\n", + " - `response`: Agent response with tool calls (List[dict])\n", + " - `tool_definitions`: Available tools, only file_search supported (List[dict])\n", + "\n", + "3. **Conversation Evaluation:**\n", + " - `conversation`: Multi-turn conversation with context (Conversation object)" + ] + }, + { + "cell_type": "markdown", + "id": "9aa68c67", + "metadata": {}, + "source": [ + "### Initialize Groundedness Evaluator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "298d067a", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import GroundednessEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "# Configure the model\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"context\": {\n", + " \"type\": \"string\"\n", + " },\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"context\": \"{{item.context}}\",\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "# Initialize the evaluator\n", + "groundedness_evaluator = GroundednessEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "18b0a169", + "metadata": {}, + "source": [ + "## Sample Evaluations" + ] + }, + { + "cell_type": "markdown", + "id": "b73b1ccd", + "metadata": {}, + "source": [ + "### Response and Context as Strings (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e5fa898", + "metadata": {}, + "outputs": [], + "source": [ + "# Example of a well-grounded response using weather context\n", + "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", + "The forecast indicates overcast skies with light precipitation typical for Pacific Northwest weather. \n", + "Humidity is at 85% with winds from the southwest at 12 mph. Visibility is reduced to 8 miles due to rain.\"\"\"\n", + "\n", + "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", + "\n", + "result = groundedness_evaluator(response=response, context=context)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "0c03f745", + "metadata": {}, + "source": [ + "### Example of Ungrounded Response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6de6a51e", + "metadata": {}, + "outputs": [], + "source": [ + "# Example of ungrounded response with unsupported claims\n", + "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", + "The forecast indicates overcast skies with light precipitation typical for Pacific Northwest weather.\"\"\"\n", + "\n", + "response = \"The current weather in Seattle is rainy with a temperature of 14°C. The city is experiencing its wettest month in 50 years, and the mayor has declared a weather emergency due to flooding concerns.\"\n", + "\n", + "result = groundedness_evaluator(response=response, context=context)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "9052eb13", + "metadata": {}, + "source": [ + "### Query, Response, and Context as Strings (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c77316c4", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "\n", + "context = \"\"\"Weather report for Seattle, Washington: Currently experiencing rainy weather with temperature at 14°C. \n", + "Overcast conditions with light rain are expected to continue. The current conditions are typical for the Pacific Northwest region \n", + "during this season. Wind speed is moderate at 12 mph from southwest direction.\"\"\"\n", + "\n", + "response = \"The weather in Seattle is rainy with a temperature of 14°C. These are typical Pacific Northwest conditions with overcast skies.\"\n", + "\n", + "result = groundedness_evaluator(response=response, context=context, query=query)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "fd0a420f", + "metadata": {}, + "source": [ + "### Query as String (str), Response as List[dict], Tool Definitions as List[dict]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72b922ee", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you get me the current weather information for Seattle?\"\n", + "\n", + "# Agent response with file_search tool call\n", + "agent_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_filesearch_weather_123\",\n", + " \"name\": \"file_search\",\n", + " \"arguments\": {\"query\": \"current weather Seattle temperature conditions\"}\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"role\": \"tool\",\n", + " \"tool_call_id\": \"call_filesearch_weather_123\", \n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_result\",\n", + " \"tool_result\": {\n", + " \"content\": \"Seattle weather report: Currently rainy with temperature of 14°C. Overcast skies with light precipitation. Typical Pacific Northwest weather with 85% humidity and southwest winds at 12 mph.\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Based on the weather data, Seattle is currently experiencing rainy weather with a temperature of 14°C. The conditions include overcast skies and light precipitation, which is typical for the Pacific Northwest.\"\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"file_search\",\n", + " \"description\": \"Search through uploaded files to find relevant information\",\n", + " \"parameters\": {\n", + " \"type\": \"object\", \n", + " \"properties\": {\n", + " \"query\": {\"type\": \"string\", \"description\": \"Search query\"}\n", + " }\n", + " }\n", + " }\n", + "]\n", + "\n", + "result = groundedness_evaluator(\n", + " query=query, \n", + " response=agent_response, \n", + " tool_definitions=tool_definitions\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "6dc31ae0", + "metadata": {}, + "source": [ + "### Conversation as Dict with Context and Messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f37ae51", + "metadata": {}, + "outputs": [], + "source": [ + "conversation = {\n", + " \"context\": \"Weather data shows Seattle currently has rainy conditions at 14°C with overcast skies. London shows cloudy weather at 8°C with partly cloudy conditions. Both cities are experiencing typical seasonal weather patterns.\",\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"According to the current weather data, Seattle is experiencing rainy conditions with a temperature of 14°C and overcast skies.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"How does that compare to London?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"London is currently cloudier but drier than Seattle, with a temperature of 8°C and partly cloudy conditions. Seattle is warmer but rainier at 14°C.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Can you email me a summary of both cities' weather?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I can provide you with a weather summary: Seattle has rainy weather at 14°C with overcast skies, while London has partly cloudy conditions at 8°C. However, I would need email access to send this information to you.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = groundedness_evaluator(conversation=conversation)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb new file mode 100644 index 00000000..4debe16c --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb @@ -0,0 +1,401 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Intent Resolution Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates to how to use intent resolution evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Intent Resolution evaluator measures how well an agent has identified and resolved the user intent.\n", + "The scoring is on a 1-5 integer scale and is as follows:\n", + "\n", + " - Score 1: Response completely unrelated to user intent\n", + " - Score 2: Response minimally relates to user intent\n", + " - Score 3: Response partially addresses the user intent but lacks complete details\n", + " - Score 4: Response addresses the user intent with moderate accuracy but has minor inaccuracies or omissions\n", + " - Score 5: Response directly addresses the user intent and fully resolves it\n", + "\n", + "The evaluation requires the following inputs:\n", + "\n", + " - Query : The user query. Either a string with a user request or a list of messages with previous requests from the user and responses from the assistant, potentially including a system message.\n", + " - Response : The response to be evaluated. Either a string or a message with the response from the agent to the last user query.\n", + "\n", + "There is a third optional parameter:\n", + " - ToolDefinitions : The list of tool definitions the agent can call. This may be useful for the evaluator to better assess if the right tool was called to resolve a given intent." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize Intent Resolution Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from azure.ai.evaluation import IntentResolutionEvaluator\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "intent_resolution_evaluator = IntentResolutionEvaluator(model_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluating query and response as string" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Success example. Intent is identified and understood and the response correctly resolves user intent\n", + "result = intent_resolution_evaluator(\n", + " query=\"What are the opening hours of the Eiffel Tower?\",\n", + " response=\"Opening hours of the Eiffel Tower are 9:00 AM to 11:00 PM.\",\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Failure example. Even though intent is correctly identified, the response does not resolve the user intent\n", + "result = intent_resolution_evaluator(\n", + " query=\"What is the opening hours of the Eiffel Tower?\",\n", + " response=\"Please check the official website for the up-to-date information on Eiffel Tower opening hours.\",\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Query and Response as String with Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example with a single tool definition dict instead of a list\n", + "query = \"What's the status of order #789?\"\n", + "response = \"Order #789 has been shipped and will arrive on March 18, 2025.\"\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"get_order\",\n", + " \"description\": \"Get the details of a specific order.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"order_id\": {\"type\": \"string\", \"description\": \"The order ID to get the details for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = intent_resolution_evaluator(\n", + " query=query,\n", + " response=response,\n", + " tool_definitions=tool_definition_dict,\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluating query and response as list of messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = [\n", + " {\"role\": \"system\", \"content\": \"You are a friendly and helpful customer service agent.\"},\n", + " {\n", + " \"createdAt\": \"2025-03-14T06:14:20Z\",\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"Hi, I need help with the last 2 orders on my account #888. Could you please update me on their status?\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-14T06:14:30Z\",\n", + " \"run_id\": \"0\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"text\", \"text\": \"Hello! Let me quickly look up your account details.\"}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T06:14:35Z\",\n", + " \"run_id\": \"0\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"tool_call_20250310_001\",\n", + " \"name\": \"get_orders\",\n", + " \"arguments\": {\"account_number\": \"888\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T06:14:40Z\",\n", + " \"run_id\": \"0\",\n", + " \"tool_call_id\": \"tool_call_20250310_001\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": '[{ \"order_id\": \"123\" }, { \"order_id\": \"124\" }]'}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T06:14:45Z\",\n", + " \"run_id\": \"0\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"Thanks for your patience. I see two orders on your account. Let me fetch the details for both.\",\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T06:14:50Z\",\n", + " \"run_id\": \"0\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"tool_call_20250310_002\",\n", + " \"name\": \"get_order\",\n", + " \"arguments\": {\"order_id\": \"123\"},\n", + " },\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"tool_call_20250310_003\",\n", + " \"name\": \"get_order\",\n", + " \"arguments\": {\"order_id\": \"124\"},\n", + " },\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T06:14:55Z\",\n", + " \"run_id\": \"0\",\n", + " \"tool_call_id\": \"tool_call_20250310_002\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_result\",\n", + " \"tool_result\": '{ \"order\": { \"id\": \"123\", \"status\": \"shipped\", \"delivery_date\": \"2025-03-15\" } }',\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T06:15:00Z\",\n", + " \"run_id\": \"0\",\n", + " \"tool_call_id\": \"tool_call_20250310_003\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_result\",\n", + " \"tool_result\": '{ \"order\": { \"id\": \"124\", \"status\": \"delayed\", \"expected_delivery\": \"2025-03-20\" } }',\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T06:15:05Z\",\n", + " \"run_id\": \"0\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The order with ID 123 has been shipped and is expected to be delivered on March 15, 2025. However, the order with ID 124 is delayed and should now arrive by March 20, 2025. Is there anything else I can help you with?\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "# please note that the tool definitions are not strictly required, and that some of the tools below are not used in the example above and that is ok.\n", + "# if context length is a concern you can remove the unused tool definitions or even the tool definitions altogether as the impact to the intent resolution evaluation is usual minimal.\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"get_orders\",\n", + " \"description\": \"Get the list of orders for a given account number.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"account_number\": {\"type\": \"string\", \"description\": \"The account number to get the orders for.\"}\n", + " },\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"get_order\",\n", + " \"description\": \"Get the details of a specific order.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"order_id\": {\"type\": \"string\", \"description\": \"The order ID to get the details for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"initiate_return\",\n", + " \"description\": \"Initiate the return process for an order.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"order_id\": {\"type\": \"string\", \"description\": \"The order ID for the return process.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"update_shipping_address\",\n", + " \"description\": \"Update the shipping address for a given account.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"account_number\": {\"type\": \"string\", \"description\": \"The account number to update.\"},\n", + " \"new_address\": {\"type\": \"string\", \"description\": \"The new shipping address.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = intent_resolution_evaluator(\n", + " query=query,\n", + " response=response,\n", + " tool_definitions=tool_definitions,\n", + ")\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb new file mode 100644 index 00000000..49d9f977 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb @@ -0,0 +1,237 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "bb0675a7", + "metadata": {}, + "source": [ + "# Relevance Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use the Relevance evaluator to assess the relevance of AI-generated responses. The evaluator supports multiple input formats including:\n", + "- Simple query and response pairs\n", + "- Multi-turn conversations\n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 15 minutes running this notebook.\n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities.\n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure OpenAI Endpoint to be used for evaluation\n", + "3) **AZURE_OPENAI_API_KEY** - Azure OpenAI Key to be used for evaluation\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure OpenAI API version to be used for evaluation\n" + ] + }, + { + "cell_type": "markdown", + "id": "1a21aa7e", + "metadata": {}, + "source": [ + "## What is Relevance?\n", + "\n", + "The Relevance evaluator assesses the ability of AI responses to capture the key points and address the user's query appropriately. It measures how well the response relates to and answers the specific question or request.\n", + "\n", + "**Scoring:** Relevance scores range from 1 to 5, with:\n", + "- **1**: Completely irrelevant to the query\n", + "- **2**: Partially relevant but misses key points\n", + "- **3**: Moderately relevant with some key points addressed\n", + "- **4**: Highly relevant with most key points covered\n", + "- **5**: Perfectly relevant, fully addressing the query\n", + "\n", + "High relevance scores indicate the AI system's understanding of the input and capability to produce contextually appropriate outputs." + ] + }, + { + "cell_type": "markdown", + "id": "f0dd0ebf", + "metadata": {}, + "source": [ + "## Relevance Evaluator Input Requirements\n", + "\n", + "The Relevance evaluator supports the following input formats:\n", + "\n", + "1. **Simple Query-Response Evaluation:**\n", + " - `query`: The user's question or request (str)\n", + " - `response`: The AI's response to evaluate (str)\n", + "\n", + "2. **Conversation Evaluation:**\n", + " - `conversation`: Multi-turn conversation with messages list and optional context (Conversation object)\n" + ] + }, + { + "cell_type": "markdown", + "id": "519631da", + "metadata": {}, + "source": [ + "### Initialize Relevance Evaluator" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3484e790", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import RelevanceEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "# Configure the model\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"type\": \"string\"\n", + " },\n", + " \"response\": {\n", + " \"type\": \"string\"\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", + "\n", + "# Initialize the evaluator\n", + "relevance_evaluator = RelevanceEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "4f739700", + "metadata": {}, + "source": [ + "## Sample Evaluations" + ] + }, + { + "cell_type": "markdown", + "id": "83d8d0ec", + "metadata": {}, + "source": [ + "### Query and Response as Strings (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43eaf25d", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", + "\n", + "result = relevance_evaluator(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "89266048", + "metadata": {}, + "source": [ + "### Example of Irrelevant Response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "69761825", + "metadata": {}, + "outputs": [], + "source": [ + "# Irrelevant response example\n", + "query = \"What are the symptoms of flu?\"\n", + "response = \"The stock market had significant fluctuations today, with technology stocks leading the decline.\"\n", + "\n", + "result = relevance_evaluator(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "bd158e43", + "metadata": {}, + "source": [ + "### Conversation as Dict with Messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cf17774b", + "metadata": {}, + "outputs": [], + "source": [ + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"I'm planning a trip to Japan. Can you help me with some recommendations?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I'd be happy to help you plan your trip to Japan! Here are some popular destinations: Tokyo for modern city life and culture, Kyoto for traditional temples and gardens, Osaka for amazing food, and Mount Fuji for scenic beauty. What type of experiences are you most interested in?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"I'm particularly interested in traditional culture and temples.\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Perfect! For traditional culture and temples, I highly recommend Kyoto. Visit Fushimi Inari Shrine with its thousands of red torii gates, Kinkaku-ji (Golden Pavilion), and Gion district where you might spot geishas. Nara is also wonderful with its deer park and Todai-ji Temple housing a giant Buddha statue.\"\n", + " }\n", + " ]\n", + "}\n", + "\n", + "result = relevance_evaluator(conversation=conversation)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb new file mode 100644 index 00000000..09efd837 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb @@ -0,0 +1,229 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Response Completeness Evaluator" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting Started\n", + "\n", + "This sample demonstrates how to use Response Completeness Evaluator on agent's response when ground truth is provided. This evaluator is helpful when you have ground truth to assess the quality of the agent's final response. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Response Completeness evaluator assesses the quality of an agent response by examining how well it aligns with the provided ground truth. The evaluation is based on the following scoring system:\n", + "\n", + "
\n",
+    "Score 1: Fully incomplete: The response misses all necessary and relevant information compared to the ground truth.\n",
+    "Score 2: Barely complete: The response contains only a small percentage of the necessary information.\n",
+    "Score 3: Moderately complete: The response includes about half of the necessary information.\n",
+    "Score 4: Mostly complete: The response contains most of the necessary information, with only minor omissions.\n",
+    "Score 5: Fully complete: The response perfectly matches all necessary and relevant information from the ground truth.\n",
+    "
\n", + "\n", + "The evaluation requires the following inputs:\n", + "\n", + "- Response: The response to be evaluated. (string)\n", + "- Ground Truth: The correct and complete information against which the response is compared. (string)\n", + "\n", + "The evaluator uses these inputs to determine the completeness score, ensuring that the response meaningfully addresses the query while adhering to the provided definitions and data." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize Completeness Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.evaluation import ResponseCompletenessEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "import os\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"ground_truth\": {\n", + " \"type\": \"string\"\n", + " },\n", + " \"response\": {\n", + " \"type\": \"string\"\n", + " }\n", + " },\n", + " \"required\": [\"ground_truth\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"ground_truth\": \"{{item.ground_truth}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", + "\n", + "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Response and Ground Truth as Strings (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = response_completeness_evaluator(\n", + " response=\"The capital of Japan\",\n", + " ground_truth=\"The capital of Japan is Tokyo.\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = response_completeness_evaluator(\n", + " response=\"The capital of Japan is Tokyo.\",\n", + " ground_truth=\"The capital of Japan is Tokyo.\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluate with a reasoning model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.evaluation import ResponseCompletenessEvaluator , AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "# set is_reasoning_model to True in case the model is a reasoning model (ex: o3-mini, o1-preview)\n", + "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config,\n", + " is_reasoning_model=False)\n", + "\n", + "result = response_completeness_evaluator(\n", + " response=\"The capital of Japan is Tokyo.\",\n", + " ground_truth=\"The capital of Japan is Tokyo.\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Conversation as Dict with Ground Truth in Context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Conversation format with ground truth in context\n", + "conversation = {\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What's the weather forecast for Seattle this weekend?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday.\",\n", + " \"ground_truth\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday and 17°C on Sunday, with a 20% chance of rain on Sunday afternoon.\"\n", + " }\n", + " ],\n", + " \n", + "}\n", + "\n", + "result = response_completeness_evaluator(conversation=conversation)\n", + "result" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb new file mode 100644 index 00000000..6ffd58ff --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb @@ -0,0 +1,318 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Task Adherence Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates to how to use task adherence evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 10 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Task Adherence evaluator measures how well the agent adheres to their assigned tasks or predefined goal.\n", + "\n", + "The scoring is on a 1-5 integer scale and is as follows:\n", + "\n", + " - Score 1: Fully Inadherent\n", + " - Score 2: Barely Adherent\n", + " - Score 3: Moderately Adherent\n", + " - Score 4: Mostly Adherent\n", + " - Score 5: Fully Adherent\n", + "\n", + "The evaluation requires the following inputs:\n", + "\n", + " - Query : The user query. Either a string with a user request or a list of messages with previous requests from the user and responses from the assistant, potentially including a system message.\n", + " - Response : The response to be evaluated. Either a string or a message with the response from the agent to the last user query.\n", + "\n", + "There is a third optional parameter:\n", + " - ToolDefinitions : The list of tool definitions the agent can call. This may be useful for the evaluator to better assess if the right tool was called to adhere to user intent." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize Task Adherence Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import TaskAdherenceEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "task_adherence_evaluator = TaskAdherenceEvaluator(model_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluating query and response as string" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Failure example, there's only a vague adherence to the task\n", + "result = task_adherence_evaluator(\n", + " query=\"What are the best practices for maintaining a healthy rose garden during the summer?\",\n", + " response=\"Make sure to water your roses regularly and trim them occasionally.\",\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Success example, full adherence to the task\n", + "result = task_adherence_evaluator(\n", + " query=\"What are the best practices for maintaining a healthy rose garden during the summer?\",\n", + " response=\"For optimal summer care of your rose garden, start by watering deeply early in the morning to ensure the roots are well-hydrated without encouraging fungal growth. Apply a 2-3 inch layer of organic mulch around the base of the plants to conserve moisture and regulate soil temperature. Fertilize with a balanced rose fertilizer every 4 to 6 weeks to support healthy growth. Prune away any dead or diseased wood to promote good air circulation, and inspect regularly for pests such as aphids or spider mites, treating them promptly with an appropriate organic insecticidal soap. Finally, ensure that your roses receive at least 6 hours of direct sunlight daily for robust flowering.\",\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Query and Response as String with Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example with a single tool definition dict\n", + "query = \"Can you recommend a science fiction book for me?\"\n", + "response = \"I found 'Dune' by Frank Herbert for you. It's a classic science fiction novel set in a distant future.\"\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"get_book\",\n", + " \"description\": \"Retrieve a book recommendation for a specified genre.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"genre\": {\"type\": \"string\", \"description\": \"The genre for which a book recommendation is requested.\"}\n", + " },\n", + " },\n", + "}\n", + "\n", + "result = task_adherence_evaluator(\n", + " query=query,\n", + " response=response,\n", + " tool_definitions=tool_definition_dict,\n", + ")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluating query and response as list of messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = [\n", + " {\"role\": \"system\", \"content\": \"You are an expert in literature and at provid can provide book recommendations.\"},\n", + " {\n", + " \"createdAt\": \"2025-03-14T08:00:00Z\",\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"I love historical fiction. Can you recommend a good book from that genre?\"}\n", + " ],\n", + " },\n", + "]\n", + "\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-14T08:00:05Z\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"text\", \"text\": \"Let me fetch a recommendation for historical fiction.\"}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T08:00:10Z\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"tool_call_20250314_001\",\n", + " \"name\": \"get_book\",\n", + " \"arguments\": {\"genre\": \"historical fiction\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T08:00:15Z\",\n", + " \"role\": \"tool\",\n", + " \"tool_call_id\": \"tool_call_20250314_001\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_result\",\n", + " \"tool_result\": '{ \"book\": { \"title\": \"The Pillars of the Earth\", \"author\": \"Ken Follett\", \"summary\": \"A captivating tale set in medieval England that weaves historical events with personal drama.\" } }',\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-14T08:00:20Z\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"Based on our records, I recommend 'The Pillars of the Earth' by Ken Follett. This novel is an excellent example of historical fiction with a rich narrative and well-developed characters. Would you like more details or another suggestion?\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"get_book\",\n", + " \"description\": \"Retrieve a book recommendation for a specified genre.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"genre\": {\"type\": \"string\", \"description\": \"The genre for which a book recommendation is requested.\"}\n", + " },\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = task_adherence_evaluator(\n", + " query=query,\n", + " response=response,\n", + " tool_definitions=tool_definitions,\n", + ")\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb new file mode 100644 index 00000000..3d302ca7 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb @@ -0,0 +1,411 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cd88c9a4", + "metadata": {}, + "source": [ + "# Task Completion Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use task completion evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing task responses;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "id": "fe5dda1c", + "metadata": {}, + "source": [ + "The Task Completion evaluator assesses whether an AI agent successfully completes the requested task by examining:\n", + "- Whether the task was fully completed\n", + "- Quality of task execution\n", + "- Appropriateness of the response to the original request\n", + "\n", + "The evaluator uses a binary scoring system (0 or 1):\n", + "\n", + " - Score 0: The task was not completed or only partially completed\n", + " - Score 1: The task was successfully and fully completed\n", + "\n", + "This evaluation focuses on measuring whether the agent's response indicates successful completion of the user's request, regardless of the specific methods or tools used to achieve the task." + ] + }, + { + "cell_type": "markdown", + "id": "c1642178", + "metadata": {}, + "source": [ + "Task Completion requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response.\n", + "- Tool Definitions - (Optional) Tool(s) definition used by Agent to answer the query. Providing tool definitions helps the evaluator better understand the context and capabilities available to the agent.\n" + ] + }, + { + "cell_type": "markdown", + "id": "ffad775f", + "metadata": {}, + "source": [ + "### Initialize Task Completion Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0a5fae33", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._task_completion import _TaskCompletionEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "task_completion = _TaskCompletionEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "af4839bb", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "95305bbd", + "metadata": {}, + "source": [ + "#### Evaluating Simple Task Completion" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57d4592d", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = \"The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\"\n", + "\n", + "# Basic evaluation without tool definitions\n", + "result = task_completion(query=query, response=response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "2c66a362", + "metadata": {}, + "source": [ + "#### Task Completion with Tool Context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71ea8725", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = \"I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\"\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = task_completion(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "1962234d", + "metadata": {}, + "source": [ + "#### Task Completion with Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a9cd459", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"What's the current temperature in Boston?\"\n", + "response = \"The current temperature in Boston is 22°C (72°F) with clear skies. It's a beautiful day with low humidity and a gentle breeze.\"\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = task_completion(query=query, response=response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "fc85ade7", + "metadata": {}, + "source": [ + "#### Complex Task with Multiple Steps" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "915768db", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I'll get the current weather information for Seattle and then send you an email with the details.\",\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is partly cloudy with a temperature of 15°C. You should receive the email shortly at your registered email address.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = task_completion(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "5523dfcf", + "metadata": {}, + "source": [ + "#### Query as Conversation History (List of Messages)\n", + "The evaluator also supports query as a list of messages representing conversation history. This helps evaluate task completion in the context of a full conversation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c4772be", + "metadata": {}, + "outputs": [], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, I need to plan my day. Can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Also, please send me an email summary of the weather so I can reference it later.\"\n", + " }\n", + "]\n", + "\n", + "response = \"I've checked the weather in Seattle for you. It's currently 15°C and partly cloudy with light winds. I've also sent you an email summary with these details so you can reference them throughout your day. The task has been completed successfully.\"\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = task_completion(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "100e943b", + "metadata": {}, + "source": [ + "#### Example of Incomplete Task" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9986fa70", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "incomplete_response = \"I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\"\n", + "\n", + "# This response doesn't complete the email sending task\n", + "result = task_completion(query=query, response=incomplete_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb new file mode 100644 index 00000000..7009dfc2 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb @@ -0,0 +1,442 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "28c2bc93", + "metadata": {}, + "source": [ + "# Task Navigation Efficiency Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use task navigation efficiency evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing task responses;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "This is a deterministic evaluator that compares navigation paths.\n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "No environment variables are required for this evaluator." + ] + }, + { + "cell_type": "markdown", + "id": "d61cd511", + "metadata": {}, + "source": [ + "The Task Navigation Efficiency Evaluator measures how efficiently an agent navigates through a sequence of actions compared to an optimal task completion path.\n", + "\n", + "The evaluator provides comprehensive evaluation with both binary matching results and additional detailed P\\R\\F1 results:\n", + "\n", + "**Primary Result:**\n", + "- **Binary Match Result**: Pass/Fail based on the selected matching mode\n", + "\n", + "**Available Matching Modes:**\n", + "- **Exact Match**: Agent's tool calls must exactly match the ground truth (default)\n", + "- **In-Order Match**: All ground truth steps must appear in correct order (allows extra steps)\n", + "- **Any-Order Match**: All ground truth steps must appear with sufficient frequency (most lenient)\n", + "\n", + "**Properties Bag Additional Metrics (0.0 - 1.0):**\n", + "- **Precision**: How many of the agent's steps were necessary (relevant to ground truth)\n", + "- **Recall**: How many of the required steps were executed by the agent \n", + "- **F1 Score**: Harmonic mean of precision and recall\n", + "\n", + "The evaluation requires the following inputs:\n", + "- **Response**: The agent's response containing tool calls as a list of messages or string\n", + "- **Ground Truth**: List of expected tool/action steps as strings, or tuple with parameters for matching" + ] + }, + { + "cell_type": "markdown", + "id": "5e4e0a1d", + "metadata": {}, + "source": [ + "### Initialize Task Navigation Efficiency Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc3726cd", + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.evaluation._evaluators._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator, _TaskNavigationEfficiencyMatchingMode\n", + "from pprint import pprint\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"response\": {\n", + " \"type\": \"array\"\n", + " },\n", + " \"ground_truth\": {\n", + " \"type\": \"array\"\n", + " }\n", + " },\n", + " \"required\": [\"response\", \"ground_truth\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"response\": \"{{item.response}}\",\n", + " \"ground_truth\": \"{{item.ground_truth}}\"\n", + "}\n", + "\n", + "# Initialize with exact match mode\n", + "task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(\n", + " matching_mode=_TaskNavigationEfficiencyMatchingMode.EXACT_MATCH\n", + ")\n", + "\n", + "# Other examples:\n", + "# For in-order matching (allows extra steps but requires correct order)\n", + "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", + "\n", + "# For any-order matching (most lenient - allows extra steps and different order) \n", + "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.ANY_ORDER_MATCH)\n", + "\n", + "# Or use defaults (exact match mode)\n", + "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator()" + ] + }, + { + "cell_type": "markdown", + "id": "15a53227", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "1d75457a", + "metadata": {}, + "source": [ + "#### Sample 1: Perfect Path (Exact Match)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0cae15fb", + "metadata": {}, + "outputs": [], + "source": [ + "# Agent follows the exact optimal path\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\", \n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"analyze\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"report\", \"arguments\": {}}],\n", + " },\n", + "]\n", + "\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"Perfect Path Results:\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "8a9c5e69", + "metadata": {}, + "source": [ + "#### Sample 2: Efficient Path with Extra Steps" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f53dde31", + "metadata": {}, + "outputs": [], + "source": [ + "# Agent performs all required steps but with extra unnecessary step\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"validate\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\", \n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"analyze\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"report\", \"arguments\": {}}],\n", + " },\n", + "]\n", + "\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nPath with Extra Steps Results:\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "e4fa5757", + "metadata": {}, + "source": [ + "#### Sample 3: Inefficient Path (Wrong Order)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2226715", + "metadata": {}, + "outputs": [], + "source": [ + "# Agent performs all required steps but in wrong order\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"report\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"search\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\", \n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"analyze\", \"arguments\": {}}],\n", + " },\n", + "]\n", + "\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "\n", + "# Using in-order matching mode to demonstrate the difference\n", + "in_order_task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", + "\n", + "result = in_order_task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nWrong Order Results:\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "ae74ac8e", + "metadata": {}, + "source": [ + "#### Sample 4: Incomplete Path (Missing Steps)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ccd95640", + "metadata": {}, + "outputs": [], + "source": [ + "# Agent performs only some of the required steps (incomplete)\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"analyze\", \"arguments\": {}}],\n", + " },\n", + "]\n", + "\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nMissing Steps Results:\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "2c9111ef", + "metadata": {}, + "source": [ + "#### Sample 5: Real-World Customer Service Scenario" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4bd7b47", + "metadata": {}, + "outputs": [], + "source": [ + "# Real-world example: Customer service agent handling a refund request\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"lookup_order\", \"arguments\": {\"order_id\": \"12345\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"check_inventory\", \"arguments\": {\"product_id\": \"ABC123\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\", \n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"calculate_refund\", \"arguments\": {\"order_id\": \"12345\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"process_refund\", \"arguments\": {\"order_id\": \"12345\", \"amount\": \"29.99\"}}],\n", + " },\n", + "]\n", + "\n", + "ground_truth = [\"lookup_order\", \"calculate_refund\", \"process_refund\"]\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nCustomer Service Results:\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "4bf2e0ee", + "metadata": {}, + "source": [ + "#### Sample 6: Complex Path with Duplicates" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4dcf585", + "metadata": {}, + "outputs": [], + "source": [ + "# Agent repeats some steps and includes extra ones\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"search\", \"arguments\": {}}], # duplicate\n", + " },\n", + " {\n", + " \"role\": \"assistant\", \n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"validate\", \"arguments\": {}}], # extra step\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"analyze\", \"arguments\": {}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_5\", \"name\": \"report\", \"arguments\": {}}],\n", + " },\n", + "]\n", + "\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", + "print(\"\\nComplex Path with Duplicates Results:\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "478c00fb", + "metadata": {}, + "source": [ + "#### Sample 7: Tuple Format with Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7404f830", + "metadata": {}, + "outputs": [], + "source": [ + "# TaskNavigationEfficiencyEvaluator also supports tuple format with parameters for exact parameter matching\n", + "response_with_params = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {\"query\": \"test\"}}],\n", + " },\n", + "]\n", + "\n", + "# Ground truth using tuple format: (tool_names, parameters_dict)\n", + "# Parameters must match exactly for tools to be considered matching\n", + "ground_truth_with_params = ([\"search\"], {\"search\": {\"query\": \"test\"}})\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=response_with_params, ground_truth=ground_truth_with_params)\n", + "print(\"\\nTuple Format with Parameters Results:\")\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "38d2de9b", + "metadata": {}, + "source": [ + "#### Sample 8: String Response Input Type" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a0a853b", + "metadata": {}, + "outputs": [], + "source": [ + "# Demonstrate string response input type\n", + "# The string response should contain structured tool call information that can be parsed\n", + "string_response = \"I'll help you with that. Let me search for information, then analyze the results, and finally provide a report.\"\n", + "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "\n", + "result = task_navigation_efficiency_evaluator(response=string_response, ground_truth=ground_truth)\n", + "print(\"\\nString Response Results:\")\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb new file mode 100644 index 00000000..b57a36df --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb @@ -0,0 +1,575 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tool Call Accuracy Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates to how to use tool call accuracy evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing tool calls;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Tool Call Accuracy evaluator assesses how accurately an AI uses tools by examining:\n", + "- Relevance to the conversation\n", + "- Parameter correctness according to tool definitions\n", + "- Parameter value extraction from the conversation\n", + "- Potential usefulness of the tool call\n", + "\n", + "The evaluator uses a binary scoring (0 or 1) for each tool call:\n", + "\n", + " - Score 0: The tool call is irrelevant or contains information not in the conversation/definition\n", + " - Score 1: The tool call is relevant with properly extracted parameters from the conversation\n", + "\n", + "If there are multiple call, the final score will be an **average** of individual tool calls, which can be interpreted as the **passing rate** of tool calls.\n", + "\n", + "This evaluation focuses on measuring whether tool calls meaningfully contribute to addressing query while properly following tool definitions and using information present in the conversation history." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Tool Call Accuracy requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). Latter helps to determine if Agent used the information in history to make right tool calls.\n", + "- Tool Calls - Tool Call(s) made by Agent to answer the query. Optional - if response has tool calls, if not provided evaluator will look for tool calls in response.\n", + "- Response - (Optional) Response from Agent (or any GenAI App). This can be a single text response or a list or messages generated as part of Agent Response. If tool calls are not provide Tool Call Accuracy Evaluator will look at response for tool calls.\n", + "- Tool Definitions - Tool(s) definition used by Agent to answer the query. \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize Tool Call Accuracy Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation import ToolCallAccuracyEvaluator, AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_calls\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"tool_definitions\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\",\n", + " \"tool_calls\": \"{{item.tool_calls}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", + "\n", + "tool_call_accuracy = ToolCallAccuracyEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Evaluating Single Tool Call" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle ?\"\n", + "tool_call = {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + "}\n", + "\n", + "tool_definition = {\n", + " \"id\": \"fetch_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = tool_call_accuracy(query=query, tool_calls=tool_call, tool_definitions=tool_definition)\n", + "pprint(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Multiple Tool Calls used by Agent to respond" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle ?\"\n", + "tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " },\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"London\"},\n", + " },\n", + "]\n", + "\n", + "tool_definition = {\n", + " \"id\": \"fetch_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = tool_call_accuracy(query=query, tool_calls=tool_calls, tool_definitions=tool_definition)\n", + "pprint(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Tool Calls passed as part of `Response` (common for agent case)\n", + "- Tool Call Accuracy Evaluator extracts tool calls from response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Rainy, 14\\u00b0C\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"your_email@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"The current weather in Seattle is rainy with a temperature of 14\\u00b0C.\",\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\"}}\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is rainy with a temperature of 14\\u00b0C.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = tool_call_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Book a flight to New York for tomorrow\"\n", + "\n", + "# Response as a simple string instead of a list of messages\n", + "response = \"I've found several flight options to New York for tomorrow. I'll use the booking tool to reserve your seat.\"\n", + "\n", + "tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_book_flight_456\",\n", + " \"name\": \"book_flight\",\n", + " \"arguments\": {\n", + " \"destination\": \"New York\",\n", + " \"date\": \"tomorrow\"\n", + " },\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"book_flight\",\n", + " \"description\": \"Books a flight to the specified destination on the given date.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"destination\": {\"type\": \"string\", \"description\": \"The destination city.\"},\n", + " \"date\": {\"type\": \"string\", \"description\": \"The date of the flight.\"},\n", + " },\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_call_accuracy(query=query, response=response, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Response as List[dict] with Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query = \"What's the weather in San Francisco?\"\n", + "\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T18:15:22Z\",\n", + " \"run_id\": \"run_abc123\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_sf_weather_789\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"San Francisco\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T18:15:24Z\",\n", + " \"run_id\": \"run_abc123\",\n", + " \"tool_call_id\": \"call_sf_weather_789\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Foggy, 18°C\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T18:15:25Z\",\n", + " \"run_id\": \"run_abc123\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The weather in San Francisco is currently foggy with a temperature of 18°C.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = tool_call_accuracy(query=query, response=response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Query as Conversation History (List of Messages)\n", + "The evaluator also supports query as a list of messages representing conversation history. This helps determine if the Agent used the information in the conversation history to make the right tool calls." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Actually, could you also send me an email with that weather information to john@example.com?\"\n", + " }\n", + "]\n", + "\n", + "tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " },\n", + " {\n", + " \"type\": \"tool_call\", \n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"john@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Here is the weather information you requested.\"\n", + " },\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = tool_call_accuracy(query=query_as_conversation, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(response)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb new file mode 100644 index 00000000..fc14bce5 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb @@ -0,0 +1,483 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8d6b6cf9", + "metadata": {}, + "source": [ + "# Tool Input Accuracy Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use tool input accuracy evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing tool calls;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "id": "74ac1b5c", + "metadata": {}, + "source": [ + "The Tool Input Accuracy evaluator performs a strict binary evaluation (PASS/FAIL) of parameters passed to tool calls. It ensures that ALL parameters meet ALL criteria:\n", + "\n", + "- Parameter grounding: All parameters must be derived from conversation history/query\n", + "- Type compliance: All parameters must match exact types specified in tool definitions\n", + "- Format compliance: All parameters must follow exact format and structure requirements\n", + "- Completeness: All required parameters must be provided\n", + "- No unexpected parameters: Only defined parameters are allowed\n", + "\n", + "The evaluator uses strict binary evaluation:\n", + "\n", + " - 1: Only when ALL criteria are satisfied perfectly for ALL parameters\n", + " - 0: When ANY criterion fails for ANY parameter\n", + "\n", + "This evaluation focuses on ensuring tool call parameters are completely correct without any tolerance for partial correctness." + ] + }, + { + "cell_type": "markdown", + "id": "17346632", + "metadata": {}, + "source": [ + "Tool Input Accuracy requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator extracts tool calls from the response.\n", + "- Tool Definitions - Tool(s) definition used by Agent to answer the query. Required to validate parameter types and structures.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d78194ee", + "metadata": {}, + "source": [ + "### Initialize Tool Input Accuracy Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1663a6b1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._tool_input_accuracy import _ToolInputAccuracyEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\", \"tool_definitions\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "tool_input_accuracy = _ToolInputAccuracyEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "f92e8378", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "ba5f6441", + "metadata": {}, + "source": [ + "#### Evaluating Correct Tool Input Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "486057ca", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "80fc6e82", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "626462d3", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Check the weather in Miami\"\n", + "\n", + "# Response as a simple string containing tool call information\n", + "response_str = \"\"\"I'll check the weather for you. Calling fetch_weather with location=\"Miami\".\"\"\"\n", + "\n", + "tool_definition = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + "}\n", + "\n", + "result = tool_input_accuracy(query=query, response=response_str, tool_definitions=tool_definition)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "e9275f1c", + "metadata": {}, + "source": [ + "#### Response as a List and Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91e1e4d3", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"What's the temperature in Boston?\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_boston_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Boston\"},\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + "}\n", + "\n", + "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "c351c61a", + "metadata": {}, + "source": [ + "#### Complex Tool Parameters with Multiple Fields" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b1147ab", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you send an email to john@example.com with the subject 'Weather Update' and tell him the weather in Seattle is 15°C and partly cloudy?\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"john@example.com\",\n", + " \"subject\": \"Weather Update\",\n", + " \"body\": \"The weather in Seattle is 15°C and partly cloudy.\"\n", + " },\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " \"required\": [\"recipient\", \"subject\", \"body\"]\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "2b274da2", + "metadata": {}, + "source": [ + "#### Query as Conversation History (List of Messages)\n", + "The evaluator also supports query as a list of messages representing conversation history. This helps validate parameters are grounded in the conversation context." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a90e8827", + "metadata": {}, + "outputs": [], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Please send the results to admin@company.com with the subject 'Daily Weather Report'.\"\n", + " }\n", + "]\n", + "\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\", \n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"admin@company.com\",\n", + " \"subject\": \"Daily Weather Report\",\n", + " \"body\": \"Weather information for Seattle as requested.\"\n", + " },\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " \"required\": [\"recipient\", \"subject\", \"body\"]\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_input_accuracy(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "14c41e90", + "metadata": {}, + "source": [ + "#### Example of Incorrect Tool Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59810a22", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "# Missing required parameter or wrong type\n", + "incorrect_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"city\": \"Seattle\"}, # Wrong parameter name (should be \"location\")\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " \"required\": [\"location\"]\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score 0 due to incorrect parameter name\n", + "result = tool_input_accuracy(query=query, response=incorrect_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb new file mode 100644 index 00000000..7020eca7 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb @@ -0,0 +1,591 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "d638561f", + "metadata": {}, + "source": [ + "# Tool Output Utilization Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use tool output utilization evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing agent responses;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "41d3647c", + "metadata": {}, + "source": [ + "The Tool Output Utilization Evaluator assesses how effectively an AI agent utilizes the outputs from tools and whether it accurately incorporates this information into its responses.\n", + "\n", + "Scoring is based on two levels:\n", + "1. Pass - The agent effectively utilizes tool outputs and accurately incorporates the information into its response.\n", + "2. Fail - The agent fails to properly utilize tool outputs or incorrectly incorporates the information into its response.\n", + "\n", + "The evaluation includes the score, a brief explanation, and a final pass/fail result.\n", + "\n", + "This evaluation focuses on measuring whether the agent properly uses the information returned by tools to provide accurate and complete responses to users." + ] + }, + { + "cell_type": "markdown", + "id": "246c7bcd", + "metadata": {}, + "source": [ + "Tool Output Utilization requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response.\n", + "- Tool Definitions - Tool(s) definition used by Agent to answer the query. Required to understand the context and expected outputs of the tools.\n" + ] + }, + { + "cell_type": "markdown", + "id": "d386d576", + "metadata": {}, + "source": [ + "### Initialize Tool Output Utilization Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "087dd96d", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._tool_output_utilization import _ToolOutputUtilizationEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "tool_output_utilization = _ToolOutputUtilizationEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "c7b4f98a", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "9ee00e9d", + "metadata": {}, + "source": [ + "#### Evaluating Good Tool Output Utilization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "017a7afe", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C. The humidity is at 68% and there's a light wind from the northwest at 8 mph.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "78270b9b", + "metadata": {}, + "source": [ + "#### Complex Scenario with Multiple Tool Outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e8543c52", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\", \"precipitation\": \"light rain\", \"wind\": \"12 mph SW\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"your_email@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Current weather in Seattle: 14°C and rainy with light rain. Wind is from the southwest at 12 mph. Stay dry!\",\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\", \"status\": \"delivered\"}}\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The email includes the current conditions: 14°C and rainy with light rain, plus wind details. The email was delivered successfully to your inbox.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "1c52f24c", + "metadata": {}, + "source": [ + "#### Query as Conversation History" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3b9bef5", + "metadata": {}, + "outputs": [], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, I need to know the weather in Seattle for my trip planning.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you also email me a summary so I can reference it later when packing?\"\n", + " }\n", + "]\n", + "\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\", \"forecast\": \"no rain expected\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"user@example.com\",\n", + " \"subject\": \"Seattle Weather Summary for Trip Planning\",\n", + " \"body\": \"Weather summary for Seattle: Temperature is 15°C, partly cloudy conditions, humidity at 68%, with northwest winds at 8 mph. No rain is expected, so light layers should be sufficient for your trip.\"\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email sent successfully\", \"delivery_status\": \"delivered\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"Perfect! I've checked the weather in Seattle for your trip planning. It's currently 15°C and partly cloudy with light northwest winds and no rain expected. I've also sent you an email summary with all these details including packing recommendations based on the weather conditions. The email was delivered successfully, so you can reference it when preparing for your trip.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_output_utilization(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "199a11bc", + "metadata": {}, + "source": [ + "#### Example of Poor Tool Output Utilization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6be39269", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "# Poor utilization - agent doesn't use the detailed tool output effectively\n", + "poor_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\", \"UV_index\": \"3\", \"visibility\": \"10 km\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The weather in Seattle is okay. It's not too bad outside.\", # Poor utilization of detailed weather data\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score poorly due to inadequate use of tool output\n", + "result = tool_output_utilization(query=query, response=poor_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "bcb8a2cf", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0845822f", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"What's the weather like in Seattle?\"\n", + "# Response as a simple string (not a list of messages)\n", + "response_str = \"The weather in Seattle is currently 15°C and partly cloudy with light winds from the northwest.\"\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_output_utilization(query=query, response=response_str, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "b4cb9ddd", + "metadata": {}, + "source": [ + "#### Tool Definitions as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6424b3bb", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Check the weather in Seattle\"\n", + "response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The current weather in Seattle is rainy with a temperature of 14°C.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "# Tool definition as a single dict (not a list)\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb new file mode 100644 index 00000000..7211ae72 --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb @@ -0,0 +1,644 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "862760b0", + "metadata": {}, + "source": [ + "# Tool Selection Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use tool selection evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing tool calls;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + ] + }, + { + "cell_type": "markdown", + "id": "31a75ee6", + "metadata": {}, + "source": [ + "The Tool Selection evaluator assesses the appropriateness and efficiency of tool choices made by an AI agent by examining:\n", + "- Relevance of selected tools to the conversation\n", + "- Completeness of tool selection according to task requirements\n", + "- Efficiency in avoiding unnecessary or redundant tools\n", + "\n", + "The evaluator uses a binary scoring system:\n", + "\n", + " - Score 0 (Fail): Tools selected are irrelevant, incorrect, or missing essential tools\n", + " - Score 1 (Pass): All needed tools are selected, even if there are redundant tools\n", + "\n", + "This evaluation focuses on measuring whether the right tools were chosen for the task, regardless of how those tools were executed or their parameter correctness." + ] + }, + { + "cell_type": "markdown", + "id": "353f2304", + "metadata": {}, + "source": [ + "Tool Selection requires following input:\n", + "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", + "- Tool Calls - Tool Call(s) made by Agent to answer the query. Optional - if response has tool calls, if not provided evaluator will look for tool calls in response.\n", + "- Response - (Optional) Response from Agent (or any GenAI App). This can be a single text response or a list or messages generated as part of Agent Response. If tool calls are not provided, Tool Selection Evaluator will look at response for tool calls.\n", + "- Tool Definitions - Tool(s) definition used by Agent to answer the query. Required to understand available tools and their purposes.\n" + ] + }, + { + "cell_type": "markdown", + "id": "75125e94", + "metadata": {}, + "source": [ + "### Initialize Tool Selection Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5f31144", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._tool_selection import _ToolSelectionEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_calls\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\", \"tool_definitions\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_calls\": \"{{item.tool_calls}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "tool_selection = _ToolSelectionEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "fb435367", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "344fe68e", + "metadata": {}, + "source": [ + "#### Evaluating Single Tool Selection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dcb71257", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "tool_call = {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + "}\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"id\": \"fetch_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"id\": \"send_email\",\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "response = tool_selection(query=query, tool_calls=tool_call, tool_definitions=tool_definitions)\n", + "pprint(response)" + ] + }, + { + "cell_type": "markdown", + "id": "e078b1fa", + "metadata": {}, + "source": [ + "#### Multiple Tool Selections for Complex Task" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "322efe28", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " },\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"user@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Weather data will be included here.\",\n", + " },\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"id\": \"fetch_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"id\": \"send_email\",\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + " {\n", + " \"id\": \"get_calendar\",\n", + " \"name\": \"get_calendar\",\n", + " \"description\": \"Retrieves calendar events for a specified date range.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"start_date\": {\"type\": \"string\", \"description\": \"Start date for calendar events.\"},\n", + " \"end_date\": {\"type\": \"string\", \"description\": \"End date for calendar events.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "response = tool_selection(query=query, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(response)" + ] + }, + { + "cell_type": "markdown", + "id": "19b3c8de", + "metadata": {}, + "source": [ + "#### Tool Calls passed as part of `Response` (common for agent case)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "629f3cc7", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"Can you send me an email with weather information for Seattle?\"\n", + "response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Rainy, 14°C\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"your_email@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"The current weather in Seattle is rainy with a temperature of 14°C.\",\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\"}}\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is rainy with a temperature of 14°C.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"get_calendar\",\n", + " \"description\": \"Retrieves calendar events for a specified date range.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"start_date\": {\"type\": \"string\", \"description\": \"Start date for calendar events.\"},\n", + " \"end_date\": {\"type\": \"string\", \"description\": \"End date for calendar events.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_selection(query=query, response=response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "682839b8", + "metadata": {}, + "source": [ + "#### Query as Conversation History (List of Messages)\n", + "The evaluator also supports query as a list of messages representing conversation history. This helps determine if the Agent selected appropriate tools based on the conversation context." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "218ade7a", + "metadata": {}, + "outputs": [], + "source": [ + "# Query as conversation history instead of a single string\n", + "query_as_conversation = [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Hi, can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Actually, could you also send me an email with that weather information to john@example.com?\"\n", + " }\n", + "]\n", + "\n", + "tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " },\n", + " {\n", + " \"type\": \"tool_call\", \n", + " \"tool_call_id\": \"call_email_456\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"john@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Here is the weather information you requested.\"\n", + " },\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"get_calendar\",\n", + " \"description\": \"Retrieves calendar events for a specified date range.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"start_date\": {\"type\": \"string\", \"description\": \"Start date for calendar events.\"},\n", + " \"end_date\": {\"type\": \"string\", \"description\": \"End date for calendar events.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_selection(query=query_as_conversation, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "406bc90c", + "metadata": {}, + "source": [ + "#### Example of Poor Tool Selection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c451e4e", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "# Using irrelevant tool for the task\n", + "poor_tool_calls = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_calendar_123\",\n", + " \"name\": \"get_calendar\",\n", + " \"arguments\": {\"start_date\": \"2025-01-01\", \"end_date\": \"2025-01-31\"},\n", + " },\n", + "]\n", + "\n", + "# This should score poorly as get_calendar is not relevant for weather queries\n", + "result = tool_selection(query=query, tool_calls=poor_tool_calls, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "ab4a46c7", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e34bec52", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"What's the weather in New York?\"\n", + "\n", + "# Response as a simple string (not commonly used, but supported by the API)\n", + "# When using string response, tool_calls should be provided separately\n", + "response_str = \"The current weather in New York is sunny with a temperature of 22°C.\"\n", + "\n", + "tool_calls = {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_ny_weather_789\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"New York\"},\n", + "}\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_selection(query=query, response=response_str, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "cc857725", + "metadata": {}, + "source": [ + "#### Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d94822e1", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"How is the weather in Seattle?\"\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"get_weather\",\n", + " \"description\": \"Get the current weather for a location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\"type\": \"string\", \"description\": \"The city and state, e.g. San Francisco, CA\"},\n", + " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]}\n", + " },\n", + " \"required\": [\"location\"]\n", + " }\n", + "}\n", + "\n", + "tool_calls_dict = {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_abc123\",\n", + " \"name\": \"get_weather\",\n", + " \"arguments\": {\"location\": \"Seattle, WA\", \"unit\": \"celsius\"}\n", + "}\n", + "\n", + "result = tool_selection(query=query, tool_definitions=tool_definition_dict, tool_calls=tool_calls_dict)\n", + "\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb new file mode 100644 index 00000000..8f94021e --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb @@ -0,0 +1,519 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cea11cae", + "metadata": {}, + "source": [ + "# Tool Success Evaluator\n", + "\n", + "## Objective\n", + "This sample demonstrates how to use tool success evaluator on agent data. The supported input formats include:\n", + "- simple data such as strings and `dict` describing agent responses;\n", + "- user-agent conversations in the form of list of agent messages. \n", + "\n", + "## Time\n", + "\n", + "You should expect to spend about 20 minutes running this notebook. \n", + "\n", + "## Before you begin\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "\n", + "### Prerequisite\n", + "```bash\n", + "pip install azure-ai-evaluation\n", + "```\n", + "Set these environment variables with your own values:\n", + "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", + "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", + "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", + "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + ] + }, + { + "cell_type": "markdown", + "id": "49404a44", + "metadata": {}, + "source": [ + "The Tool Success evaluator determines whether tool calls done by an AI agent includes failures or not.\n", + "\n", + "This evaluator focuses solely on tool call results and tool definitions, disregarding user's query to the agent, conversation history and agent's final response. Although tool definitions is optional, providing them can help the evaluator better understand the context of the tool calls made by the agent. Please note that this evaluator validates tool calls for potential technical failures like errors, exceptions, timeouts and empty results (only in cases where empty results could indicate a failure). It does not assess the correctness or the tool result itself, like mathematical errors and unrealistic field values like name=\"668656\".\n", + "\n", + "Scoring is binary:\n", + "- TRUE: All tool calls were successful\n", + "- FALSE: At least one tool call failed\n", + "\n", + "This evaluation focuses on measuring the technical success of tool execution, not the semantic correctness of the results." + ] + }, + { + "cell_type": "markdown", + "id": "d91cb876", + "metadata": {}, + "source": [ + "Tool Success requires following input:\n", + "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator examines tool call results within the response.\n", + "- Tool Definitions - (Optional) Tool(s) definition used by Agent. Providing tool definitions helps the evaluator better understand the context of the tool calls made by the agent.\n" + ] + }, + { + "cell_type": "markdown", + "id": "e07d293d", + "metadata": {}, + "source": [ + "### Initialize Tool Success Evaluator\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "919d1b9a", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from azure.ai.evaluation._evaluators._tool_success import _ToolSuccessEvaluator\n", + "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from pprint import pprint\n", + "\n", + "model_config = AzureOpenAIModelConfiguration(\n", + " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", + " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", + " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", + " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", + ")\n", + "\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "data_mapping = {\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", + "\n", + "tool_call_success = _ToolSuccessEvaluator(model_config=model_config)" + ] + }, + { + "cell_type": "markdown", + "id": "4739878c", + "metadata": {}, + "source": [ + "### Samples" + ] + }, + { + "cell_type": "markdown", + "id": "2352536d", + "metadata": {}, + "source": [ + "#### Evaluating Successful Tool Calls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ddc2e22", + "metadata": {}, + "outputs": [], + "source": [ + "# Successful tool execution\n", + "successful_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "result = tool_call_success(response=successful_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "c146de06", + "metadata": {}, + "source": [ + "#### Response as String (str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "575d33eb", + "metadata": {}, + "outputs": [], + "source": [ + "# Response as a simple string containing tool call information\n", + "# This format is less common but still valid for the evaluator\n", + "response_str = \"\"\"Tool call executed: fetch_weather(location=\"Paris\")\n", + "Tool result: {\"temperature\": \"18°C\", \"condition\": \"clear sky\", \"humidity\": \"55%\"}\n", + "The weather in Paris is currently clear with a temperature of 18°C.\"\"\"\n", + "\n", + "tool_definition = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = tool_call_success(response=response_str, tool_definitions=tool_definition)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "b7217ddc", + "metadata": {}, + "source": [ + "#### Tool Definition as Single Dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f89ca010", + "metadata": {}, + "outputs": [], + "source": [ + "# Successful tool execution with single tool definition dict\n", + "successful_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_nyc_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"New York\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_nyc_weather\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"22°C\", \"condition\": \"sunny\", \"humidity\": \"45%\"}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"The weather in New York is currently sunny with a temperature of 22°C.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definition_dict = {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + "}\n", + "\n", + "result = tool_call_success(response=successful_response, tool_definitions=tool_definition_dict)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "1aeb8c11", + "metadata": {}, + "source": [ + "#### Multiple Successful Tool Calls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "444ee59c", + "metadata": {}, + "outputs": [], + "source": [ + "# Multiple successful tool executions\n", + "multiple_successful_response = [\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\", \"precipitation\": \"light rain\"}}],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"name\": \"send_email\",\n", + " \"arguments\": {\n", + " \"recipient\": \"your_email@example.com\",\n", + " \"subject\": \"Weather Information for Seattle\",\n", + " \"body\": \"Current weather: 14°C and rainy with light rain.\",\n", + " },\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", + " \"role\": \"tool\",\n", + " \"content\": [\n", + " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\", \"status\": \"delivered\"}}\n", + " ],\n", + " },\n", + " {\n", + " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", + " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle.\",\n", + " }\n", + " ],\n", + " },\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"send_email\",\n", + " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", + " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", + " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "result = tool_call_success(response=multiple_successful_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "63e43736", + "metadata": {}, + "source": [ + "#### Tool Call Success without Tool Definitions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0ca062e", + "metadata": {}, + "outputs": [], + "source": [ + "# Successful execution without providing tool definitions\n", + "simple_successful_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_calendar_123\",\n", + " \"name\": \"get_calendar\",\n", + " \"arguments\": {\"start_date\": \"2025-01-01\", \"end_date\": \"2025-01-31\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_calendar_123\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"events\": [{\"date\": \"2025-01-15\", \"title\": \"Team Meeting\"}, {\"date\": \"2025-01-20\", \"title\": \"Project Review\"}]}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I found 2 events in your calendar for January: a Team Meeting on the 15th and a Project Review on the 20th.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "# Evaluation without tool definitions\n", + "result = tool_call_success(response=simple_successful_response)\n", + "pprint(result)" + ] + }, + { + "cell_type": "markdown", + "id": "97c1af0f", + "metadata": {}, + "source": [ + "#### Example of Failed Tool Call" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cafdee8e", + "metadata": {}, + "outputs": [], + "source": [ + "# Failed tool execution with error\n", + "failed_response = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"InvalidCity\"},\n", + " }\n", + " ],\n", + " },\n", + " {\n", + " \"tool_call_id\": \"call_weather_456\",\n", + " \"role\": \"tool\",\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Location not found\", \"status\": \"failed\", \"code\": 404}}],\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I'm sorry, I couldn't retrieve the weather information due to an error.\",\n", + " }\n", + " ],\n", + " }\n", + "]\n", + "\n", + "tool_definitions = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "# This should score FALSE due to tool failure\n", + "result = tool_call_success(response=failed_response, tool_definitions=tool_definitions)\n", + "pprint(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "samples", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 449f31a62340e94e536aed6dfe0c559525ccf6a2 Mon Sep 17 00:00:00 2001 From: v-mradi Date: Thu, 6 Nov 2025 18:51:23 +0200 Subject: [PATCH 6/7] Added all initialization parameters and migrated tool success evaluator to online evaluation --- .../AI_Judge_Evaluator_Coherence.ipynb | 4 + .../AI_Judge_Evaluator_Fluency.ipynb | 4 + .../AI_Judge_Evaluator_Groundedness.ipynb | 4 + ...AI_Judge_Evaluator_Intent_Resolution.ipynb | 4 + .../AI_Judge_Evaluator_Relevance.ipynb | 4 + ...udge_Evaluator_Response_Completeness.ipynb | 4 + .../AI_Judge_Evaluator_Task_Adherence.ipynb | 4 + .../AI_Judge_Evaluator_Task_Completion.ipynb | 4 + ...Evaluator_Task_Navigation_Efficiency.ipynb | 4 + ...I_Judge_Evaluator_Tool_Call_Accuracy.ipynb | 4 + ..._Judge_Evaluator_Tool_Input_Accuracy.ipynb | 4 + ...ge_Evaluator_Tool_Output_Utilization.ipynb | 4 + .../AI_Judge_Evaluator_Tool_Selection.ipynb | 4 + .../AI_Judge_Evaluator_Tool_Success.ipynb | 222 +++++++++++++++--- .../Agent_Online_Evaluation/agent_utils.py | 92 ++++++++ 15 files changed, 328 insertions(+), 38 deletions(-) create mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/agent_utils.py diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb index 2380a14c..b662d23f 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb @@ -105,6 +105,10 @@ " \"response\": \"{{item.response}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "coherence_evaluator = CoherenceEvaluator(model_config=model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb index 7d0ce118..c015da44 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb @@ -102,6 +102,10 @@ " \"response\": \"{{item.response}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "fluency_evaluator = FluencyEvaluator(model_config=model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb index 060c7ee3..27188e03 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb @@ -161,6 +161,10 @@ " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "# Initialize the evaluator\n", "groundedness_evaluator = GroundednessEvaluator(model_config=model_config)" ] diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb index 4debe16c..4d3d2ad2 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb @@ -126,6 +126,10 @@ " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "intent_resolution_evaluator = IntentResolutionEvaluator(model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb index 49d9f977..fd0811ad 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb @@ -115,6 +115,10 @@ " \"response\": \"{{item.response}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "# Initialize the evaluator\n", "relevance_evaluator = RelevanceEvaluator(model_config=model_config)" ] diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb index 09efd837..de5587e4 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb @@ -101,6 +101,10 @@ " \"response\": \"{{item.response}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb index 6ffd58ff..8389fa34 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb @@ -126,6 +126,10 @@ " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "task_adherence_evaluator = TaskAdherenceEvaluator(model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb index 3d302ca7..51d2460e 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb @@ -136,6 +136,10 @@ " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "task_completion = _TaskCompletionEvaluator(model_config=model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb index 7009dfc2..1e01e6a3 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb @@ -93,6 +93,10 @@ " \"ground_truth\": \"{{item.ground_truth}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"matching_mode\" : \"exact_match\" # Can be \"exact_match\", \"in_order_match\", or \"any_order_match\"\n", + "}\n", + "\n", "# Initialize with exact match mode\n", "task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(\n", " matching_mode=_TaskNavigationEfficiencyMatchingMode.EXACT_MATCH\n", diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb index b57a36df..b0e32540 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb @@ -146,6 +146,10 @@ " \"response\": \"{{item.response}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "tool_call_accuracy = ToolCallAccuracyEvaluator(model_config=model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb index fc14bce5..260a8b08 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb @@ -139,6 +139,10 @@ " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "tool_input_accuracy = _ToolInputAccuracyEvaluator(model_config=model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb index 7020eca7..7a6a39f8 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb @@ -134,6 +134,10 @@ " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "tool_output_utilization = _ToolOutputUtilizationEvaluator(model_config=model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb index 7211ae72..2f43123c 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb @@ -149,6 +149,10 @@ " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", "}\n", "\n", + "initialization_parameters = {\n", + " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", + "}\n", + "\n", "tool_selection = _ToolSelectionEvaluator(model_config=model_config)" ] }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb index 8f94021e..91788ef2 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb @@ -21,13 +21,11 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)\n" ] }, { @@ -72,17 +70,14 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation._evaluators._tool_success import _ToolSuccessEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from azure.ai.projects.models import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", "data_source_config = {\n", " \"type\": \"custom\",\n", " \"item_schema\": {\n", @@ -116,12 +111,19 @@ " \"include_sample_schema\": True\n", "}\n", "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", "data_mapping = {\n", " \"tool_definitions\": \"{{item.tool_definitions}}\",\n", " \"response\": \"{{item.response}}\"\n", "}\n", "\n", - "tool_call_success = _ToolSuccessEvaluator(model_config=model_config)" + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", + "\n", + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []\n" ] }, { @@ -129,7 +131,9 @@ "id": "4739878c", "metadata": {}, "source": [ - "### Samples" + "### Sample Test Cases\n", + "\n", + "In the following cells, we'll define various test cases and append them to the `evaluation_contents` list. Each test case demonstrates a different scenario for tool success evaluation.\n" ] }, { @@ -147,7 +151,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Successful tool execution\n", + "# Test Case 1: Successful tool execution\n", "successful_response = [\n", " {\n", " \"role\": \"assistant\",\n", @@ -187,8 +191,15 @@ " }\n", "]\n", "\n", - "result = tool_call_success(response=successful_response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"tool_definitions\": tool_definitions,\n", + " \"response\": successful_response\n", + " }\n", + " )\n", + ")" ] }, { @@ -206,8 +217,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Response as a simple string containing tool call information\n", - "# This format is less common but still valid for the evaluator\n", + "# Test Case 2: Response as a simple string containing tool call information\n", "response_str = \"\"\"Tool call executed: fetch_weather(location=\"Paris\")\n", "Tool result: {\"temperature\": \"18°C\", \"condition\": \"clear sky\", \"humidity\": \"55%\"}\n", "The weather in Paris is currently clear with a temperature of 18°C.\"\"\"\n", @@ -221,8 +231,15 @@ " },\n", "}\n", "\n", - "result = tool_call_success(response=response_str, tool_definitions=tool_definition)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"tool_definitions\": tool_definition,\n", + " \"response\": response_str\n", + " }\n", + " )\n", + ")" ] }, { @@ -240,8 +257,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Successful tool execution with single tool definition dict\n", - "successful_response = [\n", + "# Test Case 3: Successful tool execution with single tool definition dict\n", + "successful_response_nyc = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [\n", @@ -278,8 +295,15 @@ " },\n", "}\n", "\n", - "result = tool_call_success(response=successful_response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"tool_definitions\": tool_definition_dict,\n", + " \"response\": successful_response_nyc\n", + " }\n", + " )\n", + ")" ] }, { @@ -297,7 +321,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Multiple successful tool executions\n", + "# Test Case 4: Multiple successful tool executions\n", "multiple_successful_response = [\n", " {\n", " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", @@ -358,7 +382,7 @@ " },\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_multiple = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -381,8 +405,13 @@ " },\n", "]\n", "\n", - "result = tool_call_success(response=multiple_successful_response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " tool_definitions=tool_definitions_multiple,\n", + " response=multiple_successful_response\n", + " )\n", + ")" ] }, { @@ -400,7 +429,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Successful execution without providing tool definitions\n", + "# Test Case 5: Successful execution without providing tool definitions\n", "simple_successful_response = [\n", " {\n", " \"role\": \"assistant\",\n", @@ -429,9 +458,13 @@ " }\n", "]\n", "\n", - "# Evaluation without tool definitions\n", - "result = tool_call_success(response=simple_successful_response)\n", - "pprint(result)" + "# Append without tool definitions - evaluator should still work\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " tool_definitions=None,\n", + " response=simple_successful_response\n", + " )\n", + ")" ] }, { @@ -449,7 +482,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Failed tool execution with error\n", + "# Test Case 6: Failed tool execution with error\n", "failed_response = [\n", " {\n", " \"role\": \"assistant\",\n", @@ -478,7 +511,7 @@ " }\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_failed = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -490,8 +523,121 @@ "]\n", "\n", "# This should score FALSE due to tool failure\n", - "result = tool_call_success(response=failed_response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " tool_definitions=tool_definitions_failed,\n", + " response=failed_response\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "4bdea268", + "metadata": {}, + "source": [ + "### Run Evaluation on All Test Cases\n", + "\n", + "Now that we've defined all test cases, let's run the evaluation once on all of them.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f9783c0", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"\\n{'='*80}\")\n", + "print(f\"Running evaluation on {len(evaluation_contents)} test cases...\")\n", + "print(f\"{'='*80}\\n\")\n", + "\n", + "# Run the evaluator on all test cases at once\n", + "results = run_evaluator(\n", + " evaluator_name=\"tool-success\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")\n", + "\n", + "print(f\"\\n{'='*80}\")\n", + "print(f\"Evaluation Complete! Results for {len(results)} test cases:\")\n", + "print(f\"{'='*80}\\n\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "8d7fecd5", + "metadata": {}, + "source": [ + "### Display Results\n", + "\n", + "View the evaluation results for each test case.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e4fdb3a", + "metadata": {}, + "outputs": [], + "source": [ + "# Define test case descriptions for better readability\n", + "test_case_descriptions = [\n", + " \"Successful tool execution\",\n", + " \"Response as string\",\n", + " \"Tool definition as single dict\",\n", + " \"Multiple successful tool calls\",\n", + " \"Success without tool definitions\",\n", + " \"Failed tool call with error\"\n", + "]\n", + "\n", + "# Display results for each test case\n", + "for idx, result in enumerate(results, 1):\n", + " print(f\"Test Case {idx}: {test_case_descriptions[idx-1]}\")\n", + " print(\"-\" * 80)\n", + " pprint(result)\n", + " print(\"\\n\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "4937b4b5", + "metadata": {}, + "source": [ + "### Summary Statistics\n", + "\n", + "Get an overview of the evaluation results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36a06ff1", + "metadata": {}, + "outputs": [], + "source": [ + "# Calculate summary statistics\n", + "total_cases = len(results)\n", + "successful_cases = sum(1 for r in results if r.get(\"tool_success\") == True)\n", + "failed_cases = total_cases - successful_cases\n", + "\n", + "print(f\"\\n{'='*80}\")\n", + "print(f\"EVALUATION SUMMARY\")\n", + "print(f\"{'='*80}\")\n", + "print(f\"Total test cases: {total_cases}\")\n", + "print(f\"Successful tool calls: {successful_cases} ({successful_cases/total_cases*100:.1f}%)\")\n", + "print(f\"Failed tool calls: {failed_cases} ({failed_cases/total_cases*100:.1f}%)\")\n", + "print(f\"{'='*80}\\n\")\n", + "\n", + "# Show which cases failed\n", + "if failed_cases > 0:\n", + " print(\"Failed test cases:\")\n", + " for idx, result in enumerate(results, 1):\n", + " if result.get(\"tool_success\") == False:\n", + " print(f\" • Test Case {idx}: {test_case_descriptions[idx-1]}\")\n", + " print(f\" Reason: {result.get('tool_success_reason', 'N/A')}\")\n" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/agent_utils.py b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/agent_utils.py new file mode 100644 index 00000000..dcf75aee --- /dev/null +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/agent_utils.py @@ -0,0 +1,92 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +import os +import time +from pprint import pprint + +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +from openai.types.evals.create_eval_jsonl_run_data_source_param import ( + CreateEvalJSONLRunDataSourceParam, + SourceFileContent, + SourceFileContentContent, +) + + +def run_evaluator( + evaluator_name: str, + evaluation_contents: list[SourceFileContentContent], + data_source_config: dict, + initialization_parameters: dict[str, str], + data_mapping: dict[str, str], +) -> list: + """ + Run an evaluator on the provided evaluation contents. + + Args: + evaluator_name: Name of the evaluator (e.g., "tool-success") + evaluation_contents: List of test cases to evaluate + data_source_config: Configuration for the data source schema + initialization_parameters: Parameters for initializing the evaluator + data_mapping: Mapping of data fields to evaluator inputs + + Returns: + List of evaluation output items with results + """ + endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] + + with DefaultAzureCredential() as credential: + with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: + print("Creating an OpenAI client from the AI Project client") + client = project_client.get_openai_client() + + testing_criteria = [ + { + "type": "azure_ai_evaluator", + "name": f"{evaluator_name}", + "evaluator_name": f"builtin.{evaluator_name}", + "initialization_parameters": initialization_parameters, + "data_mapping": data_mapping, + } + ] + + print("Creating Eval Group") + eval_object = client.evals.create( + name=f"Test {evaluator_name} Evaluator with inline data", + data_source_config=data_source_config, + testing_criteria=testing_criteria, + ) + print(f"Eval Group created with ID: {eval_object.id}") + + print("Creating Eval Run with Inline Data") + eval_run_object = client.evals.runs.create( + eval_id=eval_object.id, + name="inline_data_run", + metadata={"team": "eval-exp", "scenario": "inline-data-v1"}, + data_source=CreateEvalJSONLRunDataSourceParam( + type="jsonl", + source=SourceFileContent(type="file_content", content=evaluation_contents) + ), + ) + + print(f"Eval Run created with ID: {eval_run_object.id}") + print("Waiting for eval run to complete...") + + # Poll for completion + while True: + run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) + if run.status == "completed" or run.status == "failed": + output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id)) + print(f"✓ Eval Run Status: {run.status}") + print(f"📊 Eval Run Report URL: {run.report_url}") + + # Extract just the output from each item + results = [item.output for item in output_items] + return results + + time.sleep(5) + print(f" Status: {run.status}...") From ac409ca59e0185ad2d54a1b0ce19cc934bfc37e9 Mon Sep 17 00:00:00 2001 From: v-mradi Date: Thu, 13 Nov 2025 15:02:38 +0200 Subject: [PATCH 7/7] Modifying all sample notebook to support online evaluation lib --- .../AI_Judge_Evaluator_Coherence.ipynb | 150 ++-- .../AI_Judge_Evaluator_Fluency.ipynb | 195 ++---- .../AI_Judge_Evaluator_Groundedness.ipynb | 265 +++++-- ...AI_Judge_Evaluator_Intent_Resolution.ipynb | 216 ++++-- .../AI_Judge_Evaluator_Relevance.ipynb | 154 +++-- ...udge_Evaluator_Response_Completeness.ipynb | 146 ++-- .../AI_Judge_Evaluator_Task_Adherence.ipynb | 229 +++++-- .../AI_Judge_Evaluator_Task_Completion.ipynb | 254 +++++-- ...Evaluator_Task_Navigation_Efficiency.ipynb | 226 ++++-- ...I_Judge_Evaluator_Tool_Call_Accuracy.ipynb | 349 +++++++--- ...I_Judge_Evaluator_Tool_Call_Success.ipynb} | 121 +--- ..._Judge_Evaluator_Tool_Input_Accuracy.ipynb | 247 +++++-- ...ge_Evaluator_Tool_Output_Utilization.ipynb | 231 +++++-- .../AI_Judge_Evaluator_Tool_Selection.ipynb | 318 +++++++-- .../AI_Judge_Evaluator_Tool_Success.ipynb | 482 ------------- .../agent_utils.py | 12 +- .../AI_Judge_Evaluator_Coherence.ipynb | 239 ------- .../AI_Judge_Evaluator_Fluency.ipynb | 316 --------- .../AI_Judge_Evaluator_Groundedness.ipynb | 398 ----------- ...AI_Judge_Evaluator_Intent_Resolution.ipynb | 405 ----------- .../AI_Judge_Evaluator_Relevance.ipynb | 241 ------- ...udge_Evaluator_Response_Completeness.ipynb | 233 ------- .../AI_Judge_Evaluator_Task_Adherence.ipynb | 322 --------- .../AI_Judge_Evaluator_Task_Completion.ipynb | 415 ----------- ...Evaluator_Task_Navigation_Efficiency.ipynb | 446 ------------ ...I_Judge_Evaluator_Tool_Call_Accuracy.ipynb | 579 ---------------- ..._Judge_Evaluator_Tool_Input_Accuracy.ipynb | 487 ------------- ...ge_Evaluator_Tool_Output_Utilization.ipynb | 595 ---------------- .../AI_Judge_Evaluator_Tool_Selection.ipynb | 648 ------------------ 29 files changed, 2158 insertions(+), 6761 deletions(-) rename scenarios/evaluate/Supported_Evaluation_Metrics/{Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb => Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb} (83%) delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb rename scenarios/evaluate/Supported_Evaluation_Metrics/{Agent_Online_Evaluation => Agent_Evaluation}/agent_utils.py (86%) delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb delete mode 100644 scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb index 48194df1..99fa87ee 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Coherence.ipynb @@ -15,17 +15,15 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4o` or `gpt-4.1`. \n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)" ] }, { @@ -45,14 +43,11 @@ "Score 5: Excellent - The response is perfectly coherent with smooth, natural flow and clear logical progression.\n", "\n", "\n", - "The evaluation requires one of the following input patterns:\n", + "The evaluation requires the following input pattern:\n", "\n", "**Pattern 1: Query-Response Evaluation**\n", "- Query: The user's question or prompt. (string)\n", - "- Response: The AI-generated response to be evaluated for coherence. (string)\n", - "\n", - "**Pattern 2: Conversation Evaluation**\n", - "- Conversation: A multi-turn conversation where each turn is evaluated for coherence. (dict with \"messages\" key)" + "- Response: The AI-generated response to be evaluated for coherence. (string)" ] }, { @@ -71,17 +66,44 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation import CoherenceEvaluator, AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", + "\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", + "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"type\": \"string\"\n", + " },\n", + " \"response\": {\n", + " \"type\": \"string\"\n", + " }\n", + " },\n", + " \"required\": []\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", "\n", - "coherence_evaluator = CoherenceEvaluator(model_config=model_config)" + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -107,11 +129,19 @@ "metadata": {}, "outputs": [], "source": [ + "# Test Case 1: High coherence response with logical flow and clear connections\n", "query = \"Can you explain how machine learning works?\"\n", "response = \"Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed. It works by analyzing data patterns to make predictions or decisions. First, the system is trained on a dataset, which allows it to identify patterns and relationships. Then, it uses these learned patterns to make predictions on new, unseen data. The accuracy improves over time as the model processes more data and refines its understanding.\"\n", "\n", - "result = coherence_evaluator(query=query, response=response)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query,\n", + " \"response\": response\n", + " }\n", + " )\n", + ")" ] }, { @@ -129,61 +159,65 @@ "metadata": {}, "outputs": [], "source": [ - "# Poor coherence example\n", + "# Test Case 2: Poor coherence example with disjointed ideas and unclear transitions\n", "query = \"What is blockchain technology?\"\n", "response = \"Blockchain is decentralized. Bitcoin uses it. Blocks contain data. Cryptography is involved. Banks might use blockchain. It's secure because of the chain. Mining is how new blocks are added. Digital currencies exist. The ledger is distributed.\"\n", "\n", - "result = coherence_evaluator(query=query, response=response)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query,\n", + " \"response\": response\n", + " }\n", + " )\n", + ")" ] }, { "cell_type": "markdown", - "id": "1d22e3d6", + "id": "ad4b46bd", "metadata": {}, "source": [ - "#### Conversation as Dict with Messages" + "### Run Evaluation on All Test Cases\n", + "\n", + "Now that we've defined all test cases, let's run the evaluation once on all of them." ] }, { "cell_type": "code", "execution_count": null, - "id": "556c197b", + "id": "141e26ab", "metadata": {}, "outputs": [], "source": [ - "# Multi-turn conversation\n", - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"I'm planning a trip to Japan. What should I know?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Japan is a fascinating destination with a rich cultural heritage. First, you should know that the best times to visit are spring (March-May) for cherry blossoms and fall (September-November) for autumn colors. The country has an excellent public transportation system, particularly the Shinkansen bullet train network. Additionally, learning a few basic Japanese phrases will be appreciated by locals, though many signs in major cities have English translations.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What about the food?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Japanese cuisine is incredibly diverse and delicious. Beyond the well-known sushi and ramen, you should try regional specialties like okonomiyaki in Osaka or motsunabe in Fukuoka. Most restaurants display plastic food models in their windows, which makes ordering easier even with a language barrier. It's also important to note that tipping is not customary in Japan and can sometimes be considered rude.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Are there any cultural customs I should be aware of?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Yes, there are several important customs to observe. Always remove your shoes when entering homes, temples, and some traditional restaurants. Bowing is a common greeting, though a slight nod is acceptable for foreigners. When using chopsticks, never stick them upright in rice as this resembles a funeral ritual. Finally, be mindful of noise levels on public transportation, as talking on phones or having loud conversations is considered impolite.\"\n", - " }\n", - " ]\n", - "}\n", + "results = run_evaluator(\n", + " evaluator_name=\"coherence\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "2e15dccc", + "metadata": {}, + "source": [ + "### Display Results\n", "\n", - "result = coherence_evaluator(conversation=conversation)\n", - "pprint(result)" + "View the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c164710e", + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb index 4c3f6809..e9422f37 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Fluency.ipynb @@ -15,17 +15,15 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4o` or `gpt-4.1`. \n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)" ] }, { @@ -45,13 +43,10 @@ "Score 5: Excellent - The response is perfectly fluent with proper grammar, syntax, and vocabulary usage.\n", "\n", "\n", - "The evaluation requires one of the following input patterns:\n", + "The evaluation requires the following input format:\n", "\n", - "**Pattern 1: Single Response Evaluation**\n", - "- Response: The text response to be evaluated for fluency. (string)\n", - "\n", - "**Pattern 2: Conversation Evaluation**\n", - "- Conversation: A multi-turn conversation where each turn is evaluated for fluency. (List[dict])" + "**Query-Response Evaluation**\n", + "- Response: The AI's response to be evaluated for fluency (string)" ] }, { @@ -70,17 +65,37 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation import FluencyEvaluator, AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", + "\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", + "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"query\": {\"type\": \"string\"}, \"response\": {\"type\": \"string\"}},\n", + " \"required\": [],\n", + " },\n", + " \"include_sample_schema\": True,\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", "\n", - "fluency_evaluator = FluencyEvaluator(model_config=model_config)" + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -108,8 +123,14 @@ "source": [ "response = \"The weather in Seattle is currently partly cloudy with a temperature of 15°C. The forecast indicates that conditions will remain stable throughout the day, with a gentle breeze from the northwest.\"\n", "\n", - "result = fluency_evaluator(response=response)\n", - "pprint(result)" + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": None,\n", + " \"response\": response\n", + " }\n", + " )\n", + ")" ] }, { @@ -130,139 +151,61 @@ "# Poor fluency example\n", "response = \"Email draft attach is. You review and giving feedback must. Important very for project success it being.\"\n", "\n", - "result = fluency_evaluator(response=response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "b26dd4bb", - "metadata": {}, - "source": [ - "#### Conversation as Dict with Messages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9ccaff56", - "metadata": {}, - "outputs": [], - "source": [ - "# Multi-turn conversation\n", - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Can you help me understand the weather forecast for this week?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Of course! I'd be happy to help you understand this week's weather forecast. The outlook shows a mix of sunny and cloudy conditions.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What about the temperature?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Temperatures will ranging from 12°C to 18°C throughout the week, with the warmest days being Wednesday and Thursday.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Should I bring an umbrella?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Yes, it would be advisable to carry an umbrella, particularly on Friday when there is a 60% chance of precipitation.\"\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": None,\n", + " \"response\": response\n", " }\n", - " ]\n", - "}\n", - "\n", - "result = fluency_evaluator(conversation=conversation)\n", - "pprint(result)" + " )\n", + ")" ] }, { "cell_type": "markdown", - "id": "0ba89e73", + "id": "91b113ad", "metadata": {}, "source": [ - "#### Conversation as List[dict] - Agent with Tool Calls" + "### Run Evaluation on All Test Cases\n", + "\n", + "Now that we've defined all test cases, let's run the evaluation once on all of them." ] }, { "cell_type": "code", "execution_count": null, - "id": "2658f610", + "id": "b1d14a1c", "metadata": {}, "outputs": [], "source": [ - "# Conversation with tool calls - evaluates fluency of assistant responses\n", - "conversation_with_tools = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Can you send me an email with the weather report for Seattle?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'll fetch the current weather information for Seattle and send you an email with the details.\"\n", - " },\n", - " {\n", - " \"role\": \"tool\",\n", - " \"content\": \"Temperature: 15°C, Condition: Partly cloudy, Wind: 8 mph NW\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I have successfully sent you an email containing the weather report for Seattle. The current conditions are partly cloudy with a temperature of 15°C and light winds from the northwest.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = fluency_evaluator(conversation=conversation_with_tools)\n", - "pprint(result)" + "results = run_evaluator(\n", + " evaluator_name=\"fluency\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" ] }, { "cell_type": "markdown", - "id": "ff71d193", + "id": "80270783", "metadata": {}, "source": [ - "#### Conversation as List[dict] - Mixed Fluency Quality" + "### Display Results\n", + "\n", + "View the evaluation results for each test case." ] }, { "cell_type": "code", "execution_count": null, - "id": "c0d8bf46", + "id": "ecc32e9a", "metadata": {}, "outputs": [], "source": [ - "# Conversation with varying fluency levels across turns\n", - "conversation_mixed = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What's the status of my order?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Your order is being process currently and will be ship very soon to you.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"When will it arrive?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The estimated delivery date for your order is March 15, 2025. You will receive tracking information via email once the package has been dispatched.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = fluency_evaluator(conversation=conversation_mixed)\n", - "pprint(result)" + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb index 6c78ac1d..5d8e6dcb 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb @@ -21,17 +21,15 @@ "You should expect to spend about 20 minutes running this notebook.\n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities.\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4o`.\n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure OpenAI Endpoint to be used for evaluation\n", - "3) **AZURE_OPENAI_API_KEY** - Azure OpenAI Key to be used for evaluation\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure OpenAI API version to be used for evaluation" + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)" ] }, { @@ -95,19 +93,77 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation import GroundednessEvaluator, AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "# Configure the model\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", "\n", - "# Initialize the evaluator\n", - "groundedness_evaluator = GroundednessEvaluator(model_config=model_config)" + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"context\": {\n", + " \"type\": \"string\"\n", + " },\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"context\": \"{{item.context}}\",\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", + "\n", + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -133,15 +189,24 @@ "metadata": {}, "outputs": [], "source": [ - "# Example of a well-grounded response using weather context\n", + "# Test Case 1: Well-grounded response using weather context\n", "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", "The forecast indicates overcast skies with light precipitation typical for Pacific Northwest weather. \n", "Humidity is at 85% with winds from the southwest at 12 mph. Visibility is reduced to 8 miles due to rain.\"\"\"\n", "\n", "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", "\n", - "result = groundedness_evaluator(response=response, context=context)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": None,\n", + " \"tool_definitions\": None,\n", + " \"response\": response,\n", + " \"context\": context\n", + " }\n", + " )\n", + ")" ] }, { @@ -159,14 +224,23 @@ "metadata": {}, "outputs": [], "source": [ - "# Example of ungrounded response with unsupported claims\n", + "# Test Case 2: Ungrounded response with unsupported claims\n", "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", "The forecast indicates overcast skies with light precipitation typical for Pacific Northwest weather.\"\"\"\n", "\n", "response = \"The current weather in Seattle is rainy with a temperature of 14°C. The city is experiencing its wettest month in 50 years, and the mayor has declared a weather emergency due to flooding concerns.\"\n", "\n", - "result = groundedness_evaluator(response=response, context=context)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": None,\n", + " \"tool_definitions\": None,\n", + " \"response\": response,\n", + " \"context\": context\n", + " }\n", + " )\n", + ")" ] }, { @@ -184,6 +258,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Test Case 3: Query, response, and context evaluation\n", "query = \"How is the weather in Seattle?\"\n", "\n", "context = \"\"\"Weather report for Seattle, Washington: Currently experiencing rainy weather with temperature at 14°C. \n", @@ -192,8 +267,17 @@ "\n", "response = \"The weather in Seattle is rainy with a temperature of 14°C. These are typical Pacific Northwest conditions with overcast skies.\"\n", "\n", - "result = groundedness_evaluator(response=response, context=context, query=query)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"tool_definitions\": None,\n", + " \"query\": query,\n", + " \"response\": response,\n", + " \"context\": context\n", + " }\n", + " )\n", + ")" ] }, { @@ -211,6 +295,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Test Case 4: Agent response with file_search tool call\n", "query = \"Can you get me the current weather information for Seattle?\"\n", "\n", "# Agent response with file_search tool call\n", @@ -257,12 +342,17 @@ " }\n", "]\n", "\n", - "result = groundedness_evaluator(\n", - " query=query, \n", - " response=agent_response, \n", - " tool_definitions=tool_definitions\n", - ")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"context\": None,\n", + " \"query\": query,\n", + " \"response\": agent_response,\n", + " \"tool_definitions\": tool_definitions\n", + " }\n", + " )\n", + ")" ] }, { @@ -280,38 +370,93 @@ "metadata": {}, "outputs": [], "source": [ - "conversation = {\n", - " \"context\": \"Weather data shows Seattle currently has rainy conditions at 14°C with overcast skies. London shows cloudy weather at 8°C with partly cloudy conditions. Both cities are experiencing typical seasonal weather patterns.\",\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Can you check the weather in Seattle for me?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"According to the current weather data, Seattle is experiencing rainy conditions with a temperature of 14°C and overcast skies.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"How does that compare to London?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"London is currently cloudier but drier than Seattle, with a temperature of 8°C and partly cloudy conditions. Seattle is warmer but rainier at 14°C.\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"Can you email me a summary of both cities' weather?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I can provide you with a weather summary: Seattle has rainy weather at 14°C with overcast skies, while London has partly cloudy conditions at 8°C. However, I would need email access to send this information to you.\"\n", + "# Test Case 5: Multi-turn conversation evaluation\n", + "conversation_context = \"Weather data shows Seattle currently has rainy conditions at 14°C with overcast skies. London shows cloudy weather at 8°C with partly cloudy conditions. Both cities are experiencing typical seasonal weather patterns.\"\n", + "\n", + "conversation_messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Can you check the weather in Seattle for me?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"According to the current weather data, Seattle is experiencing rainy conditions with a temperature of 14°C and overcast skies.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"How does that compare to London?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"London is currently cloudier but drier than Seattle, with a temperature of 8°C and partly cloudy conditions. Seattle is warmer but rainier at 14°C.\"\n", + " },\n", + " {\n", + " \"role\": \"user\", \n", + " \"content\": \"Can you email me a summary of both cities' weather?\"\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"I can provide you with a weather summary: Seattle has rainy weather at 14°C with overcast skies, while London has partly cloudy conditions at 8°C. However, I would need email access to send this information to you.\"\n", + " }\n", + "]\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": None,\n", + " \"tool_definitions\": None,\n", + " \"context\": conversation_context,\n", + " \"response\": conversation_messages\n", " }\n", - " ]\n", - "}\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "3adcae5d", + "metadata": {}, + "source": [ + "### Run Evaluation on All Test Cases\n", "\n", - "result = groundedness_evaluator(conversation=conversation)\n", - "pprint(result)" + "Now that we've defined all test cases, let's run the evaluation once on all of them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e543aa00", + "metadata": {}, + "outputs": [], + "source": [ + "results = run_evaluator(\n", + " evaluator_name=\"groundedness\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "2f4d7c8e", + "metadata": {}, + "source": [ + "### Display Results\n", + "\n", + "View the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "87aadc9a", + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb index 0b73807b..ab70531d 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb @@ -16,17 +16,15 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4o` or `gpt-4.1`. \n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)" ] }, { @@ -65,18 +63,72 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", - "from azure.ai.evaluation import IntentResolutionEvaluator\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", "\n", - "intent_resolution_evaluator = IntentResolutionEvaluator(model_config)" + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", + "\n", + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -99,12 +151,20 @@ "metadata": {}, "outputs": [], "source": [ - "# Success example. Intent is identified and understood and the response correctly resolves user intent\n", - "result = intent_resolution_evaluator(\n", - " query=\"What are the opening hours of the Eiffel Tower?\",\n", - " response=\"Opening hours of the Eiffel Tower are 9:00 AM to 11:00 PM.\",\n", - ")\n", - "pprint(result)" + "# Test Case 1: Successful intent resolution (should score high)\n", + "query1 = \"What are the opening hours of the Eiffel Tower?\"\n", + "response1 = \"Opening hours of the Eiffel Tower are 9:00 AM to 11:00 PM.\"\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query1,\n", + " \"response\": response1,\n", + " \"tool_definitions\": None\n", + " }\n", + " )\n", + ")" ] }, { @@ -113,12 +173,20 @@ "metadata": {}, "outputs": [], "source": [ - "# Failure example. Even though intent is correctly identified, the response does not resolve the user intent\n", - "result = intent_resolution_evaluator(\n", - " query=\"What is the opening hours of the Eiffel Tower?\",\n", - " response=\"Please check the official website for the up-to-date information on Eiffel Tower opening hours.\",\n", - ")\n", - "pprint(result)" + "# Test Case 2: Intent identified but not resolved (should score low)\n", + "query2 = \"What is the opening hours of the Eiffel Tower?\"\n", + "response2 = \"Please check the official website for the up-to-date information on Eiffel Tower opening hours.\"\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query2,\n", + " \"response\": response2,\n", + " \"tool_definitions\": None\n", + " }\n", + " )\n", + ")" ] }, { @@ -134,25 +202,29 @@ "metadata": {}, "outputs": [], "source": [ - "# Example with a single tool definition dict instead of a list\n", - "query = \"What's the status of order #789?\"\n", - "response = \"Order #789 has been shipped and will arrive on March 18, 2025.\"\n", + "# Test Case 3: Order status query with tool definition (should score high)\n", + "query3 = \"What's the status of order #789?\"\n", + "response3 = \"Order #789 has been shipped and will arrive on March 18, 2025.\"\n", "\n", - "tool_definition_dict = {\n", + "tool_definition_dict = [{\n", " \"name\": \"get_order\",\n", " \"description\": \"Get the details of a specific order.\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\"order_id\": {\"type\": \"string\", \"description\": \"The order ID to get the details for.\"}},\n", " },\n", - "}\n", + "}]\n", "\n", - "result = intent_resolution_evaluator(\n", - " query=query,\n", - " response=response,\n", - " tool_definitions=tool_definition_dict,\n", - ")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query3,\n", + " \"response\": response3,\n", + " \"tool_definitions\": tool_definition_dict\n", + " }\n", + " )\n", + ")" ] }, { @@ -168,7 +240,8 @@ "metadata": {}, "outputs": [], "source": [ - "query = [\n", + "# Test Case 4: Complex conversation with multiple orders (should score high)\n", + "query4 = [\n", " {\"role\": \"system\", \"content\": \"You are a friendly and helpful customer service agent.\"},\n", " {\n", " \"createdAt\": \"2025-03-14T06:14:20Z\",\n", @@ -182,7 +255,7 @@ " },\n", "]\n", "\n", - "response = [\n", + "response4 = [\n", " {\n", " \"createdAt\": \"2025-03-14T06:14:30Z\",\n", " \"run_id\": \"0\",\n", @@ -276,9 +349,8 @@ " },\n", "]\n", "\n", - "# please note that the tool definitions are not strictly required, and that some of the tools below are not used in the example above and that is ok.\n", - "# if context length is a concern you can remove the unused tool definitions or even the tool definitions altogether as the impact to the intent resolution evaluation is usual minimal.\n", - "tool_definitions = [\n", + "# Tool definitions for customer service operations\n", + "tool_definitions4 = [\n", " {\n", " \"name\": \"get_orders\",\n", " \"description\": \"Get the list of orders for a given account number.\",\n", @@ -318,12 +390,58 @@ " },\n", "]\n", "\n", - "result = intent_resolution_evaluator(\n", - " query=query,\n", - " response=response,\n", - " tool_definitions=tool_definitions,\n", - ")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query4,\n", + " \"response\": response4,\n", + " \"tool_definitions\": tool_definitions4\n", + " }\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run Evaluation on All Test Cases\n", + "\n", + "Now that we've defined all test cases, let's run the evaluation once on all of them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results = run_evaluator(\n", + " evaluator_name=\"intent_resolution\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Display Results\n", + "\n", + "View the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb index efe311f3..3672638f 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Relevance.ipynb @@ -8,26 +8,23 @@ "# Relevance Evaluator\n", "\n", "## Objective\n", - "This sample demonstrates how to use the Relevance evaluator to assess the relevance of AI-generated responses. The evaluator supports multiple input formats including:\n", + "This sample demonstrates how to use the Relevance evaluator to assess the relevance of AI-generated responses. The evaluator supports the following input format:\n", "- Simple query and response pairs\n", - "- Multi-turn conversations\n", "\n", "## Time\n", "\n", "You should expect to spend about 15 minutes running this notebook.\n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities.\n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4.1`. It is not recommended to use `gpt-4.1-nano`.\n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure OpenAI Endpoint to be used for evaluation\n", - "3) **AZURE_OPENAI_API_KEY** - Azure OpenAI Key to be used for evaluation\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure OpenAI API version to be used for evaluation\n" + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)\n" ] }, { @@ -56,14 +53,11 @@ "source": [ "## Relevance Evaluator Input Requirements\n", "\n", - "The Relevance evaluator supports the following input formats:\n", + "The Relevance evaluator supports the following input format:\n", "\n", "1. **Simple Query-Response Evaluation:**\n", " - `query`: The user's question or request (str)\n", - " - `response`: The AI's response to evaluate (str)\n", - "\n", - "2. **Conversation Evaluation:**\n", - " - `conversation`: Multi-turn conversation with messages list and optional context (Conversation object)\n" + " - `response`: The AI's response to evaluate (str)\n" ] }, { @@ -82,19 +76,40 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation import RelevanceEvaluator, AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", + "\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", + "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"array\", \"items\": {\"type\": \"object\"}}]},\n", + " \"response\": {\"anyOf\": [{\"type\": \"string\"}, {\"type\": \"array\", \"items\": {\"type\": \"object\"}}]},\n", + " },\n", + " \"required\": [\"query\", \"response\"],\n", + " },\n", + " \"include_sample_schema\": True,\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", "\n", - "# Configure the model\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", "\n", - "# Initialize the evaluator\n", - "relevance_evaluator = RelevanceEvaluator(model_config=model_config)" + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -120,11 +135,19 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", - "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", - "\n", - "result = relevance_evaluator(query=query, response=response)\n", - "pprint(result)" + "# Test Case 1: Highly relevant weather response (should score high)\n", + "query1 = \"How is the weather in Seattle?\"\n", + "response1 = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query1,\n", + " \"response\": response1\n", + " }\n", + " )\n", + ")" ] }, { @@ -142,52 +165,65 @@ "metadata": {}, "outputs": [], "source": [ - "# Irrelevant response example\n", - "query = \"What are the symptoms of flu?\"\n", - "response = \"The stock market had significant fluctuations today, with technology stocks leading the decline.\"\n", - "\n", - "result = relevance_evaluator(query=query, response=response)\n", - "pprint(result)" + "# Test Case 2: Completely irrelevant response (should score low)\n", + "query2 = \"What are the symptoms of flu?\"\n", + "response2 = \"The stock market had significant fluctuations today, with technology stocks leading the decline.\"\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query2,\n", + " \"response\": response2\n", + " }\n", + " )\n", + ")" ] }, { "cell_type": "markdown", - "id": "bd158e43", + "id": "56752df0", "metadata": {}, "source": [ - "### Conversation as Dict with Messages" + "### Run Evaluation on All Test Cases\n", + "\n", + "Now that we've defined all test cases, let's run the evaluation once on all of them." ] }, { "cell_type": "code", "execution_count": null, - "id": "cf17774b", + "id": "d7519cb0", "metadata": {}, "outputs": [], "source": [ - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"I'm planning a trip to Japan. Can you help me with some recommendations?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'd be happy to help you plan your trip to Japan! Here are some popular destinations: Tokyo for modern city life and culture, Kyoto for traditional temples and gardens, Osaka for amazing food, and Mount Fuji for scenic beauty. What type of experiences are you most interested in?\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"I'm particularly interested in traditional culture and temples.\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Perfect! For traditional culture and temples, I highly recommend Kyoto. Visit Fushimi Inari Shrine with its thousands of red torii gates, Kinkaku-ji (Golden Pavilion), and Gion district where you might spot geishas. Nara is also wonderful with its deer park and Todai-ji Temple housing a giant Buddha statue.\"\n", - " }\n", - " ]\n", - "}\n", + "results = run_evaluator(\n", + " evaluator_name=\"relevance\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "1b97b1e7", + "metadata": {}, + "source": [ + "### Display Results\n", "\n", - "result = relevance_evaluator(conversation=conversation)\n", - "pprint(result)" + "View the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7343ff8", + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb index 00cd4eaa..685663ce 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb @@ -20,17 +20,15 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4o` or `gpt-4.1`. \n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)\n" ] }, { @@ -68,27 +66,45 @@ "metadata": {}, "outputs": [], "source": [ - "from azure.ai.evaluation import ResponseCompletenessEvaluator, AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", "import os\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", + "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", + "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"ground_truth\": {\n", + " \"type\": \"string\"\n", + " },\n", + " \"response\": {\n", + " \"type\": \"string\"\n", + " }\n", + " },\n", + " \"required\": [\"ground_truth\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"ground_truth\": \"{{item.ground_truth}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", + "\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", "\n", - "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config)" + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -111,11 +127,19 @@ "metadata": {}, "outputs": [], "source": [ - "result = response_completeness_evaluator(\n", - " response=\"The capital of Japan\",\n", - " ground_truth=\"The capital of Japan is Tokyo.\"\n", - ")\n", - "result" + "# Test Case 1: Incomplete response (should score low)\n", + "response1 = \"The capital of Japan\"\n", + "ground_truth1 = \"The capital of Japan is Tokyo.\"\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"response\": response1,\n", + " \"ground_truth\": ground_truth1\n", + " }\n", + " )\n", + ")" ] }, { @@ -124,18 +148,28 @@ "metadata": {}, "outputs": [], "source": [ - "result = response_completeness_evaluator(\n", - " response=\"The capital of Japan is Tokyo.\",\n", - " ground_truth=\"The capital of Japan is Tokyo.\"\n", - ")\n", - "result" + "# Test Case 2: Complete response (should score high)\n", + "response2 = \"The capital of Japan is Tokyo.\"\n", + "ground_truth2 = \"The capital of Japan is Tokyo.\"\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"response\": response2,\n", + " \"ground_truth\": ground_truth2\n", + " }\n", + " )\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### Evaluate with a reasoning model" + "### Run Evaluation on All Test Cases\n", + "\n", + "Now that we've defined all test cases, let's run the evaluation once on all of them." ] }, { @@ -144,25 +178,22 @@ "metadata": {}, "outputs": [], "source": [ - "from azure.ai.evaluation import ResponseCompletenessEvaluator , AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "# set is_reasoning_model to True in case the model is a reasoning model (ex: o3-mini, o1-preview)\n", - "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config,\n", - " is_reasoning_model=False)\n", - "\n", - "result = response_completeness_evaluator(\n", - " response=\"The capital of Japan is Tokyo.\",\n", - " ground_truth=\"The capital of Japan is Tokyo.\"\n", - ")\n", - "result" + "results = run_evaluator(\n", + " evaluator_name=\"response_completeness\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### Conversation as Dict with Ground Truth in Context" + "### Display Results\n", + "\n", + "View the evaluation results for each test case." ] }, { @@ -171,24 +202,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Conversation format with ground truth in context\n", - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What's the weather forecast for Seattle this weekend?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday.\",\n", - " \"ground_truth\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday and 17°C on Sunday, with a 20% chance of rain on Sunday afternoon.\"\n", - " }\n", - " ],\n", - " \n", - "}\n", - "\n", - "result = response_completeness_evaluator(conversation=conversation)\n", - "result" + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb index 1a260070..9bdb691e 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb @@ -16,17 +16,15 @@ "You should expect to spend about 10 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4o` or `gpt-4.1`. \n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)" ] }, { @@ -66,16 +64,72 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation import TaskAdherenceEvaluator, AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "task_adherence_evaluator = TaskAdherenceEvaluator(model_config)" + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", + "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", + "\n", + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -98,12 +152,20 @@ "metadata": {}, "outputs": [], "source": [ - "# Failure example, there's only a vague adherence to the task\n", - "result = task_adherence_evaluator(\n", - " query=\"What are the best practices for maintaining a healthy rose garden during the summer?\",\n", - " response=\"Make sure to water your roses regularly and trim them occasionally.\",\n", - ")\n", - "pprint(result)" + "# Test Case 1: Vague adherence example (should score low)\n", + "query1 = \"What are the best practices for maintaining a healthy rose garden during the summer?\"\n", + "response1 = \"Make sure to water your roses regularly and trim them occasionally.\"\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query1,\n", + " \"response\": response1,\n", + " \"tool_definitions\": None,\n", + " }\n", + " )\n", + ")" ] }, { @@ -112,12 +174,20 @@ "metadata": {}, "outputs": [], "source": [ - "# Success example, full adherence to the task\n", - "result = task_adherence_evaluator(\n", - " query=\"What are the best practices for maintaining a healthy rose garden during the summer?\",\n", - " response=\"For optimal summer care of your rose garden, start by watering deeply early in the morning to ensure the roots are well-hydrated without encouraging fungal growth. Apply a 2-3 inch layer of organic mulch around the base of the plants to conserve moisture and regulate soil temperature. Fertilize with a balanced rose fertilizer every 4 to 6 weeks to support healthy growth. Prune away any dead or diseased wood to promote good air circulation, and inspect regularly for pests such as aphids or spider mites, treating them promptly with an appropriate organic insecticidal soap. Finally, ensure that your roses receive at least 6 hours of direct sunlight daily for robust flowering.\",\n", - ")\n", - "pprint(result)" + "# Test Case 2: Full adherence example (should score high)\n", + "query2 = \"What are the best practices for maintaining a healthy rose garden during the summer?\"\n", + "response2 = \"For optimal summer care of your rose garden, start by watering deeply early in the morning to ensure the roots are well-hydrated without encouraging fungal growth. Apply a 2-3 inch layer of organic mulch around the base of the plants to conserve moisture and regulate soil temperature. Fertilize with a balanced rose fertilizer every 4 to 6 weeks to support healthy growth. Prune away any dead or diseased wood to promote good air circulation, and inspect regularly for pests such as aphids or spider mites, treating them promptly with an appropriate organic insecticidal soap. Finally, ensure that your roses receive at least 6 hours of direct sunlight daily for robust flowering.\"\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query2,\n", + " \"response\": response2,\n", + " \"tool_definitions\": None,\n", + " }\n", + " )\n", + ")" ] }, { @@ -133,27 +203,33 @@ "metadata": {}, "outputs": [], "source": [ - "# Example with a single tool definition dict\n", - "query = \"Can you recommend a science fiction book for me?\"\n", - "response = \"I found 'Dune' by Frank Herbert for you. It's a classic science fiction novel set in a distant future.\"\n", + "# Test Case 3: Book recommendation with tool definition as single dict\n", + "query3 = \"Can you recommend a science fiction book for me?\"\n", + "response3 = \"I found 'Dune' by Frank Herbert for you. It's a classic science fiction novel set in a distant future.\"\n", "\n", - "tool_definition_dict = {\n", - " \"name\": \"get_book\",\n", - " \"description\": \"Retrieve a book recommendation for a specified genre.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"genre\": {\"type\": \"string\", \"description\": \"The genre for which a book recommendation is requested.\"}\n", + "tool_definition_dict = [\n", + " {\n", + " \"name\": \"get_book\",\n", + " \"description\": \"Retrieve a book recommendation for a specified genre.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"genre\": {\"type\": \"string\", \"description\": \"The genre for which a book recommendation is requested.\"}\n", + " },\n", " },\n", - " },\n", - "}\n", + " }\n", + "]\n", "\n", - "result = task_adherence_evaluator(\n", - " query=query,\n", - " response=response,\n", - " tool_definitions=tool_definition_dict,\n", - ")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query3,\n", + " \"response\": response3,\n", + " \"tool_definitions\": tool_definition_dict\n", + " }\n", + " )\n", + ")" ] }, { @@ -169,7 +245,8 @@ "metadata": {}, "outputs": [], "source": [ - "query = [\n", + "# Test Case 4: Historical fiction recommendation with conversation messages and tool usage\n", + "query4 = [\n", " {\"role\": \"system\", \"content\": \"You are an expert in literature and at provid can provide book recommendations.\"},\n", " {\n", " \"createdAt\": \"2025-03-14T08:00:00Z\",\n", @@ -180,7 +257,7 @@ " },\n", "]\n", "\n", - "response = [\n", + "response4 = [\n", " {\n", " \"createdAt\": \"2025-03-14T08:00:05Z\",\n", " \"role\": \"assistant\",\n", @@ -221,7 +298,7 @@ " },\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions4 = [\n", " {\n", " \"name\": \"get_book\",\n", " \"description\": \"Retrieve a book recommendation for a specified genre.\",\n", @@ -234,12 +311,58 @@ " }\n", "]\n", "\n", - "result = task_adherence_evaluator(\n", - " query=query,\n", - " response=response,\n", - " tool_definitions=tool_definitions,\n", - ")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query4,\n", + " \"response\": response4,\n", + " \"tool_definitions\": tool_definitions4\n", + " }\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run Evaluation on All Test Cases\n", + "\n", + "Now that we've defined all test cases, let's run the evaluation once on all of them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results = run_evaluator(\n", + " evaluator_name=\"task_adherence\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Display Results\n", + "\n", + "View the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb index 05958d1e..c5514e9b 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb @@ -17,17 +17,15 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4.1` or `gpt-4.1-mini`. \n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)" ] }, { @@ -75,19 +73,72 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation._evaluators._task_completion import _TaskCompletionEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", "\n", - "task_completion = _TaskCompletionEvaluator(model_config=model_config)" + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", + "\n", + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -113,12 +164,20 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", - "response = \"The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\"\n", + "# Test Case 1: Simple Task Completion - Basic weather query\n", + "query_simple = \"How is the weather in Seattle?\"\n", + "response_simple = \"The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\"\n", "\n", - "# Basic evaluation without tool definitions\n", - "result = task_completion(query=query, response=response)\n", - "pprint(result)" + "# Append to evaluation_contents - Basic evaluation without tool definitions\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_simple,\n", + " \"response\": response_simple,\n", + " \"tool_definitions\": None\n", + " }\n", + " )\n", + ")" ] }, { @@ -136,10 +195,11 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", - "response = \"I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\"\n", + "# Test Case 2: Task Completion with Tool Context\n", + "query_tool_context = \"How is the weather in Seattle?\"\n", + "response_tool_context = \"I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\"\n", "\n", - "tool_definitions = [\n", + "tool_definitions_context = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -150,8 +210,16 @@ " }\n", "]\n", "\n", - "result = task_completion(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_tool_context,\n", + " \"response\": response_tool_context,\n", + " \"tool_definitions\": tool_definitions_context\n", + " }\n", + " )\n", + ")" ] }, { @@ -169,20 +237,31 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"What's the current temperature in Boston?\"\n", - "response = \"The current temperature in Boston is 22°C (72°F) with clear skies. It's a beautiful day with low humidity and a gentle breeze.\"\n", + "# Test Case 3: Tool Definition as Single Dict\n", + "query_dict = \"What's the current temperature in Boston?\"\n", + "response_dict = \"The current temperature in Boston is 22°C (72°F) with clear skies. It's a beautiful day with low humidity and a gentle breeze.\"\n", "\n", - "tool_definition_dict = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", + "tool_definition_dict = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", "\n", - "result = task_completion(query=query, response=response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_dict,\n", + " \"response\": response_dict,\n", + " \"tool_definitions\": tool_definition_dict\n", + " }\n", + " )\n", + ")" ] }, { @@ -200,8 +279,9 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "response = [\n", + "# Test Case 4: Complex Task with Multiple Steps\n", + "query_complex = \"Can you send me an email with weather information for Seattle?\"\n", + "response_complex = [\n", " {\n", " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", @@ -226,7 +306,7 @@ " },\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_complex = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -249,8 +329,16 @@ " },\n", "]\n", "\n", - "result = task_completion(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_complex,\n", + " \"response\": response_complex,\n", + " \"tool_definitions\": tool_definitions_complex\n", + " }\n", + " )\n", + ")" ] }, { @@ -269,8 +357,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Query as conversation history instead of a single string\n", - "query_as_conversation = [\n", + "# Test Case 5: Query as Conversation History\n", + "query_conversation = [\n", " {\n", " \"role\": \"system\",\n", " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", @@ -285,9 +373,9 @@ " }\n", "]\n", "\n", - "response = \"I've checked the weather in Seattle for you. It's currently 15°C and partly cloudy with light winds. I've also sent you an email summary with these details so you can reference them throughout your day. The task has been completed successfully.\"\n", + "response_conversation = \"I've checked the weather in Seattle for you. It's currently 15°C and partly cloudy with light winds. I've also sent you an email summary with these details so you can reference them throughout your day. The task has been completed successfully.\"\n", "\n", - "tool_definitions = [\n", + "tool_definitions_conversation = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -310,8 +398,16 @@ " },\n", "]\n", "\n", - "result = task_completion(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_conversation,\n", + " \"response\": response_conversation,\n", + " \"tool_definitions\": tool_definitions_conversation\n", + " }\n", + " )\n", + ")" ] }, { @@ -329,12 +425,64 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "incomplete_response = \"I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\"\n", + "# Test Case 6: Incomplete Task Example\n", + "query_incomplete = \"Can you send me an email with weather information for Seattle?\"\n", + "response_incomplete = \"I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\"\n", "\n", - "# This response doesn't complete the email sending task\n", - "result = task_completion(query=query, response=incomplete_response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents - This response doesn't complete the email sending task\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_incomplete,\n", + " \"response\": response_incomplete,\n", + " \"tool_definitions\": None,\n", + " }\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "b3a30f59", + "metadata": {}, + "source": [ + "### Execute Batch Evaluation\n", + "Run all test cases together using the batch evaluation approach." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c36b9471", + "metadata": {}, + "outputs": [], + "source": [ + "results = run_evaluator(\n", + " evaluator_name=\"task_completion\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "6d403a51", + "metadata": {}, + "source": [ + "### Results Display\n", + "Display the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35c806e1", + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb index 1d8615a0..4e06d97c 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb @@ -21,9 +21,11 @@ "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", - "No environment variables are required for this evaluator." + "Set these environment variables with your own values:\n", + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)" ] }, { @@ -68,23 +70,45 @@ "metadata": {}, "outputs": [], "source": [ - "from azure.ai.evaluation._evaluators._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator, _TaskNavigationEfficiencyMatchingMode\n", + "import os\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "# Initialize with exact match mode\n", - "task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(\n", - " matching_mode=_TaskNavigationEfficiencyMatchingMode.EXACT_MATCH\n", - ")\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", "\n", - "# Other examples:\n", - "# For in-order matching (allows extra steps but requires correct order)\n", - "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"response\": {\n", + " \"type\": \"array\"\n", + " },\n", + " \"ground_truth\": {\n", + " \"type\": \"array\"\n", + " }\n", + " },\n", + " \"required\": [\"response\", \"ground_truth\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"response\": \"{{item.response}}\",\n", + " \"ground_truth\": \"{{item.ground_truth}}\"\n", + "}\n", "\n", - "# For any-order matching (most lenient - allows extra steps and different order) \n", - "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.ANY_ORDER_MATCH)\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"matching_mode\": \"exact_match\" # Can be \"exact_match\", \"in_order_match\", or \"any_order_match\"\n", + "}\n", "\n", - "# Or use defaults (exact match mode)\n", - "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator()" + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -110,8 +134,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Agent follows the exact optimal path\n", - "response = [\n", + "# Test Case 1: Perfect Path (Exact Match) - Agent follows the exact optimal path\n", + "response_perfect = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", @@ -126,11 +150,17 @@ " },\n", "]\n", "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "ground_truth_perfect = [\"search\", \"analyze\", \"report\"]\n", "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"Perfect Path Results:\")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"response\": response_perfect,\n", + " \"ground_truth\": ground_truth_perfect\n", + " }\n", + " )\n", + ")" ] }, { @@ -148,8 +178,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Agent performs all required steps but with extra unnecessary step\n", - "response = [\n", + "# Test Case 2: Path with Extra Steps - Agent performs all required steps but with extra unnecessary step\n", + "response_extra = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", @@ -168,11 +198,17 @@ " },\n", "]\n", "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "ground_truth_extra = [\"search\", \"analyze\", \"report\"]\n", "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nPath with Extra Steps Results:\")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"response\": response_extra,\n", + " \"ground_truth\": ground_truth_extra\n", + " }\n", + " )\n", + ")" ] }, { @@ -190,8 +226,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Agent performs all required steps but in wrong order\n", - "response = [\n", + "# Test Case 3: Wrong Order - Agent performs all required steps but in wrong order\n", + "response_wrong_order = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"report\", \"arguments\": {}}],\n", @@ -206,14 +242,17 @@ " },\n", "]\n", "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", - "\n", - "# Using in-order matching mode to demonstrate the difference\n", - "in_order_task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", + "ground_truth_wrong_order = [\"search\", \"analyze\", \"report\"]\n", "\n", - "result = in_order_task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nWrong Order Results:\")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"response\": response_wrong_order,\n", + " \"ground_truth\": ground_truth_wrong_order\n", + " }\n", + " )\n", + ")" ] }, { @@ -231,8 +270,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Agent performs only some of the required steps (incomplete)\n", - "response = [\n", + "# Test Case 4: Missing Steps - Agent performs only some of the required steps (incomplete)\n", + "response_missing = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", @@ -243,11 +282,17 @@ " },\n", "]\n", "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "ground_truth_missing = [\"search\", \"analyze\", \"report\"]\n", "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nMissing Steps Results:\")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"response\": response_missing,\n", + " \"ground_truth\": ground_truth_missing\n", + " }\n", + " )\n", + ")" ] }, { @@ -265,8 +310,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Real-world example: Customer service agent handling a refund request\n", - "response = [\n", + "# Test Case 5: Customer Service Scenario - Real-world example: Customer service agent handling a refund request\n", + "response_service = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"lookup_order\", \"arguments\": {\"order_id\": \"12345\"}}],\n", @@ -285,11 +330,17 @@ " },\n", "]\n", "\n", - "ground_truth = [\"lookup_order\", \"calculate_refund\", \"process_refund\"]\n", + "ground_truth_service = [\"lookup_order\", \"check_inventory\", \"calculate_refund\", \"process_refund\"]\n", "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nCustomer Service Results:\")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"response\": response_service,\n", + " \"ground_truth\": ground_truth_service\n", + " }\n", + " )\n", + ")" ] }, { @@ -307,8 +358,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Agent repeats some steps and includes extra ones\n", - "response = [\n", + "# Test Case 6: Complex Path with Duplicates - Agent repeats some steps and includes extra ones\n", + "response_duplicates = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", @@ -318,10 +369,6 @@ " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"search\", \"arguments\": {}}], # duplicate\n", " },\n", " {\n", - " \"role\": \"assistant\", \n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"validate\", \"arguments\": {}}], # extra step\n", - " },\n", - " {\n", " \"role\": \"assistant\",\n", " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"analyze\", \"arguments\": {}}],\n", " },\n", @@ -331,11 +378,17 @@ " },\n", "]\n", "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", + "ground_truth_duplicates = [\"search\", \"analyze\", \"report\"]\n", "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nComplex Path with Duplicates Results:\")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"response\": response_duplicates,\n", + " \"ground_truth\": ground_truth_duplicates\n", + " }\n", + " )\n", + ")" ] }, { @@ -353,8 +406,8 @@ "metadata": {}, "outputs": [], "source": [ - "# TaskNavigationEfficiencyEvaluator also supports tuple format with parameters for exact parameter matching\n", - "response_with_params = [\n", + "# Test Case 7: Tuple Format with Parameters - Supports exact parameter matching\n", + "response_params = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {\"query\": \"test\"}}],\n", @@ -363,36 +416,61 @@ "\n", "# Ground truth using tuple format: (tool_names, parameters_dict)\n", "# Parameters must match exactly for tools to be considered matching\n", - "ground_truth_with_params = ([\"search\"], {\"search\": {\"query\": \"test\"}})\n", + "ground_truth_params = [[\"search\"], {\"search\": {\"query\": \"test\"}}]\n", "\n", - "result = task_navigation_efficiency_evaluator(response=response_with_params, ground_truth=ground_truth_with_params)\n", - "print(\"\\nTuple Format with Parameters Results:\")\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"response\": response_params,\n", + " \"ground_truth\": ground_truth_params\n", + " }\n", + " )\n", + ")" ] }, { "cell_type": "markdown", - "id": "38d2de9b", + "id": "16a11522", "metadata": {}, "source": [ - "#### Sample 8: String Response Input Type" + "### Execute Batch Evaluation\n", + "Run all test cases together using the batch evaluation approach." ] }, { "cell_type": "code", "execution_count": null, - "id": "1a0a853b", + "id": "1185d9f2", "metadata": {}, "outputs": [], "source": [ - "# Demonstrate string response input type\n", - "# The string response should contain structured tool call information that can be parsed\n", - "string_response = \"I'll help you with that. Let me search for information, then analyze the results, and finally provide a report.\"\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", - "\n", - "result = task_navigation_efficiency_evaluator(response=string_response, ground_truth=ground_truth)\n", - "print(\"\\nString Response Results:\")\n", - "pprint(result)" + "results = run_evaluator(\n", + " evaluator_name=\"task_navigation_efficiency\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "6ed16d6c", + "metadata": {}, + "source": [ + "### Results Display\n", + "Display the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9c0bd93", + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb index f55620fc..fc77b828 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb @@ -16,17 +16,15 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4.1`. \n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)" ] }, { @@ -74,18 +72,84 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation import ToolCallAccuracyEvaluator, AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_calls\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"tool_definitions\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\",\n", + " \"tool_calls\": \"{{item.tool_calls}}\",\n", + " \"response\": \"{{item.response}}\"\n", + "}\n", "\n", - "tool_call_accuracy = ToolCallAccuracyEvaluator(model_config=model_config)" + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", + "\n", + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -108,33 +172,40 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle ?\"\n", - "tool_call = {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - "}\n", + "# Test Case 1: Single tool call\n", + "query_single = \"How is the weather in Seattle ?\"\n", + "tool_call_single = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + "]\n", "\n", - "tool_definition = {\n", - " \"id\": \"fetch_weather\",\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = tool_call_accuracy(query=query, tool_calls=tool_call, tool_definitions=tool_definition)\n", - "pprint(response)" + "tool_definition_single = [\n", + " {\n", + " \"id\": \"fetch_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\", \n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_single,\n", + " \"tool_calls\": tool_call_single,\n", + " \"tool_definitions\": tool_definition_single,\n", + " \"response\": None\n", + " }\n", + " )\n", + ")" ] }, { @@ -150,8 +221,9 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle ?\"\n", - "tool_calls = [\n", + "# Test Case 2: Multiple tool calls for single query\n", + "query_multiple = \"How is the weather in Seattle ?\"\n", + "tool_calls_multiple = [\n", " {\n", " \"type\": \"tool_call\",\n", " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", @@ -166,25 +238,29 @@ " },\n", "]\n", "\n", - "tool_definition = {\n", - " \"id\": \"fetch_weather\",\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = tool_call_accuracy(query=query, tool_calls=tool_calls, tool_definitions=tool_definition)\n", - "pprint(response)" + "tool_definition_multiple = [\n", + " {\n", + " \"id\": \"fetch_weather\",\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_multiple,\n", + " \"tool_calls\": tool_calls_multiple,\n", + " \"tool_definitions\": tool_definition_multiple,\n", + " \"response\": None\n", + " }\n", + " )\n", + ")" ] }, { @@ -201,8 +277,9 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "response = [\n", + "# Test Case 3: Tool calls extracted from response\n", + "query_response = \"Can you send me an email with weather information for Seattle?\"\n", + "response_messages = [\n", " {\n", " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", @@ -221,7 +298,7 @@ " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Rainy, 14\\u00b0C\"}}],\n", + " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Rainy, 14°C\"}}],\n", " },\n", " {\n", " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", @@ -235,7 +312,7 @@ " \"arguments\": {\n", " \"recipient\": \"your_email@example.com\",\n", " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"The current weather in Seattle is rainy with a temperature of 14\\u00b0C.\",\n", + " \"body\": \"The current weather in Seattle is rainy with a temperature of 14°C.\",\n", " },\n", " }\n", " ],\n", @@ -256,13 +333,13 @@ " \"content\": [\n", " {\n", " \"type\": \"text\",\n", - " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is rainy with a temperature of 14\\u00b0C.\",\n", + " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is rainy with a temperature of 14°C.\",\n", " }\n", " ],\n", " },\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_response = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -283,17 +360,19 @@ " },\n", " },\n", " },\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = tool_call_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(response)" + "]\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_response,\n", + " \"response\": response_messages,\n", + " \"tool_definitions\": tool_definitions_response,\n", + " \"tool_calls\": None\n", + " }\n", + " )\n", + ")" ] }, { @@ -309,12 +388,13 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Book a flight to New York for tomorrow\"\n", + "# Test Case 4: Response as string\n", + "query_str = \"Book a flight to New York for tomorrow\"\n", "\n", "# Response as a simple string instead of a list of messages\n", - "response = \"I've found several flight options to New York for tomorrow. I'll use the booking tool to reserve your seat.\"\n", + "response_str = \"I've found several flight options to New York for tomorrow. I'll use the booking tool to reserve your seat.\"\n", "\n", - "tool_calls = [\n", + "tool_calls_str = [\n", " {\n", " \"type\": \"tool_call\",\n", " \"tool_call_id\": \"call_book_flight_456\",\n", @@ -326,7 +406,7 @@ " }\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_str = [\n", " {\n", " \"name\": \"book_flight\",\n", " \"description\": \"Books a flight to the specified destination on the given date.\",\n", @@ -340,8 +420,17 @@ " }\n", "]\n", "\n", - "result = tool_call_accuracy(query=query, response=response, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_str,\n", + " \"response\": response_str,\n", + " \"tool_calls\": tool_calls_str,\n", + " \"tool_definitions\": tool_definitions_str\n", + " }\n", + " )\n", + ")" ] }, { @@ -357,9 +446,10 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"What's the weather in San Francisco?\"\n", + "# Test Case 5: Tool definition as single dict\n", + "query_dict = \"What's the weather in San Francisco?\"\n", "\n", - "response = [\n", + "response_dict = [\n", " {\n", " \"createdAt\": \"2025-03-26T18:15:22Z\",\n", " \"run_id\": \"run_abc123\",\n", @@ -393,17 +483,28 @@ " },\n", "]\n", "\n", - "tool_definition_dict = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", + "tool_definition_dict = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", "\n", - "result = tool_call_accuracy(query=query, response=response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_dict,\n", + " \"response\": response_dict,\n", + " \"tool_definitions\": tool_definition_dict,\n", + " \"tool_calls\": None\n", + " }\n", + " )\n", + ")" ] }, { @@ -420,8 +521,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Query as conversation history instead of a single string\n", - "query_as_conversation = [\n", + "# Test Case 6: Query as conversation history\n", + "query_conversation = [\n", " {\n", " \"role\": \"system\",\n", " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", @@ -436,7 +537,7 @@ " }\n", "]\n", "\n", - "tool_calls = [\n", + "tool_calls_conversation = [\n", " {\n", " \"type\": \"tool_call\",\n", " \"tool_call_id\": \"call_weather_123\",\n", @@ -455,7 +556,7 @@ " },\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_conversation = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -476,7 +577,50 @@ " },\n", " },\n", " },\n", - "]" + "]\n", + "\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_conversation,\n", + " \"tool_calls\": tool_calls_conversation,\n", + " \"tool_definitions\": tool_definitions_conversation,\n", + " \"response\": None\n", + " }\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Execute Batch Evaluation\n", + "Run all test cases together using the batch evaluation approach." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results = run_evaluator(\n", + " evaluator_name=\"tool_call_accuracy\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Results Display\n", + "Display the evaluation results for each test case." ] }, { @@ -485,8 +629,7 @@ "metadata": {}, "outputs": [], "source": [ - "response = tool_call_accuracy(query=query_as_conversation, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(response)" + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb similarity index 83% rename from scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb rename to scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb index 91788ef2..6020dd32 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Success.ipynb @@ -5,7 +5,7 @@ "id": "cea11cae", "metadata": {}, "source": [ - "# Tool Success Evaluator\n", + "# Tool Call Success Evaluator\n", "\n", "## Objective\n", "This sample demonstrates how to use tool success evaluator on agent data. The supported input formats include:\n", @@ -17,7 +17,7 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4.1-mini`. \n", "\n", "### Prerequisite\n", "```bash\n", @@ -33,7 +33,7 @@ "id": "49404a44", "metadata": {}, "source": [ - "The Tool Success evaluator determines whether tool calls done by an AI agent includes failures or not.\n", + "The Tool Call Success evaluator determines whether tool calls done by an AI agent includes failures or not.\n", "\n", "This evaluator focuses solely on tool call results and tool definitions, disregarding user's query to the agent, conversation history and agent's final response. Although tool definitions is optional, providing them can help the evaluator better understand the context of the tool calls made by the agent. Please note that this evaluator validates tool calls for potential technical failures like errors, exceptions, timeouts and empty results (only in cases where empty results could indicate a failure). It does not assess the correctness or the tool result itself, like mathematical errors and unrealistic field values like name=\"668656\".\n", "\n", @@ -49,7 +49,7 @@ "id": "d91cb876", "metadata": {}, "source": [ - "Tool Success requires following input:\n", + "Tool Call Success requires following input:\n", "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator examines tool call results within the response.\n", "- Tool Definitions - (Optional) Tool(s) definition used by Agent. Providing tool definitions helps the evaluator better understand the context of the tool calls made by the agent.\n" ] @@ -59,7 +59,7 @@ "id": "e07d293d", "metadata": {}, "source": [ - "### Initialize Tool Success Evaluator\n" + "### Initialize Tool Call Success Evaluator\n" ] }, { @@ -70,7 +70,7 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.projects.models import SourceFileContentContent\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", "from agent_utils import run_evaluator\n", "\n", @@ -131,9 +131,7 @@ "id": "4739878c", "metadata": {}, "source": [ - "### Sample Test Cases\n", - "\n", - "In the following cells, we'll define various test cases and append them to the `evaluation_contents` list. Each test case demonstrates a different scenario for tool success evaluation.\n" + "### Samples\n" ] }, { @@ -222,14 +220,16 @@ "Tool result: {\"temperature\": \"18°C\", \"condition\": \"clear sky\", \"humidity\": \"55%\"}\n", "The weather in Paris is currently clear with a temperature of 18°C.\"\"\"\n", "\n", - "tool_definition = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", + "tool_definition = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", "\n", "# Append to evaluation_contents\n", "evaluation_contents.append(\n", @@ -286,14 +286,16 @@ " }\n", "]\n", "\n", - "tool_definition_dict = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", + "tool_definition_dict = [\n", + " {\n", + " \"name\": \"fetch_weather\",\n", + " \"description\": \"Fetches the weather information for the specified location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", + " },\n", + " }\n", + "]\n", "\n", "# Append to evaluation_contents\n", "evaluation_contents.append(\n", @@ -548,22 +550,13 @@ "metadata": {}, "outputs": [], "source": [ - "print(f\"\\n{'='*80}\")\n", - "print(f\"Running evaluation on {len(evaluation_contents)} test cases...\")\n", - "print(f\"{'='*80}\\n\")\n", - "\n", - "# Run the evaluator on all test cases at once\n", "results = run_evaluator(\n", - " evaluator_name=\"tool-success\",\n", + " evaluator_name=\"tool_call_success\",\n", " evaluation_contents=evaluation_contents,\n", " data_source_config=data_source_config,\n", " initialization_parameters=initialization_parameters,\n", " data_mapping=data_mapping\n", - ")\n", - "\n", - "print(f\"\\n{'='*80}\")\n", - "print(f\"Evaluation Complete! Results for {len(results)} test cases:\")\n", - "print(f\"{'='*80}\\n\")\n" + ")" ] }, { @@ -583,61 +576,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Define test case descriptions for better readability\n", - "test_case_descriptions = [\n", - " \"Successful tool execution\",\n", - " \"Response as string\",\n", - " \"Tool definition as single dict\",\n", - " \"Multiple successful tool calls\",\n", - " \"Success without tool definitions\",\n", - " \"Failed tool call with error\"\n", - "]\n", - "\n", - "# Display results for each test case\n", - "for idx, result in enumerate(results, 1):\n", - " print(f\"Test Case {idx}: {test_case_descriptions[idx-1]}\")\n", - " print(\"-\" * 80)\n", - " pprint(result)\n", - " print(\"\\n\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "4937b4b5", - "metadata": {}, - "source": [ - "### Summary Statistics\n", - "\n", - "Get an overview of the evaluation results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36a06ff1", - "metadata": {}, - "outputs": [], - "source": [ - "# Calculate summary statistics\n", - "total_cases = len(results)\n", - "successful_cases = sum(1 for r in results if r.get(\"tool_success\") == True)\n", - "failed_cases = total_cases - successful_cases\n", - "\n", - "print(f\"\\n{'='*80}\")\n", - "print(f\"EVALUATION SUMMARY\")\n", - "print(f\"{'='*80}\")\n", - "print(f\"Total test cases: {total_cases}\")\n", - "print(f\"Successful tool calls: {successful_cases} ({successful_cases/total_cases*100:.1f}%)\")\n", - "print(f\"Failed tool calls: {failed_cases} ({failed_cases/total_cases*100:.1f}%)\")\n", - "print(f\"{'='*80}\\n\")\n", - "\n", - "# Show which cases failed\n", - "if failed_cases > 0:\n", - " print(\"Failed test cases:\")\n", - " for idx, result in enumerate(results, 1):\n", - " if result.get(\"tool_success\") == False:\n", - " print(f\" • Test Case {idx}: {test_case_descriptions[idx-1]}\")\n", - " print(f\" Reason: {result.get('tool_success_reason', 'N/A')}\")\n" + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb index a1cdb299..16d6e4b8 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb @@ -17,17 +17,15 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4o`. \n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)" ] }, { @@ -78,19 +76,72 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation._evaluators._tool_input_accuracy import _ToolInputAccuracyEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\", \"tool_definitions\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", "\n", - "tool_input_accuracy = _ToolInputAccuracyEvaluator(model_config=model_config)" + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", + "\n", + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -116,6 +167,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Test Case 1: Correct tool input parameters\n", "query = \"How is the weather in Seattle?\"\n", "response = [\n", " {\n", @@ -143,8 +195,16 @@ " }\n", "]\n", "\n", - "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query,\n", + " \"response\": response,\n", + " \"tool_definitions\": tool_definitions\n", + " }\n", + " )\n", + ")" ] }, { @@ -162,12 +222,13 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Check the weather in Miami\"\n", + "# Test Case 2: Response as string\n", + "query_str = \"Check the weather in Miami\"\n", "\n", "# Response as a simple string containing tool call information\n", - "response_str = \"\"\"I'll check the weather for you. Calling fetch_weather with location=\"Miami\".\"\"\"\n", + "response_str = \"I'll check the weather for you. Calling fetch_weather with location=\\\"Miami\\\".\"\n", "\n", - "tool_definition = {\n", + "tool_definition = [{\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", " \"parameters\": {\n", @@ -175,10 +236,18 @@ " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", " \"required\": [\"location\"]\n", " },\n", - "}\n", + "}]\n", "\n", - "result = tool_input_accuracy(query=query, response=response_str, tool_definitions=tool_definition)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_str,\n", + " \"response\": response_str,\n", + " \"tool_definitions\": tool_definition\n", + " }\n", + " )\n", + ")" ] }, { @@ -196,8 +265,9 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"What's the temperature in Boston?\"\n", - "response = [\n", + "# Test Case 3: Tool definition as single dict\n", + "query_dict = \"What's the temperature in Boston?\"\n", + "response_dict = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [\n", @@ -211,7 +281,7 @@ " }\n", "]\n", "\n", - "tool_definition_dict = {\n", + "tool_definition_dict = [{\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", " \"parameters\": {\n", @@ -219,10 +289,18 @@ " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", " \"required\": [\"location\"]\n", " },\n", - "}\n", + "}]\n", "\n", - "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_dict,\n", + " \"response\": response_dict,\n", + " \"tool_definitions\": tool_definition_dict\n", + " }\n", + " )\n", + ")" ] }, { @@ -240,8 +318,9 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Can you send an email to john@example.com with the subject 'Weather Update' and tell him the weather in Seattle is 15°C and partly cloudy?\"\n", - "response = [\n", + "# Test Case 4: Complex tool parameters with multiple fields\n", + "query_complex = \"Can you send an email to john@example.com with the subject 'Weather Update' and tell him the weather in Seattle is 15°C and partly cloudy?\"\n", + "response_complex = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [\n", @@ -259,7 +338,7 @@ " }\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_complex = [\n", " {\n", " \"name\": \"send_email\",\n", " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", @@ -275,8 +354,16 @@ " }\n", "]\n", "\n", - "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_complex,\n", + " \"response\": response_complex,\n", + " \"tool_definitions\": tool_definitions_complex\n", + " }\n", + " )\n", + ")" ] }, { @@ -295,8 +382,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Query as conversation history instead of a single string\n", - "query_as_conversation = [\n", + "# Test Case 5: Query as conversation history with multiple tools\n", + "query_conversation = [\n", " {\n", " \"role\": \"system\",\n", " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", @@ -311,7 +398,7 @@ " }\n", "]\n", "\n", - "response = [\n", + "response_conversation = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [\n", @@ -340,7 +427,7 @@ " }\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_conversation = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -365,8 +452,16 @@ " },\n", "]\n", "\n", - "result = tool_input_accuracy(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_conversation,\n", + " \"response\": response_conversation,\n", + " \"tool_definitions\": tool_definitions_conversation\n", + " }\n", + " )\n", + ")" ] }, { @@ -384,9 +479,10 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", + "# Test Case 6: Incorrect tool parameters (should score 0)\n", + "query_incorrect = \"How is the weather in Seattle?\"\n", "# Missing required parameter or wrong type\n", - "incorrect_response = [\n", + "response_incorrect = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [\n", @@ -400,7 +496,7 @@ " }\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_incorrect = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -412,9 +508,60 @@ " }\n", "]\n", "\n", - "# This should score 0 due to incorrect parameter name\n", - "result = tool_input_accuracy(query=query, response=incorrect_response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents - This should score 0 due to incorrect parameter name\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_incorrect,\n", + " \"response\": response_incorrect,\n", + " \"tool_definitions\": tool_definitions_incorrect\n", + " }\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e4c7dd3f", + "metadata": {}, + "source": [ + "### Execute Batch Evaluation\n", + "Run all test cases together using the batch evaluation approach." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fffe6f9f", + "metadata": {}, + "outputs": [], + "source": [ + "results = run_evaluator(\n", + " evaluator_name=\"tool_input_accuracy\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "16b2b2ea", + "metadata": {}, + "source": [ + "### Results Display\n", + "Display the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f7c5195", + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], @@ -425,7 +572,15 @@ "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", "version": "3.10.19" } }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb index c0c9dfb8..18dd5b48 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb @@ -17,17 +17,15 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4o`. \n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)\n" ] }, { @@ -73,19 +71,72 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation._evaluators._tool_output_utilization import _ToolOutputUtilizationEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", "\n", - "tool_output_utilization = _ToolOutputUtilizationEvaluator(model_config=model_config)" + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", + "\n", + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -111,6 +162,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Test Case 1: Good tool output utilization\n", "query = \"How is the weather in Seattle?\"\n", "response = [\n", " {\n", @@ -151,8 +203,16 @@ " }\n", "]\n", "\n", - "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query,\n", + " \"response\": response,\n", + " \"tool_definitions\": tool_definitions\n", + " }\n", + " )\n", + ")" ] }, { @@ -170,8 +230,9 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "response = [\n", + "# Test Case 2: Complex scenario with multiple tool outputs\n", + "query_complex = \"Can you send me an email with weather information for Seattle?\"\n", + "response_complex = [\n", " {\n", " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", @@ -231,7 +292,7 @@ " },\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_multiple = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -254,8 +315,16 @@ " },\n", "]\n", "\n", - "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_complex,\n", + " \"response\": response_complex,\n", + " \"tool_definitions\": tool_definitions_multiple\n", + " }\n", + " )\n", + ")" ] }, { @@ -273,7 +342,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Query as conversation history instead of a single string\n", + "# Test Case 3: Query as conversation history instead of a single string\n", "query_as_conversation = [\n", " {\n", " \"role\": \"system\",\n", @@ -289,7 +358,7 @@ " }\n", "]\n", "\n", - "response = [\n", + "response_conversation = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [\n", @@ -337,7 +406,7 @@ " }\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_conversation = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -360,8 +429,16 @@ " },\n", "]\n", "\n", - "result = tool_output_utilization(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_as_conversation,\n", + " \"response\": response_conversation,\n", + " \"tool_definitions\": tool_definitions_conversation\n", + " }\n", + " )\n", + ")" ] }, { @@ -379,7 +456,8 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", + "# Test Case 4: Example of poor tool output utilization\n", + "query_poor = \"How is the weather in Seattle?\"\n", "# Poor utilization - agent doesn't use the detailed tool output effectively\n", "poor_response = [\n", " {\n", @@ -409,7 +487,7 @@ " }\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_poor = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -421,8 +499,15 @@ "]\n", "\n", "# This should score poorly due to inadequate use of tool output\n", - "result = tool_output_utilization(query=query, response=poor_response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_poor,\n", + " \"response\": poor_response,\n", + " \"tool_definitions\": tool_definitions_poor\n", + " }\n", + " )\n", + ")" ] }, { @@ -440,11 +525,12 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"What's the weather like in Seattle?\"\n", + "# Test Case 5: Response as string\n", + "query_str = \"What's the weather like in Seattle?\"\n", "# Response as a simple string (not a list of messages)\n", "response_str = \"The weather in Seattle is currently 15°C and partly cloudy with light winds from the northwest.\"\n", "\n", - "tool_definitions = [\n", + "tool_definitions_str = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -455,8 +541,16 @@ " }\n", "]\n", "\n", - "result = tool_output_utilization(query=query, response=response_str, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_str,\n", + " \"response\": response_str,\n", + " \"tool_definitions\": tool_definitions_str\n", + " }\n", + " )\n", + ")" ] }, { @@ -474,8 +568,9 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"Check the weather in Seattle\"\n", - "response = [\n", + "# Test Case 6: Tool definitions as single dict\n", + "query_dict = \"Check the weather in Seattle\"\n", + "response_dict = [\n", " {\n", " \"role\": \"assistant\",\n", " \"content\": [\n", @@ -504,7 +599,8 @@ "]\n", "\n", "# Tool definition as a single dict (not a list)\n", - "tool_definition_dict = {\n", + "tool_definition_dict = [\n", + " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", " \"parameters\": {\n", @@ -512,9 +608,64 @@ " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", " },\n", "}\n", + "]\n", "\n", - "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_dict,\n", + " \"response\": response_dict,\n", + " \"tool_definitions\": tool_definition_dict\n", + " }\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "9dcbf1cd", + "metadata": {}, + "source": [ + "### Run Evaluation on All Test Cases\n", + "\n", + "Now that we've defined all test cases, let's run the evaluation once on all of them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f6af4c2", + "metadata": {}, + "outputs": [], + "source": [ + "results = run_evaluator(\n", + " evaluator_name=\"tool_output_utilization\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "abfb12c7", + "metadata": {}, + "source": [ + "### Display Results\n", + "\n", + "View the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce47c4cb", + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb index aadb165b..83cc295a 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb @@ -17,17 +17,15 @@ "You should expect to spend about 20 minutes running this notebook. \n", "\n", "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", + "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend using `gpt-4.1` or `gpt-4.1-mini`.\n", "\n", "### Prerequisite\n", "```bash\n", - "pip install azure-ai-evaluation\n", + "pip install azure-ai-projects azure-identity openai\n", "```\n", "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" + "1) **AZURE_AI_PROJECT_ENDPOINT** - Your Azure AI project endpoint in format: `https://.services.ai.azure.com/api/projects/`\n", + "2) **AZURE_AI_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator (e.g., gpt-4o-mini)\n" ] }, { @@ -76,19 +74,84 @@ "outputs": [], "source": [ "import os\n", - "from azure.ai.evaluation._evaluators._tool_selection import _ToolSelectionEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", + "from openai.types.evals.create_eval_jsonl_run_data_source_param import SourceFileContentContent\n", "from pprint import pprint\n", + "from agent_utils import run_evaluator\n", "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", + "# Get environment variables\n", + "deployment_name = os.environ[\"AZURE_AI_MODEL_DEPLOYMENT_NAME\"]\n", "\n", + "# Data source configuration (defines the schema for evaluation inputs)\n", + "data_source_config = {\n", + " \"type\": \"custom\",\n", + " \"item_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"response\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"string\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_calls\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " },\n", + " \"tool_definitions\": {\n", + " \"anyOf\": [\n", + " {\"type\": \"object\"},\n", + " {\n", + " \"type\": \"array\",\n", + " \"items\": {\n", + " \"type\": \"object\"\n", + " }\n", + " }\n", + " ]\n", + " }\n", + " },\n", + " \"required\": [\"query\", \"response\", \"tool_definitions\"]\n", + " },\n", + " \"include_sample_schema\": True\n", + "}\n", + "\n", + "# Data mapping (maps evaluation inputs to evaluator parameters)\n", + "data_mapping = {\n", + " \"query\": \"{{item.query}}\",\n", + " \"response\": \"{{item.response}}\",\n", + " \"tool_calls\": \"{{item.tool_calls}}\",\n", + " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", + "}\n", + "\n", + "# Initialization parameters for the evaluator\n", + "initialization_parameters = {\n", + " \"deployment_name\": deployment_name\n", + "}\n", "\n", - "tool_selection = _ToolSelectionEvaluator(model_config=model_config)" + "# Initialize the evaluation_contents list - we'll append all test cases here\n", + "evaluation_contents = []" ] }, { @@ -114,13 +177,16 @@ "metadata": {}, "outputs": [], "source": [ + "# Test Case 1: Single tool selection for weather query\n", "query = \"How is the weather in Seattle?\"\n", - "tool_call = {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - "}\n", + "tool_call = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"Seattle\"},\n", + " }\n", + "]\n", "\n", "tool_definitions = [\n", " {\n", @@ -147,8 +213,17 @@ " },\n", "]\n", "\n", - "response = tool_selection(query=query, tool_calls=tool_call, tool_definitions=tool_definitions)\n", - "pprint(response)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query,\n", + " \"response\": None,\n", + " \"tool_calls\": tool_call,\n", + " \"tool_definitions\": tool_definitions\n", + " }\n", + " )\n", + ")" ] }, { @@ -166,6 +241,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Test Case 2: Multiple tool selections for complex task\n", "query = \"Can you send me an email with weather information for Seattle?\"\n", "tool_calls = [\n", " {\n", @@ -186,7 +262,7 @@ " },\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_multiple = [\n", " {\n", " \"id\": \"fetch_weather\",\n", " \"name\": \"fetch_weather\",\n", @@ -223,8 +299,17 @@ " },\n", "]\n", "\n", - "response = tool_selection(query=query, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(response)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query,\n", + " \"response\": None,\n", + " \"tool_calls\": tool_calls,\n", + " \"tool_definitions\": tool_definitions_multiple\n", + " }\n", + " )\n", + ")" ] }, { @@ -242,6 +327,7 @@ "metadata": {}, "outputs": [], "source": [ + "# Test Case 3: Tool calls passed as part of Response (common for agent case)\n", "query = \"Can you send me an email with weather information for Seattle?\"\n", "response = [\n", " {\n", @@ -303,7 +389,7 @@ " },\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_agent = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -337,8 +423,17 @@ " },\n", "]\n", "\n", - "result = tool_selection(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents (no separate tool_calls since they're in response)\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query,\n", + " \"response\": response,\n", + " \"tool_calls\": None,\n", + " \"tool_definitions\": tool_definitions_agent\n", + " }\n", + " )\n", + ")" ] }, { @@ -357,7 +452,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Query as conversation history instead of a single string\n", + "# Test Case 4: Query as conversation history instead of a single string\n", "query_as_conversation = [\n", " {\n", " \"role\": \"system\",\n", @@ -373,7 +468,7 @@ " }\n", "]\n", "\n", - "tool_calls = [\n", + "tool_calls_conversation = [\n", " {\n", " \"type\": \"tool_call\",\n", " \"tool_call_id\": \"call_weather_123\",\n", @@ -392,7 +487,7 @@ " },\n", "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_conversation = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -426,8 +521,17 @@ " },\n", "]\n", "\n", - "result = tool_selection(query=query_as_conversation, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_as_conversation,\n", + " \"response\": None,\n", + " \"tool_calls\": tool_calls_conversation,\n", + " \"tool_definitions\": tool_definitions_conversation\n", + " }\n", + " )\n", + ")" ] }, { @@ -445,7 +549,8 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", + "# Test Case 5: Example of poor tool selection\n", + "query_poor = \"How is the weather in Seattle?\"\n", "# Using irrelevant tool for the task\n", "poor_tool_calls = [\n", " {\n", @@ -457,8 +562,16 @@ "]\n", "\n", "# This should score poorly as get_calendar is not relevant for weather queries\n", - "result = tool_selection(query=query, tool_calls=poor_tool_calls, tool_definitions=tool_definitions)\n", - "pprint(result)" + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_poor,\n", + " \"response\": None,\n", + " \"tool_calls\": poor_tool_calls,\n", + " \"tool_definitions\": tool_definitions_conversation\n", + " }\n", + " )\n", + ")" ] }, { @@ -476,20 +589,23 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"What's the weather in New York?\"\n", + "# Test Case 6: Response as string\n", + "query_str = \"What's the weather in New York?\"\n", "\n", "# Response as a simple string (not commonly used, but supported by the API)\n", "# When using string response, tool_calls should be provided separately\n", "response_str = \"The current weather in New York is sunny with a temperature of 22°C.\"\n", "\n", - "tool_calls = {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_ny_weather_789\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"New York\"},\n", - "}\n", + "tool_calls_str = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_ny_weather_789\",\n", + " \"name\": \"fetch_weather\",\n", + " \"arguments\": {\"location\": \"New York\"},\n", + " }\n", + "]\n", "\n", - "tool_definitions = [\n", + "tool_definitions_str = [\n", " {\n", " \"name\": \"fetch_weather\",\n", " \"description\": \"Fetches the weather information for the specified location.\",\n", @@ -512,8 +628,17 @@ " },\n", "]\n", "\n", - "result = tool_selection(query=query, response=response_str, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(result)" + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_str,\n", + " \"response\": response_str,\n", + " \"tool_calls\": tool_calls_str,\n", + " \"tool_definitions\": tool_definitions_str\n", + " }\n", + " )\n", + ")" ] }, { @@ -531,31 +656,90 @@ "metadata": {}, "outputs": [], "source": [ - "query = \"How is the weather in Seattle?\"\n", + "# Test Case 7: Tool definition as single dict\n", + "query_dict = \"How is the weather in Seattle?\"\n", "\n", - "tool_definition_dict = {\n", - " \"name\": \"get_weather\",\n", - " \"description\": \"Get the current weather for a location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\"type\": \"string\", \"description\": \"The city and state, e.g. San Francisco, CA\"},\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]}\n", - " },\n", - " \"required\": [\"location\"]\n", + "tool_definition_dict = [\n", + " {\n", + " \"name\": \"get_weather\",\n", + " \"description\": \"Get the current weather for a location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\"type\": \"string\", \"description\": \"The city and state, e.g. San Francisco, CA\"},\n", + " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]}\n", + " },\n", + " \"required\": [\"location\"]\n", + " }\n", " }\n", - "}\n", + "]\n", "\n", - "tool_calls_dict = {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_abc123\",\n", - " \"name\": \"get_weather\",\n", - " \"arguments\": {\"location\": \"Seattle, WA\", \"unit\": \"celsius\"}\n", - "}\n", + "tool_calls_dict = [\n", + " {\n", + " \"type\": \"tool_call\",\n", + " \"tool_call_id\": \"call_abc123\",\n", + " \"name\": \"get_weather\",\n", + " \"arguments\": {\"location\": \"Seattle, WA\", \"unit\": \"celsius\"}\n", + " }\n", + "]\n", "\n", - "result = tool_selection(query=query, tool_definitions=tool_definition_dict, tool_calls=tool_calls_dict)\n", + "# Append to evaluation_contents\n", + "evaluation_contents.append(\n", + " SourceFileContentContent(\n", + " item={\n", + " \"query\": query_dict,\n", + " \"response\": None,\n", + " \"tool_calls\": tool_calls_dict,\n", + " \"tool_definitions\": tool_definition_dict\n", + " }\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "922df035", + "metadata": {}, + "source": [ + "### Run Evaluation on All Test Cases\n", "\n", - "pprint(result)" + "Now that we've defined all test cases, let's run the evaluation once on all of them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0266598", + "metadata": {}, + "outputs": [], + "source": [ + "results = run_evaluator(\n", + " evaluator_name=\"tool_selection\",\n", + " evaluation_contents=evaluation_contents,\n", + " data_source_config=data_source_config,\n", + " initialization_parameters=initialization_parameters,\n", + " data_mapping=data_mapping\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "eb2dc7d1", + "metadata": {}, + "source": [ + "### Display Results\n", + "\n", + "View the evaluation results for each test case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7929cda3", + "metadata": {}, + "outputs": [], + "source": [ + "pprint(results)" ] } ], diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb deleted file mode 100644 index ca8d8e3b..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Success.ipynb +++ /dev/null @@ -1,482 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "cea11cae", - "metadata": {}, - "source": [ - "# Tool Success Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use tool success evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings and `dict` describing agent responses;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." - ] - }, - { - "cell_type": "markdown", - "id": "49404a44", - "metadata": {}, - "source": [ - "The Tool Success evaluator determines whether tool calls done by an AI agent includes failures or not.\n", - "\n", - "This evaluator focuses solely on tool call results and tool definitions, disregarding user's query to the agent, conversation history and agent's final response. Although tool definitions is optional, providing them can help the evaluator better understand the context of the tool calls made by the agent. Please note that this evaluator validates tool calls for potential technical failures like errors, exceptions, timeouts and empty results (only in cases where empty results could indicate a failure). It does not assess the correctness or the tool result itself, like mathematical errors and unrealistic field values like name=\"668656\".\n", - "\n", - "Scoring is binary:\n", - "- TRUE: All tool calls were successful\n", - "- FALSE: At least one tool call failed\n", - "\n", - "This evaluation focuses on measuring the technical success of tool execution, not the semantic correctness of the results." - ] - }, - { - "cell_type": "markdown", - "id": "d91cb876", - "metadata": {}, - "source": [ - "Tool Success requires following input:\n", - "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator examines tool call results within the response.\n", - "- Tool Definitions - (Optional) Tool(s) definition used by Agent. Providing tool definitions helps the evaluator better understand the context of the tool calls made by the agent.\n" - ] - }, - { - "cell_type": "markdown", - "id": "e07d293d", - "metadata": {}, - "source": [ - "### Initialize Tool Success Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "919d1b9a", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation._evaluators._tool_success import _ToolSuccessEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "\n", - "tool_call_success = _ToolSuccessEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "4739878c", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "id": "2352536d", - "metadata": {}, - "source": [ - "#### Evaluating Successful Tool Calls" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9ddc2e22", - "metadata": {}, - "outputs": [], - "source": [ - "# Successful tool execution\n", - "successful_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " }\n", - "]\n", - "\n", - "result = tool_call_success(response=successful_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "c146de06", - "metadata": {}, - "source": [ - "#### Response as String (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "575d33eb", - "metadata": {}, - "outputs": [], - "source": [ - "# Response as a simple string containing tool call information\n", - "# This format is less common but still valid for the evaluator\n", - "response_str = \"\"\"Tool call executed: fetch_weather(location=\"Paris\")\n", - "Tool result: {\"temperature\": \"18°C\", \"condition\": \"clear sky\", \"humidity\": \"55%\"}\n", - "The weather in Paris is currently clear with a temperature of 18°C.\"\"\"\n", - "\n", - "tool_definition = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", - "\n", - "result = tool_call_success(response=response_str, tool_definitions=tool_definition)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "b7217ddc", - "metadata": {}, - "source": [ - "#### Tool Definition as Single Dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f89ca010", - "metadata": {}, - "outputs": [], - "source": [ - "# Successful tool execution with single tool definition dict\n", - "successful_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_nyc_weather\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"New York\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_nyc_weather\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"22°C\", \"condition\": \"sunny\", \"humidity\": \"45%\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The weather in New York is currently sunny with a temperature of 22°C.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definition_dict = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", - "\n", - "result = tool_call_success(response=successful_response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "1aeb8c11", - "metadata": {}, - "source": [ - "#### Multiple Successful Tool Calls" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "444ee59c", - "metadata": {}, - "outputs": [], - "source": [ - "# Multiple successful tool executions\n", - "multiple_successful_response = [\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\", \"precipitation\": \"light rain\"}}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"your_email@example.com\",\n", - " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"Current weather: 14°C and rainy with light rain.\",\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"role\": \"tool\",\n", - " \"content\": [\n", - " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\", \"status\": \"delivered\"}}\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I have successfully sent you an email with the weather information for Seattle.\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = tool_call_success(response=multiple_successful_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "63e43736", - "metadata": {}, - "source": [ - "#### Tool Call Success without Tool Definitions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a0ca062e", - "metadata": {}, - "outputs": [], - "source": [ - "# Successful execution without providing tool definitions\n", - "simple_successful_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_calendar_123\",\n", - " \"name\": \"get_calendar\",\n", - " \"arguments\": {\"start_date\": \"2025-01-01\", \"end_date\": \"2025-01-31\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_calendar_123\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"events\": [{\"date\": \"2025-01-15\", \"title\": \"Team Meeting\"}, {\"date\": \"2025-01-20\", \"title\": \"Project Review\"}]}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I found 2 events in your calendar for January: a Team Meeting on the 15th and a Project Review on the 20th.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "# Evaluation without tool definitions\n", - "result = tool_call_success(response=simple_successful_response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "97c1af0f", - "metadata": {}, - "source": [ - "#### Example of Failed Tool Call" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cafdee8e", - "metadata": {}, - "outputs": [], - "source": [ - "# Failed tool execution with error\n", - "failed_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_456\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"InvalidCity\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_456\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"error\": \"Location not found\", \"status\": \"failed\", \"code\": 404}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I'm sorry, I couldn't retrieve the weather information due to an error.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " }\n", - "]\n", - "\n", - "# This should score FALSE due to tool failure\n", - "result = tool_call_success(response=failed_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/agent_utils.py b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/agent_utils.py similarity index 86% rename from scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/agent_utils.py rename to scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/agent_utils.py index dcf75aee..4003784c 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/agent_utils.py +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/agent_utils.py @@ -23,7 +23,7 @@ def run_evaluator( data_source_config: dict, initialization_parameters: dict[str, str], data_mapping: dict[str, str], -) -> list: +) -> list:#list of type """ Run an evaluator on the provided evaluation contents. @@ -41,7 +41,6 @@ def run_evaluator( with DefaultAzureCredential() as credential: with AIProjectClient(endpoint=endpoint, credential=credential) as project_client: - print("Creating an OpenAI client from the AI Project client") client = project_client.get_openai_client() testing_criteria = [ @@ -54,15 +53,12 @@ def run_evaluator( } ] - print("Creating Eval Group") eval_object = client.evals.create( name=f"Test {evaluator_name} Evaluator with inline data", data_source_config=data_source_config, testing_criteria=testing_criteria, ) - print(f"Eval Group created with ID: {eval_object.id}") - print("Creating Eval Run with Inline Data") eval_run_object = client.evals.runs.create( eval_id=eval_object.id, name="inline_data_run", @@ -76,7 +72,6 @@ def run_evaluator( print(f"Eval Run created with ID: {eval_run_object.id}") print("Waiting for eval run to complete...") - # Poll for completion while True: run = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id) if run.status == "completed" or run.status == "failed": @@ -84,9 +79,6 @@ def run_evaluator( print(f"✓ Eval Run Status: {run.status}") print(f"📊 Eval Run Report URL: {run.report_url}") - # Extract just the output from each item - results = [item.output for item in output_items] - return results + return output_items time.sleep(5) - print(f" Status: {run.status}...") diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb deleted file mode 100644 index b662d23f..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Coherence.ipynb +++ /dev/null @@ -1,239 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "da22a6e7", - "metadata": {}, - "source": [ - "# Coherence Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use the Coherence evaluator to assess the quality and readability of AI-generated responses. The evaluator measures the ability of the language model to generate text that reads naturally, flows smoothly, and resembles human-like language.\n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." - ] - }, - { - "cell_type": "markdown", - "id": "48f8cb64", - "metadata": {}, - "source": [ - "The Coherence evaluator assesses the ability of the language model to generate text that reads naturally, flows smoothly, and resembles human-like language in its responses. It measures the readability and user-friendliness of the model's generated responses.\n", - "\n", - "Coherence scores range from 1 to 5:\n", - "\n", - "
\n",
-    "Score 1: Very Poor - The response is completely incoherent with no logical flow or connection between ideas.\n",
-    "Score 2: Poor - The response has significant coherence issues with disjointed ideas and unclear transitions.\n",
-    "Score 3: Fair - The response is somewhat coherent but has noticeable gaps in logical flow or organization.\n",
-    "Score 4: Good - The response is mostly coherent with good flow and logical connections between ideas.\n",
-    "Score 5: Excellent - The response is perfectly coherent with smooth, natural flow and clear logical progression.\n",
-    "
\n", - "\n", - "The evaluation requires one of the following input patterns:\n", - "\n", - "**Pattern 1: Query-Response Evaluation**\n", - "- Query: The user's question or prompt. (string)\n", - "- Response: The AI-generated response to be evaluated for coherence. (string)\n", - "\n", - "**Pattern 2: Conversation Evaluation**\n", - "- Conversation: A multi-turn conversation where each turn is evaluated for coherence. (dict with \"messages\" key)" - ] - }, - { - "cell_type": "markdown", - "id": "542c1340", - "metadata": {}, - "source": [ - "### Initialize Coherence Evaluator" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35799119", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation import CoherenceEvaluator, AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"type\": \"string\"\n", - " },\n", - " \"response\": {\n", - " \"type\": \"string\"\n", - " }\n", - " },\n", - " \"required\": []\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "coherence_evaluator = CoherenceEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "b10aff00", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "id": "724adfc8", - "metadata": {}, - "source": [ - "#### Query and Response as Strings (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82a1be93", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you explain how machine learning works?\"\n", - "response = \"Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed. It works by analyzing data patterns to make predictions or decisions. First, the system is trained on a dataset, which allows it to identify patterns and relationships. Then, it uses these learned patterns to make predictions on new, unseen data. The accuracy improves over time as the model processes more data and refines its understanding.\"\n", - "\n", - "result = coherence_evaluator(query=query, response=response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "bc250127", - "metadata": {}, - "source": [ - "#### Example of Poor Coherence" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "62ad5478", - "metadata": {}, - "outputs": [], - "source": [ - "# Poor coherence example\n", - "query = \"What is blockchain technology?\"\n", - "response = \"Blockchain is decentralized. Bitcoin uses it. Blocks contain data. Cryptography is involved. Banks might use blockchain. It's secure because of the chain. Mining is how new blocks are added. Digital currencies exist. The ledger is distributed.\"\n", - "\n", - "result = coherence_evaluator(query=query, response=response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "1d22e3d6", - "metadata": {}, - "source": [ - "#### Conversation as Dict with Messages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "556c197b", - "metadata": {}, - "outputs": [], - "source": [ - "# Multi-turn conversation\n", - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"I'm planning a trip to Japan. What should I know?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Japan is a fascinating destination with a rich cultural heritage. First, you should know that the best times to visit are spring (March-May) for cherry blossoms and fall (September-November) for autumn colors. The country has an excellent public transportation system, particularly the Shinkansen bullet train network. Additionally, learning a few basic Japanese phrases will be appreciated by locals, though many signs in major cities have English translations.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What about the food?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Japanese cuisine is incredibly diverse and delicious. Beyond the well-known sushi and ramen, you should try regional specialties like okonomiyaki in Osaka or motsunabe in Fukuoka. Most restaurants display plastic food models in their windows, which makes ordering easier even with a language barrier. It's also important to note that tipping is not customary in Japan and can sometimes be considered rude.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Are there any cultural customs I should be aware of?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Yes, there are several important customs to observe. Always remove your shoes when entering homes, temples, and some traditional restaurants. Bowing is a common greeting, though a slight nod is acceptable for foreigners. When using chopsticks, never stick them upright in rice as this resembles a funeral ritual. Finally, be mindful of noise levels on public transportation, as talking on phones or having loud conversations is considered impolite.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = coherence_evaluator(conversation=conversation)\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb deleted file mode 100644 index c015da44..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Fluency.ipynb +++ /dev/null @@ -1,316 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "21599c07", - "metadata": {}, - "source": [ - "# Fluency Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use the Fluency evaluator to assess the linguistic quality of AI-generated responses. The evaluator measures how well generated text conforms to grammatical rules, syntactic structures, and appropriate vocabulary usage.\n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." - ] - }, - { - "cell_type": "markdown", - "id": "7add5ec1", - "metadata": {}, - "source": [ - "The Fluency evaluator assesses the extent to which generated text conforms to grammatical rules, syntactic structures, and appropriate vocabulary usage, resulting in linguistically correct responses.\n", - "\n", - "Fluency scores range from 1 to 5:\n", - "\n", - "
\n",
-    "Score 1: Very Poor - The response is incomprehensible with severe grammatical errors and improper vocabulary.\n",
-    "Score 2: Poor - The response has frequent grammatical errors and awkward phrasing that hinder understanding.\n",
-    "Score 3: Fair - The response is understandable but contains noticeable grammatical errors or awkward expressions.\n",
-    "Score 4: Good - The response is mostly fluent with minor grammatical issues that don't significantly impact readability.\n",
-    "Score 5: Excellent - The response is perfectly fluent with proper grammar, syntax, and vocabulary usage.\n",
-    "
\n", - "\n", - "The evaluation requires one of the following input patterns:\n", - "\n", - "**Pattern 1: Single Response Evaluation**\n", - "- Response: The text response to be evaluated for fluency. (string)\n", - "\n", - "**Pattern 2: Conversation Evaluation**\n", - "- Conversation: A multi-turn conversation where each turn is evaluated for fluency. (List[dict])" - ] - }, - { - "cell_type": "markdown", - "id": "2abd4d05", - "metadata": {}, - "source": [ - "### Initialize Fluency Evaluator" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bfd097f8", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation import FluencyEvaluator, AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"type\": \"string\"\n", - " },\n", - " \"response\": {\n", - " \"type\": \"string\"\n", - " }\n", - " },\n", - " \"required\": []\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "fluency_evaluator = FluencyEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "4793759a", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "id": "46dc8414", - "metadata": {}, - "source": [ - "#### Response as String (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3c2df39a", - "metadata": {}, - "outputs": [], - "source": [ - "response = \"The weather in Seattle is currently partly cloudy with a temperature of 15°C. The forecast indicates that conditions will remain stable throughout the day, with a gentle breeze from the northwest.\"\n", - "\n", - "result = fluency_evaluator(response=response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "59709c65", - "metadata": {}, - "source": [ - "#### Example of Poor Fluency" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d9c624ab", - "metadata": {}, - "outputs": [], - "source": [ - "# Poor fluency example\n", - "response = \"Email draft attach is. You review and giving feedback must. Important very for project success it being.\"\n", - "\n", - "result = fluency_evaluator(response=response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "b26dd4bb", - "metadata": {}, - "source": [ - "#### Conversation as Dict with Messages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9ccaff56", - "metadata": {}, - "outputs": [], - "source": [ - "# Multi-turn conversation\n", - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Can you help me understand the weather forecast for this week?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Of course! I'd be happy to help you understand this week's weather forecast. The outlook shows a mix of sunny and cloudy conditions.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What about the temperature?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Temperatures will ranging from 12°C to 18°C throughout the week, with the warmest days being Wednesday and Thursday.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Should I bring an umbrella?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Yes, it would be advisable to carry an umbrella, particularly on Friday when there is a 60% chance of precipitation.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = fluency_evaluator(conversation=conversation)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "0ba89e73", - "metadata": {}, - "source": [ - "#### Conversation as List[dict] - Agent with Tool Calls" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2658f610", - "metadata": {}, - "outputs": [], - "source": [ - "# Conversation with tool calls - evaluates fluency of assistant responses\n", - "conversation_with_tools = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Can you send me an email with the weather report for Seattle?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'll fetch the current weather information for Seattle and send you an email with the details.\"\n", - " },\n", - " {\n", - " \"role\": \"tool\",\n", - " \"content\": \"Temperature: 15°C, Condition: Partly cloudy, Wind: 8 mph NW\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I have successfully sent you an email containing the weather report for Seattle. The current conditions are partly cloudy with a temperature of 15°C and light winds from the northwest.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = fluency_evaluator(conversation=conversation_with_tools)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "ff71d193", - "metadata": {}, - "source": [ - "#### Conversation as List[dict] - Mixed Fluency Quality" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c0d8bf46", - "metadata": {}, - "outputs": [], - "source": [ - "# Conversation with varying fluency levels across turns\n", - "conversation_mixed = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What's the status of my order?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Your order is being process currently and will be ship very soon to you.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"When will it arrive?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The estimated delivery date for your order is March 15, 2025. You will receive tracking information via email once the package has been dispatched.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = fluency_evaluator(conversation=conversation_mixed)\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb deleted file mode 100644 index 27188e03..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Groundedness.ipynb +++ /dev/null @@ -1,398 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "6a3f10c0", - "metadata": {}, - "source": [ - "# Groundedness Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use the Groundedness evaluator to assess whether AI-generated responses are grounded in the provided context. The evaluator supports multiple input formats including:\n", - "- Simple response and context evaluation\n", - "- Query, response, and context evaluation\n", - "- Agent responses with tool calls (file_search)\n", - "- Multi-turn conversations\n", - "\n", - "This notebook uses consistent examples with the ToolCallAccuracyEvaluator for better comparison across evaluators.\n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook.\n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities.\n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure OpenAI Endpoint to be used for evaluation\n", - "3) **AZURE_OPENAI_API_KEY** - Azure OpenAI Key to be used for evaluation\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure OpenAI API version to be used for evaluation" - ] - }, - { - "cell_type": "markdown", - "id": "fe3af2a1", - "metadata": {}, - "source": [ - "## What is Groundedness?\n", - "\n", - "The Groundedness evaluator assesses the correspondence between claims in an AI-generated response and the source context. It ensures that responses are substantiated by the provided context, preventing hallucinations and unsupported claims.\n", - "\n", - "**Key Points:**\n", - "- Even factually correct responses are considered ungrounded if they can't be verified against the provided context\n", - "- Essential for RAG (Retrieval-Augmented Generation) applications\n", - "- Helps ensure AI responses are trustworthy and verifiable\n", - "\n", - "**Scoring:** Groundedness scores range from 1 to 5, with:\n", - "- **1**: Completely ungrounded - no claims supported by context\n", - "- **2**: Mostly ungrounded - few claims supported\n", - "- **3**: Partially grounded - some claims supported\n", - "- **4**: Mostly grounded - most claims supported\n", - "- **5**: Fully grounded - all claims supported by context" - ] - }, - { - "cell_type": "markdown", - "id": "bd8a00fb", - "metadata": {}, - "source": [ - "## Groundedness Evaluator Input Requirements\n", - "\n", - "The Groundedness evaluator supports multiple input formats:\n", - "\n", - "1. **Basic Context Evaluation:**\n", - " - `response`: The AI response to evaluate (str)\n", - " - `context`: The source context/documents (str)\n", - " - `query`: Optional query for enhanced evaluation (str)\n", - "\n", - "2. **Agent Tool Evaluation:**\n", - " - `query`: The user query (str)\n", - " - `response`: Agent response with tool calls (List[dict])\n", - " - `tool_definitions`: Available tools, only file_search supported (List[dict])\n", - "\n", - "3. **Conversation Evaluation:**\n", - " - `conversation`: Multi-turn conversation with context (Conversation object)" - ] - }, - { - "cell_type": "markdown", - "id": "9aa68c67", - "metadata": {}, - "source": [ - "### Initialize Groundedness Evaluator" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "298d067a", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation import GroundednessEvaluator, AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "# Configure the model\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"context\": {\n", - " \"type\": \"string\"\n", - " },\n", - " \"query\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"response\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_definitions\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " },\n", - " \"required\": [\"response\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"context\": \"{{item.context}}\",\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\",\n", - " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "# Initialize the evaluator\n", - "groundedness_evaluator = GroundednessEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "18b0a169", - "metadata": {}, - "source": [ - "## Sample Evaluations" - ] - }, - { - "cell_type": "markdown", - "id": "b73b1ccd", - "metadata": {}, - "source": [ - "### Response and Context as Strings (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4e5fa898", - "metadata": {}, - "outputs": [], - "source": [ - "# Example of a well-grounded response using weather context\n", - "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", - "The forecast indicates overcast skies with light precipitation typical for Pacific Northwest weather. \n", - "Humidity is at 85% with winds from the southwest at 12 mph. Visibility is reduced to 8 miles due to rain.\"\"\"\n", - "\n", - "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", - "\n", - "result = groundedness_evaluator(response=response, context=context)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "0c03f745", - "metadata": {}, - "source": [ - "### Example of Ungrounded Response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6de6a51e", - "metadata": {}, - "outputs": [], - "source": [ - "# Example of ungrounded response with unsupported claims\n", - "context = \"\"\"Current weather data for Seattle shows rainy conditions with a temperature of 14°C. \n", - "The forecast indicates overcast skies with light precipitation typical for Pacific Northwest weather.\"\"\"\n", - "\n", - "response = \"The current weather in Seattle is rainy with a temperature of 14°C. The city is experiencing its wettest month in 50 years, and the mayor has declared a weather emergency due to flooding concerns.\"\n", - "\n", - "result = groundedness_evaluator(response=response, context=context)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "9052eb13", - "metadata": {}, - "source": [ - "### Query, Response, and Context as Strings (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c77316c4", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "\n", - "context = \"\"\"Weather report for Seattle, Washington: Currently experiencing rainy weather with temperature at 14°C. \n", - "Overcast conditions with light rain are expected to continue. The current conditions are typical for the Pacific Northwest region \n", - "during this season. Wind speed is moderate at 12 mph from southwest direction.\"\"\"\n", - "\n", - "response = \"The weather in Seattle is rainy with a temperature of 14°C. These are typical Pacific Northwest conditions with overcast skies.\"\n", - "\n", - "result = groundedness_evaluator(response=response, context=context, query=query)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "fd0a420f", - "metadata": {}, - "source": [ - "### Query as String (str), Response as List[dict], Tool Definitions as List[dict]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72b922ee", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you get me the current weather information for Seattle?\"\n", - "\n", - "# Agent response with file_search tool call\n", - "agent_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_filesearch_weather_123\",\n", - " \"name\": \"file_search\",\n", - " \"arguments\": {\"query\": \"current weather Seattle temperature conditions\"}\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"role\": \"tool\",\n", - " \"tool_call_id\": \"call_filesearch_weather_123\", \n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_result\",\n", - " \"tool_result\": {\n", - " \"content\": \"Seattle weather report: Currently rainy with temperature of 14°C. Overcast skies with light precipitation. Typical Pacific Northwest weather with 85% humidity and southwest winds at 12 mph.\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Based on the weather data, Seattle is currently experiencing rainy weather with a temperature of 14°C. The conditions include overcast skies and light precipitation, which is typical for the Pacific Northwest.\"\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"file_search\",\n", - " \"description\": \"Search through uploaded files to find relevant information\",\n", - " \"parameters\": {\n", - " \"type\": \"object\", \n", - " \"properties\": {\n", - " \"query\": {\"type\": \"string\", \"description\": \"Search query\"}\n", - " }\n", - " }\n", - " }\n", - "]\n", - "\n", - "result = groundedness_evaluator(\n", - " query=query, \n", - " response=agent_response, \n", - " tool_definitions=tool_definitions\n", - ")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "6dc31ae0", - "metadata": {}, - "source": [ - "### Conversation as Dict with Context and Messages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0f37ae51", - "metadata": {}, - "outputs": [], - "source": [ - "conversation = {\n", - " \"context\": \"Weather data shows Seattle currently has rainy conditions at 14°C with overcast skies. London shows cloudy weather at 8°C with partly cloudy conditions. Both cities are experiencing typical seasonal weather patterns.\",\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Can you check the weather in Seattle for me?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"According to the current weather data, Seattle is experiencing rainy conditions with a temperature of 14°C and overcast skies.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"How does that compare to London?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"London is currently cloudier but drier than Seattle, with a temperature of 8°C and partly cloudy conditions. Seattle is warmer but rainier at 14°C.\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"Can you email me a summary of both cities' weather?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I can provide you with a weather summary: Seattle has rainy weather at 14°C with overcast skies, while London has partly cloudy conditions at 8°C. However, I would need email access to send this information to you.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = groundedness_evaluator(conversation=conversation)\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb deleted file mode 100644 index 4d3d2ad2..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb +++ /dev/null @@ -1,405 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Intent Resolution Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates to how to use intent resolution evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The Intent Resolution evaluator measures how well an agent has identified and resolved the user intent.\n", - "The scoring is on a 1-5 integer scale and is as follows:\n", - "\n", - " - Score 1: Response completely unrelated to user intent\n", - " - Score 2: Response minimally relates to user intent\n", - " - Score 3: Response partially addresses the user intent but lacks complete details\n", - " - Score 4: Response addresses the user intent with moderate accuracy but has minor inaccuracies or omissions\n", - " - Score 5: Response directly addresses the user intent and fully resolves it\n", - "\n", - "The evaluation requires the following inputs:\n", - "\n", - " - Query : The user query. Either a string with a user request or a list of messages with previous requests from the user and responses from the assistant, potentially including a system message.\n", - " - Response : The response to be evaluated. Either a string or a message with the response from the agent to the last user query.\n", - "\n", - "There is a third optional parameter:\n", - " - ToolDefinitions : The list of tool definitions the agent can call. This may be useful for the evaluator to better assess if the right tool was called to resolve a given intent." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Initialize Intent Resolution Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", - "from azure.ai.evaluation import IntentResolutionEvaluator\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"response\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_definitions\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " },\n", - " \"required\": [\"query\", \"response\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\",\n", - " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "intent_resolution_evaluator = IntentResolutionEvaluator(model_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Evaluating query and response as string" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Success example. Intent is identified and understood and the response correctly resolves user intent\n", - "result = intent_resolution_evaluator(\n", - " query=\"What are the opening hours of the Eiffel Tower?\",\n", - " response=\"Opening hours of the Eiffel Tower are 9:00 AM to 11:00 PM.\",\n", - ")\n", - "pprint(result)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Failure example. Even though intent is correctly identified, the response does not resolve the user intent\n", - "result = intent_resolution_evaluator(\n", - " query=\"What is the opening hours of the Eiffel Tower?\",\n", - " response=\"Please check the official website for the up-to-date information on Eiffel Tower opening hours.\",\n", - ")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Query and Response as String with Tool Definition as Single Dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example with a single tool definition dict instead of a list\n", - "query = \"What's the status of order #789?\"\n", - "response = \"Order #789 has been shipped and will arrive on March 18, 2025.\"\n", - "\n", - "tool_definition_dict = {\n", - " \"name\": \"get_order\",\n", - " \"description\": \"Get the details of a specific order.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"order_id\": {\"type\": \"string\", \"description\": \"The order ID to get the details for.\"}},\n", - " },\n", - "}\n", - "\n", - "result = intent_resolution_evaluator(\n", - " query=query,\n", - " response=response,\n", - " tool_definitions=tool_definition_dict,\n", - ")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Evaluating query and response as list of messages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = [\n", - " {\"role\": \"system\", \"content\": \"You are a friendly and helpful customer service agent.\"},\n", - " {\n", - " \"createdAt\": \"2025-03-14T06:14:20Z\",\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"Hi, I need help with the last 2 orders on my account #888. Could you please update me on their status?\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "response = [\n", - " {\n", - " \"createdAt\": \"2025-03-14T06:14:30Z\",\n", - " \"run_id\": \"0\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"text\", \"text\": \"Hello! Let me quickly look up your account details.\"}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T06:14:35Z\",\n", - " \"run_id\": \"0\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"tool_call_20250310_001\",\n", - " \"name\": \"get_orders\",\n", - " \"arguments\": {\"account_number\": \"888\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T06:14:40Z\",\n", - " \"run_id\": \"0\",\n", - " \"tool_call_id\": \"tool_call_20250310_001\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": '[{ \"order_id\": \"123\" }, { \"order_id\": \"124\" }]'}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T06:14:45Z\",\n", - " \"run_id\": \"0\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"Thanks for your patience. I see two orders on your account. Let me fetch the details for both.\",\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T06:14:50Z\",\n", - " \"run_id\": \"0\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"tool_call_20250310_002\",\n", - " \"name\": \"get_order\",\n", - " \"arguments\": {\"order_id\": \"123\"},\n", - " },\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"tool_call_20250310_003\",\n", - " \"name\": \"get_order\",\n", - " \"arguments\": {\"order_id\": \"124\"},\n", - " },\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T06:14:55Z\",\n", - " \"run_id\": \"0\",\n", - " \"tool_call_id\": \"tool_call_20250310_002\",\n", - " \"role\": \"tool\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_result\",\n", - " \"tool_result\": '{ \"order\": { \"id\": \"123\", \"status\": \"shipped\", \"delivery_date\": \"2025-03-15\" } }',\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T06:15:00Z\",\n", - " \"run_id\": \"0\",\n", - " \"tool_call_id\": \"tool_call_20250310_003\",\n", - " \"role\": \"tool\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_result\",\n", - " \"tool_result\": '{ \"order\": { \"id\": \"124\", \"status\": \"delayed\", \"expected_delivery\": \"2025-03-20\" } }',\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T06:15:05Z\",\n", - " \"run_id\": \"0\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The order with ID 123 has been shipped and is expected to be delivered on March 15, 2025. However, the order with ID 124 is delayed and should now arrive by March 20, 2025. Is there anything else I can help you with?\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "# please note that the tool definitions are not strictly required, and that some of the tools below are not used in the example above and that is ok.\n", - "# if context length is a concern you can remove the unused tool definitions or even the tool definitions altogether as the impact to the intent resolution evaluation is usual minimal.\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"get_orders\",\n", - " \"description\": \"Get the list of orders for a given account number.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"account_number\": {\"type\": \"string\", \"description\": \"The account number to get the orders for.\"}\n", - " },\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"get_order\",\n", - " \"description\": \"Get the details of a specific order.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"order_id\": {\"type\": \"string\", \"description\": \"The order ID to get the details for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"initiate_return\",\n", - " \"description\": \"Initiate the return process for an order.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"order_id\": {\"type\": \"string\", \"description\": \"The order ID for the return process.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"update_shipping_address\",\n", - " \"description\": \"Update the shipping address for a given account.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"account_number\": {\"type\": \"string\", \"description\": \"The account number to update.\"},\n", - " \"new_address\": {\"type\": \"string\", \"description\": \"The new shipping address.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = intent_resolution_evaluator(\n", - " query=query,\n", - " response=response,\n", - " tool_definitions=tool_definitions,\n", - ")\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb deleted file mode 100644 index fd0811ad..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Relevance.ipynb +++ /dev/null @@ -1,241 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "bb0675a7", - "metadata": {}, - "source": [ - "# Relevance Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use the Relevance evaluator to assess the relevance of AI-generated responses. The evaluator supports multiple input formats including:\n", - "- Simple query and response pairs\n", - "- Multi-turn conversations\n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 15 minutes running this notebook.\n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities.\n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure OpenAI Endpoint to be used for evaluation\n", - "3) **AZURE_OPENAI_API_KEY** - Azure OpenAI Key to be used for evaluation\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure OpenAI API version to be used for evaluation\n" - ] - }, - { - "cell_type": "markdown", - "id": "1a21aa7e", - "metadata": {}, - "source": [ - "## What is Relevance?\n", - "\n", - "The Relevance evaluator assesses the ability of AI responses to capture the key points and address the user's query appropriately. It measures how well the response relates to and answers the specific question or request.\n", - "\n", - "**Scoring:** Relevance scores range from 1 to 5, with:\n", - "- **1**: Completely irrelevant to the query\n", - "- **2**: Partially relevant but misses key points\n", - "- **3**: Moderately relevant with some key points addressed\n", - "- **4**: Highly relevant with most key points covered\n", - "- **5**: Perfectly relevant, fully addressing the query\n", - "\n", - "High relevance scores indicate the AI system's understanding of the input and capability to produce contextually appropriate outputs." - ] - }, - { - "cell_type": "markdown", - "id": "f0dd0ebf", - "metadata": {}, - "source": [ - "## Relevance Evaluator Input Requirements\n", - "\n", - "The Relevance evaluator supports the following input formats:\n", - "\n", - "1. **Simple Query-Response Evaluation:**\n", - " - `query`: The user's question or request (str)\n", - " - `response`: The AI's response to evaluate (str)\n", - "\n", - "2. **Conversation Evaluation:**\n", - " - `conversation`: Multi-turn conversation with messages list and optional context (Conversation object)\n" - ] - }, - { - "cell_type": "markdown", - "id": "519631da", - "metadata": {}, - "source": [ - "### Initialize Relevance Evaluator" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3484e790", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation import RelevanceEvaluator, AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "# Configure the model\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"type\": \"string\"\n", - " },\n", - " \"response\": {\n", - " \"type\": \"string\"\n", - " }\n", - " },\n", - " \"required\": [\"query\", \"response\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "# Initialize the evaluator\n", - "relevance_evaluator = RelevanceEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "4f739700", - "metadata": {}, - "source": [ - "## Sample Evaluations" - ] - }, - { - "cell_type": "markdown", - "id": "83d8d0ec", - "metadata": {}, - "source": [ - "### Query and Response as Strings (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "43eaf25d", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "response = \"The current weather in Seattle is rainy with a temperature of 14°C. It's typical Pacific Northwest weather for this time of year with overcast skies and light precipitation.\"\n", - "\n", - "result = relevance_evaluator(query=query, response=response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "89266048", - "metadata": {}, - "source": [ - "### Example of Irrelevant Response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "69761825", - "metadata": {}, - "outputs": [], - "source": [ - "# Irrelevant response example\n", - "query = \"What are the symptoms of flu?\"\n", - "response = \"The stock market had significant fluctuations today, with technology stocks leading the decline.\"\n", - "\n", - "result = relevance_evaluator(query=query, response=response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "bd158e43", - "metadata": {}, - "source": [ - "### Conversation as Dict with Messages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cf17774b", - "metadata": {}, - "outputs": [], - "source": [ - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"I'm planning a trip to Japan. Can you help me with some recommendations?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'd be happy to help you plan your trip to Japan! Here are some popular destinations: Tokyo for modern city life and culture, Kyoto for traditional temples and gardens, Osaka for amazing food, and Mount Fuji for scenic beauty. What type of experiences are you most interested in?\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"I'm particularly interested in traditional culture and temples.\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Perfect! For traditional culture and temples, I highly recommend Kyoto. Visit Fushimi Inari Shrine with its thousands of red torii gates, Kinkaku-ji (Golden Pavilion), and Gion district where you might spot geishas. Nara is also wonderful with its deer park and Todai-ji Temple housing a giant Buddha statue.\"\n", - " }\n", - " ]\n", - "}\n", - "\n", - "result = relevance_evaluator(conversation=conversation)\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb deleted file mode 100644 index de5587e4..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Response_Completeness.ipynb +++ /dev/null @@ -1,233 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Response Completeness Evaluator" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Getting Started\n", - "\n", - "This sample demonstrates how to use Response Completeness Evaluator on agent's response when ground truth is provided. This evaluator is helpful when you have ground truth to assess the quality of the agent's final response. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The Response Completeness evaluator assesses the quality of an agent response by examining how well it aligns with the provided ground truth. The evaluation is based on the following scoring system:\n", - "\n", - "
\n",
-    "Score 1: Fully incomplete: The response misses all necessary and relevant information compared to the ground truth.\n",
-    "Score 2: Barely complete: The response contains only a small percentage of the necessary information.\n",
-    "Score 3: Moderately complete: The response includes about half of the necessary information.\n",
-    "Score 4: Mostly complete: The response contains most of the necessary information, with only minor omissions.\n",
-    "Score 5: Fully complete: The response perfectly matches all necessary and relevant information from the ground truth.\n",
-    "
\n", - "\n", - "The evaluation requires the following inputs:\n", - "\n", - "- Response: The response to be evaluated. (string)\n", - "- Ground Truth: The correct and complete information against which the response is compared. (string)\n", - "\n", - "The evaluator uses these inputs to determine the completeness score, ensuring that the response meaningfully addresses the query while adhering to the provided definitions and data." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Initialize Completeness Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.evaluation import ResponseCompletenessEvaluator, AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "import os\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"ground_truth\": {\n", - " \"type\": \"string\"\n", - " },\n", - " \"response\": {\n", - " \"type\": \"string\"\n", - " }\n", - " },\n", - " \"required\": [\"ground_truth\", \"response\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"ground_truth\": \"{{item.ground_truth}}\",\n", - " \"response\": \"{{item.response}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Response and Ground Truth as Strings (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result = response_completeness_evaluator(\n", - " response=\"The capital of Japan\",\n", - " ground_truth=\"The capital of Japan is Tokyo.\"\n", - ")\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result = response_completeness_evaluator(\n", - " response=\"The capital of Japan is Tokyo.\",\n", - " ground_truth=\"The capital of Japan is Tokyo.\"\n", - ")\n", - "result" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Evaluate with a reasoning model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.evaluation import ResponseCompletenessEvaluator , AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "# set is_reasoning_model to True in case the model is a reasoning model (ex: o3-mini, o1-preview)\n", - "response_completeness_evaluator = ResponseCompletenessEvaluator(model_config=model_config,\n", - " is_reasoning_model=False)\n", - "\n", - "result = response_completeness_evaluator(\n", - " response=\"The capital of Japan is Tokyo.\",\n", - " ground_truth=\"The capital of Japan is Tokyo.\"\n", - ")\n", - "result" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Conversation as Dict with Ground Truth in Context" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Conversation format with ground truth in context\n", - "conversation = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"What's the weather forecast for Seattle this weekend?\"\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday.\",\n", - " \"ground_truth\": \"The weather in Seattle this weekend will be partly cloudy with temperatures around 15°C on Saturday and 17°C on Sunday, with a 20% chance of rain on Sunday afternoon.\"\n", - " }\n", - " ],\n", - " \n", - "}\n", - "\n", - "result = response_completeness_evaluator(conversation=conversation)\n", - "result" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb deleted file mode 100644 index 8389fa34..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb +++ /dev/null @@ -1,322 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Task Adherence Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates to how to use task adherence evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 10 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The Task Adherence evaluator measures how well the agent adheres to their assigned tasks or predefined goal.\n", - "\n", - "The scoring is on a 1-5 integer scale and is as follows:\n", - "\n", - " - Score 1: Fully Inadherent\n", - " - Score 2: Barely Adherent\n", - " - Score 3: Moderately Adherent\n", - " - Score 4: Mostly Adherent\n", - " - Score 5: Fully Adherent\n", - "\n", - "The evaluation requires the following inputs:\n", - "\n", - " - Query : The user query. Either a string with a user request or a list of messages with previous requests from the user and responses from the assistant, potentially including a system message.\n", - " - Response : The response to be evaluated. Either a string or a message with the response from the agent to the last user query.\n", - "\n", - "There is a third optional parameter:\n", - " - ToolDefinitions : The list of tool definitions the agent can call. This may be useful for the evaluator to better assess if the right tool was called to adhere to user intent." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Initialize Task Adherence Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation import TaskAdherenceEvaluator, AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"response\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_definitions\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " },\n", - " \"required\": [\"query\", \"response\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\",\n", - " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "task_adherence_evaluator = TaskAdherenceEvaluator(model_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Evaluating query and response as string" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Failure example, there's only a vague adherence to the task\n", - "result = task_adherence_evaluator(\n", - " query=\"What are the best practices for maintaining a healthy rose garden during the summer?\",\n", - " response=\"Make sure to water your roses regularly and trim them occasionally.\",\n", - ")\n", - "pprint(result)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Success example, full adherence to the task\n", - "result = task_adherence_evaluator(\n", - " query=\"What are the best practices for maintaining a healthy rose garden during the summer?\",\n", - " response=\"For optimal summer care of your rose garden, start by watering deeply early in the morning to ensure the roots are well-hydrated without encouraging fungal growth. Apply a 2-3 inch layer of organic mulch around the base of the plants to conserve moisture and regulate soil temperature. Fertilize with a balanced rose fertilizer every 4 to 6 weeks to support healthy growth. Prune away any dead or diseased wood to promote good air circulation, and inspect regularly for pests such as aphids or spider mites, treating them promptly with an appropriate organic insecticidal soap. Finally, ensure that your roses receive at least 6 hours of direct sunlight daily for robust flowering.\",\n", - ")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Query and Response as String with Tool Definition as Single Dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example with a single tool definition dict\n", - "query = \"Can you recommend a science fiction book for me?\"\n", - "response = \"I found 'Dune' by Frank Herbert for you. It's a classic science fiction novel set in a distant future.\"\n", - "\n", - "tool_definition_dict = {\n", - " \"name\": \"get_book\",\n", - " \"description\": \"Retrieve a book recommendation for a specified genre.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"genre\": {\"type\": \"string\", \"description\": \"The genre for which a book recommendation is requested.\"}\n", - " },\n", - " },\n", - "}\n", - "\n", - "result = task_adherence_evaluator(\n", - " query=query,\n", - " response=response,\n", - " tool_definitions=tool_definition_dict,\n", - ")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Evaluating query and response as list of messages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = [\n", - " {\"role\": \"system\", \"content\": \"You are an expert in literature and at provid can provide book recommendations.\"},\n", - " {\n", - " \"createdAt\": \"2025-03-14T08:00:00Z\",\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"text\", \"text\": \"I love historical fiction. Can you recommend a good book from that genre?\"}\n", - " ],\n", - " },\n", - "]\n", - "\n", - "response = [\n", - " {\n", - " \"createdAt\": \"2025-03-14T08:00:05Z\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"text\", \"text\": \"Let me fetch a recommendation for historical fiction.\"}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T08:00:10Z\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"tool_call_20250314_001\",\n", - " \"name\": \"get_book\",\n", - " \"arguments\": {\"genre\": \"historical fiction\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T08:00:15Z\",\n", - " \"role\": \"tool\",\n", - " \"tool_call_id\": \"tool_call_20250314_001\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_result\",\n", - " \"tool_result\": '{ \"book\": { \"title\": \"The Pillars of the Earth\", \"author\": \"Ken Follett\", \"summary\": \"A captivating tale set in medieval England that weaves historical events with personal drama.\" } }',\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-14T08:00:20Z\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"Based on our records, I recommend 'The Pillars of the Earth' by Ken Follett. This novel is an excellent example of historical fiction with a rich narrative and well-developed characters. Would you like more details or another suggestion?\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"get_book\",\n", - " \"description\": \"Retrieve a book recommendation for a specified genre.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"genre\": {\"type\": \"string\", \"description\": \"The genre for which a book recommendation is requested.\"}\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "result = task_adherence_evaluator(\n", - " query=query,\n", - " response=response,\n", - " tool_definitions=tool_definitions,\n", - ")\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb deleted file mode 100644 index 51d2460e..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Completion.ipynb +++ /dev/null @@ -1,415 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "cd88c9a4", - "metadata": {}, - "source": [ - "# Task Completion Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use task completion evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings and `dict` describing task responses;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." - ] - }, - { - "cell_type": "markdown", - "id": "fe5dda1c", - "metadata": {}, - "source": [ - "The Task Completion evaluator assesses whether an AI agent successfully completes the requested task by examining:\n", - "- Whether the task was fully completed\n", - "- Quality of task execution\n", - "- Appropriateness of the response to the original request\n", - "\n", - "The evaluator uses a binary scoring system (0 or 1):\n", - "\n", - " - Score 0: The task was not completed or only partially completed\n", - " - Score 1: The task was successfully and fully completed\n", - "\n", - "This evaluation focuses on measuring whether the agent's response indicates successful completion of the user's request, regardless of the specific methods or tools used to achieve the task." - ] - }, - { - "cell_type": "markdown", - "id": "c1642178", - "metadata": {}, - "source": [ - "Task Completion requires following input:\n", - "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", - "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response.\n", - "- Tool Definitions - (Optional) Tool(s) definition used by Agent to answer the query. Providing tool definitions helps the evaluator better understand the context and capabilities available to the agent.\n" - ] - }, - { - "cell_type": "markdown", - "id": "ffad775f", - "metadata": {}, - "source": [ - "### Initialize Task Completion Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0a5fae33", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation._evaluators._task_completion import _TaskCompletionEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"response\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_definitions\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " },\n", - " \"required\": [\"query\", \"response\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\",\n", - " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "task_completion = _TaskCompletionEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "af4839bb", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "id": "95305bbd", - "metadata": {}, - "source": [ - "#### Evaluating Simple Task Completion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57d4592d", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "response = \"The current weather in Seattle is partly cloudy with a temperature of 15°C (59°F). There's a light breeze from the northwest at 8 mph, and the humidity is at 68%. No precipitation is expected for the rest of the day.\"\n", - "\n", - "# Basic evaluation without tool definitions\n", - "result = task_completion(query=query, response=response)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "2c66a362", - "metadata": {}, - "source": [ - "#### Task Completion with Tool Context" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "71ea8725", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "response = \"I've checked the weather for Seattle and found that it's currently partly cloudy with a temperature of 15°C. There's a light breeze and no rain expected today.\"\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " }\n", - "]\n", - "\n", - "result = task_completion(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "1962234d", - "metadata": {}, - "source": [ - "#### Task Completion with Tool Definition as Single Dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a9cd459", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"What's the current temperature in Boston?\"\n", - "response = \"The current temperature in Boston is 22°C (72°F) with clear skies. It's a beautiful day with low humidity and a gentle breeze.\"\n", - "\n", - "tool_definition_dict = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", - "\n", - "result = task_completion(query=query, response=response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "fc85ade7", - "metadata": {}, - "source": [ - "#### Complex Task with Multiple Steps" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "915768db", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "response = [\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I'll get the current weather information for Seattle and then send you an email with the details.\",\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is partly cloudy with a temperature of 15°C. You should receive the email shortly at your registered email address.\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = task_completion(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "5523dfcf", - "metadata": {}, - "source": [ - "#### Query as Conversation History (List of Messages)\n", - "The evaluator also supports query as a list of messages representing conversation history. This helps evaluate task completion in the context of a full conversation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2c4772be", - "metadata": {}, - "outputs": [], - "source": [ - "# Query as conversation history instead of a single string\n", - "query_as_conversation = [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"Hi, I need to plan my day. Can you check the weather in Seattle for me?\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Also, please send me an email summary of the weather so I can reference it later.\"\n", - " }\n", - "]\n", - "\n", - "response = \"I've checked the weather in Seattle for you. It's currently 15°C and partly cloudy with light winds. I've also sent you an email summary with these details so you can reference them throughout your day. The task has been completed successfully.\"\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = task_completion(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "100e943b", - "metadata": {}, - "source": [ - "#### Example of Incomplete Task" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9986fa70", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "incomplete_response = \"I can see that you want weather information for Seattle. The weather there is usually quite nice this time of year, with temperatures ranging from mild to warm depending on the season.\"\n", - "\n", - "# This response doesn't complete the email sending task\n", - "result = task_completion(query=query, response=incomplete_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb deleted file mode 100644 index 1e01e6a3..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Task_Navigation_Efficiency.ipynb +++ /dev/null @@ -1,446 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "28c2bc93", - "metadata": {}, - "source": [ - "# Task Navigation Efficiency Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use task navigation efficiency evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings and `dict` describing task responses;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "This is a deterministic evaluator that compares navigation paths.\n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "No environment variables are required for this evaluator." - ] - }, - { - "cell_type": "markdown", - "id": "d61cd511", - "metadata": {}, - "source": [ - "The Task Navigation Efficiency Evaluator measures how efficiently an agent navigates through a sequence of actions compared to an optimal task completion path.\n", - "\n", - "The evaluator provides comprehensive evaluation with both binary matching results and additional detailed P\\R\\F1 results:\n", - "\n", - "**Primary Result:**\n", - "- **Binary Match Result**: Pass/Fail based on the selected matching mode\n", - "\n", - "**Available Matching Modes:**\n", - "- **Exact Match**: Agent's tool calls must exactly match the ground truth (default)\n", - "- **In-Order Match**: All ground truth steps must appear in correct order (allows extra steps)\n", - "- **Any-Order Match**: All ground truth steps must appear with sufficient frequency (most lenient)\n", - "\n", - "**Properties Bag Additional Metrics (0.0 - 1.0):**\n", - "- **Precision**: How many of the agent's steps were necessary (relevant to ground truth)\n", - "- **Recall**: How many of the required steps were executed by the agent \n", - "- **F1 Score**: Harmonic mean of precision and recall\n", - "\n", - "The evaluation requires the following inputs:\n", - "- **Response**: The agent's response containing tool calls as a list of messages or string\n", - "- **Ground Truth**: List of expected tool/action steps as strings, or tuple with parameters for matching" - ] - }, - { - "cell_type": "markdown", - "id": "5e4e0a1d", - "metadata": {}, - "source": [ - "### Initialize Task Navigation Efficiency Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bc3726cd", - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.evaluation._evaluators._task_navigation_efficiency import _TaskNavigationEfficiencyEvaluator, _TaskNavigationEfficiencyMatchingMode\n", - "from pprint import pprint\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"response\": {\n", - " \"type\": \"array\"\n", - " },\n", - " \"ground_truth\": {\n", - " \"type\": \"array\"\n", - " }\n", - " },\n", - " \"required\": [\"response\", \"ground_truth\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"response\": \"{{item.response}}\",\n", - " \"ground_truth\": \"{{item.ground_truth}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"matching_mode\" : \"exact_match\" # Can be \"exact_match\", \"in_order_match\", or \"any_order_match\"\n", - "}\n", - "\n", - "# Initialize with exact match mode\n", - "task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(\n", - " matching_mode=_TaskNavigationEfficiencyMatchingMode.EXACT_MATCH\n", - ")\n", - "\n", - "# Other examples:\n", - "# For in-order matching (allows extra steps but requires correct order)\n", - "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", - "\n", - "# For any-order matching (most lenient - allows extra steps and different order) \n", - "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.ANY_ORDER_MATCH)\n", - "\n", - "# Or use defaults (exact match mode)\n", - "# task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator()" - ] - }, - { - "cell_type": "markdown", - "id": "15a53227", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "id": "1d75457a", - "metadata": {}, - "source": [ - "#### Sample 1: Perfect Path (Exact Match)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0cae15fb", - "metadata": {}, - "outputs": [], - "source": [ - "# Agent follows the exact optimal path\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\", \n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"analyze\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"report\", \"arguments\": {}}],\n", - " },\n", - "]\n", - "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", - "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"Perfect Path Results:\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "8a9c5e69", - "metadata": {}, - "source": [ - "#### Sample 2: Efficient Path with Extra Steps" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f53dde31", - "metadata": {}, - "outputs": [], - "source": [ - "# Agent performs all required steps but with extra unnecessary step\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"validate\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\", \n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"analyze\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"report\", \"arguments\": {}}],\n", - " },\n", - "]\n", - "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", - "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nPath with Extra Steps Results:\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "e4fa5757", - "metadata": {}, - "source": [ - "#### Sample 3: Inefficient Path (Wrong Order)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c2226715", - "metadata": {}, - "outputs": [], - "source": [ - "# Agent performs all required steps but in wrong order\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"report\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"search\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\", \n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"analyze\", \"arguments\": {}}],\n", - " },\n", - "]\n", - "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", - "\n", - "# Using in-order matching mode to demonstrate the difference\n", - "in_order_task_navigation_efficiency_evaluator = _TaskNavigationEfficiencyEvaluator(matching_mode=_TaskNavigationEfficiencyMatchingMode.IN_ORDER_MATCH)\n", - "\n", - "result = in_order_task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nWrong Order Results:\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "ae74ac8e", - "metadata": {}, - "source": [ - "#### Sample 4: Incomplete Path (Missing Steps)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ccd95640", - "metadata": {}, - "outputs": [], - "source": [ - "# Agent performs only some of the required steps (incomplete)\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"analyze\", \"arguments\": {}}],\n", - " },\n", - "]\n", - "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", - "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nMissing Steps Results:\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "2c9111ef", - "metadata": {}, - "source": [ - "#### Sample 5: Real-World Customer Service Scenario" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b4bd7b47", - "metadata": {}, - "outputs": [], - "source": [ - "# Real-world example: Customer service agent handling a refund request\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"lookup_order\", \"arguments\": {\"order_id\": \"12345\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"check_inventory\", \"arguments\": {\"product_id\": \"ABC123\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\", \n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"calculate_refund\", \"arguments\": {\"order_id\": \"12345\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"process_refund\", \"arguments\": {\"order_id\": \"12345\", \"amount\": \"29.99\"}}],\n", - " },\n", - "]\n", - "\n", - "ground_truth = [\"lookup_order\", \"calculate_refund\", \"process_refund\"]\n", - "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nCustomer Service Results:\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "4bf2e0ee", - "metadata": {}, - "source": [ - "#### Sample 6: Complex Path with Duplicates" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e4dcf585", - "metadata": {}, - "outputs": [], - "source": [ - "# Agent repeats some steps and includes extra ones\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_2\", \"name\": \"search\", \"arguments\": {}}], # duplicate\n", - " },\n", - " {\n", - " \"role\": \"assistant\", \n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_3\", \"name\": \"validate\", \"arguments\": {}}], # extra step\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_4\", \"name\": \"analyze\", \"arguments\": {}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_5\", \"name\": \"report\", \"arguments\": {}}],\n", - " },\n", - "]\n", - "\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", - "\n", - "result = task_navigation_efficiency_evaluator(response=response, ground_truth=ground_truth)\n", - "print(\"\\nComplex Path with Duplicates Results:\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "478c00fb", - "metadata": {}, - "source": [ - "#### Sample 7: Tuple Format with Parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7404f830", - "metadata": {}, - "outputs": [], - "source": [ - "# TaskNavigationEfficiencyEvaluator also supports tuple format with parameters for exact parameter matching\n", - "response_with_params = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [{\"type\": \"tool_call\", \"tool_call_id\": \"call_1\", \"name\": \"search\", \"arguments\": {\"query\": \"test\"}}],\n", - " },\n", - "]\n", - "\n", - "# Ground truth using tuple format: (tool_names, parameters_dict)\n", - "# Parameters must match exactly for tools to be considered matching\n", - "ground_truth_with_params = ([\"search\"], {\"search\": {\"query\": \"test\"}})\n", - "\n", - "result = task_navigation_efficiency_evaluator(response=response_with_params, ground_truth=ground_truth_with_params)\n", - "print(\"\\nTuple Format with Parameters Results:\")\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "38d2de9b", - "metadata": {}, - "source": [ - "#### Sample 8: String Response Input Type" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1a0a853b", - "metadata": {}, - "outputs": [], - "source": [ - "# Demonstrate string response input type\n", - "# The string response should contain structured tool call information that can be parsed\n", - "string_response = \"I'll help you with that. Let me search for information, then analyze the results, and finally provide a report.\"\n", - "ground_truth = [\"search\", \"analyze\", \"report\"]\n", - "\n", - "result = task_navigation_efficiency_evaluator(response=string_response, ground_truth=ground_truth)\n", - "print(\"\\nString Response Results:\")\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb deleted file mode 100644 index b0e32540..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb +++ /dev/null @@ -1,579 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Tool Call Accuracy Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates to how to use tool call accuracy evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings and `dict` describing tool calls;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The Tool Call Accuracy evaluator assesses how accurately an AI uses tools by examining:\n", - "- Relevance to the conversation\n", - "- Parameter correctness according to tool definitions\n", - "- Parameter value extraction from the conversation\n", - "- Potential usefulness of the tool call\n", - "\n", - "The evaluator uses a binary scoring (0 or 1) for each tool call:\n", - "\n", - " - Score 0: The tool call is irrelevant or contains information not in the conversation/definition\n", - " - Score 1: The tool call is relevant with properly extracted parameters from the conversation\n", - "\n", - "If there are multiple call, the final score will be an **average** of individual tool calls, which can be interpreted as the **passing rate** of tool calls.\n", - "\n", - "This evaluation focuses on measuring whether tool calls meaningfully contribute to addressing query while properly following tool definitions and using information present in the conversation history." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Tool Call Accuracy requires following input:\n", - "- Query - This can be a single query or a list of messages(conversation history with agent). Latter helps to determine if Agent used the information in history to make right tool calls.\n", - "- Tool Calls - Tool Call(s) made by Agent to answer the query. Optional - if response has tool calls, if not provided evaluator will look for tool calls in response.\n", - "- Response - (Optional) Response from Agent (or any GenAI App). This can be a single text response or a list or messages generated as part of Agent Response. If tool calls are not provide Tool Call Accuracy Evaluator will look at response for tool calls.\n", - "- Tool Definitions - Tool(s) definition used by Agent to answer the query. \n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Initialize Tool Call Accuracy Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation import ToolCallAccuracyEvaluator, AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_definitions\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_calls\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"response\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " },\n", - " \"required\": [\"query\", \"tool_definitions\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"tool_definitions\": \"{{item.tool_definitions}}\",\n", - " \"tool_calls\": \"{{item.tool_calls}}\",\n", - " \"response\": \"{{item.response}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "tool_call_accuracy = ToolCallAccuracyEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Evaluating Single Tool Call" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle ?\"\n", - "tool_call = {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - "}\n", - "\n", - "tool_definition = {\n", - " \"id\": \"fetch_weather\",\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = tool_call_accuracy(query=query, tool_calls=tool_call, tool_definitions=tool_definition)\n", - "pprint(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Multiple Tool Calls used by Agent to respond" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle ?\"\n", - "tool_calls = [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " },\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"London\"},\n", - " },\n", - "]\n", - "\n", - "tool_definition = {\n", - " \"id\": \"fetch_weather\",\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = tool_call_accuracy(query=query, tool_calls=tool_calls, tool_definitions=tool_definition)\n", - "pprint(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Tool Calls passed as part of `Response` (common for agent case)\n", - "- Tool Call Accuracy Evaluator extracts tool calls from response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "response = [\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Rainy, 14\\u00b0C\"}}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"your_email@example.com\",\n", - " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"The current weather in Seattle is rainy with a temperature of 14\\u00b0C.\",\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"role\": \"tool\",\n", - " \"content\": [\n", - " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\"}}\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is rainy with a temperature of 14\\u00b0C.\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = tool_call_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Response as String (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Book a flight to New York for tomorrow\"\n", - "\n", - "# Response as a simple string instead of a list of messages\n", - "response = \"I've found several flight options to New York for tomorrow. I'll use the booking tool to reserve your seat.\"\n", - "\n", - "tool_calls = [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_book_flight_456\",\n", - " \"name\": \"book_flight\",\n", - " \"arguments\": {\n", - " \"destination\": \"New York\",\n", - " \"date\": \"tomorrow\"\n", - " },\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"book_flight\",\n", - " \"description\": \"Books a flight to the specified destination on the given date.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"destination\": {\"type\": \"string\", \"description\": \"The destination city.\"},\n", - " \"date\": {\"type\": \"string\", \"description\": \"The date of the flight.\"},\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "result = tool_call_accuracy(query=query, response=response, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Response as List[dict] with Tool Definition as Single Dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "query = \"What's the weather in San Francisco?\"\n", - "\n", - "response = [\n", - " {\n", - " \"createdAt\": \"2025-03-26T18:15:22Z\",\n", - " \"run_id\": \"run_abc123\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_sf_weather_789\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"San Francisco\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T18:15:24Z\",\n", - " \"run_id\": \"run_abc123\",\n", - " \"tool_call_id\": \"call_sf_weather_789\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Foggy, 18°C\"}}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T18:15:25Z\",\n", - " \"run_id\": \"run_abc123\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The weather in San Francisco is currently foggy with a temperature of 18°C.\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "tool_definition_dict = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", - "\n", - "result = tool_call_accuracy(query=query, response=response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Query as Conversation History (List of Messages)\n", - "The evaluator also supports query as a list of messages representing conversation history. This helps determine if the Agent used the information in the conversation history to make the right tool calls." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Query as conversation history instead of a single string\n", - "query_as_conversation = [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"Hi, can you check the weather in Seattle for me?\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Actually, could you also send me an email with that weather information to john@example.com?\"\n", - " }\n", - "]\n", - "\n", - "tool_calls = [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " },\n", - " {\n", - " \"type\": \"tool_call\", \n", - " \"tool_call_id\": \"call_email_456\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"john@example.com\",\n", - " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"Here is the weather information you requested.\"\n", - " },\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = tool_call_accuracy(query=query_as_conversation, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(response)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb deleted file mode 100644 index 260a8b08..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Input_Accuracy.ipynb +++ /dev/null @@ -1,487 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "8d6b6cf9", - "metadata": {}, - "source": [ - "# Tool Input Accuracy Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use tool input accuracy evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings and `dict` describing tool calls;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." - ] - }, - { - "cell_type": "markdown", - "id": "74ac1b5c", - "metadata": {}, - "source": [ - "The Tool Input Accuracy evaluator performs a strict binary evaluation (PASS/FAIL) of parameters passed to tool calls. It ensures that ALL parameters meet ALL criteria:\n", - "\n", - "- Parameter grounding: All parameters must be derived from conversation history/query\n", - "- Type compliance: All parameters must match exact types specified in tool definitions\n", - "- Format compliance: All parameters must follow exact format and structure requirements\n", - "- Completeness: All required parameters must be provided\n", - "- No unexpected parameters: Only defined parameters are allowed\n", - "\n", - "The evaluator uses strict binary evaluation:\n", - "\n", - " - 1: Only when ALL criteria are satisfied perfectly for ALL parameters\n", - " - 0: When ANY criterion fails for ANY parameter\n", - "\n", - "This evaluation focuses on ensuring tool call parameters are completely correct without any tolerance for partial correctness." - ] - }, - { - "cell_type": "markdown", - "id": "17346632", - "metadata": {}, - "source": [ - "Tool Input Accuracy requires following input:\n", - "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", - "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response. The evaluator extracts tool calls from the response.\n", - "- Tool Definitions - Tool(s) definition used by Agent to answer the query. Required to validate parameter types and structures.\n" - ] - }, - { - "cell_type": "markdown", - "id": "d78194ee", - "metadata": {}, - "source": [ - "### Initialize Tool Input Accuracy Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1663a6b1", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation._evaluators._tool_input_accuracy import _ToolInputAccuracyEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"response\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_definitions\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " },\n", - " \"required\": [\"query\", \"response\", \"tool_definitions\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\",\n", - " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "tool_input_accuracy = _ToolInputAccuracyEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "f92e8378", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "id": "ba5f6441", - "metadata": {}, - "source": [ - "#### Evaluating Correct Tool Input Parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "486057ca", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " \"required\": [\"location\"]\n", - " },\n", - " }\n", - "]\n", - "\n", - "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "80fc6e82", - "metadata": {}, - "source": [ - "#### Response as String (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "626462d3", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Check the weather in Miami\"\n", - "\n", - "# Response as a simple string containing tool call information\n", - "response_str = \"\"\"I'll check the weather for you. Calling fetch_weather with location=\"Miami\".\"\"\"\n", - "\n", - "tool_definition = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " \"required\": [\"location\"]\n", - " },\n", - "}\n", - "\n", - "result = tool_input_accuracy(query=query, response=response_str, tool_definitions=tool_definition)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "e9275f1c", - "metadata": {}, - "source": [ - "#### Response as a List and Tool Definition as Single Dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "91e1e4d3", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"What's the temperature in Boston?\"\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_boston_weather\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Boston\"},\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definition_dict = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " \"required\": [\"location\"]\n", - " },\n", - "}\n", - "\n", - "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "c351c61a", - "metadata": {}, - "source": [ - "#### Complex Tool Parameters with Multiple Fields" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6b1147ab", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you send an email to john@example.com with the subject 'Weather Update' and tell him the weather in Seattle is 15°C and partly cloudy?\"\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_email_456\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"john@example.com\",\n", - " \"subject\": \"Weather Update\",\n", - " \"body\": \"The weather in Seattle is 15°C and partly cloudy.\"\n", - " },\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " \"required\": [\"recipient\", \"subject\", \"body\"]\n", - " },\n", - " }\n", - "]\n", - "\n", - "result = tool_input_accuracy(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "2b274da2", - "metadata": {}, - "source": [ - "#### Query as Conversation History (List of Messages)\n", - "The evaluator also supports query as a list of messages representing conversation history. This helps validate parameters are grounded in the conversation context." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a90e8827", - "metadata": {}, - "outputs": [], - "source": [ - "# Query as conversation history instead of a single string\n", - "query_as_conversation = [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"Hi, can you check the weather in Seattle for me?\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Please send the results to admin@company.com with the subject 'Daily Weather Report'.\"\n", - " }\n", - "]\n", - "\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\", \n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_email_456\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"admin@company.com\",\n", - " \"subject\": \"Daily Weather Report\",\n", - " \"body\": \"Weather information for Seattle as requested.\"\n", - " },\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " \"required\": [\"location\"]\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " \"required\": [\"recipient\", \"subject\", \"body\"]\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = tool_input_accuracy(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "14c41e90", - "metadata": {}, - "source": [ - "#### Example of Incorrect Tool Parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59810a22", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "# Missing required parameter or wrong type\n", - "incorrect_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"city\": \"Seattle\"}, # Wrong parameter name (should be \"location\")\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " \"required\": [\"location\"]\n", - " },\n", - " }\n", - "]\n", - "\n", - "# This should score 0 due to incorrect parameter name\n", - "result = tool_input_accuracy(query=query, response=incorrect_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb deleted file mode 100644 index 7a6a39f8..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Output_Utilization.ipynb +++ /dev/null @@ -1,595 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "d638561f", - "metadata": {}, - "source": [ - "# Tool Output Utilization Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use tool output utilization evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings and `dict` describing agent responses;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" - ] - }, - { - "cell_type": "markdown", - "id": "41d3647c", - "metadata": {}, - "source": [ - "The Tool Output Utilization Evaluator assesses how effectively an AI agent utilizes the outputs from tools and whether it accurately incorporates this information into its responses.\n", - "\n", - "Scoring is based on two levels:\n", - "1. Pass - The agent effectively utilizes tool outputs and accurately incorporates the information into its response.\n", - "2. Fail - The agent fails to properly utilize tool outputs or incorrectly incorporates the information into its response.\n", - "\n", - "The evaluation includes the score, a brief explanation, and a final pass/fail result.\n", - "\n", - "This evaluation focuses on measuring whether the agent properly uses the information returned by tools to provide accurate and complete responses to users." - ] - }, - { - "cell_type": "markdown", - "id": "246c7bcd", - "metadata": {}, - "source": [ - "Tool Output Utilization requires following input:\n", - "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", - "- Response - Response from Agent (or any GenAI App). This can be a single text response or a list of messages generated as part of Agent Response.\n", - "- Tool Definitions - Tool(s) definition used by Agent to answer the query. Required to understand the context and expected outputs of the tools.\n" - ] - }, - { - "cell_type": "markdown", - "id": "d386d576", - "metadata": {}, - "source": [ - "### Initialize Tool Output Utilization Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "087dd96d", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation._evaluators._tool_output_utilization import _ToolOutputUtilizationEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"response\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_definitions\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " },\n", - " \"required\": [\"query\", \"response\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\",\n", - " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "tool_output_utilization = _ToolOutputUtilizationEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "c7b4f98a", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "id": "9ee00e9d", - "metadata": {}, - "source": [ - "#### Evaluating Good Tool Output Utilization" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "017a7afe", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The current weather in Seattle is partly cloudy with a temperature of 15°C. The humidity is at 68% and there's a light wind from the northwest at 8 mph.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " }\n", - "]\n", - "\n", - "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "78270b9b", - "metadata": {}, - "source": [ - "#### Complex Scenario with Multiple Tool Outputs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e8543c52", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "response = [\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\", \"precipitation\": \"light rain\", \"wind\": \"12 mph SW\"}}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"your_email@example.com\",\n", - " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"Current weather in Seattle: 14°C and rainy with light rain. Wind is from the southwest at 12 mph. Stay dry!\",\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"role\": \"tool\",\n", - " \"content\": [\n", - " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\", \"status\": \"delivered\"}}\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The email includes the current conditions: 14°C and rainy with light rain, plus wind details. The email was delivered successfully to your inbox.\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "1c52f24c", - "metadata": {}, - "source": [ - "#### Query as Conversation History" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b3b9bef5", - "metadata": {}, - "outputs": [], - "source": [ - "# Query as conversation history instead of a single string\n", - "query_as_conversation = [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"Hi, I need to know the weather in Seattle for my trip planning.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Can you also email me a summary so I can reference it later when packing?\"\n", - " }\n", - "]\n", - "\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\", \"forecast\": \"no rain expected\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_email_456\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"user@example.com\",\n", - " \"subject\": \"Seattle Weather Summary for Trip Planning\",\n", - " \"body\": \"Weather summary for Seattle: Temperature is 15°C, partly cloudy conditions, humidity at 68%, with northwest winds at 8 mph. No rain is expected, so light layers should be sufficient for your trip.\"\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_email_456\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email sent successfully\", \"delivery_status\": \"delivered\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"Perfect! I've checked the weather in Seattle for your trip planning. It's currently 15°C and partly cloudy with light northwest winds and no rain expected. I've also sent you an email summary with all these details including packing recommendations based on the weather conditions. The email was delivered successfully, so you can reference it when preparing for your trip.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = tool_output_utilization(query=query_as_conversation, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "199a11bc", - "metadata": {}, - "source": [ - "#### Example of Poor Tool Output Utilization" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6be39269", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "# Poor utilization - agent doesn't use the detailed tool output effectively\n", - "poor_response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"15°C\", \"condition\": \"partly cloudy\", \"humidity\": \"68%\", \"wind\": \"8 mph NW\", \"UV_index\": \"3\", \"visibility\": \"10 km\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The weather in Seattle is okay. It's not too bad outside.\", # Poor utilization of detailed weather data\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " }\n", - "]\n", - "\n", - "# This should score poorly due to inadequate use of tool output\n", - "result = tool_output_utilization(query=query, response=poor_response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "bcb8a2cf", - "metadata": {}, - "source": [ - "#### Response as String (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0845822f", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"What's the weather like in Seattle?\"\n", - "# Response as a simple string (not a list of messages)\n", - "response_str = \"The weather in Seattle is currently 15°C and partly cloudy with light winds from the northwest.\"\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " }\n", - "]\n", - "\n", - "result = tool_output_utilization(query=query, response=response_str, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "b4cb9ddd", - "metadata": {}, - "source": [ - "#### Tool Definitions as Single Dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6424b3bb", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Check the weather in Seattle\"\n", - "response = [\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_456\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"tool_call_id\": \"call_weather_456\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"temperature\": \"14°C\", \"condition\": \"rainy\"}}],\n", - " },\n", - " {\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"The current weather in Seattle is rainy with a temperature of 14°C.\",\n", - " }\n", - " ],\n", - " }\n", - "]\n", - "\n", - "# Tool definition as a single dict (not a list)\n", - "tool_definition_dict = {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - "}\n", - "\n", - "result = tool_output_utilization(query=query, response=response, tool_definitions=tool_definition_dict)\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb deleted file mode 100644 index 2f43123c..00000000 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Online_Evaluation/AI_Judge_Evaluator_Tool_Selection.ipynb +++ /dev/null @@ -1,648 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "862760b0", - "metadata": {}, - "source": [ - "# Tool Selection Evaluator\n", - "\n", - "## Objective\n", - "This sample demonstrates how to use tool selection evaluator on agent data. The supported input formats include:\n", - "- simple data such as strings and `dict` describing tool calls;\n", - "- user-agent conversations in the form of list of agent messages. \n", - "\n", - "## Time\n", - "\n", - "You should expect to spend about 20 minutes running this notebook. \n", - "\n", - "## Before you begin\n", - "For quality evaluation, you need to deploy a `gpt` model supporting JSON mode. We recommend a model `gpt-4o` or `gpt-4o-mini` for their strong reasoning capabilities. \n", - "\n", - "### Prerequisite\n", - "```bash\n", - "pip install azure-ai-evaluation\n", - "```\n", - "Set these environment variables with your own values:\n", - "1) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", - "2) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", - "3) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "4) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" - ] - }, - { - "cell_type": "markdown", - "id": "31a75ee6", - "metadata": {}, - "source": [ - "The Tool Selection evaluator assesses the appropriateness and efficiency of tool choices made by an AI agent by examining:\n", - "- Relevance of selected tools to the conversation\n", - "- Completeness of tool selection according to task requirements\n", - "- Efficiency in avoiding unnecessary or redundant tools\n", - "\n", - "The evaluator uses a binary scoring system:\n", - "\n", - " - Score 0 (Fail): Tools selected are irrelevant, incorrect, or missing essential tools\n", - " - Score 1 (Pass): All needed tools are selected, even if there are redundant tools\n", - "\n", - "This evaluation focuses on measuring whether the right tools were chosen for the task, regardless of how those tools were executed or their parameter correctness." - ] - }, - { - "cell_type": "markdown", - "id": "353f2304", - "metadata": {}, - "source": [ - "Tool Selection requires following input:\n", - "- Query - This can be a single query or a list of messages(conversation history with agent). The original task request from the user.\n", - "- Tool Calls - Tool Call(s) made by Agent to answer the query. Optional - if response has tool calls, if not provided evaluator will look for tool calls in response.\n", - "- Response - (Optional) Response from Agent (or any GenAI App). This can be a single text response or a list or messages generated as part of Agent Response. If tool calls are not provided, Tool Selection Evaluator will look at response for tool calls.\n", - "- Tool Definitions - Tool(s) definition used by Agent to answer the query. Required to understand available tools and their purposes.\n" - ] - }, - { - "cell_type": "markdown", - "id": "75125e94", - "metadata": {}, - "source": [ - "### Initialize Tool Selection Evaluator\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b5f31144", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from azure.ai.evaluation._evaluators._tool_selection import _ToolSelectionEvaluator\n", - "from azure.ai.evaluation import AzureOpenAIModelConfiguration\n", - "from pprint import pprint\n", - "\n", - "model_config = AzureOpenAIModelConfiguration(\n", - " azure_endpoint=os.environ[\"AZURE_OPENAI_ENDPOINT\"],\n", - " api_key=os.environ[\"AZURE_OPENAI_API_KEY\"],\n", - " api_version=os.environ[\"AZURE_OPENAI_API_VERSION\"],\n", - " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", - ")\n", - "\n", - "data_source_config = {\n", - " \"type\": \"custom\",\n", - " \"item_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"query\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"response\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"string\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_calls\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " },\n", - " \"tool_definitions\": {\n", - " \"anyOf\": [\n", - " {\"type\": \"object\"},\n", - " {\n", - " \"type\": \"array\",\n", - " \"items\": {\n", - " \"type\": \"object\"\n", - " }\n", - " }\n", - " ]\n", - " }\n", - " },\n", - " \"required\": [\"query\", \"response\", \"tool_definitions\"]\n", - " },\n", - " \"include_sample_schema\": True\n", - "}\n", - "\n", - "data_mapping = {\n", - " \"query\": \"{{item.query}}\",\n", - " \"response\": \"{{item.response}}\",\n", - " \"tool_calls\": \"{{item.tool_calls}}\",\n", - " \"tool_definitions\": \"{{item.tool_definitions}}\"\n", - "}\n", - "\n", - "initialization_parameters = {\n", - " \"deployment_name\" : os.environ.get(\"AZURE_AI_MODEL_DEPLOYMENT_NAME\", \"\") # Sample : gpt-4o-mini\n", - "}\n", - "\n", - "tool_selection = _ToolSelectionEvaluator(model_config=model_config)" - ] - }, - { - "cell_type": "markdown", - "id": "fb435367", - "metadata": {}, - "source": [ - "### Samples" - ] - }, - { - "cell_type": "markdown", - "id": "344fe68e", - "metadata": {}, - "source": [ - "#### Evaluating Single Tool Selection" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dcb71257", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "tool_call = {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - "}\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"id\": \"fetch_weather\",\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"id\": \"send_email\",\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "response = tool_selection(query=query, tool_calls=tool_call, tool_definitions=tool_definitions)\n", - "pprint(response)" - ] - }, - { - "cell_type": "markdown", - "id": "e078b1fa", - "metadata": {}, - "source": [ - "#### Multiple Tool Selections for Complex Task" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "322efe28", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "tool_calls = [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " },\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"user@example.com\",\n", - " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"Weather data will be included here.\",\n", - " },\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"id\": \"fetch_weather\",\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"id\": \"send_email\",\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - " {\n", - " \"id\": \"get_calendar\",\n", - " \"name\": \"get_calendar\",\n", - " \"description\": \"Retrieves calendar events for a specified date range.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"start_date\": {\"type\": \"string\", \"description\": \"Start date for calendar events.\"},\n", - " \"end_date\": {\"type\": \"string\", \"description\": \"End date for calendar events.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "response = tool_selection(query=query, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(response)" - ] - }, - { - "cell_type": "markdown", - "id": "19b3c8de", - "metadata": {}, - "source": [ - "#### Tool Calls passed as part of `Response` (common for agent case)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "629f3cc7", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"Can you send me an email with weather information for Seattle?\"\n", - "response = [\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:35Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:37Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_CUdbkBfvVBla2YP3p24uhElJ\",\n", - " \"role\": \"tool\",\n", - " \"content\": [{\"type\": \"tool_result\", \"tool_result\": {\"weather\": \"Rainy, 14°C\"}}],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:38Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"your_email@example.com\",\n", - " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"The current weather in Seattle is rainy with a temperature of 14°C.\",\n", - " },\n", - " }\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:41Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"tool_call_id\": \"call_iq9RuPxqzykebvACgX8pqRW2\",\n", - " \"role\": \"tool\",\n", - " \"content\": [\n", - " {\"type\": \"tool_result\", \"tool_result\": {\"message\": \"Email successfully sent to your_email@example.com.\"}}\n", - " ],\n", - " },\n", - " {\n", - " \"createdAt\": \"2025-03-26T17:27:42Z\",\n", - " \"run_id\": \"run_zblZyGCNyx6aOYTadmaqM4QN\",\n", - " \"role\": \"assistant\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": \"I have successfully sent you an email with the weather information for Seattle. The current weather is rainy with a temperature of 14°C.\",\n", - " }\n", - " ],\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"get_calendar\",\n", - " \"description\": \"Retrieves calendar events for a specified date range.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"start_date\": {\"type\": \"string\", \"description\": \"Start date for calendar events.\"},\n", - " \"end_date\": {\"type\": \"string\", \"description\": \"End date for calendar events.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = tool_selection(query=query, response=response, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "682839b8", - "metadata": {}, - "source": [ - "#### Query as Conversation History (List of Messages)\n", - "The evaluator also supports query as a list of messages representing conversation history. This helps determine if the Agent selected appropriate tools based on the conversation context." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "218ade7a", - "metadata": {}, - "outputs": [], - "source": [ - "# Query as conversation history instead of a single string\n", - "query_as_conversation = [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are a helpful assistant that can fetch weather information and send emails.\"\n", - " },\n", - " {\n", - " \"role\": \"user\", \n", - " \"content\": \"Hi, can you check the weather in Seattle for me?\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Actually, could you also send me an email with that weather information to john@example.com?\"\n", - " }\n", - "]\n", - "\n", - "tool_calls = [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_weather_123\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"Seattle\"},\n", - " },\n", - " {\n", - " \"type\": \"tool_call\", \n", - " \"tool_call_id\": \"call_email_456\",\n", - " \"name\": \"send_email\",\n", - " \"arguments\": {\n", - " \"recipient\": \"john@example.com\",\n", - " \"subject\": \"Weather Information for Seattle\",\n", - " \"body\": \"Here is the weather information you requested.\"\n", - " },\n", - " },\n", - "]\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"get_calendar\",\n", - " \"description\": \"Retrieves calendar events for a specified date range.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"start_date\": {\"type\": \"string\", \"description\": \"Start date for calendar events.\"},\n", - " \"end_date\": {\"type\": \"string\", \"description\": \"End date for calendar events.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = tool_selection(query=query_as_conversation, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "406bc90c", - "metadata": {}, - "source": [ - "#### Example of Poor Tool Selection" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2c451e4e", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "# Using irrelevant tool for the task\n", - "poor_tool_calls = [\n", - " {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_calendar_123\",\n", - " \"name\": \"get_calendar\",\n", - " \"arguments\": {\"start_date\": \"2025-01-01\", \"end_date\": \"2025-01-31\"},\n", - " },\n", - "]\n", - "\n", - "# This should score poorly as get_calendar is not relevant for weather queries\n", - "result = tool_selection(query=query, tool_calls=poor_tool_calls, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "ab4a46c7", - "metadata": {}, - "source": [ - "#### Response as String (str)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e34bec52", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"What's the weather in New York?\"\n", - "\n", - "# Response as a simple string (not commonly used, but supported by the API)\n", - "# When using string response, tool_calls should be provided separately\n", - "response_str = \"The current weather in New York is sunny with a temperature of 22°C.\"\n", - "\n", - "tool_calls = {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_ny_weather_789\",\n", - " \"name\": \"fetch_weather\",\n", - " \"arguments\": {\"location\": \"New York\"},\n", - "}\n", - "\n", - "tool_definitions = [\n", - " {\n", - " \"name\": \"fetch_weather\",\n", - " \"description\": \"Fetches the weather information for the specified location.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The location to fetch weather for.\"}},\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"send_email\",\n", - " \"description\": \"Sends an email with the specified subject and body to the recipient.\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"recipient\": {\"type\": \"string\", \"description\": \"Email address of the recipient.\"},\n", - " \"subject\": {\"type\": \"string\", \"description\": \"Subject of the email.\"},\n", - " \"body\": {\"type\": \"string\", \"description\": \"Body content of the email.\"},\n", - " },\n", - " },\n", - " },\n", - "]\n", - "\n", - "result = tool_selection(query=query, response=response_str, tool_calls=tool_calls, tool_definitions=tool_definitions)\n", - "pprint(result)" - ] - }, - { - "cell_type": "markdown", - "id": "cc857725", - "metadata": {}, - "source": [ - "#### Tool Definition as Single Dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d94822e1", - "metadata": {}, - "outputs": [], - "source": [ - "query = \"How is the weather in Seattle?\"\n", - "\n", - "tool_definition_dict = {\n", - " \"name\": \"get_weather\",\n", - " \"description\": \"Get the current weather for a location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\"type\": \"string\", \"description\": \"The city and state, e.g. San Francisco, CA\"},\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]}\n", - " },\n", - " \"required\": [\"location\"]\n", - " }\n", - "}\n", - "\n", - "tool_calls_dict = {\n", - " \"type\": \"tool_call\",\n", - " \"tool_call_id\": \"call_abc123\",\n", - " \"name\": \"get_weather\",\n", - " \"arguments\": {\"location\": \"Seattle, WA\", \"unit\": \"celsius\"}\n", - "}\n", - "\n", - "result = tool_selection(query=query, tool_definitions=tool_definition_dict, tool_calls=tool_calls_dict)\n", - "\n", - "pprint(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "samples", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.19" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}