diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb index 2dcd8d01..ab69e4fc 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb @@ -23,14 +23,11 @@ "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n", - "6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n", - "7) **PROJECT_NAME** - Azure AI Project Name\n", - "8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n" + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." ] }, { @@ -324,11 +321,7 @@ " evaluators={\n", " \"intent_resolution\": intent_resolution_evaluator,\n", " },\n", - " azure_ai_project={\n", - " \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n", - " \"project_name\": os.environ[\"PROJECT_NAME\"],\n", - " \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n", - " },\n", + " azure_ai_project=os.environ[\"AZURE_AI_PROJECT\"],\n", ")\n", "pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')" ] @@ -336,7 +329,7 @@ ], "metadata": { "kernelspec": { - "display_name": "test_agent_eval_sample", + "display_name": "azureai-sample", "language": "python", "name": "python3" }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb index 8a1f9350..075676b8 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb @@ -25,14 +25,11 @@ "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "1) **AZURE_AI_PROJECT** - The project url, as found in the overview page of your Azure AI Foundry project.\n", "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n", - "6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n", - "7) **PROJECT_NAME** - Azure AI Project Name\n", - "8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n" + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" ] }, { @@ -47,14 +44,11 @@ "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "1) **AZURE_AI_PROJECT** - The project url, as found in the overview page of your Azure AI Foundry project.\n", "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the AI model, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n", - "6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n", - "7) **PROJECT_NAME** - Azure AI Project Name\n", - "8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name" + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation." ] }, { @@ -262,11 +256,7 @@ " evaluators={\n", " \"task_adherence\": task_adherence_evaluator,\n", " },\n", - " azure_ai_project={\n", - " \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n", - " \"project_name\": os.environ[\"PROJECT_NAME\"],\n", - " \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n", - " },\n", + " azure_ai_project=os.environ[\"AZURE_AI_PROJECT\"],\n", ")\n", "pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')" ] @@ -274,7 +264,7 @@ ], "metadata": { "kernelspec": { - "display_name": "test_agent_eval_sample", + "display_name": "azureai-sample", "language": "python", "name": "python3" }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb index c43b1aca..0aa60d2d 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb @@ -23,14 +23,11 @@ "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n", - "6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n", - "7) **PROJECT_NAME** - Azure AI Project Name\n", - "8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n" + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" ] }, { @@ -328,11 +325,7 @@ " evaluators={\n", " \"tool_call_accuracy\": tool_call_accuracy,\n", " },\n", - " azure_ai_project={\n", - " \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n", - " \"project_name\": os.environ[\"PROJECT_NAME\"],\n", - " \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n", - " },\n", + " azure_ai_project=os.environ[\"AZURE_AI_PROJECT\"],\n", ")\n", "pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')" ] @@ -340,7 +333,7 @@ ], "metadata": { "kernelspec": { - "display_name": "test_agent_eval_sample", + "display_name": "azureai-sample", "language": "python", "name": "python3" }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb index 8139a32f..06142292 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb @@ -27,14 +27,11 @@ "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", - "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n", - "6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n", - "7) **PROJECT_NAME** - Azure AI Project Name\n", - "8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n" + "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n" ] }, { @@ -197,11 +194,7 @@ "source": [ "from azure.ai.evaluation import evaluate\n", "\n", - "azure_ai_project = {\n", - " \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n", - " \"project_name\": os.environ[\"PROJECT_NAME\"],\n", - " \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n", - "}\n", + "azure_ai_project = os.environ[\"AZURE_AI_PROJECT\"]\n", "\n", "response = evaluate(\n", " data=file_path,\n", @@ -217,7 +210,7 @@ ], "metadata": { "kernelspec": { - "display_name": "test_agent_eval_sample", + "display_name": "azureai-sample", "language": "python", "name": "python3" }, diff --git a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/Evaluate_Azure_AI_Agent_Quality.ipynb b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/Evaluate_Azure_AI_Agent_Quality.ipynb index 6b1930a3..c8954b42 100644 --- a/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/Evaluate_Azure_AI_Agent_Quality.ipynb +++ b/scenarios/evaluate/Supported_Evaluation_Metrics/Agent_Evaluation/Evaluate_Azure_AI_Agent_Quality.ipynb @@ -46,15 +46,12 @@ "pip install azure-ai-projects azure-identity azure-ai-evaluation\n", "```\n", "Set these environment variables with your own values:\n", - "1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n", + "1) **AZURE_AI_PROJECT** - The project url, as found in the overview page of your Azure AI Foundry project.\n", "2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for AI-assisted evaluators, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n", "3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n", "4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n", "5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n", - "6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n", - "7) **PROJECT_NAME** - Azure AI Project Name\n", - "8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n", - "9) **AGENT_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for your Azure AI agent, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project." + "6) **AGENT_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for your Azure AI agent, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project." ] }, { @@ -73,14 +70,14 @@ "import os\n", "from azure.ai.projects import AIProjectClient\n", "from azure.identity import DefaultAzureCredential\n", - "from azure.ai.projects.models import FunctionTool, ToolSet\n", + "from azure.ai.agents.models import FunctionTool, ToolSet\n", "\n", "# Import your custom functions to be used as Tools for the Agent\n", "from user_functions import user_functions\n", "\n", - "project_client = AIProjectClient.from_connection_string(\n", + "project_client = AIProjectClient(\n", + " endpoint=os.environ[\"AZURE_AI_PROJECT\"],\n", " credential=DefaultAzureCredential(),\n", - " conn_str=os.environ[\"PROJECT_CONNECTION_STRING\"],\n", ")\n", "\n", "AGENT_NAME = \"Seattle Tourist Assistant\"\n", @@ -92,7 +89,7 @@ "toolset.add(functions)\n", "\n", "# To enable tool calls executed automatically\n", - "project_client.agents.enable_auto_function_calls(toolset=toolset)" + "project_client.agents.enable_auto_function_calls(toolset)" ] }, { @@ -131,7 +128,7 @@ "metadata": {}, "outputs": [], "source": [ - "thread = project_client.agents.create_thread()\n", + "thread = project_client.agents.threads.create()\n", "print(f\"Created thread, ID: {thread.id}\")" ] }, @@ -162,7 +159,7 @@ "\n", "MESSAGE = \"Can you email me weather info for Seattle ?\"\n", "\n", - "message = project_client.agents.create_message(\n", + "message = project_client.agents.messages.create(\n", " thread_id=thread.id,\n", " role=\"user\",\n", " content=MESSAGE,\n", @@ -183,7 +180,7 @@ "metadata": {}, "outputs": [], "source": [ - "run = project_client.agents.create_and_process_run(thread_id=thread.id, agent_id=agent.id)\n", + "run = project_client.agents.runs.create_and_process(thread_id=thread.id, agent_id=agent.id)\n", "\n", "print(f\"Run finished with status: {run.status}\")\n", "\n", @@ -206,7 +203,7 @@ "metadata": {}, "outputs": [], "source": [ - "for message in project_client.agents.list_messages(thread.id, order=\"asc\").data:\n", + "for message in project_client.agents.messages.list(thread.id, order=\"asc\"):\n", " print(f\"Role: {message.role}\")\n", " print(f\"Content: {message.content[0].text.value}\")\n", " print(\"-\" * 40)" @@ -284,11 +281,7 @@ " azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n", ")\n", "# Needed to use content safety evaluators\n", - "azure_ai_project = {\n", - " \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n", - " \"project_name\": os.environ[\"PROJECT_NAME\"],\n", - " \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n", - "}\n", + "azure_ai_project = os.environ[\"AZURE_AI_PROJECT\"]\n", "\n", "intent_resolution = IntentResolutionEvaluator(model_config=model_config)\n", "\n", @@ -319,11 +312,7 @@ " \"intent_resolution\": intent_resolution,\n", " \"task_adherence\": task_adherence,\n", " },\n", - " azure_ai_project={\n", - " \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n", - " \"project_name\": os.environ[\"PROJECT_NAME\"],\n", - " \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n", - " },\n", + " azure_ai_project=azure_ai_project,\n", ")\n", "pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')" ] @@ -352,7 +341,7 @@ ], "metadata": { "kernelspec": { - "display_name": "evaluate-agents-test", + "display_name": "azureai-sample", "language": "python", "name": "python3" },