Skip to content

Updating Agent Samples to use new Foundry Project #281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
slister1001 merged 3 commits into from
Oct 15, 2025
Merged

Updating Agent Samples to use new Foundry Project #281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
73f663a
Add files via upload
singankit Oct 15, 2025
ec89aac
Removing loading env code
singankit Oct 15, 2025
1db3bb7
Running nb-clean
singankit Oct 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 4 additions & 11 deletions .../Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Intent_Resolution.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,11 @@
"pip install azure-ai-projects azure-identity azure-ai-evaluation\n",
"```\n",
"Set these environment variables with your own values:\n",
"1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
"1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
"2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n",
"3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n",
"4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n",
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n",
"6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n",
"7) **PROJECT_NAME** - Azure AI Project Name\n",
"8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n"
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation."
]
},
{
Expand Down Expand Up @@ -324,19 +321,15 @@
" evaluators={\n",
" \"intent_resolution\": intent_resolution_evaluator,\n",
" },\n",
" azure_ai_project={\n",
" \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n",
" \"project_name\": os.environ[\"PROJECT_NAME\"],\n",
" \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n",
" },\n",
" azure_ai_project=os.environ[\"AZURE_AI_PROJECT\"],\n",
")\n",
"pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "test_agent_eval_sample",
"display_name": "azureai-sample",
"language": "python",
"name": "python3"
},
Expand Down
22 changes: 6 additions & 16 deletions ...ate/Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Task_Adherence.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,11 @@
"pip install azure-ai-projects azure-identity azure-ai-evaluation\n",
"```\n",
"Set these environment variables with your own values:\n",
"1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
"1) **AZURE_AI_PROJECT** - The project url, as found in the overview page of your Azure AI Foundry project.\n",
"2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n",
"3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n",
"4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n",
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n",
"6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n",
"7) **PROJECT_NAME** - Azure AI Project Name\n",
"8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n"
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n"
]
},
{
Expand All @@ -47,14 +44,11 @@
"pip install azure-ai-projects azure-identity azure-ai-evaluation\n",
"```\n",
"Set these environment variables with your own values:\n",
"1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
"1) **AZURE_AI_PROJECT** - The project url, as found in the overview page of your Azure AI Foundry project.\n",
"2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the AI model, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n",
"3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n",
"4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n",
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n",
"6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n",
"7) **PROJECT_NAME** - Azure AI Project Name\n",
"8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name"
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation."
]
},
{
Expand Down Expand Up @@ -262,19 +256,15 @@
" evaluators={\n",
" \"task_adherence\": task_adherence_evaluator,\n",
" },\n",
" azure_ai_project={\n",
" \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n",
" \"project_name\": os.environ[\"PROJECT_NAME\"],\n",
" \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n",
" },\n",
" azure_ai_project=os.environ[\"AZURE_AI_PROJECT\"],\n",
")\n",
"pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "test_agent_eval_sample",
"display_name": "azureai-sample",
"language": "python",
"name": "python3"
},
Expand Down
15 changes: 4 additions & 11 deletions ...Supported_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluator_Tool_Call_Accuracy.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,11 @@
"pip install azure-ai-projects azure-identity azure-ai-evaluation\n",
"```\n",
"Set these environment variables with your own values:\n",
"1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
"1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
"2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n",
"3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n",
"4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n",
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n",
"6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n",
"7) **PROJECT_NAME** - Azure AI Project Name\n",
"8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n"
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n"
]
},
{
Expand Down Expand Up @@ -328,19 +325,15 @@
" evaluators={\n",
" \"tool_call_accuracy\": tool_call_accuracy,\n",
" },\n",
" azure_ai_project={\n",
" \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n",
" \"project_name\": os.environ[\"PROJECT_NAME\"],\n",
" \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n",
" },\n",
" azure_ai_project=os.environ[\"AZURE_AI_PROJECT\"],\n",
")\n",
"pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "test_agent_eval_sample",
"display_name": "azureai-sample",
"language": "python",
"name": "python3"
},
Expand Down
15 changes: 4 additions & 11 deletions ...orted_Evaluation_Metrics/Agent_Evaluation/AI_Judge_Evaluators_Response_Completeness.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,11 @@
"pip install azure-ai-projects azure-identity azure-ai-evaluation\n",
"```\n",
"Set these environment variables with your own values:\n",
"1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
"1) **AZURE_AI_PROJECT** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
"2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for this AI-assisted evaluator, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n",
"3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n",
"4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n",
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n",
"6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n",
"7) **PROJECT_NAME** - Azure AI Project Name\n",
"8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n"
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n"
]
},
{
Expand Down Expand Up @@ -197,11 +194,7 @@
"source": [
"from azure.ai.evaluation import evaluate\n",
"\n",
"azure_ai_project = {\n",
" \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n",
" \"project_name\": os.environ[\"PROJECT_NAME\"],\n",
" \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n",
"}\n",
"azure_ai_project = os.environ[\"AZURE_AI_PROJECT\"]\n",
"\n",
"response = evaluate(\n",
" data=file_path,\n",
Expand All @@ -217,7 +210,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "test_agent_eval_sample",
"display_name": "azureai-sample",
"language": "python",
"name": "python3"
},
Expand Down
37 changes: 13 additions & 24 deletions ...luate/Supported_Evaluation_Metrics/Agent_Evaluation/Evaluate_Azure_AI_Agent_Quality.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,12 @@
"pip install azure-ai-projects azure-identity azure-ai-evaluation\n",
"```\n",
"Set these environment variables with your own values:\n",
"1) **PROJECT_CONNECTION_STRING** - The project connection string, as found in the overview page of your Azure AI Foundry project.\n",
"1) **AZURE_AI_PROJECT** - The project url, as found in the overview page of your Azure AI Foundry project.\n",
"2) **MODEL_DEPLOYMENT_NAME** - The deployment name of the model for AI-assisted evaluators, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project.\n",
"3) **AZURE_OPENAI_ENDPOINT** - Azure Open AI Endpoint to be used for evaluation.\n",
"4) **AZURE_OPENAI_API_KEY** - Azure Open AI Key to be used for evaluation.\n",
"5) **AZURE_OPENAI_API_VERSION** - Azure Open AI Api version to be used for evaluation.\n",
"6) **AZURE_SUBSCRIPTION_ID** - Azure Subscription Id of Azure AI Project\n",
"7) **PROJECT_NAME** - Azure AI Project Name\n",
"8) **RESOURCE_GROUP_NAME** - Azure AI Project Resource Group Name\n",
"9) **AGENT_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for your Azure AI agent, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project."
"6) **AGENT_MODEL_DEPLOYMENT_NAME** - The deployment name of the model for your Azure AI agent, as found under the \"Name\" column in the \"Models + endpoints\" tab in your Azure AI Foundry project."
]
},
{
Expand All @@ -73,14 +70,14 @@
"import os\n",
"from azure.ai.projects import AIProjectClient\n",
"from azure.identity import DefaultAzureCredential\n",
"from azure.ai.projects.models import FunctionTool, ToolSet\n",
"from azure.ai.agents.models import FunctionTool, ToolSet\n",
"\n",
"# Import your custom functions to be used as Tools for the Agent\n",
"from user_functions import user_functions\n",
"\n",
"project_client = AIProjectClient.from_connection_string(\n",
"project_client = AIProjectClient(\n",
" endpoint=os.environ[\"AZURE_AI_PROJECT\"],\n",
" credential=DefaultAzureCredential(),\n",
" conn_str=os.environ[\"PROJECT_CONNECTION_STRING\"],\n",
")\n",
"\n",
"AGENT_NAME = \"Seattle Tourist Assistant\"\n",
Expand All @@ -92,7 +89,7 @@
"toolset.add(functions)\n",
"\n",
"# To enable tool calls executed automatically\n",
"project_client.agents.enable_auto_function_calls(toolset=toolset)"
"project_client.agents.enable_auto_function_calls(toolset)"
]
},
{
Expand Down Expand Up @@ -131,7 +128,7 @@
"metadata": {},
"outputs": [],
"source": [
"thread = project_client.agents.create_thread()\n",
"thread = project_client.agents.threads.create()\n",
"print(f\"Created thread, ID: {thread.id}\")"
]
},
Expand Down Expand Up @@ -162,7 +159,7 @@
"\n",
"MESSAGE = \"Can you email me weather info for Seattle ?\"\n",
"\n",
"message = project_client.agents.create_message(\n",
"message = project_client.agents.messages.create(\n",
" thread_id=thread.id,\n",
" role=\"user\",\n",
" content=MESSAGE,\n",
Expand All @@ -183,7 +180,7 @@
"metadata": {},
"outputs": [],
"source": [
"run = project_client.agents.create_and_process_run(thread_id=thread.id, agent_id=agent.id)\n",
"run = project_client.agents.runs.create_and_process(thread_id=thread.id, agent_id=agent.id)\n",
"\n",
"print(f\"Run finished with status: {run.status}\")\n",
"\n",
Expand All @@ -206,7 +203,7 @@
"metadata": {},
"outputs": [],
"source": [
"for message in project_client.agents.list_messages(thread.id, order=\"asc\").data:\n",
"for message in project_client.agents.messages.list(thread.id, order=\"asc\"):\n",
" print(f\"Role: {message.role}\")\n",
" print(f\"Content: {message.content[0].text.value}\")\n",
" print(\"-\" * 40)"
Expand Down Expand Up @@ -284,11 +281,7 @@
" azure_deployment=os.environ[\"MODEL_DEPLOYMENT_NAME\"],\n",
")\n",
"# Needed to use content safety evaluators\n",
"azure_ai_project = {\n",
" \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n",
" \"project_name\": os.environ[\"PROJECT_NAME\"],\n",
" \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n",
"}\n",
"azure_ai_project = os.environ[\"AZURE_AI_PROJECT\"]\n",
"\n",
"intent_resolution = IntentResolutionEvaluator(model_config=model_config)\n",
"\n",
Expand Down Expand Up @@ -319,11 +312,7 @@
" \"intent_resolution\": intent_resolution,\n",
" \"task_adherence\": task_adherence,\n",
" },\n",
" azure_ai_project={\n",
" \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n",
" \"project_name\": os.environ[\"PROJECT_NAME\"],\n",
" \"resource_group_name\": os.environ[\"RESOURCE_GROUP_NAME\"],\n",
" },\n",
" azure_ai_project=azure_ai_project,\n",
")\n",
"pprint(f'AI Foundary URL: {response.get(\"studio_url\")}')"
]
Expand Down Expand Up @@ -352,7 +341,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "evaluate-agents-test",
"display_name": "azureai-sample",
"language": "python",
"name": "python3"
},
Expand Down
Loading