diff --git a/docs/labs-config.json b/docs/labs-config.json index 9eee7626..c91cb867 100644 --- a/docs/labs-config.json +++ b/docs/labs-config.json @@ -85,6 +85,26 @@ "githubPath": "https://github.com/Azure-Samples/AI-Gateway/tree/main/labs/ai-foundry-hosted-agents", "lastCommitDate": "2026-03-02T11:38:30.603Z" }, + { + "id": "ai-foundry-hosted-agents-custom-framework", + "name": "AI Foundry Hosted Agents (Custom Frameworks)", + "architectureDiagram": "images/ai-foundry-hosted-agents.gif", + "categories": [ + "AI Agents" + ], + "services": [ + "Azure AI Foundry", + "Azure API Management" + ], + "shortDescription": "Deploy AI Foundry Hosted Agents built with custom frameworks, including Pydantic AI and Strands.", + "detailedDescription": "This lab provides custom framework examples for AI Foundry Hosted Agents, showing how to package and deploy hosted agents built with Pydantic AI and Strands to Azure Container Apps. It includes a Bicep deployment for Azure API Management, Azure AI Foundry, and a GPT-5-Mini model deployment, plus end-to-end setup notebooks and Dockerfiles.", + "authors": [ + "georgeollis" + ], + "tags": [], + "githubPath": "https://github.com/Azure-Samples/AI-Gateway/tree/main/labs/ai-foundry-hosted-agents-custom-framework", + "lastCommitDate": "2026-07-08T00:00:00.000Z" + }, { "id": "ai-foundry-model-gateway", "name": "AI Foundry Model Gateway", diff --git a/images/ai-foundry-hosted-agents.gif b/images/ai-foundry-hosted-agents.gif index 33dcd4e7..03d0b15f 100644 Binary files a/images/ai-foundry-hosted-agents.gif and b/images/ai-foundry-hosted-agents.gif differ diff --git a/labs/ai-foundry-hosted-agents-custom-framework/.gitignore b/labs/ai-foundry-hosted-agents-custom-framework/.gitignore new file mode 100644 index 00000000..cfc63089 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/.gitignore @@ -0,0 +1,5 @@ +.venv/ +__pycache__/ +*.pyc +.env +params.json \ No newline at end of file diff --git a/labs/ai-foundry-hosted-agents-custom-framework/README.md b/labs/ai-foundry-hosted-agents-custom-framework/README.md new file mode 100644 index 00000000..bb9a204c --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/README.md @@ -0,0 +1,173 @@ +--- +name: "AI Foundry Hosted Agents (Custom Frameworks)" +architectureDiagram: images/ai-foundry-hosted-agents.gif +categories: + - AI Agents +services: + - Microsoft Foundry + - Hosted Agents + - Azure API Management + - Container Registry +shortDescription: Deploy AI Foundry Hosted Agents built with custom frameworks, including Pydantic AI and Strands. +detailedDescription: This lab provides custom framework examples for AI Foundry Hosted Agents, showing how to package and deploy hosted agents built with Pydantic AI and Strands. It includes a Bicep deployment for Azure API Management, Azure AI Foundry resources, and a GPT-5-Mini model deployment, plus a single end-to-end notebook and Dockerfiles. +authors: + - georgeollis +tags: [] +--- + +# APIM ❤️ AI Foundry + +## AI Foundry Hosted Agents with Custom Frameworks + +This lab extends the Hosted Agents scenario with framework-specific examples so you can build agents using your preferred runtime while keeping a common deployment pattern. + +### Why run custom frameworks on Foundry Hosted Agents? + +1. Built-in observability, tracing, and monitoring. +Foundry provides a standard operational surface for agent runs, telemetry, and diagnostics so teams can troubleshoot faster and keep a consistent monitoring model across different agent runtimes. + +2. Agent Identity and RBAC by default. +Your runtime is registered as a Foundry Agent and gets an Agent Identity, enabling least-privilege access to downstream Azure resources (for example, storage, search, or APIs) through RBAC instead of embedded secrets. + +3. Foundry guardrails and governance. +Hosted agents can inherit platform safety controls and governance policies, helping you enforce security and compliance consistently even when the runtime framework is custom. + +4. Discovery through Agent365. +Publishing into the Foundry ecosystem makes agents easier to discover and reuse across teams, reducing duplicate implementations. + +5. Native evaluation and risk testing integration. +You can plug directly into Foundry evaluations, red teaming, and cost-estimation workflows to compare quality, safety, and spend using the same platform tooling. + +6. Control plane and platform operations. +Agents are managed as platform assets in the Foundry control plane, with operational benefits such as managed hosting lifecycle, scaling, and centralized administration. + +### When custom frameworks are a good fit + +- You need framework-specific capabilities (for example, Pydantic AI or Strands primitives) not available in a default runtime. +- You want framework flexibility without giving up Foundry governance and enterprise operations. +- You need to standardize deployment and operations across multiple agent implementations. + +## Lab Notebook + +The lab runs end-to-end from a single notebook. Set the `framework` variable (`strands` or `pydantic`) in the initialization cell to choose which custom-framework agent to build and deploy: + +- [ai-foundry-hosted-agents-custom-framework.ipynb](ai-foundry-hosted-agents-custom-framework.ipynb) + +It deploys the infrastructure with Bicep, builds and pushes the agent image, registers the hosted agent in Foundry, and tests it both directly and through APIM. + +### What gets deployed + +Core services: +- Azure API Management (APIM) as gateway and reverse proxy +- Microsoft Foundry resources (two AI Services resources): + - foundry-models: hosts inference models used by agents + - foundry-agents: hosts your custom framework runtimes via Hosted Agents Responses protocol v1.0.0 +- Azure Container Registry (ACR) for agent container images +- Application Insights and Log Analytics for observability + +RBAC and access control: +- ACR repository permissions for the hosted-agent Foundry resource (Container Registry Repository Reader + AcrPull) +- ACR repository permissions for the deploying user (Container Registry Repository Writer + Container Registry Repository Catalog Lister) +- Foundry User role assignments for configured user object IDs across Foundry resources + +APIM proxy configuration (optional, enabled when enableHostedAgentResponsesApi=true): +- Dedicated API endpoint proxying Foundry Hosted Agent Responses +- Managed identity token injection for Foundry authentication +- Header injection/enforcement: + - Content-Type: application/json + - Foundry-Features: HostedAgents=V1Preview +- Uses APIM subscription key for client authentication +- Supports multiple agents without APIM reconfiguration by using agent-specific URL paths + +Policy definition is in [hosted-agent-policy.xml](hosted-agent-policy.xml). + +The deployment template is in [main.bicep](main.bicep). + +## Framework Samples + +- [Frameworks Overview](src/responses/agents/frameworks/README.md) +- [Strands Framework](src/responses/agents/frameworks/strands/README.md) +- [Pydantic AI Framework](src/responses/agents/frameworks/pydantic/README.md) + +## Prerequisites + +- [Python 3.12 or later version](https://www.python.org/) installed +- [VS Code](https://code.visualstudio.com/) installed with the [Jupyter notebook extension](https://marketplace.visualstudio.com/items?itemName=ms-toolsai.jupyter) enabled +- [uv](https://docs.astral.sh/uv/) — run `uv sync` from the repo root to install dependencies +- [An Azure Subscription](https://azure.microsoft.com/free/) with [Contributor](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/privileged#contributor) + [RBAC Administrator](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/privileged#role-based-access-control-administrator) or [Owner](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/privileged#owner) roles +- [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) installed and [Signed into your Azure subscription](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively) + +> The agent container image is built in Azure Container Registry with `az acr build`, so a local Docker installation is not required. + +## Get Started + +Proceed by opening the [Jupyter notebook](ai-foundry-hosted-agents-custom-framework.ipynb) and follow the steps provided. + +The notebook runs top-to-bottom and: + +1. Deploys the infrastructure with Bicep (Foundry resources, APIM, ACR, monitoring). +2. Builds and pushes the selected framework's agent image to ACR with `az acr build`. +3. Registers the container image as a Foundry Hosted Agent. +4. Tests the agent directly against Foundry and through APIM. + +Set the `framework` variable in the initialization cell to `strands` or `pydantic` to switch frameworks, then re-run the build, deploy, and test cells. + +Invoke a deployed agent through APIM using agent-specific URL path routing: + +```http +POST https://apim-{suffix}.azure-api.net/hosted-agent-responses/agents/{agentName}/endpoint/protocols/openai/responses?api-version=v1 +api-key: {subscription-key} +Content-Type: application/json +``` + +Example body: + +```json +{ + "input": "Hello! What can you help me with?", + "stream": false +} +``` + +- Replace {agentName} to target a different deployed agent. +- Do not use agent_reference in the request body for hosted-agent routing. + +## Test Flow Explained + +Direct test: +- Calls the Foundry Hosted Agent Responses API directly +- Uses a bearer token from the Azure CLI credential with audience https://ai.azure.com/.default +- Best baseline for runtime/connectivity troubleshooting + +APIM test: +- Routes through APIM using the subscription key (api-key) +- APIM injects the managed identity token and required headers +- Validates the production-like client -> APIM -> Foundry path + +## Clean Up Resources + +Use [clean-up-resources.ipynb](clean-up-resources.ipynb) when finished. + +## Troubleshooting + +If direct test fails: +- Verify agent status is Running in Foundry +- Verify Azure CLI authentication (az login) +- Confirm PROJECT_ENDPOINT is correct +- Check networking/firewall constraints + +If APIM test fails but direct test succeeds: +- Verify APIM suffix and subscription key +- Check [hosted-agent-policy.xml](hosted-agent-policy.xml) token audience (https://ai.azure.com) +- Use APIM Trace in Azure Portal to inspect policy and backend flow + +If container startup fails: +- Verify Foundry identity has AcrPull access +- Verify pushed image URI matches agent definition +- Check Foundry diagnostics and Application Insights + +## References + +- [Microsoft Foundry Documentation](https://learn.microsoft.com/en-us/foundry/) +- [Hosted Agents Responses Protocol](https://learn.microsoft.com/en-us/azure/ai-studio/ai-services/agents/protocols/responses) +- [APIM Authentication Policies](https://learn.microsoft.com/en-us/azure/api-management/policies/authentication-policies) diff --git a/labs/ai-foundry-hosted-agents-custom-framework/ai-foundry-hosted-agents-custom-framework.ipynb b/labs/ai-foundry-hosted-agents-custom-framework/ai-foundry-hosted-agents-custom-framework.ipynb new file mode 100644 index 00000000..e99b7d76 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/ai-foundry-hosted-agents-custom-framework.ipynb @@ -0,0 +1,466 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "be6ee817", + "metadata": {}, + "source": [ + "# APIM ❤️ Microsoft Foundry\n", + "\n", + "## Microsoft Foundry Hosted Agents (Custom Frameworks) lab\n", + "![flow](../../images/ai-foundry-hosted-agents.gif)\n", + "\n", + "This lab deploys a **custom-framework agent** ([Strands](https://strandsagents.com/) or [Pydantic AI](https://ai.pydantic.dev/)) as a **Microsoft Foundry Hosted Agent** and exposes it through Azure API Management (APIM).\n", + "\n", + "The lab is backed by two Microsoft Foundry resources:\n", + "- **foundry-models**: hosts the `gpt-5-mini` model deployment, consumed by the agent through APIM.\n", + "- **foundry-agents**: hosts the custom-framework runtime as a Hosted Agent (Responses protocol v1.0.0).\n", + "\n", + "### Why run custom frameworks on Foundry Hosted Agents?\n", + "\n", + "- Built-in observability, tracing, and monitoring across agent runtimes.\n", + "- Agent Identity and RBAC by default (least-privilege access without embedded secrets).\n", + "- Foundry guardrails, governance, evaluations, and red-teaming integration.\n", + "- Discovery and reuse across teams through the Foundry control plane.\n", + "\n", + "Clients invoke a hosted agent by specifying the **agent name in the URL path**, so multiple agents are served without changing the APIM configuration:\n", + "\n", + "```\n", + "POST {apim-gateway}/hosted-agent-responses/agents/{agentName}/endpoint/protocols/openai/responses?api-version=v1\n", + "```\n", + "\n", + "### Prerequisites\n", + "\n", + "- [Python 3.12 or later version](https://www.python.org/) installed\n", + "- [VS Code](https://code.visualstudio.com/) installed with the [Jupyter notebook extension](https://marketplace.visualstudio.com/items?itemName=ms-toolsai.jupyter) enabled\n", + "- [uv](https://docs.astral.sh/uv/) — run `uv sync` from the repo root to install dependencies\n", + "- [An Azure Subscription](https://azure.microsoft.com/free/) with [Contributor](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/privileged#contributor) + [RBAC Administrator](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/privileged#role-based-access-control-administrator) or [Owner](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/privileged#owner) roles\n", + "- [Azure CLI](https://learn.microsoft.com/cli/azure/install-azure-cli) installed and [Signed into your Azure subscription](https://learn.microsoft.com/cli/azure/authenticate-azure-cli-interactively)\n", + "\n", + "▶️ Click `Run All` to execute all steps sequentially, or execute them `Step by Step`..." + ] + }, + { + "cell_type": "markdown", + "id": "3268dff6", + "metadata": {}, + "source": [ + "\n", + "### 0️⃣ Initialize notebook variables\n", + "\n", + "Configure resource names, regions, the model deployment, and the hosted-agent framework used throughout this lab. Set `framework` to `'strands'` or `'pydantic'` to choose which custom-framework agent to build and deploy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86f857ab", + "metadata": {}, + "outputs": [], + "source": [ + "import os, sys, json\n", + "sys.path.insert(1, '../../shared') # add the shared directory to the Python path\n", + "import utils\n", + "\n", + "deployment_name = os.path.basename(os.path.dirname(globals()['__vsc_ipynb_file__']))\n", + "resource_group_name = f\"lab-{deployment_name}\"\n", + "resource_group_location = 'swedencentral'\n", + "\n", + "# Two Microsoft Foundry resources:\n", + "# foundry-models: hosts the inference model consumed by the agent\n", + "# foundry-agents: hosts the custom-framework runtime as a Hosted Agent (index 1)\n", + "aiservices_config = [\n", + " {\"name\": \"foundry-models\", \"location\": \"swedencentral\"},\n", + " {\"name\": \"foundry-agents\", \"location\": \"swedencentral\"},\n", + "]\n", + "\n", + "# Model deployed to foundry-models and consumed by the hosted agent through APIM\n", + "models_config = [\n", + " {\"name\": \"gpt-5-mini\", \"publisher\": \"OpenAI\", \"version\": \"2025-08-07\",\n", + " \"sku\": \"GlobalStandard\", \"capacity\": 10, \"aiservice\": \"foundry-models\"}\n", + "]\n", + "\n", + "# APIM configuration\n", + "apim_sku = 'Basicv2'\n", + "apim_subscriptions_config = [{\"name\": \"subscription1\", \"displayName\": \"Subscription 1\"}]\n", + "\n", + "# API paths\n", + "inference_api_path = 'inference'\n", + "inference_api_type = 'AzureAI'\n", + "hosted_agent_responses_api_path = 'hosted-agent-responses'\n", + "foundry_project_name = 'default'\n", + "foundry_agent_ai_service_index = 1\n", + "\n", + "frameworks = {\n", + " 'strands': {'agent_name': 'strands-agent', 'image': 'strands-agent'},\n", + " 'pydantic': {'agent_name': 'pydantic-agent', 'image': 'pydantic-agent'},\n", + "}\n", + "\n", + "build_version = 1\n", + "\n", + "utils.print_ok('Notebook initialized')" + ] + }, + { + "cell_type": "markdown", + "id": "d15a64d3", + "metadata": {}, + "source": [ + "\n", + "### 1️⃣ Verify the Azure CLI and the connected Azure subscription\n", + "\n", + "The following commands ensure that you have the latest version of the Azure CLI and that the Azure CLI is connected to your Azure subscription." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b4bdd3a", + "metadata": {}, + "outputs": [], + "source": [ + "output = utils.run('az account show', 'Retrieved az account', 'Failed to get the current az account')\n", + "\n", + "if output.success and output.json_data:\n", + " current_user = output.json_data['user']['name']\n", + " tenant_id = output.json_data['tenantId']\n", + " subscription_id = output.json_data['id']\n", + " utils.print_info(f'Current user: {current_user}')\n", + " utils.print_info(f'Tenant ID: {tenant_id}')\n", + " utils.print_info(f'Subscription: {subscription_id}')\n", + "\n", + "# Signed-in user object id is granted the Foundry User (Azure AI User) role on the Foundry resources\n", + "output = utils.run('az ad signed-in-user show', 'Retrieved signed-in user', 'Failed to retrieve signed-in user')\n", + "foundry_user_object_ids = [output.json_data['id']] if output.success and output.json_data else []" + ] + }, + { + "cell_type": "markdown", + "id": "4f38fce7", + "metadata": {}, + "source": [ + "\n", + "### 2️⃣ Create deployment using 🦾 Bicep\n", + "\n", + "This lab uses [Bicep](https://learn.microsoft.com/azure/azure-resource-manager/bicep/overview?tabs=bicep) to declaratively define all the resources deployed in the resource group. Change the parameters or the [main.bicep](main.bicep) directly to try different configurations.\n", + "\n", + "Deployed resources include:\n", + "- Log Analytics Workspace + Application Insights\n", + "- Azure API Management (Basicv2) with the inference API and the hosted-agent responses API\n", + "- Two Microsoft Foundry resources (`foundry-models` with `gpt-5-mini`, `foundry-agents` for the hosted agent)\n", + "- Azure Container Registry for the agent container image\n", + "- Role assignments for ACR and Foundry access" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b89daab8", + "metadata": {}, + "outputs": [], + "source": [ + "# Create the resource group if it doesn't exist\n", + "utils.create_resource_group(resource_group_name, resource_group_location)\n", + "\n", + "# Define the Bicep parameters\n", + "bicep_parameters = {\n", + " '$schema': 'https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#',\n", + " 'contentVersion': '1.0.0.0',\n", + " 'parameters': {\n", + " 'apimSku': {'value': apim_sku},\n", + " 'aiServicesConfig': {'value': aiservices_config},\n", + " 'modelsConfig': {'value': models_config},\n", + " 'apimSubscriptionsConfig': {'value': apim_subscriptions_config},\n", + " 'inferenceAPIPath': {'value': inference_api_path},\n", + " 'inferenceAPIType': {'value': inference_api_type},\n", + " 'foundryProjectName': {'value': foundry_project_name},\n", + " 'foundryAgentAiServiceIndex': {'value': foundry_agent_ai_service_index},\n", + " 'foundryUserObjectIds': {'value': foundry_user_object_ids},\n", + " 'enableHostedAgentResponsesApi': {'value': True},\n", + " 'hostedAgentResponsesApiPath': {'value': hosted_agent_responses_api_path},\n", + " }\n", + "}\n", + "\n", + "# Write the parameters to the params.json file\n", + "with open('params.json', 'w') as bicep_parameters_file:\n", + " bicep_parameters_file.write(json.dumps(bicep_parameters))\n", + "\n", + "# Run the deployment\n", + "output = utils.run(\n", + " f'az deployment group create --name {deployment_name} --resource-group {resource_group_name} --template-file main.bicep --parameters params.json',\n", + " f\"Deployment '{deployment_name}' succeeded\",\n", + " f\"Deployment '{deployment_name}' failed\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "71c43a83", + "metadata": {}, + "source": [ + "\n", + "### 3️⃣ Get the deployment outputs\n", + "\n", + "Retrieve the gateway URL, the subscription key, the Container Registry name, and the Foundry agent project endpoint used by the remaining steps." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f6cc281", + "metadata": {}, + "outputs": [], + "source": [ + "output = utils.run(\n", + " f'az deployment group show --name {deployment_name} -g {resource_group_name}',\n", + " f'Retrieved deployment: {deployment_name}',\n", + " f'Failed to retrieve deployment: {deployment_name}'\n", + ")\n", + "\n", + "if output.success and output.json_data:\n", + " apim_resource_gateway_url = utils.get_deployment_output(output, 'apimResourceGatewayURL', 'APIM Gateway URL')\n", + " container_registry_name = utils.get_deployment_output(output, 'containerRegistryName', 'Container Registry Name')\n", + " foundry_agent_project_endpoint = utils.get_deployment_output(output, 'foundryAgentProjectEndpoint', 'Foundry Agent Project Endpoint')\n", + "\n", + " apim_subscriptions = json.loads(utils.get_deployment_output(output, 'apimSubscriptions').replace(\"\\'\", '\"'))\n", + " for subscription in apim_subscriptions:\n", + " utils.print_info(f\"Subscription Name: {subscription['name']}\")\n", + " utils.print_info(f\"Subscription Key: ****{subscription['key'][-4:]}\")\n", + " api_key = apim_subscriptions[0].get('key')\n", + "\n", + " # Inference endpoint the hosted agent uses to call the model through APIM\n", + " inference_endpoint = f'{apim_resource_gateway_url}/{inference_api_path}/models'" + ] + }, + { + "cell_type": "markdown", + "id": "c52019af", + "metadata": {}, + "source": [ + "\n", + "### 4️⃣ Build and push the agent image to the Container Registry\n", + "\n", + "Build the selected framework's Docker image directly in Azure Container Registry with `az acr build` (no local Docker required). The source is in [`src/frameworks/`](src/frameworks/README.md)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9eafa83b", + "metadata": {}, + "outputs": [], + "source": [ + "# Choose your framework: 'strands' or 'pydantic'\n", + "# For Hosted agent framework to build and deploy\n", + "framework = 'pydantic'\n", + "\n", + "build_version = build_version + 1\n", + "agent_name = frameworks[framework]['agent_name']\n", + "agent_image_tag = f\"{frameworks[framework]['image']}:{build_version}\"\n", + "framework_src = f\"src/frameworks/{framework}\"\n", + "model_deployment_name = models_config[0]['name']\n", + "\n", + "output = utils.run(\n", + " f'az acr build --registry {container_registry_name} --image {agent_image_tag} {framework_src}',\n", + " f\"'{framework}' agent image built and pushed: {agent_image_tag}\",\n", + " f\"Failed to build and push the '{framework}' agent image\"\n", + ")\n", + "\n", + "image_uri = f'{container_registry_name}.azurecr.io/{agent_image_tag}'\n", + "utils.print_info(f'Image URI: {image_uri}')" + ] + }, + { + "cell_type": "markdown", + "id": "8fa67ea3", + "metadata": {}, + "source": [ + "\n", + "### 5️⃣ Create the hosted agent version in Microsoft Foundry\n", + "\n", + "Register the container image as a **Foundry Hosted Agent** (Responses protocol v1.0.0). Foundry auto-assigns a version suffix (`:1`, `:2`, ...). The agent calls the model through the APIM inference API using the injected environment variables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25c56c67", + "metadata": {}, + "outputs": [], + "source": [ + "output = utils.run(\n", + " 'pip install azure-ai-projects==2.3.0 azure-identity -q',\n", + " 'Foundry SDK installed',\n", + " 'Failed to install the Foundry SDK'\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2db292e2", + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.projects import AIProjectClient\n", + "from azure.ai.projects.models import (\n", + " HostedAgentDefinition,\n", + " ProtocolVersionRecord,\n", + " AgentEndpointProtocol,\n", + " ContainerConfiguration,\n", + ")\n", + "from azure.identity import AzureCliCredential\n", + "\n", + "credential = AzureCliCredential()\n", + "\n", + "project = AIProjectClient(\n", + " endpoint=foundry_agent_project_endpoint,\n", + " credential=credential,\n", + " allow_preview=True,\n", + ")\n", + "\n", + "agent = project.agents.create_version(\n", + " agent_name=agent_name,\n", + " definition=HostedAgentDefinition(\n", + " protocol_versions=[\n", + " ProtocolVersionRecord(protocol=AgentEndpointProtocol.RESPONSES, version='1.0.0')\n", + " ],\n", + " cpu='1',\n", + " memory='2Gi',\n", + " container_configuration=ContainerConfiguration(image=image_uri),\n", + " environment_variables={\n", + " 'AZURE_OPENAI_ENDPOINT': inference_endpoint,\n", + " 'AZURE_OPENAI_API_VERSION': '2024-05-01-preview',\n", + " 'AZURE_OPENAI_DEPLOYMENT': model_deployment_name,\n", + " 'APIM_SUBSCRIPTION_KEY': api_key,\n", + " 'LOG_LEVEL': 'INFO',\n", + " #'OTEL_SDK_DISABLED': 'true'\n", + " },\n", + " ),\n", + ")\n", + "\n", + "utils.print_ok(f'Agent created: {agent.name}, version: {agent.version}, id: {agent.id}')" + ] + }, + { + "cell_type": "markdown", + "id": "a3e074e6", + "metadata": {}, + "source": [ + "\n", + "### 🧪 Test the hosted agent directly\n", + "\n", + "Call the Foundry Responses API directly using your Azure CLI credential. This is the baseline test — if it fails, the APIM test will fail too. Wait until the agent reaches the `Running` state before testing.\n", + "\n", + "> **Tip**: Use the [tracing tool](../../tools/tracing.ipynb) to monitor token usage and troubleshoot the [policy](hosted-agent-policy.xml)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39f41137", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "query = 'Hello! What can you help me with?'\n", + "url = f'{foundry_agent_project_endpoint}/agents/{agent_name}/endpoint/protocols/openai/responses'\n", + "\n", + "token = credential.get_token('https://ai.azure.com/.default')\n", + "headers = {\n", + " 'Authorization': f'Bearer {token.token}',\n", + " 'Content-Type': 'application/json',\n", + " 'Foundry-Features': 'HostedAgents=V1Preview',\n", + "}\n", + "\n", + "utils.print_message(f'Q: {query}')\n", + "response = requests.post(url, headers=headers, json={'input': query, 'stream': False}, params={'api-version': 'v1'}, timeout=120)\n", + "\n", + "if response.status_code == 200:\n", + " utils.print_ok('Agent responded successfully (direct)')\n", + " print(json.dumps(response.json(), indent=2))\n", + "else:\n", + " utils.print_error(f'Agent returned HTTP {response.status_code}: {response.text}')" + ] + }, + { + "cell_type": "markdown", + "id": "8b8c9d90", + "metadata": {}, + "source": [ + "\n", + "### 🧪 Test the hosted agent through APIM\n", + "\n", + "Route the request through Azure API Management using only the subscription key. APIM injects the managed-identity bearer token, forces `Content-Type: application/json`, and adds the `Foundry-Features` header. The target agent is selected by the `{agentName}` segment in the URL path." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b8f8455", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "query = 'Hello! What can you help me with?'\n", + "url = f'{apim_resource_gateway_url}/{hosted_agent_responses_api_path}/agents/{agent_name}/endpoint/protocols/openai/responses'\n", + "\n", + "headers = {\n", + " 'api-key': api_key,\n", + " 'Content-Type': 'application/json',\n", + " 'Foundry-Features': 'HostedAgents=V1Preview',\n", + "}\n", + "\n", + "utils.print_message(f'Q: {query}')\n", + "response = requests.post(url, headers=headers, json={'input': query, 'stream': False}, params={'api-version': 'v1'}, timeout=120)\n", + "\n", + "if response.status_code == 200:\n", + " utils.print_ok('Agent responded successfully (via APIM)')\n", + " result = response.json()\n", + " agent_text = None\n", + " if result.get('output'):\n", + " content = result['output'][0].get('content', [])\n", + " if content:\n", + " agent_text = content[0].get('text')\n", + " utils.print_ok(f'A: {agent_text}') if agent_text else print(json.dumps(result, indent=2))\n", + "else:\n", + " utils.print_error(f'Agent returned HTTP {response.status_code}: {response.text}')" + ] + }, + { + "cell_type": "markdown", + "id": "1641aeaf", + "metadata": {}, + "source": [ + "\n", + "### 🗑️ Clean up resources\n", + "\n", + "When you're finished with the lab, you should remove all your deployed resources from Azure to avoid extra charges and keep your Azure subscription uncluttered.\n", + "Use the [clean-up-resources notebook](clean-up-resources.ipynb) for that." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/labs/ai-foundry-hosted-agents-custom-framework/clean-up-resources.ipynb b/labs/ai-foundry-hosted-agents-custom-framework/clean-up-resources.ipynb new file mode 100644 index 00000000..bab5ea71 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/clean-up-resources.ipynb @@ -0,0 +1,54 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f27aae93", + "metadata": {}, + "source": [ + "### 🗑️ Clean up resources\n", + "\n", + "When you're finished with the lab, you should remove all your deployed resources from Azure to avoid extra charges and keep your Azure subscription uncluttered.\n", + "\n", + "The cell below deletes the resource group and everything in it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b90b2598", + "metadata": {}, + "outputs": [], + "source": [ + "import os, sys\n", + "sys.path.insert(1, '../../shared') # add the shared directory to the Python path\n", + "import utils\n", + "\n", + "deployment_name = os.path.basename(os.path.dirname(globals()['__vsc_ipynb_file__']))\n", + "resource_group_name = f\"lab-{deployment_name}\"\n", + "\n", + "utils.cleanup_resources(deployment_name, resource_group_name=resource_group_name)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pip2uv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/labs/ai-foundry-hosted-agents-custom-framework/hosted-agent-policy.xml b/labs/ai-foundry-hosted-agents-custom-framework/hosted-agent-policy.xml new file mode 100644 index 00000000..0ea1e5ea --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/hosted-agent-policy.xml @@ -0,0 +1,31 @@ + + + + + + + + + @("Bearer " + (string)context.Variables["managed-id-access-token"]) + + + + + application/json + + + + + HostedAgents=V1Preview + + + + + + + + + + + + \ No newline at end of file diff --git a/labs/ai-foundry-hosted-agents-custom-framework/main.bicep b/labs/ai-foundry-hosted-agents-custom-framework/main.bicep new file mode 100644 index 00000000..178a7b07 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/main.bicep @@ -0,0 +1,318 @@ +// ------------------ +// PARAMETERS +// ------------------ + +@description('Configuration array for AI Services. Each item needs name and location.') +param aiServicesConfig array = [] + +@description('Configuration array for model deployments.') +param modelsConfig array = [] + +@description('SKU for the API Management instance') +param apimSku string = 'Basicv2' + +@description('Configuration array for APIM subscriptions') +param apimSubscriptionsConfig array = [] + +@description('Path for the inference API exposed in APIM') +param inferenceAPIPath string = 'inference' + +@description('Type of inference API') +param inferenceAPIType string = 'AzureAI' + +@description('Name of the AI Foundry project') +param foundryProjectName string = 'default' + +@description('Index of the AI Services config entry that hosts the Foundry hosted agents (default: second entry).') +param foundryAgentAiServiceIndex int = 1 + +@description('Microsoft Entra object IDs to assign Foundry User (Azure AI User) across all Foundry resources in this deployment.') +param foundryUserObjectIds array = [] + +@description('Enable APIM proxy API for Foundry Hosted Agent Responses. Clients specify agent via agent_reference in request body.') +param enableHostedAgentResponsesApi bool = false + +@description('APIM path for hosted agent responses API.') +param hostedAgentResponsesApiPath string = 'hosted-agent-responses' + +// ------------------ +// VARIABLES +// ------------------ + +var resourceSuffix = uniqueString(subscription().id, resourceGroup().id) +var apiManagementName = 'apim-${resourceSuffix}' +var azureAIUserRoleDefinitionId = resourceId('Microsoft.Authorization/roleDefinitions', '53ca6127-db72-4b80-b1b0-d745d6d5456d') + +// ------------------ +// RESOURCES +// ------------------ + +// 1. Log Analytics Workspace +module lawModule '../../modules/operational-insights/v1/workspaces.bicep' = { + name: 'lawModule' +} + +// 2. Application Insights +module appInsightsModule '../../modules/monitor/v1/appinsights.bicep' = { + name: 'appInsightsModule' + params: { + lawId: lawModule.outputs.id + customMetricsOptedInType: 'WithDimensions' + } +} + +// 3. API Management +module apimModule '../../modules/apim/v3/apim.bicep' = { + name: 'apimModule' + params: { + apimSku: apimSku + apimSubscriptionsConfig: apimSubscriptionsConfig + lawId: lawModule.outputs.id + appInsightsId: appInsightsModule.outputs.id + appInsightsInstrumentationKey: appInsightsModule.outputs.instrumentationKey + } +} + +// 4. AI Foundry with model deployments (for example, gpt-5-mini) +module foundryModule '../../modules/cognitive-services/v3/foundry.bicep' = { + name: 'foundryModule' + params: { + aiServicesConfig: aiServicesConfig + modelsConfig: modelsConfig + apimPrincipalId: apimModule.outputs.principalId + foundryProjectName: foundryProjectName + appInsightsId: appInsightsModule.outputs.id + appInsightsInstrumentationKey: appInsightsModule.outputs.instrumentationKey + appInsightsConnectionString: appInsightsModule.outputs.connectionString + } +} + +// Foundry accounts created by the module, referenced here for RBAC assignments. +resource aiFoundryAccounts 'Microsoft.CognitiveServices/accounts@2025-06-01' existing = [for config in aiServicesConfig: { + name: '${config.name}-${resourceSuffix}' + dependsOn: [foundryModule] +}] + +// 5. APIM Inference API pointing to Foundry/AI Services +module inferenceAPIModule '../../modules/apim/v3/inference-api.bicep' = { + name: 'inferenceAPIModule' + params: { + policyXml: loadTextContent('policy.xml') + apimLoggerId: apimModule.outputs.loggerId + appInsightsId: appInsightsModule.outputs.id + appInsightsInstrumentationKey: appInsightsModule.outputs.instrumentationKey + aiServicesConfig: [foundryModule.outputs.extendedAIServicesConfig[0]] + inferenceAPIType: inferenceAPIType + inferenceAPIPath: inferenceAPIPath + } +} + +// 6. Container Registry (for hosted agent images) +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-11-01-preview' = { + name: 'acr${resourceSuffix}' + location: resourceGroup().location + sku: { + name: 'Basic' + } + properties: { + adminUserEnabled: true + anonymousPullEnabled: false + publicNetworkAccess: 'Enabled' + } +} + +// Assign Foundry User role for all provided users on the model-hosting Foundry resource. +resource modelFoundryUserRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for principalId in foundryUserObjectIds: { + name: guid(resourceGroup().id, aiFoundryAccounts[0].name, principalId, azureAIUserRoleDefinitionId) + scope: aiFoundryAccounts[0] + properties: { + roleDefinitionId: azureAIUserRoleDefinitionId + principalId: principalId + principalType: 'User' + } +}] + +// Assign Foundry User role for all provided users on the hosted-agent Foundry resource. +resource agentFoundryUserRoleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [for principalId in foundryUserObjectIds: { + name: guid(resourceGroup().id, aiFoundryAccounts[foundryAgentAiServiceIndex].name, principalId, azureAIUserRoleDefinitionId) + scope: aiFoundryAccounts[foundryAgentAiServiceIndex] + properties: { + roleDefinitionId: azureAIUserRoleDefinitionId + principalId: principalId + principalType: 'User' + } +}] + +// Reference the Foundry projects created by the module to assign ACR roles to their managed identities. +resource modelFoundryProject 'Microsoft.CognitiveServices/accounts/projects@2025-04-01-preview' existing = { + parent: aiFoundryAccounts[0] + name: '${foundryProjectName}-foundry-models' +} + +resource agentFoundryProject 'Microsoft.CognitiveServices/accounts/projects@2025-04-01-preview' existing = { + parent: aiFoundryAccounts[foundryAgentAiServiceIndex] + name: '${foundryProjectName}-foundry-agents' +} + +// Repository-level ACR roles (ABAC-enabled roles) +var acrRepositoryReaderRoleId = resourceId('Microsoft.Authorization/roleDefinitions', 'b93aa761-3e63-49ed-ac28-beffa264f7ac') +var acrRepositoryWriterRoleId = resourceId('Microsoft.Authorization/roleDefinitions', '2a1e307c-b015-4ebd-883e-5b7698a07328') +var acrRepositoryCatalogListerRoleId = resourceId('Microsoft.Authorization/roleDefinitions', 'bfdb9389-c9a5-478a-bb2f-ba9ca092c3c7') +var acrPullRoleId = resourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') +var foundryModelAccountName = '${aiServicesConfig[0].name}-${resourceSuffix}' +var foundryAgentAccountName = '${aiServicesConfig[foundryAgentAiServiceIndex].name}-${resourceSuffix}' + +// Assign AcrPull to Foundry models account for any container image pulls +resource foundryModelAcrPullRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, foundryModelAccountName, acrPullRoleId) + scope: containerRegistry + properties: { + roleDefinitionId: acrPullRoleId + principalId: foundryModule.outputs.extendedAIServicesConfig[0].principalId + principalType: 'ServicePrincipal' + } +} + +// Assign AcrPull to Foundry models project for container image pulls +resource foundryModelProjectAcrPullRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, modelFoundryProject.id, acrPullRoleId) + scope: containerRegistry + properties: { + roleDefinitionId: acrPullRoleId + principalId: modelFoundryProject.identity.principalId + principalType: 'ServicePrincipal' + } +} + +resource foundryAgentAcrRepositoryReaderRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, foundryAgentAccountName, acrRepositoryReaderRoleId) + scope: containerRegistry + properties: { + roleDefinitionId: acrRepositoryReaderRoleId + principalId: foundryModule.outputs.extendedAIServicesConfig[foundryAgentAiServiceIndex].principalId + principalType: 'ServicePrincipal' + } +} + +// Assign AcrPull to Foundry hosted-agent account for container image pulls +resource foundryAgentAcrPullRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, foundryAgentAccountName, acrPullRoleId) + scope: containerRegistry + properties: { + roleDefinitionId: acrPullRoleId + principalId: foundryModule.outputs.extendedAIServicesConfig[foundryAgentAiServiceIndex].principalId + principalType: 'ServicePrincipal' + } +} + +// Assign AcrPull to Foundry hosted-agent project for container image pulls +resource foundryAgentProjectAcrPullRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, agentFoundryProject.id, acrPullRoleId) + scope: containerRegistry + properties: { + roleDefinitionId: acrPullRoleId + principalId: agentFoundryProject.identity.principalId + principalType: 'ServicePrincipal' + } +} + +resource deployerAcrRepositoryWriterRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, deployer().objectId, containerRegistry.id, acrRepositoryWriterRoleId) + scope: containerRegistry + properties: { + roleDefinitionId: acrRepositoryWriterRoleId + principalId: deployer().objectId + } +} + +resource deployerAcrRepositoryCatalogListerRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, deployer().objectId, containerRegistry.id, acrRepositoryCatalogListerRoleId) + scope: containerRegistry + properties: { + roleDefinitionId: acrRepositoryCatalogListerRoleId + principalId: deployer().objectId + } +} + +// Existing APIM service reference for custom API resources. +resource apimService 'Microsoft.ApiManagement/service@2024-06-01-preview' existing = { + name: apiManagementName + dependsOn: [ + apimModule + ] +} + +// Proxy Foundry Responses API through APIM for any deployed hosted agent. +// Clients specify the target agent via the agent name in the URL path: /agents/{agentName}/endpoint/protocols/openai/responses +resource hostedAgentResponsesApi 'Microsoft.ApiManagement/service/apis@2024-06-01-preview' = if(enableHostedAgentResponsesApi) { + name: 'hosted-agent-responses-api' + parent: apimService + properties: { + apiType: 'http' + description: 'Proxy for Azure AI Foundry Responses API - routes requests to specific hosted agents by agent name in URL path' + displayName: 'Foundry Responses API' + path: hostedAgentResponsesApiPath + protocols: [ + 'https' + ] + serviceUrl: '${foundryModule.outputs.extendedAIServicesConfig[foundryAgentAiServiceIndex].foundryProjectEndpoint}' + subscriptionKeyParameterNames: { + header: 'api-key' + query: 'api-key' + } + subscriptionRequired: true + type: 'http' + } +} + +resource hostedAgentResponsesOperation 'Microsoft.ApiManagement/service/apis/operations@2024-06-01-preview' = if(enableHostedAgentResponsesApi) { + name: 'create-response' + parent: hostedAgentResponsesApi + properties: { + displayName: 'Create Response' + description: 'Create a model response using a specific hosted agent. Agent name must be specified in the URL path.' + method: 'POST' + urlTemplate: '/agents/{agentName}/endpoint/protocols/openai/responses' + templateParameters: [ + { + name: 'agentName' + description: 'Name of the hosted agent to invoke' + type: 'string' + required: true + values: [] + } + ] + responses: [ + { + statusCode: 200 + description: 'Successful response from the agent' + } + ] + } +} + +resource hostedAgentResponsesApiPolicy 'Microsoft.ApiManagement/service/apis/policies@2024-06-01-preview' = if(enableHostedAgentResponsesApi) { + name: 'policy' + parent: hostedAgentResponsesApi + properties: { + format: 'rawxml' + value: loadTextContent('hosted-agent-policy.xml') + } +} + +// ------------------ +// OUTPUTS +// ------------------ + +output logAnalyticsWorkspaceId string = lawModule.outputs.customerId +output apimServiceId string = apimModule.outputs.id +output apimResourceGatewayURL string = apimModule.outputs.gatewayUrl +output apimSubscriptions array = apimModule.outputs.apimSubscriptions +output aiGatewayUrl string = '${apimModule.outputs.gatewayUrl}/${inferenceAPIPath}' +output foundryProjectEndpoint string = foundryModule.outputs.extendedAIServicesConfig[0].foundryProjectEndpoint +output foundryAiServicesEndpoint string = foundryModule.outputs.extendedAIServicesConfig[0].endpoint +output foundryAgentProjectEndpoint string = foundryModule.outputs.extendedAIServicesConfig[foundryAgentAiServiceIndex].foundryProjectEndpoint +output foundryAgentAiServicesEndpoint string = foundryModule.outputs.extendedAIServicesConfig[foundryAgentAiServiceIndex].endpoint +output containerRegistryName string = containerRegistry.name +output containerRegistryLoginServer string = containerRegistry.properties.loginServer +output hostedAgentResponsesApimPath string = enableHostedAgentResponsesApi ? '${apimModule.outputs.gatewayUrl}/${hostedAgentResponsesApiPath}/responses' : '' diff --git a/labs/ai-foundry-hosted-agents-custom-framework/policy.xml b/labs/ai-foundry-hosted-agents-custom-framework/policy.xml new file mode 100644 index 00000000..b06e225f --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/policy.xml @@ -0,0 +1,19 @@ + + + + + + @("Bearer " + (string)context.Variables["managed-id-access-token"]) + + + + + + + + + + + + + diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/README.md b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/README.md new file mode 100644 index 00000000..37278baf --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/README.md @@ -0,0 +1,213 @@ +# Hosted Agent Frameworks + +This directory contains multiple framework implementations for deploying custom agents on **Microsoft Foundry Hosted Agents**. + +## Quick Start + +1. **Choose your framework** in the lab notebook by setting the `framework` variable to `strands` or `pydantic`: + - [**Strands Framework**](#strands-framework) + - [**Pydantic AI Framework**](#pydantic-ai-framework) + +2. **Run the lab notebook** [ai-foundry-hosted-agents-custom-framework.ipynb](../../../../ai-foundry-hosted-agents-custom-framework.ipynb) top-to-bottom. It deploys the infrastructure, builds the selected framework's image, registers the hosted agent, and tests it directly and through APIM. + +## Framework Overview + +### Strands Agents + +https://strandsagents.com/ + +### Pydantic AI + +https://pydantic.dev/docs/ai/core-concepts/agent/ + +## Folder Structure + +``` +frameworks/ +├── strands/ # Strands framework implementation +│ ├── main.py # Agent entry point +│ ├── Dockerfile # Container image definition +│ ├── requirements-strands.txt # Strands framework dependencies +│ ├── README.md # Strands-specific documentation +│ ├── example.env # Environment variables template +│ ├── .gitignore +│ └── .dockerignore +│ +└── pydantic/ # Pydantic AI framework implementation + ├── main.py # Agent entry point + ├── Dockerfile # Container image definition + ├── requirements-pydantic.txt # Pydantic dependencies + ├── README.md # Pydantic-specific documentation + ├── example.env # Environment variables template + ├── .gitignore + └── .dockerignore +``` + +## Strands Framework + +**Best for**: Custom agent logic with full control over request/response handling + +### What it is +- Custom framework for building agents with structured responses +- Direct control over agent behavior and response formatting +- Ideal for domain-specific agent implementations + +### How it works (in the lab notebook, `framework = 'strands'`) +1. **Builds the image in ACR** from the `strands/` directory with `az acr build` +2. **Deploys to Foundry** using the image from ACR +3. **Tests direct API** - calls Foundry Responses API with bearer token +4. **Tests via APIM** - routes through API Management gateway with subscription key + +### Key Files +- `main.py` - Agent execution and response protocol handler +- `requirements-strands.txt` - Framework-specific dependencies +- `Dockerfile` - Container image with `FROM python:3.12-slim` + +### Getting Started +```python +# Run notebook cells in order: +# 1. Build and push Docker image to ACR +# 2. Create hosted agent version in Foundry +# 3. Test direct API call (validates agent is running) +# 4. Test through APIM (validates gateway routing) +``` + +## Pydantic AI Framework + +**Best for**: LLM-powered agents with automatic input/output validation + +### What it is +- Lightweight LLM framework with built-in type safety +- Automatic request/response validation via Pydantic models +- Streamlined agent development with minimal boilerplate + +### How it works (in the lab notebook, `framework = 'pydantic'`) +1. **Builds the image in ACR** from the `pydantic/` directory with `az acr build` +2. **Deploys to Foundry** using the image from ACR +3. **Tests direct API** - calls Foundry Responses API with bearer token +4. **Tests via APIM** - routes through API Management gateway with subscription key + +### Key Files +- `main.py` - Pydantic-based agent with validation +- `requirements-pydantic.txt` - Pydantic and related dependencies +- `Dockerfile` - Container image with `FROM python:3.12-slim` + +### Getting Started +```python +# Run notebook cells in order: +# 1. Build and push Docker image to ACR +# 2. Create hosted agent version in Foundry +# 3. Test direct API call (validates agent is running) +# 4. Test through APIM (validates gateway routing) +``` + +## Important: Foundry Hosted Agent URL Format + +**Agent-specific routing is required** — there is no generic `/responses` endpoint that accepts `agent_reference` in the request body. + +Each agent must be invoked with its name in the URL path: + +``` +POST {FOUNDRY_ENDPOINT}/agents/{AGENT_NAME}/endpoint/protocols/openai/responses?api-version=v1 +``` + +**Through APIM Gateway:** +``` +POST https://apim-{APIM_SUFFIX}.azure-api.net/hosted-agent-responses/agents/{AGENT_NAME}/endpoint/protocols/openai/responses?api-version=v1 +``` + +- `AGENT_NAME` - The name you choose when creating the agent (e.g., `strands-agent`, `pydantic-agent`) +- `api-version=v1` - Required query parameter for Responses protocol v1.0.0 + +## Environment Configuration + +Both frameworks expect these environment variables (set in Foundry agent definition): + +| Variable | Purpose | Example | +|----------|---------|---------| +| `AZURE_OPENAI_ENDPOINT` | APIM inference API for model calls | `https://apim-xyz.azure-api.net/inference/models` | +| `AZURE_OPENAI_API_VERSION` | OpenAI API version | `2024-05-01-preview` | +| `AZURE_OPENAI_DEPLOYMENT` | Model name | `gpt-5-mini` | +| `APIM_SUBSCRIPTION_KEY` | APIM subscription key | (from deployment outputs) | + +## Workflow: Direct vs APIM + +### Direct Call +- **URL**: `https://foundry-agents-{suffix}.services.ai.azure.com/api/projects/default-foundry-agents/agents/{AGENT_NAME}/endpoint/protocols/openai/responses?api-version=v1` +- **Auth**: Bearer token (from `az login` → `https://ai.azure.com/.default`) +- **Use case**: Development, validation that agent is running correctly +- **Pros**: Direct connection, best for debugging +- **Cons**: Requires Azure credentials on client + +### APIM Gateway +- **URL**: `https://apim-{APIM_SUFFIX}.azure-api.net/hosted-agent-responses/agents/{AGENT_NAME}/endpoint/protocols/openai/responses?api-version=v1` +- **Auth**: API key header (`api-key: `) +- **Use case**: Production, client applications, rate limiting, monitoring +- **Pros**: Gateway features (caching, rate limiting, analytics), simpler auth +- **Cons**: One extra hop through APIM + +## Adding a New Framework + +To add another framework (e.g., CrewAI, AutoGen): + +1. **Create folder**: `mkdir frameworks/crew-ai` +2. **Copy template files**: + - Copy `Dockerfile` from existing framework + - Create `main.py` with your framework initialization + - Create `requirements-crew-ai.txt` with framework dependencies + - Create `README.md` documenting the framework +3. **Register the framework in the lab notebook**: add an entry to the `frameworks` map in the initialization cell, for example `'crew-ai': {'agent_name': 'crew-ai-agent', 'image': 'crew-ai-agent'}`, then set `framework = 'crew-ai'` and re-run the build, deploy, and test cells. +4. **Update deployment**: If deploying multiple agents, register them with different names. + +## Testing Your Agent + +After deployment, verify your agent works: + +```bash +# 1. Ensure agent is in "Running" state (check Foundry portal) +# 2. Run Section 3 (Direct) test +# 3. Run Section 4 (APIM) test +# 4. Compare responses - they should be identical +``` + +Successful response structure: +```json +{ + "id": "caresp_...", + "object": "response", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{ + "type": "output_text", + "text": "Agent's response text..." + }] + }], + "status": "completed" +} +``` + +## Troubleshooting + +### "Agent endpoint not found" +- Verify agent name in URL matches the agent name created in Foundry +- Ensure agent is in "Running" state (not "Creating", "Failed", etc.) + +### "api-version query parameter is not allowed" +- Ensure you're using `/v1` path, not `/v2` +- Don't include `api-version` in APIM policy - use it only in URL + +### "Managed identity does not have access to Azure AI" +- Verify Foundry project's managed identity has proper role assignment +- Check APIM managed identity has `Contributor` or `Cognitive Services User` role + +### Agent not receiving requests +- Check the image builds successfully in ACR: `az acr build --registry {registry} --image test:1.0 .` +- Verify APIM policy correctly injects authentication headers +- Check ACR image is pullable by Foundry (check image pull errors in Foundry portal) + +## References + +- [Microsoft Foundry Hosted Agents Documentation](https://learn.microsoft.com/en-us/azure/ai-studio/ai-services/agents/) +- [Responses Protocol v1.0.0 Specification](https://learn.microsoft.com/en-us/azure/ai-studio/ai-services/agents/protocols/responses) +- [APIM Policies for Authentication](https://learn.microsoft.com/en-us/azure/api-management/policies/authentication-policies) diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/.dockerignore b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/.dockerignore new file mode 100644 index 00000000..1a7b9960 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/.dockerignore @@ -0,0 +1,26 @@ +**/__pycache__/ +**/*.py[cod] +**/*.egg-info/ +.eggs/ + +# Virtual environments +.venv/ +venv/ +env/ + +# IDE settings +.vscode/ +.idea/ + +# Version control +.git/ +.gitignore + +# Docker files +.dockerignore + +# Docs +README.md + +# Local environment (never bake credentials into the image) +.env \ No newline at end of file diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/.gitignore b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/.gitignore new file mode 100644 index 00000000..a98520db --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/.gitignore @@ -0,0 +1,4 @@ +.venv/ +__pycache__/ +*.pyc +.env diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/Dockerfile b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/Dockerfile new file mode 100644 index 00000000..c8fd2526 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12-slim +WORKDIR /app +COPY . user_agent/ +WORKDIR /app/user_agent +RUN if [ -f requirements-pydantic.txt ]; then pip install -r requirements-pydantic.txt; elif [ -f requirements.txt ]; then pip install -r requirements.txt; fi +EXPOSE 8088 +CMD ["python", "main.py"] \ No newline at end of file diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/README.md b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/README.md new file mode 100644 index 00000000..ed957700 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/README.md @@ -0,0 +1,69 @@ +# Pydantic AI Agent on Foundry Hosted Agents + +This sample shows how to build, deploy, and test a Pydantic AI-based custom agent on Microsoft Foundry Hosted Agents using the Responses protocol v1.0.0. + +## Framework Overview (Pydantic AI) + +According to the official Pydantic AI agent docs, an agent is the primary abstraction and can be treated as a container for instructions, tools/toolsets, structured output typing, dependency typing, model settings, and reusable capabilities. + +Why that matters for this sample: +- Good fit when output shape and validation matter to downstream systems. +- Good fit when you want typed dependencies and strong IDE/static-checker feedback. +- Good fit when you want to compose reusable behavior via capabilities. + +Official reference: +- https://pydantic.dev/docs/ai/core-concepts/agent/ + +![Pydantic AI Hosted Agent Sample](image.png) + +## What Is Included + +- `main.py`: Pydantic AI agent server implementation (Responses protocol + tool calling + streaming). +- `Dockerfile`: Container definition for hosting the agent in Foundry Hosted Agents. +- `requirements-pydantic.txt`: Python dependencies for this runtime. +- `../../../../ai-foundry-hosted-agents-custom-framework.ipynb`: End-to-end lab notebook (set `framework = 'pydantic'`). + +## Get Started + +1. Open the lab notebook `../../../../ai-foundry-hosted-agents-custom-framework.ipynb` and set `framework = 'pydantic'` in the initialization cell. +2. Run the notebook from top to bottom. +3. It builds and pushes the container image to Azure Container Registry with `az acr build`. +4. It creates a hosted agent version in your Foundry project. +5. It validates the agent through: + - Direct Foundry call (baseline) + - APIM call (production-like path) + +## Prerequisites + +- Azure subscription with access to Microsoft Foundry and Azure Container Registry. +- Permission to push images to the target ACR repository. +- Permission to create hosted agent versions in Foundry. +- Azure CLI installed and authenticated (`az login`). +- Python environment with dependencies required by this sample. + +## Invocation and Routing + +Hosted agents are invoked by agent-specific URL path. + +Direct Foundry endpoint: + +```http +POST {PROJECT_ENDPOINT}/agents/{AGENT_NAME}/endpoint/protocols/openai/responses?api-version=v1 +``` + +APIM endpoint: + +```http +POST https://apim-{APIM_SUFFIX}.azure-api.net/hosted-agent-responses/agents/{AGENT_NAME}/endpoint/protocols/openai/responses?api-version=v1 +``` + +- Do not use `agent_reference` in the request body for hosted-agent routing. +- Use `api-key` header when calling APIM. +- Include `Content-Type: application/json`. + +## Notes + +- This sample routes model calls through APIM inference (`/inference/models`). +- Configure `AZURE_OPENAI_ENDPOINT` to your APIM inference URL and set `APIM_SUBSCRIPTION_KEY`. +- The runtime sends the APIM subscription key in the `api-key` header for model calls. +- Keep notebook placeholder values aligned with your infrastructure deployment outputs. diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/example.env b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/example.env new file mode 100644 index 00000000..d7135309 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/example.env @@ -0,0 +1,6 @@ +AZURE_OPENAI_ENDPOINT=https://XXXXXXXXX.azure-api.net/inference/models +AZURE_OPENAI_DEPLOYMENT=gpt-5-mini +AZURE_OPENAI_API_VERSION=2024-05-01-preview +LOG_LEVEL=INFO +STRANDS_LOG_LEVEL=INFO +APIM_SUBSCRIPTION_KEY=XXXXXXXXX \ No newline at end of file diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/image.png b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/image.png new file mode 100644 index 00000000..63cfd9bd Binary files /dev/null and b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/image.png differ diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/main.py b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/main.py new file mode 100644 index 00000000..a3f0699f --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/main.py @@ -0,0 +1,191 @@ +"""Azure AI Agent Server (Responses protocol), +embedding Pydantic AI as the agent framework. + +Requires the following environment variables (see .env): + + AZURE_OPENAI_ENDPOINT APIM base URL (recommended): + https://.azure-api.net/inference/models + or full chat-completions URL: + https://.azure-api.net/inference/models/chat/completions?api-version=2024-05-01-preview + AZURE_OPENAI_DEPLOYMENT model name sent as the chat-completions "model" field + AZURE_OPENAI_API_KEY required API key used for model calls (or OPENAI_API_KEY) + AZURE_OPENAI_API_VERSION optional, defaults to 2024-05-01-preview + APIM_SUBSCRIPTION_KEY optional; when set, sent as api-key + LOG_LEVEL optional, defaults to INFO + +Authentication: + + This app uses API key authentication for model calls. + It does not use managed identity for OpenAI chat-completions requests. +""" + +import asyncio +import logging +import os +import random +from typing import Sequence + +from dotenv import load_dotenv +from openai import AsyncOpenAI + +from pydantic_ai import Agent +from pydantic_ai.models.openai import OpenAIChatModel +from pydantic_ai.providers.openai import OpenAIProvider + +from azure.ai.agentserver.responses import ( + CreateResponse, + ResponseContext, + ResponsesAgentServerHost, + TextResponse, +) +from azure.ai.agentserver.responses.models import ( + Item, + MessageContentInputTextContent, + MessageContentRefusalContent, + OutputMessageContentOutputTextContent, + get_content_expanded, +) + +load_dotenv() + +logging.basicConfig( + level=os.environ.get("LOG_LEVEL", "INFO"), + format="%(asctime)s %(levelname)-8s %(name)s: %(message)s", +) +logging.getLogger("httpx").setLevel(logging.WARNING) + +logger = logging.getLogger("pydantic_ai_responses_app") + +app = ResponsesAgentServerHost() + +_AGENT: Agent | None = None + + +def _message_text(item: Item) -> list[str]: + texts: list[str] = [] + for part in get_content_expanded(item): + if isinstance(part, (MessageContentInputTextContent, OutputMessageContentOutputTextContent)): + text = getattr(part, "text", None) + if text: + texts.append(text) + elif isinstance(part, MessageContentRefusalContent): + if part.refusal: + texts.append(f"[refused to answer: {part.refusal}]") + return texts + + +def _build_prompt(history: Sequence[Item], input_text: str) -> str: + history_lines: list[str] = [] + for item in history: + role = getattr(item, "role", None) + if role not in ("user", "assistant"): + continue + role_str = getattr(role, "value", role) + for text in _message_text(item): + history_lines.append(f"{role_str}: {text}") + + prompt_parts: list[str] = [] + if history_lines: + prompt_parts.append("Conversation so far:") + prompt_parts.extend(history_lines) + if input_text: + prompt_parts.append(f"user: {input_text}") + return "\n".join(prompt_parts) if prompt_parts else "Hello" + + +def build_agent() -> Agent: + global _AGENT + if _AGENT is None: + endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] + deployment = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "gpt-5-mini") + api_key = ( + os.environ.get("AZURE_OPENAI_API_KEY") + or os.environ.get("OPENAI_API_KEY") + or os.environ.get("APIM_SUBSCRIPTION_KEY") + ) + if not api_key: + raise RuntimeError( + "Missing API key. Set AZURE_OPENAI_API_KEY (or OPENAI_API_KEY / APIM_SUBSCRIPTION_KEY)." + ) + + api_version = os.environ.get("AZURE_OPENAI_API_VERSION", "2024-05-01-preview") + base_url = endpoint.split("?", 1)[0].rstrip("/") + if base_url.endswith("/chat/completions"): + base_url = base_url[: -len("/chat/completions")] + + client = AsyncOpenAI( + base_url=base_url, + api_key=api_key, + default_query={"api-version": api_version}, + default_headers={"api-key": api_key}, + ) + + model = OpenAIChatModel( + deployment, + provider=OpenAIProvider(openai_client=client), + ) + + _AGENT = Agent( + model, + instructions=( + "You are a helpful assistant who can explain concepts, answer questions, and " + "reason through problems. You have access to one tool: get_weather for weather " + "questions." + ), + ) + + @_AGENT.tool_plain + def get_weather(city: str) -> str: + temperature_c = random.randint(-5, 35) + logger.info("tool_call=get_weather city=%s result_c=%s", city, temperature_c) + return f"The current temperature in {city} is {temperature_c} deg C." + + + return _AGENT + + +@app.response_handler +async def handler( + request: CreateResponse, + context: ResponseContext, + cancellation_signal: asyncio.Event, +) -> TextResponse: + input_text = await context.get_input_text() + history_items = await context.get_history() + + logger.info( + "response_id=%s input_chars=%d history_items=%d", + context.response_id, + len(input_text or ""), + len(history_items), + ) + + agent = build_agent() + prompt = _build_prompt(history_items, input_text or "") + + async def _generate_tokens(): + previous = "" + async with agent.run_stream(prompt) as run: + async for chunk in run.stream_text(): + if cancellation_signal.is_set(): + logger.warning("response_id=%s cancellation requested", context.response_id) + break + + if not isinstance(chunk, str) or not chunk: + continue + + # Convert cumulative stream_text output into incremental deltas. + if chunk.startswith(previous): + delta = chunk[len(previous) :] + else: + delta = chunk + + previous = chunk + if delta: + yield delta + + return TextResponse(context, request, text=_generate_tokens()) + + +if __name__ == "__main__": + app.run() diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/requirements-pydantic.txt b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/requirements-pydantic.txt new file mode 100644 index 00000000..f7656d40 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/pydantic/requirements-pydantic.txt @@ -0,0 +1,6 @@ +azure-ai-agentserver-responses==1.0.0b8 +azure-ai-projects==2.3.0 +azure-identity>=1.20.0 +openai>=1.50.0 +pydantic-ai[openai]>=0.0.40 +python-dotenv>=1.0.0 diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/.dockerignore b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/.dockerignore new file mode 100644 index 00000000..1a7b9960 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/.dockerignore @@ -0,0 +1,26 @@ +**/__pycache__/ +**/*.py[cod] +**/*.egg-info/ +.eggs/ + +# Virtual environments +.venv/ +venv/ +env/ + +# IDE settings +.vscode/ +.idea/ + +# Version control +.git/ +.gitignore + +# Docker files +.dockerignore + +# Docs +README.md + +# Local environment (never bake credentials into the image) +.env \ No newline at end of file diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/.gitignore b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/.gitignore new file mode 100644 index 00000000..a98520db --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/.gitignore @@ -0,0 +1,4 @@ +.venv/ +__pycache__/ +*.pyc +.env diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/Dockerfile b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/Dockerfile new file mode 100644 index 00000000..9b31e8a4 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.12-slim +WORKDIR /app +COPY . user_agent/ +WORKDIR /app/user_agent +RUN if [ -f requirements-strands.txt ]; then pip install -r requirements-strands.txt; elif [ -f requirements.txt ]; then pip install -r requirements.txt; fi +EXPOSE 8088 +CMD ["python", "main.py"] \ No newline at end of file diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/README.md b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/README.md new file mode 100644 index 00000000..787806bd --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/README.md @@ -0,0 +1,144 @@ +# Strands Agent on Foundry Hosted Agents + +This sample shows how to build, deploy, and test a **Strands-based custom agent** on Microsoft Foundry Hosted Agents using the **Responses protocol v1.0.0**. + +## Framework Overview (Strands) + +From the official Strands site, Strands is an open-source toolkit focused on building production agents with model/provider flexibility, built-in context management, execution limits, observability, and hook-based runtime control. + +Why that matters for this sample: +- Good fit for tool-heavy workflow automation. +- Good fit when you want to intercept/steer runtime behavior with hooks. +- Good fit when operational visibility and control of the agent loop are top priorities. + +Official reference: +- https://strandsagents.com/ + +The agent runs in a containerized environment managed by Foundry, with full observability, RBAC identity, and optional APIM gateway integration. + +![Strands Hosted Agent Sample](image.png) + +## What Is Included + +- **`main.py`**: Strands agent server implementing the Responses protocol with tool calling and streaming. +- **`Dockerfile`**: Container image definition for Foundry Hosted Agents (builds for Linux amd64). +- **`requirements-strands.txt`**: Python dependencies (Strands, FastAPI, etc.). +- **`../../../../ai-foundry-hosted-agents-custom-framework.ipynb`**: End-to-end lab notebook (set `framework = 'strands'`) covering: + 1. Build and push container to ACR + 2. Create hosted agent version in Foundry + 3. Test agent directly via Foundry Responses API (baseline validation) + 4. Test agent via APIM gateway (production-like path with managed identity auth) + +## How It Works + +### Responses Protocol + +Your Strands agent implements the **Responses protocol v1.0.0**, which is Foundry's standard for hosted agents: +- HTTP-based request/response model (no WebSocket or streaming required for basic cases) +- Request: `POST /endpoint/protocols/openai/responses?api-version=v1` +- Request body: `{ "input": "", "stream": false }` +- Response: `{ "output_text": "" }` (or streaming deltas) + +Your Foundry project manages the deployment, scaling, health checks, and lifecycle. + +### Deployment Architecture + +``` +Client + ↓ +APIM Gateway (managed identity injection, header/param enforcement) + ↓ +Foundry Hosted Agent (runs your Strands container) + ↓ +APIM Inference API (agent calls models via this endpoint) + ↓ +OpenAI (GPT-5-Mini) +``` + +**Authentication flow:** +- Client → APIM: Use `api-key` header (APIM subscription key) +- APIM → Foundry: APIM's managed identity fetches bearer token for `https://ai.azure.com` +- Agent → Models: Agent uses APIM inference endpoint with its own APIM subscription key + +## Get Started + +### Step 1: Configure the lab notebook + +Open `../../../../ai-foundry-hosted-agents-custom-framework.ipynb`, set `framework = 'strands'` in the initialization cell, and run the deployment cells. The APIM gateway URL, Container Registry name, Foundry agent project endpoint, and subscription key are read automatically from the deployment outputs. + +### Step 2: Build & Push Container + +Run the build cell: +- Builds the image in Azure Container Registry: `az acr build --registry {registry} --image strands-agent:1.0.0 src/responses/agents/frameworks/strands` +- No local Docker is required; ACR builds a Linux amd64 image compatible with Foundry hosting. + +### Step 3: Create Hosted Agent + +Run the deploy cell: +- Creates a HostedAgentDefinition with your container image +- Specifies resource allocation (1 CPU, 2Gi memory) +- Sets environment variables for your agent to reach models +- Foundry automatically pulls the image and starts your container + +Once the agent transitions to "Running" state, it's ready for testing. + +### Step 4: Validate Agent Directly + +Run the direct test cell: +- Calls Foundry's Responses API directly using your Azure CLI credential +- No APIM involvement—validates agent and basic connectivity +- Helpful for troubleshooting deployment issues + +If this test fails, check agent status in Foundry or review container logs. + +### Step 5: Test via APIM Gateway (Production Path) + +Run the APIM test cell: +- Routes through APIM gateway using `api-key` header +- APIM automatically: + - Injects managed identity bearer token + - Enforces `Content-Type: application/json` + - Uses Responses API `api-version=v1` + - Injects `Foundry-Features: HostedAgents=V1Preview` (required for preview) +- Validates end-to-end production path + +Policy configuration is in `../../hosted-agent-policy.xml` for customization. + +## Prerequisites + +- Microsoft Foundry resources deployed (see the parent [lab notebook](../../../../ai-foundry-hosted-agents-custom-framework.ipynb)) +- Azure CLI installed and authenticated (`az login`) +- Python 3.12+ with dependencies from the repo root (`uv sync`) +- Azure subscription with permissions to: + - Push images to ACR + - Create agent versions in Foundry + - Assign roles (for RBAC setup) + +## Key Configuration Notes + +- **Agent name**: Use the same value configured as `agent_name` in the lab notebook. +- **Model endpoint**: Your agent calls the APIM **inference** API, not the hosted-agent API. +- **Token audience**: + - Direct tests use `https://ai.azure.com/.default` + - APIM uses `https://ai.azure.com` (managed identity) +- **No hard-coded API keys in agent**: The agent gets credentials through environment variables injected at deployment time. + +## Monitoring & Debugging + +- **Application Insights**: Monitor traces, dependencies, and performance +- **APIM Trace Tool**: In Azure Portal → APIM → Diagnose and solve problems → Trace + - Re-run a test while tracing to see detailed request/response flow + - Useful for debugging authentication, routing, or header issues +- **Foundry Agent Details**: View agent status, logs, and deployment history in Foundry portal + +## Testing Patterns + +**If Section 3 works but Section 4 fails:** +- APIM policy issue likely (verify `hosted-agent-policy.xml`) +- Use APIM Trace to inspect inbound/outbound traffic +- Check managed identity token generation + +**If both tests fail:** +- Agent may not be in "Running" state +- Check ACR image URI and permissions +- Review Foundry agent logs for startup errors diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/example.env b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/example.env new file mode 100644 index 00000000..d7135309 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/example.env @@ -0,0 +1,6 @@ +AZURE_OPENAI_ENDPOINT=https://XXXXXXXXX.azure-api.net/inference/models +AZURE_OPENAI_DEPLOYMENT=gpt-5-mini +AZURE_OPENAI_API_VERSION=2024-05-01-preview +LOG_LEVEL=INFO +STRANDS_LOG_LEVEL=INFO +APIM_SUBSCRIPTION_KEY=XXXXXXXXX \ No newline at end of file diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/image.png b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/image.png new file mode 100644 index 00000000..c1280167 Binary files /dev/null and b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/image.png differ diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/main.py b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/main.py new file mode 100644 index 00000000..cb60c729 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/main.py @@ -0,0 +1,279 @@ +"""Azure AI Agent Server (Responses protocol), +embedding the Strands Agents SDK as the agent framework. + +Requires the following environment variables (see .env): + + AZURE_OPENAI_ENDPOINT APIM base URL (recommended): + https://.azure-api.net/inference/models + or full chat-completions URL: + https://.azure-api.net/inference/models/chat/completions?api-version=2024-05-01-preview + AZURE_OPENAI_DEPLOYMENT model name sent as the chat-completions "model" field + AZURE_OPENAI_API_KEY required API key used for model calls (or OPENAI_API_KEY) + AZURE_OPENAI_API_VERSION optional, defaults to 2024-05-01-preview + APIM_SUBSCRIPTION_KEY optional; when set, sent as api-key + LOG_LEVEL optional, defaults to INFO + STRANDS_LOG_LEVEL optional, defaults to INFO (Strands SDK's own logger) + +Authentication: + + This app uses API key authentication for model calls. + It does not use managed identity for OpenAI chat-completions requests. + +Supports server-side function calling (the "Get Weather" tool and the +"show_internal_environment_variables" debug tool below, both implemented +with ``@tool`` and executed automatically by the Strands agent loop), +incremental token streaming of the model's answer (via +``agent.stream_async()``, piped into ``TextResponse`` as +``response.output_text.delta`` SSE events), image input (input_image content +parts, as inline data: URLs), and multi-turn conversation tracking (via the +Responses protocol's conversation_id / previous_response_id chaining, +surfaced through ResponseContext.get_history() and pre-loaded into the +Strands agent's message history). +""" + +import asyncio +import logging +import os +import random +from typing import Sequence + +from dotenv import load_dotenv +from openai import AsyncOpenAI + +from strands import Agent, tool +from strands.agent.conversation_manager import SlidingWindowConversationManager +from strands.models.openai import OpenAIModel +from strands.types.content import ContentBlock, Messages + +from azure.ai.agentserver.responses import ( + CreateResponse, + ResponseContext, + ResponsesAgentServerHost, + TextResponse, + data_url, +) +from azure.ai.agentserver.responses.models import ( + Item, + ItemMessage, + MessageContentInputImageContent, + MessageContentInputTextContent, + MessageContentRefusalContent, + OutputMessageContentOutputTextContent, + get_content_expanded, +) + +load_dotenv() + +# --- Logging ----------------------------------------------------------- +# A single basicConfig call surfaces logs from our own app logger, the +# Responses hosting layer, and the Strands SDK's agent loop / tool +# invocations, since they all log through the standard `logging` module. +logging.basicConfig( + level=os.environ.get("LOG_LEVEL", "INFO"), + format="%(asctime)s %(levelname)-8s %(name)s: %(message)s", +) +logging.getLogger("httpx").setLevel(logging.WARNING) # quiet noisy per-request HTTP logs +logging.getLogger("strands").setLevel(os.environ.get("STRANDS_LOG_LEVEL", "INFO")) + +logger = logging.getLogger("strands_responses_app") + +app = ResponsesAgentServerHost() + + +@tool +def show_internal_environment_variables() -> str: + """Return the internal environment variables that the Strands agent sees, for debugging.""" + env_vars = {k: v for k, v in os.environ.items()} + return f"{env_vars}" + +@tool +def get_weather(city: str) -> str: + """Get the current weather (temperature) for the given city. + + Args: + city: The city to look up the weather for. + """ + temperature_c = random.randint(-5, 35) + logger.info("tool_call=get_weather city=%s result_c=%s", city, temperature_c) + return f"The current temperature in {city} is {temperature_c}\u00b0C." + + +_MODEL: OpenAIModel | None = None + + +def build_model() -> OpenAIModel: + global _MODEL + if _MODEL is None: + endpoint = os.environ["AZURE_OPENAI_ENDPOINT"] + deployment = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "gpt-5-mini") + api_key = ( + os.environ.get("AZURE_OPENAI_API_KEY") + or os.environ.get("OPENAI_API_KEY") + or os.environ.get("APIM_SUBSCRIPTION_KEY") + ) + if not api_key: + raise RuntimeError( + "Missing API key. Set AZURE_OPENAI_API_KEY (or OPENAI_API_KEY / APIM_SUBSCRIPTION_KEY)." + ) + + api_version = os.environ.get("AZURE_OPENAI_API_VERSION", "2024-05-01-preview") + base_url = endpoint.split("?", 1)[0].rstrip("/") + if base_url.endswith("/chat/completions"): + base_url = base_url[: -len("/chat/completions")] + + client = AsyncOpenAI( + base_url=base_url, + api_key=api_key, + default_query={"api-version": api_version}, + default_headers={"api-key": api_key}, + ) + + _MODEL = OpenAIModel(client=client, model_id=deployment) + return _MODEL + + +def _message_text(item: Item) -> list[str]: + """Extract plain text parts from an item's content, handling both the + typed ``list[MessageContent]`` form and the API's plain-string shorthand + (``get_content_expanded`` normalizes either into a list). Refusals are + included too — an assistant declining to answer is still meaningful + conversation context for the model to see on the next turn. + """ + texts = [] + for part in get_content_expanded(item): + if isinstance(part, (MessageContentInputTextContent, OutputMessageContentOutputTextContent)): + text = getattr(part, "text", None) + if text: + texts.append(text) + elif isinstance(part, MessageContentRefusalContent): + if part.refusal: + texts.append(f"[refused to answer: {part.refusal}]") + return texts + + +def _history_messages(history: Sequence[Item]) -> Messages: + """Convert prior conversation turns (from ResponseContext.get_history()) into + Strands ``Messages`` so the agent starts with full multi-turn context already + loaded into its conversation history.""" + messages: Messages = [] + for item in history: + role = getattr(item, "role", None) + if role not in ("user", "assistant"): + continue + texts = _message_text(item) + if texts: + role_str = getattr(role, "value", role) + messages.append({"role": role_str, "content": [{"text": text} for text in texts]}) + return messages + + +def _extract_image_blocks(items: Sequence[Item]) -> list[ContentBlock]: + """Pull input images out of the request's input items as Strands ContentBlocks. + + Only inline base64 ``data:`` URLs are supported, since Strands content + blocks carry raw image bytes rather than remote URLs. + """ + blocks: list[ContentBlock] = [] + for item in items: + if not isinstance(item, ItemMessage): + continue + for part in item.content or []: + if not isinstance(part, MessageContentInputImageContent): + continue + url = part.image_url + if not url: + continue + if data_url.is_data_url(url): + media_type = data_url.get_media_type(url) or "image/png" + image_format = media_type.split("/", 1)[-1] or "png" + raw_bytes = data_url.decode_bytes(url) + blocks.append({"image": {"format": image_format, "source": {"bytes": raw_bytes}}}) + else: + logger.warning("Skipping remote image URL; Strands requires inline image bytes: %s", url) + return blocks + + +def build_agent(history: Messages) -> Agent: + """Wire up a Strands agent with the weather tool, the Azure OpenAI model, + and prior conversation history pre-loaded so multi-turn context is + preserved across requests.""" + return Agent( + model=build_model(), + tools=[get_weather, show_internal_environment_variables], + system_prompt=( + "You are a helpful assistant who can explain concepts, answer questions, and " + "reason through problems. You have access to two tools - a weather tool that can provide the current " + "temperature when asked about weather or temperature in a specific place and a tool that can show the internal environment variables that the Strands agent sees, for debugging. " + ), + messages=list(history), + conversation_manager=SlidingWindowConversationManager(window_size=20), + callback_handler=None, + ) + + +@app.response_handler +async def handler( + request: CreateResponse, + context: ResponseContext, + cancellation_signal: asyncio.Event, +) -> TextResponse: + """Run a Strands agent against the request's input text/images and prior + conversation turns, with server-side function/tool calling support. + + Streams the model's answer token-by-token as it's generated (via Strands' + ``agent.stream_async()``), so the caller sees incremental + ``response.output_text.delta`` SSE events instead of waiting for the full + answer before anything is returned. + """ + input_items = await context.get_input_items() + input_text = await context.get_input_text() + history_items = await context.get_history() + + logger.info( + "response_id=%s input_chars=%d history_items=%d", + context.response_id, + len(input_text or ""), + len(history_items), + ) + + history_messages = _history_messages(history_items) + image_blocks = _extract_image_blocks(input_items) + + agent = build_agent(history_messages) + + content: list[ContentBlock] = [{"text": input_text}] if input_text else [] + content.extend(image_blocks) + + async def _generate_tokens(): + # Bridge the host's cooperative cancellation signal to the agent's own + # cancel() so a client disconnect/timeout stops in-flight model/tool calls. + async def _watch_cancellation() -> None: + await cancellation_signal.wait() + logger.warning("response_id=%s cancellation requested; stopping agent", context.response_id) + agent.cancel() + + watcher = asyncio.create_task(_watch_cancellation()) + try: + async for event in agent.stream_async(content or None): + text = event.get("data") + if isinstance(text, str) and text: + yield text + + result = event.get("result") + if result is not None: + logger.info( + "response_id=%s stop_reason=%s", + context.response_id, + result.stop_reason, + ) + except Exception: + logger.exception("response_id=%s agent invocation failed", context.response_id) + raise + finally: + watcher.cancel() + + return TextResponse(context, request, text=_generate_tokens()) + + +if __name__ == "__main__": + app.run() diff --git a/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/requirements-strands.txt b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/requirements-strands.txt new file mode 100644 index 00000000..f6dc3847 --- /dev/null +++ b/labs/ai-foundry-hosted-agents-custom-framework/src/frameworks/strands/requirements-strands.txt @@ -0,0 +1,4 @@ +azure-ai-agentserver-responses==1.0.0b8 +strands-agents[openai]==1.45.0 +python-dotenv +azure-ai-projects==2.3.0 diff --git a/modules/cognitive-services/v3/foundry.bicep b/modules/cognitive-services/v3/foundry.bicep index f50b4e0f..7d3dd3b3 100644 --- a/modules/cognitive-services/v3/foundry.bicep +++ b/modules/cognitive-services/v3/foundry.bicep @@ -28,6 +28,10 @@ param foundryProjectName string = 'default' @secure() param appInsightsInstrumentationKey string = '' +@description('The connection string for Application Insights. When provided, it is used as the App Insights connection credential so runtimes (for example, Foundry Hosted Agents) can configure Azure Monitor correctly.') +@secure() +param appInsightsConnectionString string = '' + @description('The resource ID for Application Insights') param appInsightsId string = '' @@ -123,7 +127,7 @@ resource appInsightsConnection 'Microsoft.CognitiveServices/accounts/connections ResourceId: appInsightsId } credentials: { - key: appInsightsInstrumentationKey + key: empty(appInsightsConnectionString) ? appInsightsInstrumentationKey : appInsightsConnectionString } } }] @@ -161,6 +165,7 @@ output extendedAIServicesConfig array = [for (config, i) in aiServicesConfig: { cognitiveService: cognitiveServices[i] cognitiveServiceName: cognitiveServices[i].name cognitiveServicesId: cognitiveServices[i].id + principalId: cognitiveServices[i].identity.principalId endpoint: cognitiveServices[i].properties.endpoint foundryProjectEndpoint: 'https://${cognitiveServices[i].name}.services.ai.azure.com/api/projects/${aiProject[i].name}' }] diff --git a/modules/monitor/v1/appinsights.bicep b/modules/monitor/v1/appinsights.bicep index dac55cef..3e993f63 100644 --- a/modules/monitor/v1/appinsights.bicep +++ b/modules/monitor/v1/appinsights.bicep @@ -84,5 +84,6 @@ resource workbook 'Microsoft.Insights/workbooks@2022-04-01' = if (useWorkbook) { output id string = applicationInsights.id output name string = applicationInsights.name output instrumentationKey string = applicationInsights.properties.InstrumentationKey +output connectionString string = applicationInsights.properties.ConnectionString output appId string = applicationInsights.properties.AppId output applicationInsightsName string = applicationInsightsName