diff --git a/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/extensions/byom-cross-region/main.bicep b/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/extensions/byom-cross-region/main.bicep index 476cc63c3..c4e3018c3 100644 --- a/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/extensions/byom-cross-region/main.bicep +++ b/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/extensions/byom-cross-region/main.bicep @@ -209,10 +209,22 @@ module vnet 'modules/vnet-with-backend-subnet.bicep' = { // Add the apim-outbound subnet alongside the others. Separate resource to // avoid racing the vnet module on subnet collection writes. + +// APIM StandardV2 VNet integration REQUIRES the delegated subnet to have an +// NSG associated. Minimal NSG (default rules) is sufficient for the SV2 +// outbound-integration scenario; the SV2 platform brokers APIM's traffic. +resource apimNsg 'Microsoft.Network/networkSecurityGroups@2024-05-01' = { + name: 'nsg-${apimOutboundSubnetName}-${uniqueSuffix}' + location: location +} + resource apimOutboundSubnet 'Microsoft.Network/virtualNetworks/subnets@2024-05-01' = { name: '${vnetName}/${apimOutboundSubnetName}' properties: { addressPrefix: apimOutboundSubnetPrefix + networkSecurityGroup: { + id: apimNsg.id + } // defaultOutboundAccess: false closes the implicit egress path Azure // assigns to subnets that lack explicit outbound (NAT GW / LB). // APIM's outbound traffic is brokered by the SV2 platform, so the diff --git a/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/tests/TESTING-GUIDE.md b/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/tests/TESTING-GUIDE.md new file mode 100644 index 000000000..65f6e600b --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/tests/TESTING-GUIDE.md @@ -0,0 +1,324 @@ +# BYO Model via APIM AI Gateway - Testing Guide + +This guide covers testing Azure AI Foundry agents that consume a **bring-your-own-model (BYOM)** deployment through a customer-owned **Azure API Management (APIM) AI Gateway**, as deployed by template 16 and its [`extensions/byom-cross-region`](../extensions/byom-cross-region/) template. + +The agent never talks to the model account directly. It resolves a **BYOM model connection** (`/`) that points at APIM, and APIM forwards the request — over a **cross-region private endpoint**, using its **managed identity** — to a backend Foundry account in a second region. + +> **Private Foundry (default):** The Foundry account has `publicNetworkAccess = Disabled`. You need a secure path into the VNet (VPN Gateway, ExpressRoute, or Azure Bastion) to run the SDK tests. See [Connecting to a Private Foundry Resource](#connecting-to-a-private-foundry-resource). For easier local testing you can [switch the Foundry account to public access](#switching-the-foundry-resource-to-public-access) — backend resources (AI Search, Storage, Cosmos, and the cross-region model account) stay private regardless. + +--- + +## Table of Contents + +1. [Prerequisites](#prerequisites) +2. [Permissions required](#permissions-required) +3. [Connecting to a Private Foundry Resource](#connecting-to-a-private-foundry-resource) +4. [Switching the Foundry Resource to Public Access](#switching-the-foundry-resource-to-public-access) +5. [Step 1: Deploy the Template](#step-1-deploy-the-template) +6. [Step 2: Capture Deployment Outputs](#step-2-capture-deployment-outputs) +7. [Step 3: Verify APIM and Private Endpoints](#step-3-verify-apim-and-private-endpoints) +8. [Step 4: Create and Verify the BYOM (APIM) Connection](#step-4-create-and-verify-the-byom-apim-connection) +9. [Step 5: Run the SDK Test](#step-5-run-the-sdk-test) +10. [Troubleshooting](#troubleshooting) +11. [Test Results Summary](#test-results-summary) + +--- + +## Prerequisites + +- Azure CLI installed and authenticated (`az login`) +- Permissions to create resources **and role assignments** — see [Permissions required](#permissions-required) below +- Python 3.10+ with the Agents v2 SDK: + + ```bash + pip install azure-ai-projects azure-identity openai + ``` + +- The BYOM extension deployed (`extensions/byom-cross-region`), which stands up APIM, the backend Foundry account in a second region, the cross-region private endpoint, and the BYOM connection. + +### Permissions required + +This deployment **creates role assignments** (`Microsoft.Authorization/roleAssignments/write`): the agent managed identity gets roles on AI Search / Storage / Cosmos DB, and APIM's managed identity gets a role on the backend Foundry account. Plain **Contributor is not enough** — you need one of: + +- **Owner**, or +- **Contributor + User Access Administrator** + +scoped to the target resource group (or the subscription). + +> **Symptom if missing:** infrastructure (APIM, accounts, private endpoints, DNS) provisions fine, then the deployment fails late with: +> ``` +> Authorization failed ... does not have permission to perform action +> 'Microsoft.Authorization/roleAssignments/write' +> ``` + +Grant **User Access Administrator** at the RG scope (least privilege), or activate it via **PIM** (Azure portal → *Privileged Identity Management* → *My roles*) on nonprod subscriptions: + +```bash +az role assignment create \ + --assignee \ + --role "User Access Administrator" \ + --scope "/subscriptions//resourceGroups/" +``` + +--- + +## Connecting to a Private Foundry Resource + +When the Foundry account has public network access **disabled** (the default), you must reach the Foundry endpoint from inside the VNet before SDK tests will work. + +| Method | Use Case | +|--------|----------| +| **Azure VPN Gateway** | Connect from your local machine over an encrypted tunnel | +| **Azure ExpressRoute** | Private, dedicated connection from on-premises | +| **Azure Bastion** | Access a jump box VM on the VNet through the Azure portal | + +For setup steps, see [Securely connect to Azure AI Foundry](https://learn.microsoft.com/azure/ai-foundry/how-to/configure-private-link#securely-connect-to-foundry). Once connected to the VNet, every command below works as documented. + +--- + +## Switching the Foundry Resource to Public Access + +For easier testing (no VPN/ExpressRoute/Bastion required), you can enable public network access on the **Foundry account only**. The agent still runs inside the VNet (via `networkInjections`) and reaches the private backends — AI Search, Storage, Cosmos DB, and the cross-region model account — through the Data Proxy, so those stay private. Only the Foundry control/data-plane endpoint becomes reachable from the internet. + +In [`modules-network-secured/ai-account-identity.bicep`](../modules-network-secured/ai-account-identity.bicep), change the account's network settings: + +```bicep +// Change from (private, default): + networkAcls: { + defaultAction: 'Deny' + ... + } + publicNetworkAccess: 'Disabled' + +// To (public for testing): + networkAcls: { + defaultAction: 'Allow' + ... + } + publicNetworkAccess: 'Enabled' +``` + +Then redeploy the template. To revert, set `publicNetworkAccess: 'Disabled'` and `defaultAction: 'Deny'` and redeploy. Backend resources remain on private endpoints in both modes. + +> The default stays **private** on purpose — this is a private-network template. Enable public access only while testing, then revert. + +--- + +## Step 1: Deploy the Template + +The extension deploys template 16 end-to-end plus the APIM AI Gateway, the backend Foundry account, the cross-region private endpoint, and the BYOM connection. The template creates a new APIM StandardV2 instance as part of the deployment. + +APIM's `validate-azure-ad-token` policy needs the project managed-identity **app (client) ID**, which is created by this deployment. So you deploy once with a placeholder ID, then set the real value afterward (see [Set the real project identity](#set-the-real-project-identity) below). + +```bash +RESOURCE_GROUP="" +PROJECT_REGION="" # e.g. westus +BACKEND_REGION="" # must differ from PROJECT_REGION +SFX=$(date +%m%d%H%M) # helps keep names globally unique + +az group create --name $RESOURCE_GROUP --location $PROJECT_REGION + +az deployment group create \ + --resource-group $RESOURCE_GROUP \ + --template-file extensions/byom-cross-region/main.bicep \ + --parameters \ + location=$PROJECT_REGION \ + backendLocation=$BACKEND_REGION \ + apimName=apim-byom-$SFX \ + publisherEmail=you@example.com \ + "publisherName=Your Org" \ + backendAccountName=byombackend$SFX \ + projectMiClientId=00000000-0000-0000-0000-000000000000 \ + projectModelName= projectModelVersion= \ + projectModelSkuName=GlobalStandard projectModelCapacity=10 \ + backendModelDeployments='[{"name":"","format":"OpenAI","version":"","skuName":"GlobalStandard","capacity":10}]' +``` + +> Creating APIM StandardV2 takes ~30-45 min. `apimName` and `backendAccountName` must be globally unique. + +### Set the real project identity + +The deploy above configured APIM's `validate-azure-ad-token` policy with a **placeholder** client ID (`00000000-0000-0000-0000-000000000000`). Update it to the project managed identity's real **app (client) ID** so the gateway accepts the agent's token. + +> **Do not re-run the full template to change this.** Once APIM is integrated with the `apim-outbound` subnet, a full redeploy fails with `InUseSubnetCannotBeDeleted` (the VNet can no longer be rewritten). Patch the APIM policy directly instead. + +1. Get the project MI app (client) ID: + + ```bash + PROJECT_ID=$(az deployment group show -g $RESOURCE_GROUP -n \ + --query "properties.outputs.projectId.value" -o tsv) + PRINCIPAL_ID=$(az resource show --ids "$PROJECT_ID" --query "identity.principalId" -o tsv) + APP_ID=$(az ad sp show --id "$PRINCIPAL_ID" --query appId -o tsv) + echo "project MI appId: $APP_ID" + ``` + +2. Patch the APIM `/inference` policy's ``, swapping the placeholder for `$APP_ID`: + + ```bash + BASE="https://management.azure.com/subscriptions//resourceGroups//providers/Microsoft.ApiManagement/service/" + + # fetch current policy, replace the placeholder with the real app id + az rest --method get \ + --url "$BASE/apis/inference/policies/policy?api-version=2022-08-01&format=rawxml" \ + --query "properties.value" -o tsv > policy.xml + sed -i "s/00000000-0000-0000-0000-000000000000/$APP_ID/g" policy.xml + + # PUT the updated policy back + python3 -c "import json;xml=open('policy.xml').read();json.dump({'properties':{'format':'rawxml','value':xml}}, open('policy_body.json','w'))" + az rest --method put \ + --url "$BASE/apis/inference/policies/policy?api-version=2022-08-01" \ + --headers "Content-Type=application/json" --body @policy_body.json + ``` + +The gateway now validates agent tokens against the real project identity. + +--- + +## Step 2: Capture Deployment Outputs + +The test reads its configuration from the deployment outputs. + +```bash +DEPLOYMENT_NAME=$(az deployment group list -g $RESOURCE_GROUP \ + --query "[?contains(name, 'main')].name | [0]" -o tsv) + +# Project (Foundry API) endpoint - build from the project name + account endpoint +PROJECT_NAME=$(az deployment group show -g $RESOURCE_GROUP -n $DEPLOYMENT_NAME \ + --query "properties.outputs.projectName.value" -o tsv) + +# APIM gateway URL and BYOM connection name +APIM_GATEWAY_URL=$(az deployment group show -g $RESOURCE_GROUP -n $DEPLOYMENT_NAME \ + --query "properties.outputs.apimGatewayUrl.value" -o tsv) +BYOM_CONNECTION_NAME=$(az deployment group show -g $RESOURCE_GROUP -n $DEPLOYMENT_NAME \ + --query "properties.outputs.byomConnectionName.value" -o tsv) + +# AI Services account endpoint +AI_SERVICES_NAME=$(az cognitiveservices account list -g $RESOURCE_GROUP --query "[0].name" -o tsv) +AI_ENDPOINT=$(az cognitiveservices account show -g $RESOURCE_GROUP -n $AI_SERVICES_NAME \ + --query "properties.endpoint" -o tsv) + +export PROJECT_ENDPOINT="${AI_ENDPOINT%/}/api/projects/${PROJECT_NAME}" +export APIM_GATEWAY_URL +export BYOM_CONNECTION_NAME +export BYOM_DEPLOYMENT_NAME="gpt-5" # or gpt-4o / gpt-5.1 + +echo "PROJECT_ENDPOINT=$PROJECT_ENDPOINT" +echo "APIM_GATEWAY_URL=$APIM_GATEWAY_URL" +echo "BYOM model = $BYOM_CONNECTION_NAME/$BYOM_DEPLOYMENT_NAME" +``` + +--- + +## Step 3: Verify APIM and Private Endpoints + +```bash +# APIM service (StandardV2, VNet-integrated) +az apim list -g $RESOURCE_GROUP -o table + +# Private endpoints - expect Storage, Cosmos, AI Search, Foundry, and the +# cross-region backend Foundry PE +az network private-endpoint list -g $RESOURCE_GROUP -o table +# Expected: +# *-storage-private-endpoint +# *-cosmosdb-private-endpoint +# *-search-private-endpoint +# *-private-endpoint (project Foundry) +# backend-pe... (cross-region backend Foundry) +``` + +The backend Foundry account and the project Foundry account both have `publicNetworkAccess = Disabled`; the model is never exposed on the public internet. + +--- + +## Step 4: Create and Verify the BYOM (APIM) Connection + +The agent does not talk to APIM by URL — it resolves a **project connection** that points at the APIM `/inference` API. That connection must exist **before** you run the agent test (the same way template 19 requires its A2A/MCP connections to exist before its agent tests). + +### Where the connection is created + +The [`extensions/byom-cross-region`](../extensions/byom-cross-region/) deployment creates it automatically. Its `main.bicep` calls the canonical connection module [`01-connections/apim/connection-apim.bicep`](../../01-connections/apim/connection-apim.bicep) as the `byomConnection` module and emits the name as the `byomConnectionName` output. So if you deployed via Step 1, the connection already exists — skip to verification. + +### Option A - Create it standalone (existing APIM + Foundry) + +If you already have APIM and the Foundry project deployed and only need the connection artifact, deploy the connection module directly: + +```bash +az deployment group create \ + --resource-group $RESOURCE_GROUP \ + --template-file ../../01-connections/apim/connection-apim.bicep \ + --parameters \ + projectResourceId="" \ + apimResourceId="" \ + apiName="inference" \ + connectionName="$BYOM_CONNECTION_NAME" \ + authType="ProjectManagedIdentity" \ + isSharedToAll=true \ + deploymentInPath="true" \ + inferenceAPIVersion="$INFERENCE_API_VERSION" +``` + +> The extension passes a `staticModels` array (gpt-4o / gpt-5 / gpt-5.1) so the deployments surface in the portal. See [`01-connections/apim/samples/parameters-managed-identity-static-models.json`](../../01-connections/apim/samples/parameters-managed-identity-static-models.json) for the full shape. + +### Verify the connection + +Confirm the AI Gateway connection exists and surfaces the backend deployments as `/`: + +```bash +# List project connections (the AI Gateway connection should appear) +az cognitiveservices account connection list \ + -g $RESOURCE_GROUP -n $AI_SERVICES_NAME -o table 2>/dev/null || \ +echo "Use the Foundry portal -> Project -> Management center -> Connected resources to verify '$BYOM_CONNECTION_NAME'." +``` + +In agent code the model is then referenced as, for example, `ai-gateway/gpt-5`. + +--- + +## Step 5: Run the SDK Test + +From a host with access to the VNet (or with public access temporarily enabled): + +```bash +cd tests + +# Run all tests (direct gateway probe + BYOM agent + fallback agent) +python test_byo_model_apim_gateway_agent_v2.py + +# Only the BYOM agent path (the authoritative end-to-end test) +python test_byo_model_apim_gateway_agent_v2.py --test byom + +# Only the project-region fallback model sanity check +python test_byo_model_apim_gateway_agent_v2.py --test basic +``` + +What each test validates: + +| Test | Validates | +|------|-----------| +| `gateway_direct` | Direct HTTP reachability of the APIM `/inference` endpoint. **Only PASSES when run as the project managed identity** (the identity in APIM's ``); from a developer machine the token is rejected, so it reports **SKIPPED** rather than failing. | +| `agent_byom` | The full path: project agent -> BYOM connection -> APIM (managed identity) -> cross-region private endpoint -> backend Foundry model. **This is the authoritative test.** | +| `agent_basic` | The project endpoint and a project-region (local) deployment work at all. | + +--- + +## Troubleshooting + +| Symptom | Likely cause / fix | +|---------|--------------------| +| SDK call times out or connection refused | Foundry account is private. Connect via VPN / ExpressRoute / Bastion, or run from an in-VNet jump box. | +| `agent_byom` fails with **424 / Failed Dependency** | APIM or the cross-region private endpoint cannot reach the backend account. Check the `backend-pe` private endpoint DNS (`privatelink.openai.azure.com`, `privatelink.cognitiveservices.azure.com`, `privatelink.services.ai.azure.com`) and that APIM's managed identity has the token/role on the backend account. | +| `DeploymentNotFound` / **404** | `BYOM_CONNECTION_NAME` / `BYOM_DEPLOYMENT_NAME` don't match the deployed connection. Confirm with Step 4 (default `ai-gateway/gpt-5`). | +| `gateway_direct` returns **401/403** | Expected from a developer identity — the gateway validates the **project** managed identity. Rely on `agent_byom`. | +| `401` on `agent_basic` | Your identity lacks a data-plane role (e.g. **Azure AI User**) on the project. Assign it and retry. | + +--- + +## Test Results Summary + +| Test | Status | +|------|--------| +| `gateway_direct` (direct APIM probe) | ⬜ | +| `agent_byom` (BYOM via APIM gateway) | ⬜ | +| `agent_basic` (project-region model) | ⬜ | + +Fill in ✅ / ❌ as you validate each path in your environment. diff --git a/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/tests/test_byo_model_apim_gateway_agent_v2.py b/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/tests/test_byo_model_apim_gateway_agent_v2.py new file mode 100644 index 000000000..378d3ff98 --- /dev/null +++ b/infrastructure/infrastructure-setup-bicep/16-private-network-standard-agent-apim-setup/tests/test_byo_model_apim_gateway_agent_v2.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +""" +BYO Model via APIM AI Gateway - Agents v2 Test Script + +This script tests the cross-region private bring-your-own-model (BYOM) pattern +that template 16 (private-network standard agent + APIM AI Gateway) enables via +its `extensions/byom-cross-region` deployment. + +Scenario under test: + + Foundry project (project region) + -> BYOM model connection (/) + -> Azure API Management "AI Gateway" (/inference API, managed-identity) + -> cross-region private endpoint + -> backend Foundry account (second region, publicNetworkAccess=Disabled) + +Tests: + 1. APIM Gateway Inference (Direct HTTP) - Direct REST call to the APIM + /inference endpoint (best-effort; requires a token the gateway accepts). + 2. Agent via BYOM Model - Create an agent whose `model` is the BYOM + connection reference "/" and run a prompt. This + exercises the full project -> APIM -> cross-region backend path. + 3. Basic Agent (project-region fallback model) - Sanity check that the + project endpoint and a local (project-region) deployment work. + +Uses the Agents v2 SDK pattern: + - AIProjectClient with a context manager + - project_client.get_openai_client() for the OpenAI-compatible API + - openai_client.responses.create() for the Responses API + - project_client.agents.create_version() with PromptAgentDefinition + +Note: When the Foundry account has public network access disabled (the default +for this template), SDK calls must originate from inside the VNet (VPN / +ExpressRoute / Bastion jump box). See TESTING-GUIDE.md. +""" + +import os +import sys +import json +import ssl +import logging +import argparse +import urllib.request +import urllib.error + +# ============================================================================ +# LOGGING CONFIGURATION +# ============================================================================ +LOG_LEVEL = logging.INFO + +logging.basicConfig( + level=LOG_LEVEL, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) + +logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(LOG_LEVEL) +logging.getLogger("httpx").setLevel(LOG_LEVEL) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("azure.identity").setLevel(logging.WARNING) + +# ============================================================================ + +from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import PromptAgentDefinition +from azure.identity import DefaultAzureCredential + +# ============================================================================ +# CONFIGURATION +# ============================================================================ +# Project-scoped endpoint from the Azure Portal: +# AI Services resource -> Projects -> -> Properties -> "AI Foundry API" endpoint +PROJECT_ENDPOINT = os.environ.get( + "PROJECT_ENDPOINT", + "https://aiservices.services.ai.azure.com/api/projects/project", +) + +# BYOM connection created by the byom-cross-region extension. +# In agent code the backend model is referenced as "/". +# Defaults mirror the extension's parameters (connectionName='ai-gateway', +# backend deployments gpt-4o / gpt-5 / gpt-5.1). +BYOM_CONNECTION_NAME = os.environ.get("BYOM_CONNECTION_NAME", "ai-gateway") +BYOM_DEPLOYMENT_NAME = os.environ.get("BYOM_DEPLOYMENT_NAME", "gpt-5") + +# Model reference used by the agent for the BYOM path. +BYOM_MODEL = os.environ.get( + "BYOM_MODEL", + f"{BYOM_CONNECTION_NAME}/{BYOM_DEPLOYMENT_NAME}", +) + +# Project-region local model deployment (sanity check / fallback). +FALLBACK_MODEL_NAME = os.environ.get("FALLBACK_MODEL_NAME", "gpt-4o") + +# APIM AI Gateway base URL, e.g. https://.azure-api.net +# (the `apimGatewayUrl` output of the byom-cross-region deployment). +APIM_GATEWAY_URL = os.environ.get("APIM_GATEWAY_URL", "") + +# Inference API path + version configured on APIM (extension defaults). +INFERENCE_API_PATH = os.environ.get("INFERENCE_API_PATH", "inference") +INFERENCE_API_VERSION = os.environ.get("INFERENCE_API_VERSION", "2024-10-21") + +# AAD scope for Cognitive Services / Foundry inference. +COGNITIVE_SCOPE = "https://cognitiveservices.azure.com/.default" + +# ============================================================================ + + +def log_response_info(response, label="Response"): + """Extract and log useful debugging info from OpenAI response objects.""" + logger = logging.getLogger(__name__) + try: + if hasattr(response, "_request_id"): + logger.info(f"{label} - Request ID: {response._request_id}") + if hasattr(response, "id"): + logger.info(f"{label} - Response ID: {response.id}") + if hasattr(response, "_response") and hasattr(response._response, "headers"): + headers = response._response.headers + if "x-request-id" in headers: + logger.info(f"{label} - x-request-id: {headers['x-request-id']}") + if "x-ms-request-id" in headers: + logger.info(f"{label} - x-ms-request-id: {headers['x-ms-request-id']}") + except Exception as e: + logger.debug(f"Could not extract response info: {e}") + + +def log_exception_info(exception, label="Exception"): + """Extract and log request info from OpenAI exceptions.""" + logger = logging.getLogger(__name__) + try: + if hasattr(exception, "response") and exception.response is not None: + resp = exception.response + headers = resp.headers if hasattr(resp, "headers") else {} + + request_id = headers.get("x-request-id", "N/A") + ms_request_id = headers.get("x-ms-request-id", "N/A") + + logger.error(f"{label} - x-request-id: {request_id}") + logger.error(f"{label} - x-ms-request-id: {ms_request_id}") + + print(f" 📋 Request ID (x-request-id): {request_id}") + print(f" 📋 MS Request ID (x-ms-request-id): {ms_request_id}") + + if hasattr(resp, "status_code"): + logger.error(f"{label} - HTTP Status: {resp.status_code}") + + if hasattr(exception, "request_id"): + logger.error(f"{label} - request_id attribute: {exception.request_id}") + print(f" 📋 Request ID: {exception.request_id}") + except Exception as e: + logger.debug(f"Could not extract exception info: {e}") + + +def test_apim_gateway_inference( + gateway_url: str, + deployment_name: str, + label: str = "APIM Gateway", +): + """ + Direct HTTP call to the APIM /inference endpoint (best-effort). + + NOTE: This test can only PASS when it runs as the **project managed + identity** (the identity APIM's `validate-azure-ad-token` policy is + configured to accept via ``). From a developer + machine, DefaultAzureCredential presents the developer's own token, which + the gateway rejects (401/403) - so this test reports SKIPPED there, not + failed. To actually exercise this path, run it from inside the agent + runtime / a context signed in as the project MI. The authoritative + end-to-end validation is the agent test (`--test byom`). + + Full URL: + {gateway}/{path}/deployments/{deployment}/chat/completions?api-version=... + + The APIM `/inference` API validates an Azure AD bearer token (the project + managed identity's token in production) and then rewrites the request onto + the cross-region backend Foundry account using its own managed identity. + """ + print("\n" + "=" * 60) + print(f"TEST: APIM Gateway Inference (Direct HTTP) - {label}") + print("=" * 60) + + if not gateway_url: + print(" ⚠ APIM_GATEWAY_URL not configured, skipping direct gateway test") + return None + + url = ( + f"{gateway_url.rstrip('/')}/{INFERENCE_API_PATH.strip('/')}" + f"/deployments/{deployment_name}/chat/completions" + f"?api-version={INFERENCE_API_VERSION}" + ) + print(f" Target: {url}") + + try: + credential = DefaultAzureCredential() + token = credential.get_token(COGNITIVE_SCOPE).token + + payload = json.dumps( + { + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Reply with the single word: pong."}, + ], + "max_tokens": 16, + "temperature": 0, + } + ).encode("utf-8") + + req = urllib.request.Request( + url, + data=payload, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + }, + method="POST", + ) + + ctx = ssl.create_default_context() + with urllib.request.urlopen(req, timeout=30, context=ctx) as response: + status = response.getcode() + body = json.loads(response.read().decode("utf-8")) + content = body["choices"][0]["message"]["content"] + print(f" ✓ HTTP Status: {status}") + print(f" ✓ Model: {body.get('model', 'N/A')}") + print(f" ✓ Content: {content!r}") + + print("\n" + "=" * 60) + print(f"✓ TEST PASSED: {label} inference reachable") + print("=" * 60) + return True + + except urllib.error.HTTPError as e: + if e.code in (401, 403): + print(f" ⚠ Gateway rejected the caller token (HTTP {e.code}).") + print(" This is expected from a developer identity - the gateway") + print(" validates the PROJECT managed identity. Use the agent test") + print(" (below) for the authoritative end-to-end validation.") + print(f"\n⚠ TEST SKIPPED: {label} (token not accepted by gateway)") + return None + detail = e.read().decode("utf-8", errors="ignore")[:300] + print(f"\n✗ TEST FAILED: HTTP {e.code} - {detail}") + return False + + except Exception as e: + print(f"\n✗ TEST FAILED: {str(e)}") + import traceback + + traceback.print_exc() + return False + + +def _run_agent(model: str, agent_name: str, label: str, cleanup_agent: bool = True): + """Create an agent with the given model, run one prompt, and clean up.""" + agent = None + try: + with ( + DefaultAzureCredential() as credential, + AIProjectClient(credential=credential, endpoint=PROJECT_ENDPOINT) as project_client, + project_client.get_openai_client() as openai_client, + ): + print(f"✓ Connected to AI Project at {PROJECT_ENDPOINT}") + + agent = project_client.agents.create_version( + agent_name=agent_name, + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant. Answer briefly and concisely.", + ), + ) + print(f"✓ Created agent (id: {agent.id}, name: {agent.name}, version: {agent.version})") + print(f" Model: {model}") + + response = openai_client.responses.create( + input="Say hello and confirm you are working. Keep it to one short sentence.", + extra_body={"agent_reference": {"name": agent.name, "type": "agent_reference"}}, + ) + log_response_info(response, f"{label} Response") + + print(f"\n✓ Agent response: {response.output_text}") + + if cleanup_agent: + project_client.agents.delete_version( + agent_name=agent.name, agent_version=agent.version + ) + print(f" Cleaned up agent: {agent.name}") + else: + print(f" Preserved agent version: {agent.name}:{agent.version}") + + if response.output_text: + print(f"\n✓ TEST PASSED: {label}") + return True + print(f"\n✗ TEST FAILED: {label} returned no text") + return False + + except Exception as e: + error_str = str(e) + print(f"\n✗ TEST FAILED: {error_str}") + log_exception_info(e, f"{label} Error") + + if "424" in error_str or "Failed Dependency" in error_str: + print("\n ⚠ Known Issue: cross-region backend not reachable.") + print(" The APIM gateway or the cross-region private endpoint could") + print(" not reach the backend Foundry account. Verify the backend") + print(" private endpoint DNS and the APIM managed-identity role.") + elif "DeploymentNotFound" in error_str or "404" in error_str: + print("\n ⚠ Check BYOM_CONNECTION_NAME / BYOM_DEPLOYMENT_NAME match") + print(" the deployed connection (default 'ai-gateway/gpt-5').") + + import traceback + + traceback.print_exc() + + if agent is not None and cleanup_agent: + try: + with ( + DefaultAzureCredential() as credential, + AIProjectClient(credential=credential, endpoint=PROJECT_ENDPOINT) as project_client, + ): + project_client.agents.delete_version( + agent_name=agent.name, agent_version=agent.version + ) + print(f" Cleaned up agent: {agent.name}") + except Exception: + pass + return False + + +def test_agent_via_byom_model(model: str, cleanup_agent: bool = True): + """Create an agent that uses the BYOM (APIM AI Gateway) model reference.""" + print("\n" + "=" * 60) + print("TEST: Agent via BYO Model (APIM AI Gateway)") + print("=" * 60) + return _run_agent( + model=model, + agent_name="byom-apim-gateway-test", + label="Agent via BYOM model", + cleanup_agent=cleanup_agent, + ) + + +def test_basic_agent(model: str, cleanup_agent: bool = True): + """Sanity check: a basic agent on the project-region (local) model.""" + print("\n" + "=" * 60) + print("TEST: Basic Agent (project-region fallback model)") + print("=" * 60) + return _run_agent( + model=model, + agent_name="basic-fallback-test", + label="Basic agent (fallback model)", + cleanup_agent=cleanup_agent, + ) + + +def main(): + parser = argparse.ArgumentParser( + description="BYO Model via APIM AI Gateway - Agents v2 Test Script", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python test_byo_model_apim_gateway_agent_v2.py # Run all tests + python test_byo_model_apim_gateway_agent_v2.py --test byom # Only the BYOM agent test + python test_byo_model_apim_gateway_agent_v2.py --test gateway # Only the direct APIM test + python test_byo_model_apim_gateway_agent_v2.py --test basic # Only the fallback model test + +Environment variables: + PROJECT_ENDPOINT - Azure AI project (Foundry API) endpoint + BYOM_CONNECTION_NAME - AI Gateway connection name (default: ai-gateway) + BYOM_DEPLOYMENT_NAME - Backend deployment name (default: gpt-5) + BYOM_MODEL - Full model reference (default: /) + FALLBACK_MODEL_NAME - Project-region local model (default: gpt-4o) + APIM_GATEWAY_URL - APIM gateway base URL for the direct test (optional) + INFERENCE_API_PATH - APIM inference API path (default: inference) + INFERENCE_API_VERSION - Inference API version (default: 2024-10-21) +""", + ) + parser.add_argument( + "--test", + choices=["gateway", "byom", "basic", "all"], + default="all", + help="Which test(s) to run (default: all)", + ) + parser.add_argument( + "--keep-agent", + action="store_true", + help="Preserve created agent versions instead of deleting them", + ) + args = parser.parse_args() + + cleanup = not args.keep_agent + + print("=" * 60) + print("BYO MODEL via APIM AI GATEWAY - AGENTS v2 TEST") + print("=" * 60) + print("\nConfiguration:") + print(f" Project Endpoint: {PROJECT_ENDPOINT}") + print(f" BYOM Model: {BYOM_MODEL}") + print(f" Fallback Model: {FALLBACK_MODEL_NAME}") + print(f" APIM Gateway URL: {APIM_GATEWAY_URL or '(not set - direct test skipped)'}") + print(f" Inference API Version: {INFERENCE_API_VERSION}") + + results = {} + + if args.test in ["gateway", "all"]: + result = test_apim_gateway_inference(APIM_GATEWAY_URL, BYOM_DEPLOYMENT_NAME) + if result is not None: + results["gateway_direct"] = result + + if args.test in ["byom", "all"]: + results["agent_byom"] = test_agent_via_byom_model(BYOM_MODEL, cleanup_agent=cleanup) + + if args.test in ["basic", "all"]: + results["agent_basic"] = test_basic_agent(FALLBACK_MODEL_NAME, cleanup_agent=cleanup) + + # Summary + print("\n" + "=" * 60) + print("TEST SUMMARY") + print("=" * 60) + + if not results: + print(" No tests were run.") + return 1 + + for test_name, passed in results.items(): + status = "✓ PASSED" if passed else "✗ FAILED" + print(f" {test_name}: {status}") + + all_passed = all(results.values()) + print("\n" + "=" * 60) + if all_passed: + print("ALL TESTS PASSED!") + else: + print("SOME TESTS FAILED") + print("Note: if the Foundry account is private, run this from inside the") + print(" VNet (VPN / ExpressRoute / Bastion). See TESTING-GUIDE.md.") + print("=" * 60) + + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main())