diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index 0efb83bc9..a1a34beda 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -156,7 +156,7 @@ jobs: core.setOutput("has_azure_deploy", hasAzureDeploy.toString()); // Build matrix excluding azure-deploy - const matrix = skills.filter(s => s !== "azure-deploy"); + const matrix = skills.filter(s => s !== "azure-skills/azure-deploy"); core.setOutput("matrix", JSON.stringify(matrix)); # Call the azure-deploy workflow when azure-deploy is in the skills list diff --git a/.github/workflows/test-azure-deploy.yml b/.github/workflows/test-azure-deploy.yml index 5b60dc454..4c5bc05ed 100644 --- a/.github/workflows/test-azure-deploy.yml +++ b/.github/workflows/test-azure-deploy.yml @@ -180,7 +180,7 @@ jobs: MODEL_OVERRIDE: ${{ inputs.model-override }} TEST_GROUP: ${{ matrix.test-group }} run: | - npm run test:integration -- azure-deploy "$TEST_GROUP" + npm run test:vally -- --plugin azure-skills --skill azure-deploy --tag "category=$TEST_GROUP" continue-on-error: true - name: Generate report diff --git a/evals/azure-skills/azure-deploy/deploy-eval.yaml b/evals/azure-skills/azure-deploy/deploy-eval.yaml new file mode 100644 index 000000000..eb03bc2b5 --- /dev/null +++ b/evals/azure-skills/azure-deploy/deploy-eval.yaml @@ -0,0 +1,1454 @@ +# Vally e2e eval suite for azure-deploy +# Source: tests/azure-deploy/integration.test.ts +# Covers: vanilla-static-web-apps-deploy, vanilla-app-service-deploy, +# vanilla-azure-functions-deploy, durable-task-scheduler-deploy, +# vanilla-azure-container-apps-deploy, terraform-static-web-apps-deploy, +# terraform-app-service-deploy, terraform-azure-functions-deploy, +# terraform-azure-container-apps-deploy +# +# These tests previously started from an empty workspace and asked the agent to +# both CREATE and DEPLOY an application, which conflates azure-prepare and +# azure-deploy responsibilities. Each stimulus below instead provides a +# pre-built fixture (azure.yaml + infra/) so the agent only needs to deploy. +# +# earlyTerminate fires when the agent outputs a deploy link (azurewebsites.net, +# azurestaticapps.net, azurecontainerapps.io, web.core.windows.net) — indicating +# the deployment completed and the app is live. +# NOTE: earlyTerminate tests must NOT use the `completed` grader. + +name: azure-deploy-deploy-eval +description: | + End-to-end evaluation suite for the azure-deploy skill. + Tests deployment execution quality across: + - Static Web Apps (Bicep + Terraform) + - App Service (Bicep + Terraform) + - Azure Functions (Bicep + Terraform) + - Durable Task Scheduler (Bicep) + - Azure Container Apps (Bicep + Terraform) + + Each scenario provides a fully prepared fixture (azure.yaml + infra/) so the + agent is only evaluated on its azure-deploy responsibilities, not azure-prepare. + +tags: + type: integration + skill: azure-deploy + +defaults: + runs: 1 + timeout: "55m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +stimuli: + + # ═══════════════════════════════════════════ + # vanilla-static-web-apps-deploy (Bicep) + # Source: describe("vanilla-static-web-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates whiteboard application with bicep" + # Original problem: empty workspace, agent had to create app + deploy. + # Fix: provide a prepared SWA fixture; agent only needs to deploy. + - name: "Deploy SWA - Whiteboard App (Bicep)" + prompt: "Create a static whiteboard web app and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: vanilla-static-web-apps-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates static portfolio website with bicep" + - name: "Deploy SWA - Portfolio Website (Bicep)" + prompt: "Create a static portfolio website and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: vanilla-static-web-apps-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # vanilla-app-service-deploy (Bicep) + # Source: describe("vanilla-app-service-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates discussion board" + - name: "Deploy App Service - Discussion Board (Bicep)" + prompt: "Create a discussion board application and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: vanilla-app-service-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates todo list with frontend and API" + - name: "Deploy App Service - Todo List (Bicep)" + prompt: "Create a todo list with frontend and API and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: vanilla-app-service-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # vanilla-azure-functions-deploy (Bicep) + # Source: describe("vanilla-azure-functions-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates serverless HTTP API" + - name: "Deploy Azure Functions - Serverless HTTP API (Bicep)" + prompt: "Create a serverless HTTP API using Azure Functions and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: vanilla-azure-functions-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates event-driven function app" + - name: "Deploy Azure Functions - Event-Driven (Bicep)" + prompt: "Create an event-driven function app to process messages and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: vanilla-azure-functions-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates Python function app with Service Bus trigger" + - name: "Deploy Azure Functions - Python Service Bus (Bicep)" + prompt: "Create an azure python function app that takes input from a service bus trigger and does message processing and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: vanilla-azure-functions-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # durable-task-scheduler-deploy (Bicep) + # Source: describe("durable-task-scheduler-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates and deploys workflow app with Durable Task Scheduler" + - name: "Deploy DTS - Workflow App (Bicep)" + prompt: "Create a workflow app that orchestrates a multi-step order processing pipeline. Make the app as simple as possible for demonstration purposes only. Then deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: durable-task-scheduler-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + # Verify DTS infrastructure is present in the agent-generated workspace + - type: file-matches + config: + path: "infra/**/*.bicep" + pattern: "(?i)Microsoft\\.DurableTask/schedulers" + - type: file-matches + config: + path: "infra/**/*.bicep" + pattern: "0ad04412-c4d5-4796-b79c-f76d14c8d402" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # vanilla-azure-container-apps-deploy (Bicep) + # Source: describe("vanilla-azure-container-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates containerized web application with Bicep" + - name: "Deploy Container Apps - Containerized Web App (Bicep)" + prompt: "Create a containerized web application and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: vanilla-azure-container-apps-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/**/*.bicep" + pattern: "(?i)Microsoft\\.App/containerApps" + - type: file-matches + config: + path: "infra/**/*.bicep" + pattern: "(?i)Microsoft\\.ContainerRegistry/registries" + - type: file-matches + config: + path: "infra/**/*.bicep" + pattern: "(?i)SystemAssigned" + - type: file-matches + config: + path: "infra/**/*.bicep" + pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates simple containerized Node.js app" + - name: "Deploy Container Apps - Node.js App (Bicep)" + prompt: "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: vanilla-azure-container-apps-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/**/*.bicep" + pattern: "(?i)Microsoft\\.App/containerApps" + - type: file-matches + config: + path: "infra/**/*.bicep" + pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # terraform-static-web-apps-deploy + # Source: describe("terraform-static-web-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates whiteboard application with Terraform" + - name: "Deploy SWA - Whiteboard App (Terraform)" + prompt: "Create a static whiteboard web app and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-static-web-apps-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/**/*.tf" + - type: file-not-exists + config: + path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates static portfolio website with Terraform infrastructure" + - name: "Deploy SWA - Portfolio Website (Terraform)" + prompt: "Create a static portfolio website and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-static-web-apps-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/**/*.tf" + - type: file-not-exists + config: + path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # terraform-app-service-deploy + # Source: describe("terraform-app-service-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates discussion board with Terraform" + - name: "Deploy App Service - Discussion Board (Terraform)" + prompt: "Create a discussion board application and deploy it to Azure App Service, prefer Terraform over Bicep, in my current subscription in the westus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-app-service-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/**/*.tf" + - type: file-not-exists + config: + path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates todo list with frontend and API using Terraform" + - name: "Deploy App Service - Todo List (Terraform)" + prompt: "Create a todo list with frontend and API and deploy to Azure App Service using Terraform in my current subscription in the westus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-app-service-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/**/*.tf" + - type: file-not-exists + config: + path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # terraform-azure-functions-deploy + # Source: describe("terraform-azure-functions-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates serverless HTTP API with Terraform" + - name: "Deploy Azure Functions - Serverless HTTP API (Terraform)" + prompt: "Create a serverless HTTP API using Azure Functions and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-azure-functions-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/**/*.tf" + - type: file-not-exists + config: + path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates event-driven function app with Terraform" + - name: "Deploy Azure Functions - Event-Driven (Terraform)" + prompt: "Create an event-driven function app to process messages and deploy it to Azure Functions using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-azure-functions-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/**/*.tf" + - type: file-not-exists + config: + path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates URL shortener service with Terraform infrastructure" + - name: "Deploy Azure Functions - URL Shortener (Terraform)" + prompt: "Create a URL shortener service using Azure Functions that creates short links and redirects users to the original URL and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-azure-functions-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # terraform-azure-container-apps-deploy + # Source: describe("terraform-azure-container-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates containerized web application with Terraform" + - name: "Deploy Container Apps - Containerized Web App (Terraform)" + prompt: "Create a containerized web application and deploy it to Azure Container Apps using terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-azure-container-apps-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/**/*.tf" + pattern: "(?i)azurerm_container_app[^_]" + - type: file-matches + config: + path: "infra/**/*.tf" + pattern: "(?i)azurerm_container_app_environment" + - type: file-matches + config: + path: "infra/**/*.tf" + pattern: "(?i)azurerm_container_registry" + - type: file-matches + config: + path: "infra/**/*.tf" + pattern: "(?i)SystemAssigned" + - type: file-matches + config: + path: "infra/**/*.tf" + pattern: "(?i)AcrPull" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates simple containerized Node.js app with Terraform" + - name: "Deploy Container Apps - Node.js App (Terraform)" + prompt: "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-azure-container-apps-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/**/*.tf" + pattern: "(?i)azurerm_container_app[^_]" + - type: file-matches + config: + path: "infra/**/*.tf" + pattern: "(?i)AcrPull" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "creates social media application with Terraform infrastructure" + - name: "Deploy Container Apps - Social Media App (Terraform)" + prompt: "Create a simple social media application with likes and comments and deploy to Azure using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: terraform-azure-container-apps-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/**/*.tf" + pattern: "(?i)azurerm_container_app[^_]" + - type: file-matches + config: + path: "infra/**/*.tf" + pattern: "(?i)AcrPull" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # brownfield-dotnet + # Source: describe("brownfield-dotnet") + # Brownfield tests clone real repos; the agent is expected to run the full + # azure-prepare → azure-validate → azure-deploy pipeline. + # earlyTerminate fires when azd up/provision is attempted (we don't run + # real Azure provisioning in evals). + # NOTE: earlyTerminate tests must NOT use the `completed` grader. + # ═══════════════════════════════════════════ + + # Jest: "deploys eShop" + - name: "Brownfield .NET - eShop" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + Use azd as the deployment tool. + environment: + commands: + - git clone --depth 1 https://github.com/dotnet/eShop.git . + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-dotnet + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "deploys MvcMovie 10" + - name: "Brownfield .NET - MvcMovie 10" + prompt: >- + Please deploy this application to Azure. + Use the westus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under aspnetcore/tutorials/first-mvc-app/start-mvc/sample/10.0-completed. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/AspNetCore.Docs + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set aspnetcore/tutorials/first-mvc-app/start-mvc/sample/10.0-completed + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-dotnet + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "deploys aspire azure functions" + - name: "Brownfield .NET Aspire - Azure Functions" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/aspire-with-azure-functions. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/aspire-with-azure-functions + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-dotnet + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "deploys aspire client apps integration" + - name: "Brownfield .NET Aspire - Client Apps Integration" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/client-apps-integration. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/client-apps-integration + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-dotnet + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "deploys aspire container build" + # Checks Container Apps two-phase Bicep pattern in the agent-generated workspace. + - name: "Brownfield .NET Aspire - Container Build" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/container-build. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/container-build + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-dotnet + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "does not deploy aspire custom resources" + # The agent MUST NOT attempt deployment — custom Aspire resources are unsupported. + # No earlyTerminate; let it run and verify azd up is never called. + - name: "Brownfield .NET Aspire - Custom Resources (No Deploy)" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/custom-resources. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/custom-resources + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-dotnet + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Stop if there is no further work; otherwise go with recommended options." + graders: + - type: skill-invocation + config: + required: + - azure-prepare + # Agent must NOT attempt actual deployment + - type: tool-calls + config: + disallowed: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|deploy|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys aspire database containers" + - name: "Brownfield .NET Aspire - Database Containers" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/database-containers. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/database-containers + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-dotnet + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "deploys aspire orleans-voting" + - name: "Brownfield .NET Aspire - Orleans Voting" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/orleans-voting. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/orleans-voting + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-dotnet + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + # AZD env file should have the ACR + - type: file-matches + config: + path: ".azure/**/*" + pattern: "(?i)AZURE_CONTAINER_REGISTRY_ENDPOINT|AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID|MANAGED_IDENTITY_CLIENT_ID" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # brownfield-javascript + # Source: describe("brownfield-javascript") + # ═══════════════════════════════════════════ + + # Jest: "deploys nodejs-demoapp" + - name: "Brownfield JavaScript - nodejs-demoapp" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + Use azd as the deployment tool. + environment: + commands: + - git clone --depth 1 https://github.com/benc-uk/nodejs-demoapp.git . + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-javascript + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "deploys aspire with javascript" + - name: "Brownfield .NET Aspire - JavaScript" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/aspire-with-javascript. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/aspire-with-javascript + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-javascript + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "deploys aspire with node" + - name: "Brownfield .NET Aspire - Node" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/aspire-with-node. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/aspire-with-node + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-javascript + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # ═══════════════════════════════════════════ + # brownfield-python + # Source: describe("brownfield-python") + # ═══════════════════════════════════════════ + + # Jest: "deploys flask calculator" + - name: "Brownfield Python - Flask Calculator" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + Use azd as the deployment tool. + environment: + commands: + - git clone --depth 1 https://github.com/UltiRequiem/flask-calculator.git . + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-python + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" + + # Jest: "deploys aspire with python" + - name: "Brownfield .NET Aspire - Python" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/aspire-with-python. + Use azd as the deployment tool. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/aspire-with-python + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + category: brownfield-python + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" diff --git a/evals/azure-skills/azure-deploy/eval.yaml b/evals/azure-skills/azure-deploy/output-eval.yaml similarity index 67% rename from evals/azure-skills/azure-deploy/eval.yaml rename to evals/azure-skills/azure-deploy/output-eval.yaml index 5e2b97dae..e8711386c 100644 --- a/evals/azure-skills/azure-deploy/eval.yaml +++ b/evals/azure-skills/azure-deploy/output-eval.yaml @@ -1,112 +1,156 @@ -# Vally eval config — migrated from Waza -# Source: tests/azure-deploy/eval/eval.yaml + tasks/*.yaml -# Migration: Waza → Vally (issue #1817, Phase 1) -# -# Global graders (evaluate#125 workaround): has_output and no_runtime_failure -# are duplicated into every stimulus block below. - -name: azure-deploy-eval -description: | - Evaluation suite for the azure-deploy skill. - Tests deployment guidance quality with emphasis on: - - AVM+AZD pattern-module preference - - AVM fallback when no pattern module exists - - deploy-only routing (not prepare/validate) - -tags: - type: integration - skill: azure-deploy - -defaults: - runs: 3 - timeout: "7m" - executor: integration-test-agent-runner - model: claude-sonnet-4.6 - -scoring: - threshold: 0.8 - weights: - output-contains: 0.125 - output-matches: 0.125 - output-not-matches: 0.5 - completed: 0.25 - -stimuli: - # ── avm-order-bicep-001 ── - # Waza source: tasks/avm-order-bicep.yaml - # Validates deploy guidance prefers AVM+AZD pattern modules first - - name: "AVM+AZD Priority - Bicep Deploy" - prompt: | - My app is already prepared and validated. - Give me deploy guidance and module preference order for Bicep. - Prefer AVM+AZD patterns where available, with fallback to AVM resource modules and AVM utility modules. - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - # Task: expected.output_contains - - type: output-contains - config: - substring: "AVM" - - type: output-contains - config: - substring: "deploy" - - type: output-contains - config: - substring: "pattern" - # Task grader: avm_pattern_first - - type: output-matches - config: - pattern: "(?i)AVM\\+AZD|AZD pattern|pattern modules" - # Task grader: includes_resource_and_utility_fallback - - type: output-matches - config: - pattern: "(?is)(AVM\\+AZD|AZD pattern|pattern modules).*resource modules.*utility modules" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: - pattern: "(?i)fatal error|exception occurred|crashed" - - # ── avm-fallback-no-pattern-001 ── - # Waza source: tasks/avm-fallback-no-pattern.yaml - # Validates fallback order when no AVM+AZD pattern module exists - - name: "AVM Fallback When No AZD Pattern" - prompt: | - I'm deploying with Bicep and there is no AVM+AZD pattern module for my scenario. - What module order should I follow if no pattern module exists and fallback must stay AVM resource modules then AVM utility modules? - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - # Task: expected.output_contains - - type: output-contains - config: - substring: "AVM" - - type: output-contains - config: - substring: "resource" - - type: output-contains - config: - substring: "utility" - # Task grader: explicit_no_pattern_fallback - - type: output-matches - config: - pattern: "(?is)(no .*pattern module|if no .*pattern).*AVM.*resource.*AVM.*utility" - # Task grader: avoids_non_avm_fallback - - type: output-not-matches - config: - pattern: "(?i)fallback to non-AVM|use non-AVM modules" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: +# Vally eval config — migrated from Waza +# Source: tests/azure-deploy/eval/eval.yaml + tasks/*.yaml +# Migration: Waza → Vally (issue #1817, Phase 1) +# +# Global graders (evaluate#125 workaround): has_output and no_runtime_failure +# are duplicated into every stimulus block below. + +name: azure-deploy-output-eval +description: | + Evaluation suite for the azure-deploy skill. + Tests deployment guidance quality with emphasis on: + - AVM+AZD pattern-module preference + - AVM fallback when no pattern module exists + - deploy-only routing (not prepare/validate) + +tags: + type: integration + skill: azure-deploy + +defaults: + runs: 1 + timeout: "7m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +scoring: + threshold: 0.8 + weights: + output-contains: 0.125 + output-matches: 0.125 + output-not-matches: 0.5 + completed: 0.25 + +stimuli: + # ── avm-order-bicep-001 ── + # Waza source: tasks/avm-order-bicep.yaml + # Validates deploy guidance prefers AVM+AZD pattern modules first + - name: "AVM+AZD Priority - Bicep Deploy" + prompt: | + My app is already prepared and validated. + Give me deploy guidance and module preference order for Bicep. + Prefer AVM+AZD patterns where available, with fallback to AVM resource modules and AVM utility modules. + Use azd as the deployment tool. + tags: + type: integration + skill: azure-deploy + tier: full + cost: llm + area: output + graders: + # Task: expected.output_contains + - type: output-contains + config: + substring: "AVM" + - type: output-contains + config: + substring: "deploy" + - type: output-contains + config: + substring: "pattern" + # Task grader: avm_pattern_first + - type: output-matches + config: + pattern: "(?i)AVM\\+AZD|AZD pattern|pattern modules" + # Task grader: includes_resource_and_utility_fallback + - type: output-matches + config: + pattern: "(?is)(AVM\\+AZD|AZD pattern|pattern modules).*resource modules.*utility modules" + # Global: has_output + - type: completed + # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|exception occurred|crashed" + + # ── avm-fallback-no-pattern-001 ── + # Waza source: tasks/avm-fallback-no-pattern.yaml + # Validates fallback order when no AVM+AZD pattern module exists + - name: "AVM Fallback When No AZD Pattern" + prompt: | + I'm deploying with Bicep and there is no AVM+AZD pattern module for my scenario. + What module order should I follow if no pattern module exists and fallback must stay AVM resource modules then AVM utility modules? + Use azd as the deployment tool. + tags: + type: integration + skill: azure-deploy + tier: full + cost: llm + area: output + graders: + # Task: expected.output_contains + - type: output-contains + config: + substring: "AVM" + - type: output-contains + config: + substring: "resource" + - type: output-contains + config: + substring: "utility" + # Task grader: explicit_no_pattern_fallback + - type: output-matches + config: + pattern: "(?is)(no .*pattern module|if no .*pattern).*AVM.*resource.*AVM.*utility" + # Task grader: avoids_non_avm_fallback + - type: output-not-matches + config: + pattern: "(?i)fallback to non-AVM|use non-AVM modules" + # Global: has_output + - type: completed + # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|exception occurred|crashed" + + # ── avm-azd-pattern-preference-001 ── + # Source: tests/azure-deploy/avm/integration.test.ts + # describe("avm-azd-pattern-preference") + # Validates that AZD pattern modules are preferred over AVM resource modules + # when setting up azd infrastructure with Bicep for a container app. + - name: "AZD Pattern Module Priority - Container App" + prompt: | + Set up azd infrastructure with Bicep for a container app. + Use AVM modules and prefer AZD pattern modules over resource modules. + Use azd as the deployment tool. + tags: + type: integration + skill: azure-deploy + tier: full + cost: llm + area: output + graders: + - type: output-contains + config: + substring: "AZD" + - type: output-contains + config: + substring: "pattern" + # Task grader: avm_azd_pattern_preferred — matches "AZD pattern", "AVM+AZD", or "pattern modules" + - type: output-matches + config: + pattern: "(?i)AZD\\s+pattern|AVM\\+AZD|pattern modules?" + # Task grader: context_present — AVM/Bicep/container/module present + - type: output-matches + config: + pattern: "(?i)(container|bicep|module)" + # Task grader: pattern_before_resource — pattern modules discussed before resource modules + - type: output-matches + config: + pattern: "(?is)(AZD pattern|AVM\\+AZD|pattern modules?).*resource modules?" + # Global: has_output + - type: completed + # Global: no_runtime_failure + - type: output-not-matches + config: pattern: "(?i)fatal error|exception occurred|crashed" \ No newline at end of file diff --git a/evals/azure-skills/azure-deploy/routing-eval.yaml b/evals/azure-skills/azure-deploy/routing-eval.yaml new file mode 100644 index 000000000..7014e74d6 --- /dev/null +++ b/evals/azure-skills/azure-deploy/routing-eval.yaml @@ -0,0 +1,153 @@ +# Vally eval config — migrated from Jest integration tests +# Source: tests/azure-deploy/integration.test.ts describe("skill-invocation") +# Migration: Jest skill-invocation tests → Vally routing stimuli +# +# Each stimulus provides a minimal prepared project (azure.yaml + infra/) so +# the agent has concrete context to route against. +# Stimuli with earlyTerminate tags do not use the completed grader. + +name: azure-deploy-routing-eval +description: | + Integration evaluation for azure-deploy skill — migrated from Jest skill-invocation tests. + Tests routing for Azure Deploy prompts using invocation rate measurement. + +tags: + type: integration + skill: azure-deploy + +defaults: + runs: 5 + timeout: "10m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +scoring: + threshold: 0.8 + +stimuli: + # ═══════════════════════════════════════════ + # Skill routing prompts + # Source: describe("skill-invocation") + # ═══════════════════════════════════════════ + + # Jest: "invokes azure-deploy skill for deployment prompt" + - name: "Deployment Prompt" + prompt: "Run azd up to deploy my already-prepared app to Azure. Use azd as the deployment tool." + environment: + files: + - src: fixture/swa-bicep/azure.yaml + dest: azure.yaml + - src: fixture/swa-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/swa-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/swa-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + tags: + type: integration + tier: smoke + cost: llm + area: routing + skill: azure-deploy + earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-count","count":10}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "invokes azure-deploy skill for publish to Azure prompt" + - name: "Publish to Azure Prompt" + prompt: "My app already has azure.yaml and infra/ configured. Publish it to Azure now. Use azd as the deployment tool." + environment: + files: + - src: fixture/swa-bicep/azure.yaml + dest: azure.yaml + - src: fixture/swa-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/swa-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/swa-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + tags: + type: integration + tier: smoke + cost: llm + area: routing + skill: azure-deploy + earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-count","count":10}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "invokes azure-deploy skill for Azure Functions deployment prompt" + - name: "Azure Functions Deployment Prompt" + prompt: "Deploy my existing Azure Functions project to the cloud. The infrastructure and azure.yaml are already set up. Use azd as the deployment tool." + environment: + files: + - src: fixture/functions-bicep/azure.yaml + dest: azure.yaml + - src: fixture/functions-bicep/host.json + dest: host.json + - src: fixture/functions-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/functions-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/functions-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + tags: + type: integration + tier: smoke + cost: llm + area: routing + skill: azure-deploy + earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-count","count":10}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "invokes azure-deploy skill for post-deployment role assignment check prompt" + - name: "Post-Deployment Role Assignment Prompt" + prompt: "Deploy my already-prepared Azure app and confirm the managed identity roles are properly assigned Use azd as the deployment tool." + environment: + files: + - src: fixture/container-apps-bicep/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-bicep/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/container-apps-bicep/infra/abbreviations.json + dest: infra/abbreviations.json + - src: fixture/container-apps-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/container-apps-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + tags: + type: integration + tier: smoke + cost: llm + area: routing + skill: azure-deploy + earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-count","count":10}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace"