eval: fix flaky deploy tests

evals/azure-skills/azure-deploy/deploy-eval.yaml

@@ -54,7 +54,7 @@ stimuli:
       - "Create a static whiteboard web app and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -85,7 +85,7 @@ stimuli:
       - "Create a static portfolio website and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -121,7 +121,7 @@ stimuli:
       - "Create a discussion board application and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -152,7 +152,7 @@ stimuli:
       - "Create a todo list with frontend and API and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -188,7 +188,7 @@ stimuli:
       - "Create a serverless HTTP API using Azure Functions and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -218,7 +218,7 @@ stimuli:
       - "Create an event-driven function app to process messages and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -248,7 +248,7 @@ stimuli:
       - "Create an azure python function app that takes input from a service bus trigger and does message processing and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -283,7 +283,7 @@ stimuli:
       - "Create a workflow app that orchestrates a multi-step order processing pipeline. Make the app as simple as possible for demonstration purposes only. Then deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -327,7 +327,7 @@ stimuli:
       - "Create a containerized web application and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -373,7 +373,7 @@ stimuli:
       - "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -416,7 +416,7 @@ stimuli:
       - "Create a static whiteboard web app and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -453,7 +453,7 @@ stimuli:
       - "Create a static portfolio website and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -495,7 +495,7 @@ stimuli:
       - "Create a discussion board application and deploy it to Azure App Service, prefer Terraform over Bicep, in my current subscription in the westus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -532,7 +532,7 @@ stimuli:
       - "Create a todo list with frontend and API and deploy to Azure App Service using Terraform in my current subscription in the westus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -574,7 +574,7 @@ stimuli:
       - "Create a serverless HTTP API using Azure Functions and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -610,7 +610,7 @@ stimuli:
       - "Create an event-driven function app to process messages and deploy it to Azure Functions using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -646,7 +646,7 @@ stimuli:
       - "Create a URL shortener service using Azure Functions that creates short links and redirects users to the original URL and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -681,7 +681,7 @@ stimuli:
       - "Create a containerized web application and deploy it to Azure Container Apps using terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -731,7 +731,7 @@ stimuli:
       - "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -769,7 +769,7 @@ stimuli:
       - "Create a simple social media application with likes and comments and deploy to Azure using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool."
       - "Go with recommended options and proceed with Azure deployment."
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -820,7 +820,7 @@ stimuli:
       commands:
         - git clone --depth 1 https://github.com/dotnet/eShop.git .
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -863,7 +863,7 @@ stimuli:
         - git sparse-checkout set aspnetcore/tutorials/first-mvc-app/start-mvc/sample/10.0-completed
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -906,7 +906,7 @@ stimuli:
         - git sparse-checkout set samples/aspire-with-azure-functions
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -949,7 +949,7 @@ stimuli:
         - git sparse-checkout set samples/client-apps-integration
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -993,7 +993,7 @@ stimuli:
         - git sparse-checkout set samples/container-build
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -1037,7 +1037,7 @@ stimuli:
         - git sparse-checkout set samples/custom-resources
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -1075,7 +1075,7 @@ stimuli:
         - git sparse-checkout set samples/database-containers
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -1117,7 +1117,7 @@ stimuli:
         - git sparse-checkout set samples/orleans-voting
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -1164,7 +1164,7 @@ stimuli:
       commands:
         - git clone --depth 1 https://github.com/benc-uk/nodejs-demoapp.git .
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -1207,7 +1207,7 @@ stimuli:
         - git sparse-checkout set samples/aspire-with-javascript
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -1250,7 +1250,7 @@ stimuli:
         - git sparse-checkout set samples/aspire-with-node
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -1293,7 +1293,7 @@ stimuli:
       commands:
         - git clone --depth 1 https://github.com/UltiRequiem/flask-calculator.git .
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full
@@ -1336,7 +1336,7 @@ stimuli:
         - git sparse-checkout set samples/aspire-with-python
         - git checkout -q FETCH_HEAD
     constraints:
-      max_turns: 80
+      max_turns: 100
     tags:
       type: integration
       tier: full

tests/utils/agent-runner.ts

@@ -577,8 +577,6 @@ function generateMarkdownReport(config: AgentRunConfig, agentMetadata: AgentMeta
   lines.push("# Assistant");
   lines.push("");
 
-  // Track message deltas to reconstruct full messages
-  const messageDeltas: Record<string, string> = {};
   const reasoningDeltas: Record<string, string> = {};
   const toolResults: Record<string, { success: boolean; timestamp: string; content?: string; error?: string; }> = {};
 
@@ -617,16 +615,6 @@ function generateMarkdownReport(config: AgentRunConfig, agentMetadata: AgentMeta
         break;
       }
 
-      case "assistant.message_delta": {
-        // Accumulate deltas for streaming - we'll use the final message instead
-        const messageId = event.data.messageId as string;
-        const deltaContent = event.data.deltaContent as string;
-        if (messageId && deltaContent) {
-          messageDeltas[messageId] = (messageDeltas[messageId] || "") + deltaContent;
-        }
-        break;
-      }
-
       case "assistant.reasoning": {
         const content = event.data.content as string;
         if (content) {
@@ -1277,20 +1265,14 @@ export function doesAssistantMessageIncludeKeyword(
   keyword: string,
   options: KeywordOptions = {}
 ): boolean {
-  // Merge all messages and message deltas
+  // Merge all messages
+  // message_delta events are skipped since the assistant.message events contain combined content of their corresponding assistant.message_delta events.
   const allMessages: Record<string, string> = {};
 
   agentMetadata.events.forEach(event => {
     if (event.type === "assistant.message" && event.data.messageId && event.data.content) {
       allMessages[event.data.messageId] = event.data.content;
     }
-    if (event.type === "assistant.message_delta" && event.data.messageId) {
-      if (allMessages[event.data.messageId]) {
-        allMessages[event.data.messageId] += event.data.deltaContent ?? "";
-      } else {
-        allMessages[event.data.messageId] = event.data.deltaContent ?? "";
-      }
-    }
   });
 
   return Object.values(allMessages).some(message => {

tests/utils/evaluate.ts

@@ -490,13 +490,6 @@ export function getAllAssistantMessages(agentMetadata: AgentMetadata): string {
     if (event.type === "assistant.message" && event.data.messageId && event.data.content) {
       allMessages[event.data.messageId] = event.data.content;
     }
-    if (event.type === "assistant.message_delta" && event.data.messageId) {
-      if (allMessages[event.data.messageId]) {
-        allMessages[event.data.messageId] += event.data.deltaContent ?? "";
-      } else {
-        allMessages[event.data.messageId] = event.data.deltaContent ?? "";
-      }
-    }
   });
 
   return Object.values(allMessages).join("\n");