eval: skill comparison script

.github/skills/analyze-comparison-tests/SKILL.md

@@ -0,0 +1,50 @@
+---
+name: analyze-comparison-tests
+description: "Collects comparison test run artifacts and answers the user's questions based on the trajectories of each run. WHEN TO USE: collect comparison test artifacts"
+license: MIT
+metadata:
+  author: Microsoft
+  version: "1.0.0"
+---
+
+# Steps
+
+1. Collect run artifacts
+
+Execute the collect-artifacts script to download the test run artifacts.
+
+The user must provide an JSON file to correlate each comparison test run with the GitHub Actions run. The script expects one input argument as the path to this JSON file. The JSON input is supposed to be the JSON output when queuing the comparison test runs using the `npm run compare:run` command.
+
+```bash
+cd tests/
+npm run compare:collect -- input.json
+```
+
+The collect-artifacts script will download the test run artifacts to a directory named `comparison-artifacts` in the current working directory. Before executing the script, check if there is already such an directory. If so, skip executing the script and proceed to step 2.
+
+2. Extract insights
+
+The downloaded artifacts will have the following folder structure:
+
+```text
+comparison-artifacts/
+├── <branch-name>/
+│   ├── <stimulus-name-1>/
+│   │   ├── <model>-with-skill/
+│   │   │   ├── agent-metadata-<date-string-1>.md
+│   │   │   ├── agent-metadata-<date-string-2>.md
+│   │   │   └── ...
+│   │   └── <model>-without-skill/
+│   │       ├── agent-metadata-<date-string-1>.md
+│   │       ├── agent-metadata-<date-string-2>.md
+│   │       └── ...
+│   └── <stimulus-name-2>/
+│       ├── <model>-with-skill/
+│       │   └── agent-metadata-*.md
+│       └── <model>-without-skill/
+│           └── agent-metadata-*.md
+└── <branch-name-2>/
+    └── ...
+```
+
+Each `<branch-name>/<stimulus-name>/<model>-with-skill` or `<branch-name>/<stimulus-name>/<model>-without-skill` directory contains the test run trajectories for that stimulus and model on that branch, with or without skills. Each trajectory is a markdown file that records user prompts, tool call requests, tool execution results, assistant responses that happened during the run. It also contains statistics such as token usage and turns. Based on the trajectories, answer the user's questions for each test run. Generate a report following the [report-template](./references/report-template.md) to show your answers.

.github/skills/analyze-comparison-tests/references/report-template.md

@@ -0,0 +1,19 @@
+# Comparison Report
+
+Skill: {plugin dirname/skill name}
+
+## Answers
+
+### {User question 1}
+
+{Answer to question 1}
+
+### {User question 2}
+
+{Answer to question 2}
+
+......
+
+### {User question N}
+
+{Answer to question N}

.github/workflows/test-all-integration.yml

@@ -28,10 +28,13 @@ on:
       model-override:
         description: "Model to use for testing"
         required: false
-        type: choice
-        options:
-          - claude-sonnet-4.6
-          - claude-opus-4.6
+        type: string
+        default: claude-sonnet-4.6
+      no-skills:
+        description: "Optional: whether to override the run to load no skills"
+        required: false
+        type: boolean
+        default: false
       skill-test-pattern:
         description: "Optional: pattern by name or describe block for filtering skill tests. This parameter does not apply to azure-deploy tests"
         required: false
@@ -169,6 +172,7 @@ jobs:
       model-override: ${{ inputs.model-override }}
       test-pattern: ${{ needs.resolve-inputs.outputs.deploy-test-pattern }}
       debug: ${{ needs.resolve-inputs.outputs.debug == 'true' }}
+      no-skills: ${{ inputs.no-skills }}
 
   test:
     name: Integration – ${{ matrix.skill }}
@@ -292,6 +296,7 @@ jobs:
         if: ${{ !contains(fromJson(env.JEST_SKILLS), matrix.skill) }}
         env:
           DEBUG: ${{ needs.resolve-inputs.outputs.debug == 'true' && '1' || '' }}
+          NO_SKILLS: ${{ inputs.no-skills && 'true' || '' }}
           TEST_RUN_ID: all-integration
           MODEL_OVERRIDE: ${{ inputs.model-override }}
           SKILL: ${{ matrix.skill }}

.github/workflows/test-azure-deploy.yml

@@ -30,6 +30,11 @@ on:
         required: false
         type: boolean
         default: false
+      no-skills:
+        description: 'Optional: whether to override the run to load no skills'
+        required: false
+        type: boolean
+        default: false
   workflow_call:
     inputs:
       model-override:
@@ -46,6 +51,11 @@ on:
         required: false
         type: boolean
         default: false
+      no-skills:
+        description: 'Optional: whether to override the run to load no skills'
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   setup:
@@ -177,6 +187,7 @@ jobs:
           GH_HEAD_SHA: ${{ github.sha }}
           TEST_RUN_ID: azure-deploy
           DEBUG: ${{ inputs.debug && '1' || '' }}
+          NO_SKILLS: ${{ inputs.no-skills && 'true' || '' }}
           MODEL_OVERRIDE: ${{ inputs.model-override }}
           TEST_GROUP: ${{ matrix.test-group }}
         run: |

tests/.gitignore

@@ -1,3 +1,4 @@
 reports/
 node_modules/
 coverage/
+comparison-artifacts/

tests/comparison/collect-artifacts.ts

@@ -0,0 +1,241 @@
+/**
+ * collect-artifacts.ts — download comparison test trajectories from Azure Storage.
+ *
+ * Usage: tsx collect-artifacts.ts <input.json>
+ *
+ * Exit codes:
+ *   0 = success (all runs collected)
+ *   1 = a step failed (missing dependency, Azure error, or no blobs found)
+ *   2 = usage/argument error
+ */
+
+import fs from "fs";
+import path from "path";
+import { execFileSync } from "child_process";
+import type { CompareRunOutput } from "./run-compare";
+
+const STORAGE_ACCOUNT = "strdashboarddevveobvk";
+const CONTAINER = "manual-integration-reports";
+const OUTPUT_ROOT = "comparison-artifacts";
+
+function usage(): void {
+  console.log(`Usage: collect-artifacts.ts <input.json>
+
+Exit codes:
+  0 = success (all runs collected)
+  1 = a step failed (missing dependency, Azure error, or no blobs found)
+  2 = usage/argument error`);
+}
+
+function encodeBranchName(branch: string) {
+  return branch.replaceAll("/", "_");
+}
+
+function run(): void {
+  const args = process.argv.slice(2);
+
+  if (args.length === 1 && (args[0] === "-h" || args[0] === "--help")) {
+    usage();
+    process.exit(0);
+  }
+
+  if (args.length !== 1) {
+    console.error(
+      "Error: expected exactly one argument (path to the input JSON file)."
+    );
+    usage();
+    process.exit(2);
+  }
+
+  const inputFile = args[0];
+
+  if (!fs.existsSync(inputFile)) {
+    console.error(`Error: input file not found: ${inputFile}`);
+    process.exit(2);
+  }
+
+  let input: CompareRunOutput;
+  try {
+    const content = fs.readFileSync(inputFile, "utf-8");
+    input = JSON.parse(content);
+  } catch (err: unknown) {
+    const msg = err instanceof Error ? err.message : "unknown error";
+    console.error(`Error: failed to parse input JSON: ${msg}`);
+    process.exit(2);
+  }
+
+  const date = input.date || "";
+  const skillName = input.skill?.name || "";
+
+  if (!date || !skillName) {
+    console.error("Error: input JSON must define 'date' and 'skill.name'.");
+    process.exit(2);
+  }
+
+  if (!input.results || input.results.length === 0) {
+    console.error("Error: input JSON contains no runs.");
+    process.exit(2);
+  }
+
+  // Create output directory
+  if (!fs.existsSync(OUTPUT_ROOT)) {
+    fs.mkdirSync(OUTPUT_ROOT, { recursive: true });
+  }
+
+  let failed = 0;
+
+  for (const result of input.results) {
+    const branch = result.branch;
+    const runs = result.runs;
+    for (const run of runs) {
+      const model: string = run.model;
+      const withSkill: boolean = run.withSkill;
+      const runUrl: string = run.run;
+
+      if (!model) continue;
+
+      // Extract run ID from GitHub Actions URL
+      const runId = runUrl.split("/").pop();
+      if (!runId) {
+        console.error(`Error: could not extract run id from URL: ${runUrl}`);
+        failed = 1;
+        continue;
+      }
+
+      const skillSuffix = withSkill ? "with-skill" : "without-skill";
+
+      // Discover stimuli for this run
+      const prefix = `${date}/${runId}/${skillName}/${skillName}_`;
+      console.log(
+        `Discovering stimuli for run ${runId} under ${CONTAINER}/${prefix} ...`
+      );
+
+      let discoveryResult: string;
+      try {
+        discoveryResult = execFileSync("az", [
+          "storage", "blob", "list",
+          "--account-name", STORAGE_ACCOUNT,
+          "--container-name", CONTAINER,
+          "--prefix", prefix,
+          "--auth-mode", "login",
+          "--query", "[?ends_with(name, '.md')].name",
+          "-o", "tsv"
+        ], {
+          encoding: "utf-8",
+          stdio: ["pipe", "pipe", "ignore"]
+        }) as string;
+      } catch {
+        console.error(`Error: failed to discover blobs for run ${runId}.`);
+        failed = 1;
+        continue;
+      }
+
+      // Extract unique stimuli names from blob paths
+      const blobLines = discoveryResult.trim().split("\n").filter((line: string) => line);
+      const stimuliSet: Set<string> = new Set();
+
+      for (const blob of blobLines) {
+        const regexPattern = new RegExp(`/${skillName}_([^/]+)/`);
+        const match = blob.match(regexPattern);
+        if (match) {
+          stimuliSet.add(match[1]);
+        }
+      }
+
+      if (stimuliSet.size === 0) {
+        console.warn(
+          `Warning: no stimuli directories discovered for run ${runId}`
+        );
+        continue;
+      }
+
+      console.log(
+        `Discovered stimuli for run ${runId}: ${Array.from(stimuliSet).join(", ")}`
+      );
+
+      for (const stimuliPart of stimuliSet) {
+        const stimuliOutputDir = path.join(
+          OUTPUT_ROOT,
+          encodeBranchName(branch),
+          stimuliPart,
+          `${model}-${skillSuffix}`
+        );
+        const blobPrefix = `${date}/${runId}/${skillName}/${skillName}_${stimuliPart}/agent-metadata-`;
+
+        console.log(
+          `Listing blobs for stimuli '${stimuliPart}' in run ${runId} ...`
+        );
+
+        let blobs: string;
+        try {
+          blobs = execFileSync("az", [
+            "storage", "blob", "list",
+            "--account-name", STORAGE_ACCOUNT,
+            "--container-name", CONTAINER,
+            "--prefix", blobPrefix,
+            "--auth-mode", "login",
+            "--query", "[?ends_with(name, '.md')].name",
+            "-o", "tsv"
+          ], {
+            encoding: "utf-8",
+            stdio: ["pipe", "pipe", "ignore"]
+          }) as string;
+        } catch {
+          console.error(
+            `Error: failed to list blobs for run ${runId}, stimuli ${stimuliPart}.`
+          );
+          failed = 1;
+          continue;
+        }
+
+        const blobList = blobs.trim().split("\n").filter((line: string) => line);
+        if (blobList.length === 0) {
+          console.error(
+            `Warning: no trajectory blobs found for run ${runId}, stimuli ${stimuliPart}.`
+          );
+          continue;
+        }
+
+        // Create output directory
+        if (!fs.existsSync(stimuliOutputDir)) {
+          fs.mkdirSync(stimuliOutputDir, { recursive: true });
+        }
+
+        for (const blob of blobList) {
+          const fileName = path.basename(blob);
+          console.log(`  downloading ${fileName} -> ${stimuliOutputDir}`);
+
+          try {
+            const outputPath = path.join(stimuliOutputDir, fileName);
+            execFileSync("az", [
+              "storage", "blob", "download",
+              "--account-name", STORAGE_ACCOUNT,
+              "--container-name", CONTAINER,
+              "--name", blob,
+              "--file", outputPath,
+              "--auth-mode", "login",
+              "--overwrite",
+              "--no-progress",
+              "-o", "none"
+            ], { stdio: "ignore" });
+          } catch {
+            console.error(`Error: failed to download blob ${blob}`);
+            failed = 1;
+          }
+        }
+      }
+    }
+  }
+
+  if (failed !== 0) {
+    console.error(
+      `Completed with errors. Partial artifacts are in ${OUTPUT_ROOT}`
+    );
+    process.exit(1);
+  }
+
+  console.log(`Artifacts collected in ${OUTPUT_ROOT}`);
+  process.exit(0);
+}
+
+run();