diff --git a/.github/skills/analyze-comparison-tests/SKILL.md b/.github/skills/analyze-comparison-tests/SKILL.md new file mode 100644 index 000000000..db965d0b8 --- /dev/null +++ b/.github/skills/analyze-comparison-tests/SKILL.md @@ -0,0 +1,50 @@ +--- +name: analyze-comparison-tests +description: "Collects comparison test run artifacts and answers the user's questions based on the trajectories of each run. WHEN TO USE: collect comparison test artifacts" +license: MIT +metadata: + author: Microsoft + version: "1.0.0" +--- + +# Steps + +1. Collect run artifacts + +Execute the collect-artifacts script to download the test run artifacts. + +The user must provide an JSON file to correlate each comparison test run with the GitHub Actions run. The script expects one input argument as the path to this JSON file. The JSON input is supposed to be the JSON output when queuing the comparison test runs using the `npm run compare:run` command. + +```bash +cd tests/ +npm run compare:collect -- input.json +``` + +The collect-artifacts script will download the test run artifacts to a directory named `comparison-artifacts` in the current working directory. Before executing the script, check if there is already such an directory. If so, skip executing the script and proceed to step 2. + +2. Extract insights + +The downloaded artifacts will have the following folder structure: + +```text +comparison-artifacts/ +├── / +│ ├── / +│ │ ├── -with-skill/ +│ │ │ ├── agent-metadata-.md +│ │ │ ├── agent-metadata-.md +│ │ │ └── ... +│ │ └── -without-skill/ +│ │ ├── agent-metadata-.md +│ │ ├── agent-metadata-.md +│ │ └── ... +│ └── / +│ ├── -with-skill/ +│ │ └── agent-metadata-*.md +│ └── -without-skill/ +│ └── agent-metadata-*.md +└── / + └── ... +``` + +Each `//-with-skill` or `//-without-skill` directory contains the test run trajectories for that stimulus and model on that branch, with or without skills. Each trajectory is a markdown file that records user prompts, tool call requests, tool execution results, assistant responses that happened during the run. It also contains statistics such as token usage and turns. Based on the trajectories, answer the user's questions for each test run. Generate a report following the [report-template](./references/report-template.md) to show your answers. diff --git a/.github/skills/analyze-comparison-tests/references/report-template.md b/.github/skills/analyze-comparison-tests/references/report-template.md new file mode 100644 index 000000000..b16c09a3b --- /dev/null +++ b/.github/skills/analyze-comparison-tests/references/report-template.md @@ -0,0 +1,19 @@ +# Comparison Report + +Skill: {plugin dirname/skill name} + +## Answers + +### {User question 1} + +{Answer to question 1} + +### {User question 2} + +{Answer to question 2} + +...... + +### {User question N} + +{Answer to question N} diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index 72a5b1d54..8c114b274 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -28,10 +28,13 @@ on: model-override: description: "Model to use for testing" required: false - type: choice - options: - - claude-sonnet-4.6 - - claude-opus-4.6 + type: string + default: claude-sonnet-4.6 + no-skills: + description: "Optional: whether to override the run to load no skills" + required: false + type: boolean + default: false skill-test-pattern: description: "Optional: pattern by name or describe block for filtering skill tests. This parameter does not apply to azure-deploy tests" required: false @@ -169,6 +172,7 @@ jobs: model-override: ${{ inputs.model-override }} test-pattern: ${{ needs.resolve-inputs.outputs.deploy-test-pattern }} debug: ${{ needs.resolve-inputs.outputs.debug == 'true' }} + no-skills: ${{ inputs.no-skills }} test: name: Integration – ${{ matrix.skill }} @@ -292,6 +296,7 @@ jobs: if: ${{ !contains(fromJson(env.JEST_SKILLS), matrix.skill) }} env: DEBUG: ${{ needs.resolve-inputs.outputs.debug == 'true' && '1' || '' }} + NO_SKILLS: ${{ inputs.no-skills && 'true' || '' }} TEST_RUN_ID: all-integration MODEL_OVERRIDE: ${{ inputs.model-override }} SKILL: ${{ matrix.skill }} diff --git a/.github/workflows/test-azure-deploy.yml b/.github/workflows/test-azure-deploy.yml index 4c5bc05ed..7409fefbf 100644 --- a/.github/workflows/test-azure-deploy.yml +++ b/.github/workflows/test-azure-deploy.yml @@ -30,6 +30,11 @@ on: required: false type: boolean default: false + no-skills: + description: 'Optional: whether to override the run to load no skills' + required: false + type: boolean + default: false workflow_call: inputs: model-override: @@ -46,6 +51,11 @@ on: required: false type: boolean default: false + no-skills: + description: 'Optional: whether to override the run to load no skills' + required: false + type: boolean + default: false jobs: setup: @@ -177,6 +187,7 @@ jobs: GH_HEAD_SHA: ${{ github.sha }} TEST_RUN_ID: azure-deploy DEBUG: ${{ inputs.debug && '1' || '' }} + NO_SKILLS: ${{ inputs.no-skills && 'true' || '' }} MODEL_OVERRIDE: ${{ inputs.model-override }} TEST_GROUP: ${{ matrix.test-group }} run: | diff --git a/tests/.gitignore b/tests/.gitignore index 94f77249c..2d1ec2e60 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,3 +1,4 @@ reports/ node_modules/ coverage/ +comparison-artifacts/ diff --git a/tests/comparison/collect-artifacts.ts b/tests/comparison/collect-artifacts.ts new file mode 100644 index 000000000..e99ea05e1 --- /dev/null +++ b/tests/comparison/collect-artifacts.ts @@ -0,0 +1,241 @@ +/** + * collect-artifacts.ts — download comparison test trajectories from Azure Storage. + * + * Usage: tsx collect-artifacts.ts + * + * Exit codes: + * 0 = success (all runs collected) + * 1 = a step failed (missing dependency, Azure error, or no blobs found) + * 2 = usage/argument error + */ + +import fs from "fs"; +import path from "path"; +import { execFileSync } from "child_process"; +import type { CompareRunOutput } from "./run-compare"; + +const STORAGE_ACCOUNT = "strdashboarddevveobvk"; +const CONTAINER = "manual-integration-reports"; +const OUTPUT_ROOT = "comparison-artifacts"; + +function usage(): void { + console.log(`Usage: collect-artifacts.ts + +Exit codes: + 0 = success (all runs collected) + 1 = a step failed (missing dependency, Azure error, or no blobs found) + 2 = usage/argument error`); +} + +function encodeBranchName(branch: string) { + return branch.replaceAll("/", "_"); +} + +function run(): void { + const args = process.argv.slice(2); + + if (args.length === 1 && (args[0] === "-h" || args[0] === "--help")) { + usage(); + process.exit(0); + } + + if (args.length !== 1) { + console.error( + "Error: expected exactly one argument (path to the input JSON file)." + ); + usage(); + process.exit(2); + } + + const inputFile = args[0]; + + if (!fs.existsSync(inputFile)) { + console.error(`Error: input file not found: ${inputFile}`); + process.exit(2); + } + + let input: CompareRunOutput; + try { + const content = fs.readFileSync(inputFile, "utf-8"); + input = JSON.parse(content); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : "unknown error"; + console.error(`Error: failed to parse input JSON: ${msg}`); + process.exit(2); + } + + const date = input.date || ""; + const skillName = input.skill?.name || ""; + + if (!date || !skillName) { + console.error("Error: input JSON must define 'date' and 'skill.name'."); + process.exit(2); + } + + if (!input.results || input.results.length === 0) { + console.error("Error: input JSON contains no runs."); + process.exit(2); + } + + // Create output directory + if (!fs.existsSync(OUTPUT_ROOT)) { + fs.mkdirSync(OUTPUT_ROOT, { recursive: true }); + } + + let failed = 0; + + for (const result of input.results) { + const branch = result.branch; + const runs = result.runs; + for (const run of runs) { + const model: string = run.model; + const withSkill: boolean = run.withSkill; + const runUrl: string = run.run; + + if (!model) continue; + + // Extract run ID from GitHub Actions URL + const runId = runUrl.split("/").pop(); + if (!runId) { + console.error(`Error: could not extract run id from URL: ${runUrl}`); + failed = 1; + continue; + } + + const skillSuffix = withSkill ? "with-skill" : "without-skill"; + + // Discover stimuli for this run + const prefix = `${date}/${runId}/${skillName}/${skillName}_`; + console.log( + `Discovering stimuli for run ${runId} under ${CONTAINER}/${prefix} ...` + ); + + let discoveryResult: string; + try { + discoveryResult = execFileSync("az", [ + "storage", "blob", "list", + "--account-name", STORAGE_ACCOUNT, + "--container-name", CONTAINER, + "--prefix", prefix, + "--auth-mode", "login", + "--query", "[?ends_with(name, '.md')].name", + "-o", "tsv" + ], { + encoding: "utf-8", + stdio: ["pipe", "pipe", "ignore"] + }) as string; + } catch { + console.error(`Error: failed to discover blobs for run ${runId}.`); + failed = 1; + continue; + } + + // Extract unique stimuli names from blob paths + const blobLines = discoveryResult.trim().split("\n").filter((line: string) => line); + const stimuliSet: Set = new Set(); + + for (const blob of blobLines) { + const regexPattern = new RegExp(`/${skillName}_([^/]+)/`); + const match = blob.match(regexPattern); + if (match) { + stimuliSet.add(match[1]); + } + } + + if (stimuliSet.size === 0) { + console.warn( + `Warning: no stimuli directories discovered for run ${runId}` + ); + continue; + } + + console.log( + `Discovered stimuli for run ${runId}: ${Array.from(stimuliSet).join(", ")}` + ); + + for (const stimuliPart of stimuliSet) { + const stimuliOutputDir = path.join( + OUTPUT_ROOT, + encodeBranchName(branch), + stimuliPart, + `${model}-${skillSuffix}` + ); + const blobPrefix = `${date}/${runId}/${skillName}/${skillName}_${stimuliPart}/agent-metadata-`; + + console.log( + `Listing blobs for stimuli '${stimuliPart}' in run ${runId} ...` + ); + + let blobs: string; + try { + blobs = execFileSync("az", [ + "storage", "blob", "list", + "--account-name", STORAGE_ACCOUNT, + "--container-name", CONTAINER, + "--prefix", blobPrefix, + "--auth-mode", "login", + "--query", "[?ends_with(name, '.md')].name", + "-o", "tsv" + ], { + encoding: "utf-8", + stdio: ["pipe", "pipe", "ignore"] + }) as string; + } catch { + console.error( + `Error: failed to list blobs for run ${runId}, stimuli ${stimuliPart}.` + ); + failed = 1; + continue; + } + + const blobList = blobs.trim().split("\n").filter((line: string) => line); + if (blobList.length === 0) { + console.error( + `Warning: no trajectory blobs found for run ${runId}, stimuli ${stimuliPart}.` + ); + continue; + } + + // Create output directory + if (!fs.existsSync(stimuliOutputDir)) { + fs.mkdirSync(stimuliOutputDir, { recursive: true }); + } + + for (const blob of blobList) { + const fileName = path.basename(blob); + console.log(` downloading ${fileName} -> ${stimuliOutputDir}`); + + try { + const outputPath = path.join(stimuliOutputDir, fileName); + execFileSync("az", [ + "storage", "blob", "download", + "--account-name", STORAGE_ACCOUNT, + "--container-name", CONTAINER, + "--name", blob, + "--file", outputPath, + "--auth-mode", "login", + "--overwrite", + "--no-progress", + "-o", "none" + ], { stdio: "ignore" }); + } catch { + console.error(`Error: failed to download blob ${blob}`); + failed = 1; + } + } + } + } + } + + if (failed !== 0) { + console.error( + `Completed with errors. Partial artifacts are in ${OUTPUT_ROOT}` + ); + process.exit(1); + } + + console.log(`Artifacts collected in ${OUTPUT_ROOT}`); + process.exit(0); +} + +run(); diff --git a/tests/comparison/run-compare.ts b/tests/comparison/run-compare.ts new file mode 100644 index 000000000..0a718c54c --- /dev/null +++ b/tests/comparison/run-compare.ts @@ -0,0 +1,168 @@ +import { spawn } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import { type SkillRef } from "../utils/skill-loader"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +type CompareInput = { + /** + * The skill the stimuli is for. + */ + skill: SkillRef; + + /** + * The branches to run the tests on. + */ + branches?: string[]; + + /** + * Environmental variations. + * If undefined, a hardcoded {@link defaultCompareOptions} will be used. + */ + compareOptions?: CompareOption[]; +}; + +type CompareOption = { + /** + * The base model to run the test with. + */ + model: string; + + /** + * Whether to load the skill. + */ + withSkill: boolean; +}; + +export type CompareRunOutput = { + skill: SkillRef; + date: string; + results: Array; +} + +type BranchOutput = { + branch: string; + runs: Array<{ + model: string; + withSkill: boolean; + run: string; + }>; +}; + +const defaultCompareOptions: CompareOption[] = [ + // Anthropic + { model: "claude-sonnet-5", withSkill: true }, + { model: "claude-sonnet-5", withSkill: false }, + { model: "claude-opus-4.8", withSkill: true }, + { model: "claude-opus-4.8", withSkill: false }, + { model: "claude-sonnet-4.6", withSkill: true }, + { model: "claude-sonnet-4.6", withSkill: false }, + { model: "claude-opus-4.6", withSkill: true }, + { model: "claude-opus-4.6", withSkill: false }, + // OpenAI + { model: "gpt-5.6-sol", withSkill: true }, + { model: "gpt-5.6-sol", withSkill: false }, + { model: "gpt-5.6-terra", withSkill: true }, + { model: "gpt-5.6-terra", withSkill: false }, + // // Google + { model: "gemini-3.1-pro-preview", withSkill: true }, + { model: "gemini-3.1-pro-preview", withSkill: false }, +]; + +const repo = "microsoft/GitHub-Copilot-for-Azure"; +// Id of the "Integration Tests - all" workflow +const integrationTestWorkflowId = "233698760"; + +async function queueComparisonRun(branch: string, skill: SkillRef, option: CompareOption): Promise { + const skillsInput = `${skill.pluginDirname}/${skill.name}`; + const args = ["workflow", "run", integrationTestWorkflowId, "--repo", repo, "--ref", branch, "--json"]; + const inputs = JSON.stringify({ + skills: skillsInput, + "model-override": option.model, + // Note: gh cli use string values for boolean input + "no-skills": !option.withSkill ? "true" : "false" + }); + + return await new Promise((resolve, reject) => { + const child = spawn("gh", args, { stdio: ["pipe", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { stdout += chunk; }); + child.stderr.on("data", (chunk: string) => { stderr += chunk; }); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 0) { + reject(new Error(stderr.trim() || `gh workflow run exited with code ${code}`)); + return; + } + + resolve(stdout.trim()); + }); + + child.stdin.end(inputs); + }); +} + +function readCompareInput(filePath: string): CompareInput { + const input = JSON.parse(readFileSync(filePath, "utf8")) as CompareInput; + + if (!input.skill) { + throw new Error("The input JSON must contain skill."); + } + + return input; +} + +/** + * Run a matrix of comparison runs. + * Each comparison test will feature one variation of environment, such as the model and whether skills are included. + * Each comparison test run will be scheduled to run in GitHub Actions and persist its artifacts in the manual-integration-reports blob container. + * An output file will be written to map each comparison test to its scheduled run for locating its published artifacts. + */ +async function main() { + const inputPath = process.argv[2]; + if (!inputPath) { + throw new Error("Usage: npm run compare:run -- "); + } + + const input = readCompareInput(inputPath); + const options = input.compareOptions ?? defaultCompareOptions; + const branches = input.branches ?? ["main"]; + const skill = input.skill; + const date = new Date().toISOString().slice(0, 10); // Get yyyy-mm-dd date string + const output: CompareRunOutput = { + skill: input.skill, + date: date, + results: [] + }; + for (const branch of branches) { + const branchEntry: BranchOutput = { + branch: branch, + runs: [] + }; + const results = []; + for (const option of options) { + // Each output is a url to the queued run + // e.g. https://github.com/microsoft/GitHub-Copilot-for-Azure/actions/runs/31218229738 + const output = await queueComparisonRun(branch, skill, option); + const entry = { + model: option.model, + withSkill: option.withSkill, + run: output + }; + results.push(entry); + } + branchEntry.runs = results; + output.results.push(branchEntry); + } + const outputFilename = `comparison-runs-${new Date().toISOString().replace(/[:.]/g, "-")}.json`; + writeFileSync(path.resolve(__dirname, outputFilename), JSON.stringify(output, null, 2)); +} + +void main(); \ No newline at end of file diff --git a/tests/package.json b/tests/package.json index b3e85d348..f9342aa5d 100644 --- a/tests/package.json +++ b/tests/package.json @@ -21,7 +21,9 @@ "update:snapshots": "node scripts/update-snapshots.js", "typecheck": "tsc --noEmit", "lint": "eslint", - "lint:fix": "eslint --fix" + "lint:fix": "eslint --fix", + "compare:run": "npx tsx ./comparison/run-compare.ts", + "compare:collect": "npx tsx ./comparison/collect-artifacts" }, "engines": { "node": "^22.14.0 || >=24" diff --git a/tests/vally/vally-executor.ts b/tests/vally/vally-executor.ts index 21b362035..1e49a72b4 100644 --- a/tests/vally/vally-executor.ts +++ b/tests/vally/vally-executor.ts @@ -7,6 +7,11 @@ import { getEarlyTerminateCondition, getRequiredSkillsCondition, getSkillName, g import { normalizeTestName } from "./utils.ts"; import { listPlugins, type SkillRef } from "../utils/skill-loader.ts"; +/** + * The model to use for the agent run. + */ +const modelOverride = process.env.MODEL_OVERRIDE?.trim() || undefined; + export class IntegrationTestAgentRunner implements Executor { name = "integration-test-agent-runner"; supportsMultiTurn = true; @@ -27,7 +32,7 @@ export class IntegrationTestAgentRunner implements Executor { const workDir = options.workDir; // Set the model to use - const model = options.model ?? "claude-sonnet-4.6"; + const model = modelOverride ?? options.model ?? "claude-sonnet-4.6"; const { shouldEarlyTerminate } = getEarlyTerminateCondition(tags); const systemPrompt = getSystemPrompt(tags);