diff --git a/.github/workflows/copilot-on-rails-evals.yml b/.github/workflows/copilot-on-rails-evals.yml new file mode 100644 index 000000000..c15634271 --- /dev/null +++ b/.github/workflows/copilot-on-rails-evals.yml @@ -0,0 +1,396 @@ +name: Copilot on Rails evaluations + +on: + pull_request: + branches: + - main + - feat/CoR + paths: + - .github/workflows/copilot-on-rails-evals.yml + - evals/** + - resources/agents/** + - test/copilotOnRails/** + - package.json + - package-lock.json + workflow_dispatch: + inputs: + tier: + description: Evaluation tier (contracts is offline; all others make paid model calls) + required: true + type: choice + options: + - contracts + - daily + - weekly + - release + default: contracts + run_vscode_parity: + description: Request separately gated VS Code parity evidence (currently fails closed) + required: true + type: boolean + default: false + run_live_deployment: + description: Request separately gated deployment evidence (currently fails closed) + required: true + type: boolean + default: false + schedule: + - cron: "23 5 * * *" + - cron: "41 6 * * 0" + +permissions: + contents: read + +jobs: + contracts: + name: Deterministic contracts + if: >- + github.event_name == 'pull_request' || + (github.event_name == 'workflow_dispatch' && inputs.tier == 'contracts') + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm install --global npm@11.11.1 + - run: npm ci + - run: npm run build:check + - run: npm run lint + - run: npm run eval:cor:thresholds:validate + - run: npm run eval:cor:spike:dry + - run: npm run eval:cor:graders:certify + - run: npx tsx evals/vally/native/generate.ts --check + - run: npm run eval:cor:vally:lint + - run: npm run eval:cor:vally:oracle + - run: npm run eval:cor:vally:native:lint + - run: npm run eval:cor:vally:native:oracle + - name: Load ACA backend relative to native experiments + run: >- + npx tsx -e "import path from 'node:path'; + import { createBackendRegistry, loadBackendPlugin } from '@microsoft/vally'; + void (async () => { + const registry = createBackendRegistry(); + await loadBackendPlugin('../../plugins/aca-executor/backend.js', registry, + { cwd: path.resolve('evals/vally/native/experiments') }); + const loaded = registry.getAll(); + if (loaded.length !== 1 || loaded[0].name !== 'cor-aca') + throw new Error('Expected exactly one cor-aca backend registration.'); + })().catch(error => { console.error(error); process.exitCode = 1; });" + - run: npm run eval:cor:vally:test + - name: Focused evaluator contracts + run: >- + npx mocha --require tsx/cjs --ui tdd --timeout 10000 + test/copilotOnRails/evaluationArtifactValidators.test.ts + test/copilotOnRails/baselineEvaluation.test.ts + test/copilotOnRails/baselineMeritParity.test.ts + test/copilotOnRails/evaluationMatrix.test.ts + test/copilotOnRails/evaluationDefinition.test.ts + test/copilotOnRails/releaseThresholds.test.ts + + model-evaluations: + name: ${{ github.event_name == 'schedule' && github.event.schedule == '41 6 * * 0' && 'Weekly Vally-native 3-model experiments' || github.event_name == 'schedule' && 'Daily Vally-native compatibility pilot' || format('{0} Vally-native tier', inputs.tier) }} + if: >- + (github.event_name == 'workflow_dispatch' && inputs.tier != 'contracts') || + (github.event_name == 'schedule' && + github.event.schedule == '23 5 * * *' && + vars.COR_EVAL_DAILY_ENABLED == 'true') || + (github.event_name == 'schedule' && + github.event.schedule == '41 6 * * 0' && + vars.COR_EVAL_WEEKLY_ENABLED == 'true') + runs-on: ubuntu-latest + timeout-minutes: 720 + permissions: + contents: read + id-token: write + env: + ACA_RESOURCE_GROUP: ${{ vars.COR_EVAL_RESOURCE_GROUP }} + ACA_SANDBOX_GROUP: ${{ vars.COR_EVAL_SANDBOX_GROUP }} + ACA_REGION: ${{ vars.COR_EVAL_REGION }} + GH_TOKEN: ${{ secrets.COR_EVAL_COPILOT_GITHUB_TOKEN }} + RESULT_DIRECTORY: evals/results/ci-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - uses: azure/login@v2 + with: + client-id: ${{ secrets.COR_EVAL_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.COR_EVAL_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.COR_EVAL_AZURE_SUBSCRIPTION_ID }} + - run: npm install --global npm@11.11.1 + - run: npm ci + - name: Install and verify ACA CLI + run: | + curl -fsSL https://aka.ms/aca-cli-install | bash + aca --version + az account show -o none + aca auth status >/dev/null 2>&1 || aca auth login + aca doctor + - name: Validate evaluator and generated Vally contracts + run: | + npx tsx evals/vally/native/generate.ts --check + npm run eval:cor:thresholds:validate + npm run eval:cor:vally:lint + npm run eval:cor:vally:oracle + npm run eval:cor:vally:native:lint + npm run eval:cor:vally:native:oracle + npm run eval:cor:vally:test + aca sandbox validate --file evals/sandbox.yaml + aca sandbox validate --file evals/sandbox-python.yaml + aca sandbox validate --file evals/sandbox-dotnet.yaml + - name: Load ACA backend relative to native experiments + run: >- + npx tsx -e "import path from 'node:path'; + import { createBackendRegistry, loadBackendPlugin } from '@microsoft/vally'; + void (async () => { + const registry = createBackendRegistry(); + await loadBackendPlugin('../../plugins/aca-executor/backend.js', registry, + { cwd: path.resolve('evals/vally/native/experiments') }); + const loaded = registry.getAll(); + if (loaded.length !== 1 || loaded[0].name !== 'cor-aca') + throw new Error('Expected exactly one cor-aca backend registration.'); + })().catch(error => { console.error(error); process.exitCode = 1; });" + - name: Select exact generated tier experiments + env: + DISPATCH_TIER: ${{ inputs.tier }} + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "schedule" ]; then + if [ "${{ github.event.schedule }}" = "41 6 * * 0" ]; then + TIER=weekly + else + TIER=daily + fi + else + TIER="$DISPATCH_TIER" + fi + mkdir -p "$RESULT_DIRECTORY/experiments" + case "$TIER" in + daily) + printf '%s\n' \ + evals/vally/native/experiments/compatibility-pilot-gpt-5-6-sol.experiment.yaml \ + > "$RESULT_DIRECTORY/selected-experiments.txt" + ;; + weekly) + printf '%s\n' \ + evals/vally/native/experiments/representative-gpt-5-6-sol.experiment.yaml \ + evals/vally/native/experiments/representative-claude-sonnet-5.experiment.yaml \ + evals/vally/native/experiments/representative-gpt-5-4-mini.experiment.yaml \ + > "$RESULT_DIRECTORY/selected-experiments.txt" + ;; + release) + printf '%s\n' \ + evals/vally/native/experiments/release-gpt-5-6-sol.experiment.yaml \ + > "$RESULT_DIRECTORY/selected-experiments.txt" + ;; + *) + echo "Unsupported generated Vally tier: $TIER" >&2 + exit 1 + ;; + esac + echo "EVAL_TIER=$TIER" >> "$GITHUB_ENV" + - name: Dry-run selected Vally experiments + run: | + set -euo pipefail + while IFS= read -r EXPERIMENT; do + [ -n "$EXPERIMENT" ] || continue + npx vally experiment run "$EXPERIMENT" \ + --backend ../../plugins/aca-executor/backend.js \ + --workers 2 \ + --output-dir "$RESULT_DIRECTORY/dry-run" \ + --dry-run + done < "$RESULT_DIRECTORY/selected-experiments.txt" + - name: Certify graders in ACA + if: env.EVAL_TIER == 'daily' || env.EVAL_TIER == 'weekly' + run: npm run eval:cor:graders:certify:aca -- --output "$RESULT_DIRECTORY/grader-certification" + - name: Fail closed for separately gated release integrations + if: inputs.run_vscode_parity || inputs.run_live_deployment + run: | + echo "::error::VS Code parity and live deployment require separate evidence capture. The removed native-matrix source-selection path is not valid for Vally-native experiments." + exit 1 + - name: Require release-only external evidence integrations + if: env.EVAL_TIER == 'release' + run: | + echo "::error::Release execution is disabled until provenance-bound VS Code parity and explicitly authorized live-deployment evidence are wired into this workflow. Vally-native aggregation and fail-closed release enforcement are available, but the generated release experiment will not make paid calls without those external hard gates." + exit 1 + - name: Run Vally-native ACA experiments sequentially + id: evaluation + continue-on-error: true + run: | + set -uo pipefail + STATUS=0 + while IFS= read -r EXPERIMENT; do + [ -n "$EXPERIMENT" ] || continue + NAME=$(basename "$EXPERIMENT" .experiment.yaml) + OUTPUT="$RESULT_DIRECTORY/experiments/$NAME" + mkdir -p "$OUTPUT" + npx vally experiment run "$EXPERIMENT" \ + --backend ../../plugins/aca-executor/backend.js \ + --workers 2 \ + --output-dir "$OUTPUT" || STATUS=1 + done < "$RESULT_DIRECTORY/selected-experiments.txt" + exit "$STATUS" + - name: Cleanup exact per-trial ACA owners from durable manifests + id: cleanup + if: always() + continue-on-error: true + env: + PAID_OUTCOME: ${{ steps.evaluation.outcome }} + run: | + set -uo pipefail + CLEANUP_DIRECTORY="$RESULT_DIRECTORY/cleanup" + LABELS_FILE="$CLEANUP_DIRECTORY/owner-labels.txt" + mkdir -p "$CLEANUP_DIRECTORY" + DISCOVERY_STATUS=0 + node - "$RESULT_DIRECTORY" "$LABELS_FILE" "$CLEANUP_DIRECTORY/discovery.log" <<'NODE' || DISCOVERY_STATUS=1 + const fs = require('fs'); + const path = require('path'); + const [root, labelsPath, logPath] = process.argv.slice(2); + const labels = new Set(); + const logs = []; + let invalid = false; + function visit(directory) { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + visit(entryPath); + } else if (entry.isFile() && entry.name === 'validation-manifest.json') { + try { + const manifest = JSON.parse(fs.readFileSync(entryPath, 'utf8')); + if (typeof manifest.ownerLabel !== 'string' || + !/^[a-z0-9][a-z0-9-]{0,62}$/.test(manifest.ownerLabel)) { + throw new Error('missing or invalid ownerLabel'); + } + labels.add(manifest.ownerLabel); + logs.push(`accepted ${entryPath}: ${manifest.ownerLabel}`); + } catch (error) { + invalid = true; + logs.push(`rejected ${entryPath}: ${error.message}`); + } + } + } + } + visit(root); + fs.writeFileSync(labelsPath, [...labels].sort().join('\n') + (labels.size ? '\n' : '')); + fs.writeFileSync(logPath, logs.join('\n') + (logs.length ? '\n' : '')); + if (invalid) process.exitCode = 1; + NODE + + STATUS="$DISCOVERY_STATUS" + if [ "$PAID_OUTCOME" != "skipped" ] && [ ! -s "$LABELS_FILE" ]; then + echo "No durable validation-manifest.json owner labels were found after paid execution." | + tee -a "$CLEANUP_DIRECTORY/actions.log" + STATUS=1 + fi + while IFS= read -r OWNER_LABEL; do + [ -n "$OWNER_LABEL" ] || continue + LIST_JSON="$CLEANUP_DIRECTORY/list-$OWNER_LABEL.json" + IDS_FILE="$CLEANUP_DIRECTORY/ids-$OWNER_LABEL.txt" + echo "Listing exact owner-id=$OWNER_LABEL" | tee -a "$CLEANUP_DIRECTORY/actions.log" + if ! aca sandbox list -l "owner-id=$OWNER_LABEL" -o json \ + > "$LIST_JSON" 2>> "$CLEANUP_DIRECTORY/actions.log"; then + echo "List failed for owner-id=$OWNER_LABEL" | tee -a "$CLEANUP_DIRECTORY/actions.log" + STATUS=1 + continue + fi + if ! node - "$LIST_JSON" "$IDS_FILE" <<'NODE' + const fs = require('fs'); + const [source, target] = process.argv.slice(2); + const value = JSON.parse(fs.readFileSync(source, 'utf8')); + if (!Array.isArray(value)) throw new Error('ACA sandbox list was not an array'); + const ids = [...new Set(value.map(item => item?.id).filter(id => typeof id === 'string'))]; + fs.writeFileSync(target, ids.join('\n') + (ids.length ? '\n' : '')); + NODE + then + echo "List parse failed for owner-id=$OWNER_LABEL" | tee -a "$CLEANUP_DIRECTORY/actions.log" + STATUS=1 + continue + fi + while IFS= read -r SANDBOX_ID; do + [ -n "$SANDBOX_ID" ] || continue + echo "Deleting $SANDBOX_ID for owner-id=$OWNER_LABEL" | + tee -a "$CLEANUP_DIRECTORY/actions.log" + aca sandbox delete --id "$SANDBOX_ID" --yes \ + >> "$CLEANUP_DIRECTORY/actions.log" 2>&1 || STATUS=1 + done < "$IDS_FILE" + VERIFY_JSON="$CLEANUP_DIRECTORY/verify-$OWNER_LABEL.json" + if ! aca sandbox list -l "owner-id=$OWNER_LABEL" -o json \ + > "$VERIFY_JSON" 2>> "$CLEANUP_DIRECTORY/actions.log"; then + echo "Verification list failed for owner-id=$OWNER_LABEL" | + tee -a "$CLEANUP_DIRECTORY/actions.log" + STATUS=1 + continue + fi + if ! node - "$VERIFY_JSON" <<'NODE' + const fs = require('fs'); + const value = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); + if (!Array.isArray(value)) throw new Error('ACA sandbox verification list was not an array'); + if (value.length) throw new Error(`ACA cleanup left ${value.length} sandbox(es)`); + NODE + then + echo "Residual sandboxes remain for owner-id=$OWNER_LABEL" | + tee -a "$CLEANUP_DIRECTORY/actions.log" + STATUS=1 + fi + done < "$LABELS_FILE" + exit "$STATUS" + - name: Aggregate Vally-native experiments and release gates + id: aggregate + if: always() && steps.evaluation.outcome != 'skipped' + run: >- + npm run eval:cor:vally:native:report -- + --experiment-dir "$RESULT_DIRECTORY/experiments" + --output "$RESULT_DIRECTORY/report" + - name: Summarize Vally-native evidence + if: always() + run: | + { + echo "## Vally-native evaluation evidence" + echo + echo "Tier: \`${EVAL_TIER:-not-configured}\`" + echo + echo "Generated experiments:" + sed 's/^/- `/' "$RESULT_DIRECTORY/selected-experiments.txt" 2>/dev/null | + sed 's/$/`/' || true + echo + echo "Evidence files:" + find "$RESULT_DIRECTORY/experiments" -type f -print 2>/dev/null | + sort | sed 's/^/- `/' | sed 's/$/`/' || true + if [ -f "$RESULT_DIRECTORY/report/report.md" ]; then + echo + cat "$RESULT_DIRECTORY/report/report.md" + fi + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload primary Vally-native experiment evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: cor-vally-native-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ env.RESULT_DIRECTORY }}/selected-experiments.txt + ${{ env.RESULT_DIRECTORY }}/experiments + ${{ env.RESULT_DIRECTORY }}/report + if-no-files-found: error + include-hidden-files: true + retention-days: 30 + - name: Upload cleanup evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: cor-vally-cleanup-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.RESULT_DIRECTORY }}/cleanup + if-no-files-found: error + include-hidden-files: true + retention-days: 30 + - name: Propagate paid evaluation or cleanup failure + if: >- + always() && + (steps.evaluation.outcome == 'failure' || + steps.cleanup.outcome == 'failure' || + steps.aggregate.outcome == 'failure') + run: exit 1 diff --git a/.gitignore b/.gitignore index 841af5af3..58601ab23 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,4 @@ testWorkspace test-results.xml dist stats.json +evals/results/ diff --git a/.vally.yaml b/.vally.yaml new file mode 100644 index 000000000..3173765fb --- /dev/null +++ b/.vally.yaml @@ -0,0 +1,29 @@ +paths: + skills: [] + evals: + - evals/vally + results: evals/results/vally-native + evalFilenames: + - eval.yaml + +suites: + copilot-on-rails-offline: + description: Offline contracts for archived Copilot on Rails summary evidence. + evals: + - evals/vally/eval.yaml + copilot-on-rails-hard-gates: + description: Authoritative product and scenario-specific gate contracts. + filter: + evidence: summary-only + copilot-on-rails-native-authoritative: + description: Vally-native ACA execution with fail-closed authoritative release grading. + evals: + - evals/vally/native/authoritative.eval.yaml + copilot-on-rails-native-qualitative: + description: Supplemental and uncalibrated trajectory-plus-diff user-satisfaction grading. + evals: + - evals/vally/native/qualitative.eval.yaml + copilot-on-rails-native-oracle: + description: Offline custom-metrics oracle contract with no live model execution. + evals: + - evals/vally/native/fixtures/oracle.eval.yaml diff --git a/.vscodeignore b/.vscodeignore index d328b21be..d1469db36 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -12,6 +12,7 @@ build/** dist/test/** docs/** +evals/** gulp* node_modules/** out/** diff --git a/eslint.config.mjs b/eslint.config.mjs index d2ee49bf3..d4b2f7043 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -12,6 +12,10 @@ export default defineConfig([ ignores: [ 'api/dist/**', 'api/out/**', + 'evals/results/**', + 'evals/grader-certification/reference-node-fullstack/**', + 'evals/grader-certification/reference-deployable/**', + 'evals/vscode-parity/**', 'src/webviews/copilotOnRails/views/react-shim.js', ], }, diff --git a/evals/CONTRIBUTING.md b/evals/CONTRIBUTING.md new file mode 100644 index 000000000..b29ddafc6 --- /dev/null +++ b/evals/CONTRIBUTING.md @@ -0,0 +1,380 @@ +# Contributing to Copilot on Rails evaluations + +This guide explains how to run the complete evaluation system, extend its coverage, change its +graders, and turn evaluation evidence into product changes. Start with the smallest applicable tier +and preserve the exact scenario, model, arm, attempt, and evaluator provenance whenever comparing +results. + +## What the complete suite runs + +The evaluation system has three distinct layers: + +| Layer | Purpose | Model calls | ACA Sandboxes | +|---|---|---:|---:| +| Deterministic contracts | Validate schemas, generated Vally specs, release policy, graders, and one-fault mutations | No | No | +| ACA grader certification | Prove the executable graders against a hand-authored passing project and controlled failures | No | Yes | +| Paired Vally E2E | Generate and validate projects for both Copilot on Rails and generic Copilot | Yes | Yes | + +The paired Vally experiments already include both arms: + +- `rails` is the normal Copilot on Rails end-to-end journey. +- `baseline-controlled` is generic Copilot in an empty workspace, using the same scenario, model, + endpoint, attempt, and validation. + +Do not run the standalone baseline command in addition to a Vally experiment unless you are +debugging the baseline implementation. It would duplicate paid work without producing a new matched +pair. + +The standard order is: + +1. Run deterministic contracts and offline grader certification. +2. Run ACA grader certification. Stop if it fails; product/model results are not trustworthy. +3. Dry-run the selected Vally experiment. +4. Run the paired Vally experiment. This runs Rails and baseline. +5. Aggregate the durable evidence and inspect the release assessment. +6. Verify exact-owner sandbox cleanup. + +Daily CI runs the two-scenario, primary-model compatibility pilot: two scenarios x two arms x one +attempt = four paid trials. Weekly CI runs four scenarios x two arms x two attempts for each of the +three pinned models = 48 paid trials. The local release experiment is 20 scenarios x two arms x +three attempts = 120 paid trials. + +## One-time local setup + +### Toolchain + +- Node.js 22 +- npm 11.11.1 +- Azure CLI +- An organizational Microsoft Entra account +- Access to an ACA Sandbox Group +- A GitHub token authorized for the Copilot endpoint used by the evaluator + +Install dependencies: + +```bash +npm install --global npm@11.11.1 +npm ci +``` + +Install the ACA CLI on Linux or macOS: + +```bash +curl -fsSL https://aka.ms/aca-cli-install | sh +``` + +On Windows PowerShell: + +```powershell +irm https://aka.ms/aca-cli-install-ps | iex +``` + +This same install path is also used inside sandboxes and containers for agent-driven self-installs. + +Check cached authentication before opening an interactive sign-in: + +```bash +aca --version +az account show -o none 2>/dev/null || az login +aca auth status >/dev/null 2>&1 || aca auth login +aca doctor +``` + +`aca doctor` must be green before running an ACA tier. It verifies the subscription, resource +group, default sandbox group, region, and `Container Apps SandboxGroup Data Owner` role. + +### Sandbox-group access + +Prefer the team's existing evaluation group. An administrator grants another teammate access with: + +```bash +aca sandboxgroup role create \ + --role "Container Apps SandboxGroup Data Owner" \ + --principal-id "$(az ad user show --id --query id -o tsv)" +``` + +To create a separate evaluation group instead: + +```bash +az account show -o none 2>/dev/null || az login +aca auth status >/dev/null 2>&1 || aca auth login +aca sandboxgroup create --name --location --set-config +aca doctor +``` + +`aca sandboxgroup create` grants the caller the Data Owner role. Use `role create` only for +additional principals. `--set-config` is required so evaluator commands resolve the group without a +flag on every call. + +The evaluator uses checked-in declarative manifests. The reproducible CI/CD pattern is: + +```bash +aca sandbox init +aca sandbox validate --file sandbox.yaml +aca sandbox apply --file sandbox.yaml +``` + +Use `aca sandbox init` only when adding a new manifest. Edit its `disk`, `resources`, +`lifecycle.autoSuspendPolicy`, `egressPolicy`, and any required `ports`, `env`, or `labels`. Use +`aca sandbox schema` for editor autocomplete. The manifest pattern is recommended for CI/CD and +reproducibility; do not replace it with imperative sandbox creation in evaluator code. + +### Local environment + +Select the evaluation subscription and configure the ACA defaults consumed by `aca doctor`. Export +the Copilot token without writing it to the repository: + +```bash +export GH_TOKEN="" +export COR_EVAL_OWNER_ID="yourname-local" +``` + +`COR_EVAL_OWNER_ID` must be lowercase alphanumeric/hyphen text and at most 63 characters. It lets +the evaluator and cleanup logic identify only sandboxes owned by this run. + +## Run the suite locally + +### 1. Deterministic contracts + +```bash +npm run build:check +npm run lint +npm run eval:cor:thresholds:validate +npm run eval:cor:spike:dry +npm run eval:cor:graders:certify +npm run eval:cor:vally:native:check +npm run eval:cor:vally:native:lint +npm run eval:cor:vally:native:oracle +npm run eval:cor:vally:native:test +npm run eval:cor:vally:native:pilot:dry +``` + +These commands make no model calls and create no Azure resources. + +### 2. ACA grader certification + +```bash +aca sandbox validate --file evals/sandbox.yaml +aca sandbox validate --file evals/sandbox-python.yaml +aca sandbox validate --file evals/sandbox-dotnet.yaml +npm run eval:cor:graders:certify:aca +``` + +This creates disposable sandboxes but makes no model calls. It proves real build, generated-test, +runtime, browser, accessibility, persistence, debugger-readiness, and cleanup behavior. Reports are +written under `evals/results/grader-certification/`. + +To diagnose one certification case: + +```bash +npm run eval:cor:graders:certify:aca -- --case golden-local-runtime +``` + +### 3. Paired Rails and baseline E2E + +Always inspect the dry run first: + +```bash +npm run eval:cor:vally:native:pilot:dry +``` + +Run the four-trial primary-model pilot: + +```bash +npm run eval:cor:vally:paid:pilot:gpt-5-6-sol +``` + +That one command runs both the normal Rails E2E arm and the controlled baseline arm. To test the +full pinned model set, run the corresponding Claude Sonnet 5 and GPT-5.4-mini pilot aliases listed +in `evals/README.md`. + +Aggregate one or more experiment output directories: + +```bash +npm run eval:cor:vally:native:report -- \ + --experiment-dir evals/results/vally-native/compatibility-pilot-gpt-5-6-sol \ + --output evals/results/vally-native-report +``` + +Read these files first: + +- `report.md`: human-readable outcomes, matched Rails/baseline comparison, one row per run, + failed-gate evidence, artifact links, and release recommendation. +- `vally-native-report.json`: complete machine-readable report. +- `experiment-input-manifest.json`: every accepted evidence bundle and cleanup status. +- Per-trial `artifacts/native-summary.json`: scenario/model/arm result. +- Per-trial `artifacts/cor-validation.json`: authoritative gate evidence. +- Per-trial `artifacts/validation-manifest.json`: provenance and cleanup verification. + +Generated test and lint failures do not suppress later local evidence. The attempt remains failed, +while integration, runtime, browser, persistence, and debugger gates continue when build/setup +prerequisites are available. Read each gate independently; for example, `test: failed` and +`debugger: passed` is a valid and actionable result. + +`candidate` means every configured release gate passed. `hold` means complete evidence exists but +one or more quality gates failed. `insufficient_evidence` means required coverage or external proof +is missing; it is not a passing result. + +### Cleanup check + +The evaluator deletes each sandbox by exact ID and the workflow performs a second exact-owner +sweep. Confirm no sandboxes remain for your owner label: + +```bash +aca sandbox list -l "owner-id=$COR_EVAL_OWNER_ID" -o json +``` + +If a failed run leaves a sandbox, preserve it first only when it contains state needed for +investigation: + +```bash +aca sandbox snapshot --id "$SANDBOX_ID" --name +aca sandbox delete --id "$SANDBOX_ID" --yes +``` + +Deletion is destructive. Never use a broad selector or delete sandboxes owned by another run. + +## Configure repository CI + +The workflow is `.github/workflows/copilot-on-rails-evals.yml`. + +Repository variables: + +- `COR_EVAL_DAILY_ENABLED=true` to enable the nightly primary-model pilot. +- `COR_EVAL_WEEKLY_ENABLED=true` to enable the weekly three-model representative run. +- `COR_EVAL_RESOURCE_GROUP` +- `COR_EVAL_SANDBOX_GROUP` +- `COR_EVAL_REGION` + +Repository secrets: + +- `COR_EVAL_AZURE_CLIENT_ID` +- `COR_EVAL_AZURE_TENANT_ID` +- `COR_EVAL_AZURE_SUBSCRIPTION_ID` +- `COR_EVAL_COPILOT_GITHUB_TOKEN` + +The Azure identity needs `Container Apps SandboxGroup Data Owner` on the configured group. Pull +requests run only deterministic contracts. Daily and weekly jobs run ACA grader certification +before paired model experiments and upload experiment plus cleanup evidence even when trials fail. + +Dispatch manually: + +```bash +gh workflow run copilot-on-rails-evals.yml -f tier=contracts +gh workflow run copilot-on-rails-evals.yml -f tier=daily +gh workflow run copilot-on-rails-evals.yml -f tier=weekly +``` + +The `release` workflow currently fails closed before paid calls because current provenance-bound +real VS Code breakpoint and explicitly authorized live-deployment integrations are not yet wired +into that workflow. + +## Add or modify a scenario + +Scenario sources live in `evals/scenarios/*.json`. A scenario owns the user intent and evaluator +acceptance contract; generated Vally YAML is not the source of truth. + +1. Copy the closest scenario and give it a unique kebab-case `id`. +2. Write a standalone `baselinePrompt` that does not mention Rails or assume prior context. +3. Set explicit archetype, frontend, backend, database, auth, and complexity tags. +4. Keep `requirementsAnswers.dataStores` aligned with the database tag. +5. Define build, generated-test, lint, and timeout requirements. +6. Add evaluator-owned local probes where the project should run locally. +7. For a UI, add browser actions, assertions, and an accessibility threshold. +8. Add persistence or storage-event contracts only when the scenario requires those behaviors. +9. Add debugger parity source/trigger data when applicable. +10. Regenerate Vally specs; never hand-edit generated experiment YAML. + +```bash +npm run eval:cor:vally:native:generate +npm run eval:cor:vally:native:check +npm run eval:cor:vally:native:lint +npm run eval:cor:vally:native:test +npm run eval:cor:graders:certify +``` + +If the corpus size changes, update the explicit corpus-size contract and release thresholds in the +same change. Add new scenarios to the compatibility or representative sets only deliberately; those +sets control recurring cost. + +## Add or modify a grader + +Artifact validators live in `evals/src/artifacts/`. Executable ACA validators live in +`SandboxProjectValidator.ts`, `SandboxLocalRuntimeValidator.ts`, and +`SandboxVsCodeParityValidator.ts`. Vally's authoritative aggregation bridge lives under +`evals/vally/plugins/cor-graders/`. + +Every grader change requires: + +1. A stable, specific failure code. +2. A passing golden case. +3. A one-fault mutation that triggers exactly the intended failure. +4. Evidence that unrelated gates continue to pass. +5. Fail-closed behavior when required evidence is absent. +6. A focused unit test. +7. Offline certification when possible; ACA certification when behavior requires real execution. +8. Updates to gate applicability, reporting, thresholds, and documentation when the release + contract changes. + +Add project-level mutations to `evals/grader-certification/manifest.json`. Modify the hand-authored +reference project only to model intended valid behavior; do not weaken it merely to make a grader +pass. + +Run: + +```bash +npm run eval:cor:graders:certify +npm run eval:cor:graders:certify:aca +npm run eval:cor:vally:native:oracle +npm run eval:cor:vally:native:test +``` + +## Turn results into project changes + +Treat evidence in this order: + +1. **Grader certification failure:** fix the evaluator or fixture first. Do not infer product + quality from model trials while the oracle is uncertified. +2. **Infrastructure failure:** fix or rerun the ACA/Copilot environment. Keep it outside the product + quality denominator. +3. **Harness failure:** repair evaluator orchestration, provenance, evidence collection, or cleanup. +4. **Product failure:** change Rails agents, references, tools, templates, or workflow code. + +For a product failure: + +1. Locate the exact model, scenario, arm, attempt, failed stage, and failure code in `report.md`. +2. Open that trial's `reports/run-diagnostics.md` first. It distinguishes failed gates from dependent + gates that were not attempted and includes the failing command, stdout/stderr excerpt, repair + usage, and recommended action. Use `run-result.json` for the complete unabridged stage evidence. +3. Reproduce the smallest matching scenario; do not start with the full release matrix. +4. Determine whether the same-model baseline passed. +5. Fix the product contract or implementation, not the grader, unless the expected behavior is + demonstrably wrong. +6. Add a deterministic regression test for the failure. +7. Run offline contracts and ACA grader certification. +8. Rerun the exact paired scenario/model. +9. Expand to the compatibility pilot, then representative tier, and only then the release corpus. + +Interpret matched arms carefully: + +- Baseline passes and Rails fails: high-priority Rails regression. +- Rails passes and baseline fails: evidence of Rails value for that capability. +- Both fail: likely a difficult or unsupported capability; inspect failure stages before changing + the product. +- Both pass: compare first-pass success, repair usage, latency, tokens, and nano-AIU cost. + +Never hide a recurring product failure by weakening an acceptance contract or reclassifying it as +infrastructure. Any intentional behavior change must update the scenario, grader certification, +release threshold, and product implementation together. + +## Review checklist + +- Scenario and evaluator sources changed together where required. +- Generated Vally specs have no drift. +- Golden grader fixture passes. +- Each new failure condition has a one-fault mutation and exact code. +- Rails and baseline use the same model, scenario, endpoint, and attempt. +- Reports retain candidate commit, asset hash, evaluation-definition hash, and cleanup proof. +- Paid trial count and expected cost are stated in the pull request. +- No sandbox, token, credential, generated workspace, or evaluation result is committed. +- Product fixes include a focused regression test and an exact paired rerun. diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 000000000..73a228cef --- /dev/null +++ b/evals/README.md @@ -0,0 +1,587 @@ +# Copilot on Rails evaluations + +For setup, execution order, scenario/grader authoring, result interpretation, and the workflow for +turning failures into product changes, see [CONTRIBUTING.md](CONTRIBUTING.md). + +## Vally-first ACA operator commands + +The Vally-native specs are deterministically generated from the 20 checked-in scenarios. Regenerate +after changing the corpus, or use the check in review and CI. These commands are offline: + +```sh +npm run eval:cor:vally:native:generate +npm run eval:cor:vally:native:check +npm run eval:cor:vally:native:lint +npm run eval:cor:vally:native:oracle +npm run eval:cor:vally:native:test +npm run eval:cor:vally:native:pilot:dry +``` + +`native:lint` is strict and loads the authoritative grader bridge. The oracle re-grades golden +custom metrics without an agent. Tests cover generation drift, generated specs, the executor, +backend cleanup, and fail-closed grading. The pilot dry-run resolves both experiment variants and +their schedule but makes no model call and creates no ACA sandbox. + +## Grader certification corpus + +The Vally oracle proves that authoritative evidence is aggregated correctly. The separate +grader-certification corpus proves that the underlying project validators recognize real project +behavior. Its hand-authored reference project lives under +`evals/grader-certification/reference-node-fullstack/` and is intentionally dependency-free. + +```sh +# PR-safe: artifact contracts, target discovery, and one-fault mutations. +npm run eval:cor:graders:certify + +# Nightly: real ACA build, test, runtime, browser, accessibility, persistence, +# debugger-readiness, and cleanup evidence. +npm run eval:cor:graders:certify:aca + +# Diagnose one ACA case without rerunning the whole corpus. +npm run eval:cor:graders:certify:aca -- --case golden-local-runtime +``` + +`evals/grader-certification/manifest.json` declares every validator, mutation, expected failure +code, and execution tier. The golden workspace must pass every applicable validator. Each mutation +changes exactly one condition and must produce its declared failure code and, for build/test +mutations, the exact failing command. Missing build evidence fails closed as `noBuildTargets`. +Reports are written to `evals/results/grader-certification/` as JSON and Markdown. + +The contracts workflow runs the offline corpus. Daily and weekly jobs run the ACA corpus before +model experiments. This certification does not replace Vally grading; it verifies the executable +oracles whose results Vally consumes. + +> **Paid-call warning:** every command below invokes coding models and creates ACA sandbox work. +> Confirm Azure/Copilot credentials, quotas, expected trial count, and the dry-run plan first. + +A direct, single Rails trial proves the executor and grader JavaScript bridges independently of the +experiment API: + +```sh +vally eval \ + --eval-spec evals/vally/native/canary.eval.yaml \ + --tag scenarioId=api-ts-functions-minimal \ + --model gpt-5.6-sol \ + --executor cor-aca \ + --executor-plugin ../plugins/aca-executor/index.js \ + --grader-plugin ../plugins/cor-graders/index.js \ + --runs 1 \ + --workers 1 \ + --max-retries 0 \ + --output-dir evals/results/vally-native/single-rails + +# Stable paid alias for exactly the command above. +npm run eval:cor:vally:paid:single-rails +``` + +Vally resolves local plugin paths relative to the eval spec, which is why the direct command uses +`../plugins`. `vally experiment run` resolves its backend relative to the experiment file's +`evals/vally/native/experiments` directory, so the backend path is `../../plugins`: + +Every direct run writes `executor-artifacts/reports/run-diagnostics.md` and +`run-diagnostics.json`. The failed grader line includes the same concise diagnosis: which upstream +gates passed, the primary stage/code, the exact failing task and missing executable or test input, +repair usage, and which dependent gates were not attempted. The Markdown report adds the complete +gate table, command, working directory, stdout/stderr excerpts, and recommended action. + +Vally shows two top-level graders by design. `authoritative-local-hard-gates` is one hierarchical +grader over the individual planning, scaffold, build, test, integration, runtime, browser, +accessibility, persistence, debugger, cleanup, model, and provenance gates. +`authoritative-metric` is a zero-weight integrity assertion that verifies the exported +`authoritative_hard_gates_passed` metric agrees with that evidence; it is not a second project test. + +```sh +# Offline plan: two scenarios × Rails/baseline, one run each. +vally experiment run \ + evals/vally/native/experiments/compatibility-pilot-gpt-5-6-sol.experiment.yaml \ + --backend ../../plugins/aca-executor/backend.js \ + --workers 1 \ + --dry-run + +# PAID: primary-model alias, four trials. +npm run eval:cor:vally:paid:pilot + +# PAID: four trials each; all three commands total 12 trials. +npm run eval:cor:vally:paid:pilot:gpt-5-6-sol +npm run eval:cor:vally:paid:pilot:claude-sonnet-5 +npm run eval:cor:vally:paid:pilot:gpt-5-4-mini + +# PAID: 16 trials per command; run all three for the representative model set. +npm run eval:cor:vally:paid:representative:gpt-5-6-sol +npm run eval:cor:vally:paid:representative:claude-sonnet-5 +npm run eval:cor:vally:paid:representative:gpt-5-4-mini + +# PAID RELEASE: 20 scenarios × 2 arms × 3 runs = 120 trials. +npm run eval:cor:vally:paid:release +``` + +`paid:pilot` is an alias for the `gpt-5.6-sol` pilot; use either name, not both, when running the +three-model set. + +All paid experiment aliases use +`--backend ../../plugins/aca-executor/backend.js` and cap Vally at two workers. Vally 0.12's +`experiment run` command accepts a backend plugin but has no experiment-level +`--executor-plugin`/`--grader-plugin` options. Generated experiment plans therefore deliberately +name the mock executor, while the `cor-aca` backend delegates each resolved trial to the ACA custom +executor and invokes the authoritative custom grader in-process. This is a Vally 0.12 compatibility +bridge, not mock execution. + +### Deliberate model set + +Models are explicit, pinned, and never pooled: + +| Model | Purpose | +|---|---| +| `gpt-5.6-sol` | Primary/default high-capability release candidate and longitudinal reference. | +| `claude-sonnet-5` | Cross-provider high-capability check that exposes provider-specific coupling. | +| `gpt-5.4-mini` | Smaller, lower-cost sensitivity check for capability and price/performance regressions. | + +Nightly compatibility CI uses the primary model. Operators can run the compatibility pilot for all +three models separately (12 total trials). The representative tier also runs all three separately. +The release alias uses only the primary model so release cost and evidence stay bounded. + +### Endpoint and ACA safety contract + +The ordinary authoritative endpoint is `local`: generated code runs and is tested inside a +hardware-isolated ACA Sandbox microVM. `local` never means or claims Azure deployment. +Live deployment and real VS Code F5/breakpoint parity are separate, explicit, release-only evidence +tiers; neither is implied by an ordinary Vally result. + +Sandbox manifests are the reproducible CI/CD path. Scaffold, validate, then apply them with the +authoritative `aca` CLI: + +```sh +aca sandbox init +aca sandbox validate --file sandbox.yaml +aca sandbox apply --file sandbox.yaml +``` + +Edit the generated manifest's disk, resources, `lifecycle.autoSuspendPolicy`, `egressPolicy`, +ports, environment, and labels as needed; `aca sandbox schema` provides editor schema data. The +checked-in evaluator manifests use deny-default egress with an explicit host allow-list, a +one-hour auto-delete TTL, and auto-suspend. Every trial adds a unique exact `owner-id` plus +`run-id`; cleanup selects that exact `owner-id` and deletes only the returned exact sandbox IDs. +Never use `az containerapp` for this system: it is the older Apps/Jobs surface, not ACA Sandboxes. + +## Deterministic validation + +These commands make no model calls or Azure resource changes: + +```sh +npm run eval:cor:spike:dry +npm run eval:cor:baseline -- --dry-run +npm run eval:cor:matrix -- \ + --models gpt-5.6-sol,claude-sonnet-5 \ + --scenarios api-ts-functions-minimal \ + --attempts 1 \ + --through scaffold \ + --seed compatibility-pilot-v1 \ + --output evals/results/compatibility-pilot \ + --dry-run +``` + +Use `npm run eval:cor:baseline -- --help` for all baseline options. + +## Nightly CI end state and spend guards + +`.github/workflows/copilot-on-rails-evals.yml` has four explicit tiers: + +| Tier | Trigger | Work | +|---|---|---| +| `contracts` | Pull requests or dispatch | Offline generation drift check, strict lint, oracle, focused tests, threshold validation, and experiment dry-run. No model calls or Azure changes. | +| `daily` | Nightly every day or dispatch | Bounded primary-model compatibility evidence at the ordinary local endpoint. | +| `weekly` | Sunday schedule or dispatch | Separate representative experiments for the explicit three-model set. | +| `release` | Dispatch only | Fails closed before paid calls until provenance-bound VS Code parity and explicitly authorized live-deployment evidence are wired into the workflow. | + +Scheduled model calls are disabled unless the repository variables +`COR_EVAL_DAILY_ENABLED=true` and/or `COR_EVAL_WEEKLY_ENABLED=true` are set. Required repository +variables are `COR_EVAL_RESOURCE_GROUP`, `COR_EVAL_SANDBOX_GROUP`, and `COR_EVAL_REGION`; configure +`COR_EVAL_AZURE_LOCATION` only for an explicitly authorized live-deployment tier. Required secrets +are `COR_EVAL_AZURE_CLIENT_ID`, `COR_EVAL_AZURE_TENANT_ID`, +`COR_EVAL_AZURE_SUBSCRIPTION_ID`, and `COR_EVAL_COPILOT_GITHUB_TOKEN`. The Azure identity needs ACA +Sandbox Group Data Owner access. A live-deployment subscription must be dedicated to evaluation. + +CI caps Vally/native workers at two rather than allowing unbounded fan-out. It retains +`cor-vally-native-*` experiment evidence, an aggregated release-policy report, and separate +`cor-vally-cleanup-*` cleanup evidence for 30 days, uploading them even after trial failure. Every +sandbox receives an exact workflow/trial `owner-id`; `always()` cleanup lists by that exact label, +deletes only those IDs, lists again, and fails the job unless the post-delete result is empty. The +manifest's one-hour auto-delete policy is a fail-safe, not a substitute for verified cleanup. + +Dispatch accepts only `tier` plus the separately gated parity/deployment booleans: + +```sh +# Offline PR-equivalent contracts +gh workflow run copilot-on-rails-evals.yml -f tier=contracts + +# Exact paid daily and weekly dispatches +gh workflow run copilot-on-rails-evals.yml -f tier=daily +gh workflow run copilot-on-rails-evals.yml -f tier=weekly + +# These separate evidence requests currently fail closed before paid calls. +gh workflow run copilot-on-rails-evals.yml \ + -f tier=release \ + -f run_vscode_parity=true \ + -f run_live_deployment=true +``` + +The daily `23 5 * * *` cron is nightly every day and remains disabled unless +`COR_EVAL_DAILY_ENABLED=true`; the Sunday three-model schedule independently requires +`COR_EVAL_WEEKLY_ENABLED=true`. A local `npm run eval:cor:vally:paid:release` is available to +collect raw 120-trial Vally-native evidence. Apply the checked-in policy afterward with +`npm run eval:cor:vally:native:report`; add `--enforce-release` to exit non-zero unless every gate +passes. CI release dispatch deliberately stops after dry resolution and before paid calls until +the provenance-bound VS Code parity and live-deployment integrations are available. + +**Cost warning:** one compatibility pilot makes 4 model evaluations (12 across all three models); +the three-model representative shape makes 48; a local raw-evidence release makes 120 +(20 scenarios × 3 attempts × 2 arms), before any repair calls. VS Code +parity consumes ACA time and live deployment creates billable Azure resources. Inspect the +uploaded dry plan before authorizing an expensive run. Native experiment CI produces Vally trial +records, authoritative custom-metrics grades, Vally experiment reports, same-model pass@k/pass^k +statistics, and an aggregate checked-policy recommendation. Daily and weekly jobs publish the +recommendation without treating expected product-quality failures as report-generation failures. + +## Local-runtime hard gates + +Local acceptance contracts are evaluator-owned and run inside a disposable ACA sandbox. HTTP probes +may declare validated headers and a JSON-compatible body; the harness safely serializes requests and +records response status, headers, and body evidence. + +A browser `persistence` contract is a distinct hard gate. The initial actions and assertions must +pass, then the harness terminates only the evaluator-launched backend/frontend process groups, +relaunches those application tasks without restarting dependency tasks such as PostgreSQL or +Azurite, waits for the declared readiness probes again, reloads the declared/current page, and runs +the explicit persistence assertions. Sandbox suspend/resume and an unchanged application process +do not count as persistence evidence. Results include old/new PIDs, restart commands, readiness +results, and post-restart browser evidence. + +Queue worker scenarios can declare `storageEvents`. The harness uses Azurite's documented +development account directly to create queues, enqueue the declared JSON stimulus, and poll the +output queue for the declared JSON content. This verification does not call generated tests or +generated helper code, and failures use a separate storage-event failure code. Evaluator-owned +Python Blob archival verification is also implemented: it seeds `active-documents`, selects blobs +whose `expiresAt` metadata is in the past, verifies the same name and content in +`archived-documents`, and verifies deletion of the source blob. Process liveness alone is never +reported as worker side-effect correctness. + +Evidence is bounded and sanitized: command output and response bodies are truncated, browser body +text is excerpted, and queue verification records only the declared stimulus/expectation and +observed message. These gates prove behavior in the isolated Azurite/local-service environment, not +durability or compatibility of a deployed Azure resource. + +## Controlled baseline + +Each scenario has two prompts: `prompt` drives the Rails treatment and `baselinePrompt` is one +standalone implementation request for the controlled baseline. A real paired run must pin the +same model explicitly in both arms: + +> **Paid-call warning:** both commands below invoke the selected coding model. + +```sh +MODEL= +npm run eval:cor:scaffold -- --model "$MODEL" --scenario api-ts-functions-minimal --output evals/results/treatment +npm run eval:cor:baseline -- --model "$MODEL" --scenario api-ts-functions-minimal --output evals/results/baseline +``` + +The baseline uses the SDK's generic coding agent in empty mode and an empty candidate workspace. +It loads no Rails agents or references, custom agents, skills, custom instructions, webview gates, +handoffs, MCP servers, or Rails-only artifacts. Only workspace file tools are available; shell, +network, MCP, and delegation are denied. Build and local-runtime validation happen outside the +agent. Local validation derives temporary metadata from the generated `.vscode/launch.json` and +the scenario's acceptance contract rather than requiring a Rails debug plan. Sanitized repair +evidence may be returned within the scenario's shared repair budget. + +This is a **controlled generic-agent baseline**, not a stock-Copilot arm. A future stock-Copilot +arm should separately measure Copilot with its normal ambient instructions, tools, and product +UX. Do not label controlled-baseline results as stock Copilot. + +Baseline summaries declare `evaluationArm: "baseline-controlled"` and record requested and +observed models. Treatment summaries declare `evaluationArm: "rails"`. The baseline fails closed +if the observed model differs from the requested pin. Real treatment, baseline, and local-resume +commands all require `--model`; no scenario or ambient model fallback is accepted. + +## Paired reports + +```sh +npm run eval:cor:report -- \ + --input evals/results/treatment \ + --baseline evals/results/baseline \ + --vscode-parity evals/results/vscode-parity-result.json \ + --deployment evals/results/live-deployment.json \ + --output evals/results/report +``` + +Reports pair only exact `model + scenarioId + attempt` matches from identical evaluation endpoints +(`through` must have the same set in both arms, so `scaffold` is never compared with `local`). +Modern pairs must also have an exactly equal evaluation-definition provenance object. They reject +conflicting declared arms, different per-attempt model pins, and modern definition mismatches, and +expose each arm's endpoint, model, definition, and unmatched provenance. + +For declared arms, every matched attempt must provide observed-model evidence and must have +observed exactly its requested model pin in both arms. The observed model sets must also be equal. +Reports use explicit attempt or summary `observedModels` fields; legacy Rails treatment attempts +can derive this evidence from stage `agentRun.usage.models`. Missing legacy evidence is shown as +`legacy_missing`, never as verified parity. Legacy treatment reports and reports without a +baseline remain supported. + +## Versioned release thresholds + +`evals/release-thresholds.v1.json` is the checked-in release policy for both durable Vally-native +experiment output and the historical summary adapter. +Validate the policy independently: + +```sh +npm run eval:cor:thresholds:validate + +# Aggregate one or more Vally experiment output roots. +npm run eval:cor:vally:native:report -- \ + --experiment-dir evals/results/vally-native/compatibility-pilot-gpt-5-6-sol \ + --experiment-dir evals/results/vally-native/compatibility-pilot-claude-sonnet-5 \ + --output evals/results/vally-native/compatibility-report + +# Release automation fails unless every gate passes and every durable manifest +# contains post-sweep cleanup verification. +npm run eval:cor:vally:native:report -- \ + --experiment-dir evals/results/vally-native/release-gpt-5-6-sol \ + --vscode-parity evals/results/vscode-parity-result.json \ + --deployment evals/results/live-deployment.json \ + --output evals/results/vally-native/release-report \ + --enforce-release + +# The historical offline adapter uses the same checked-in policy. +npm run eval:cor:vally -- \ + --input evals/results/treatment/summary.json \ + --baseline evals/results/baseline/summary.json \ + --vscode-parity evals/results/vscode-parity-result.json \ + --deployment evals/results/live-deployment.json \ + --output evals/results/vally-release + +# Policy experiments must be explicit. +npm run eval:cor:vally -- \ + --input evals/results/treatment/summary.json \ + --thresholds path/to/versioned-thresholds.json \ + --output evals/results/vally-policy-experiment +``` + +The adapter's Vally-shaped `report.json` records the threshold schema/set and every gate's measured +value, threshold, status, and rationale; +`releaseAssessment`, `experiments`, and `groups` provide the recommendation, same-model comparison, +and Vally reliability statistics. `report.md` renders those together with per-run Rails and baseline +diagnostic tables, failed-gate evidence, and links to each durable run result. `authoritative-evidence.json` +retains the native gate aggregation behind the custom graders. `npm run eval:cor:report` remains a +low-level compatibility/debug command for historical native consumers. + +`eval:cor:vally:native:report` discovers only durable `artifacts/native-summary.json` bundles from +`vally experiment run`, cross-checks summary, run-result, manifest, authoritative validation, and +metrics identities, rejects duplicate run IDs, and separates Rails from controlled-baseline inputs. +Its `experiment-input-manifest.json` records every accepted artifact directory and whether the +executor persisted post-sweep cleanup verification. `--enforce-release` requires that proof and +exits non-zero unless the recommendation is `candidate`. `vally-native-report.json` wraps the +input manifest and policy report as one machine-readable record; `report.md` includes the same +native-input integrity summary and per-run diagnostics. + +Gate status is strict: + +- `passed` means present evidence met the configured threshold; +- `failed` means present evidence violated it; +- `missing_evidence` means an applicable required measurement was absent or incomplete; and +- `not_applicable` means the scenario corpus declares no such contract. It is never counted as a + pass. + +Local evaluation is dependency-aware. A failed generated test or lint command keeps the attempt +failed, but Rails and controlled-baseline runs continue through integration, runtime, browser, +persistence, and debugger validation. Build, setup, infrastructure, and invalid integration +failures still stop dependent stages. Authoritative gates grade their own evidence rather than +inheriting the attempt's overall outcome, so a run can truthfully report `test: failed` and +`debugger: passed`. + +Any missing gate keeps the existing `insufficient_evidence` +recommendation. With complete evidence, any failed gate recommends `hold`; only all applicable +gates passing recommends `candidate`. Vally-native daily and weekly CI publish this assessment; +release enforcement is available but the release workflow remains pre-paid blocked on the separate +debugger/deployment evidence integrations. +The policy covers zero configured critical security/destructive/cleanup failures, full corpus +coverage, three repetitions for every model+scenario, final and first-pass success, complete UI +browser/accessibility scans with zero serious/critical violations, persistence and worker +side-effect checks where declared, provenance-bound debugger and deployment evidence, verified +deployment cleanup, controlled-baseline non-inferiority, paired latency and nano-AIU cost +multipliers, and explicit arm/commit/assets/model/cleanup provenance. A zero or absent baseline +nano-AIU measurement is missing cost evidence, not a free passing multiplier. + +Baseline release gates are evaluated per model. Pairing is exact on +`model + scenarioId + attempt + through`; every release model/scenario group must have the configured +number of pairs. Requested and observed model parity must be verified in both arms. Cross-model, +cross-endpoint, unmatched, pooled, or legacy-missing pairs cannot satisfy release gates. + +### Evaluation-definition provenance + +Every new Rails and controlled-baseline attempt records an `evaluationDefinition` object, and its +summary records the exact `evaluationDefinitions` represented by its attempts. The modern schema has +four `sha256:` hashes: + +- `scenarioCorpusHash`: raw bytes and repository-relative paths for the exact selected scenario JSON + corpus; +- `evaluatorHash`: evaluator TypeScript, ACA manifests, Vally/runtime files, release thresholds, + workflow, and locked package/runtime definitions; +- `productContractHash`: Copilot-on-Rails agents/references, product webview/controller contracts, + and shared agent-execution sources; and +- `combinedHash`: the versioned scenario IDs and all three component hashes. + +Paths are normalized and sorted before hashing, so traversal order does not affect the result while +any covered uncommitted content change does. Matrix dry manifests retain the selected-corpus +definition. Child jobs retain their one-scenario definition; aggregation rejects missing/malformed +modern data, a scenario definition that changes between attempts, or evaluator/product hashes that +change during the matrix. Rails and baseline paired reports reject unequal modern definitions. +Debugger and deployment evidence copy the source attempt definition and must match it exactly. + +Historical summaries without these fields still render and compare with explicit +`legacy_missing` provenance. They cannot satisfy `evaluation-definition-provenance` or +`cleanup-provenance`, so their release recommendation remains `insufficient_evidence`; missing +hashes are never inferred from the current checkout. + +## Multi-model matrix + +The matrix runner pins every model explicitly and schedules one Rails process and one controlled +baseline process for each `model + scenario + attempt` pairing. A seeded schedule randomizes pair +order and which arm runs first. Each child retains the normal evaluator summary and cleanup +behavior; the matrix writes the exact schedule to `matrix-manifest.json`, combined references to +`matrix-summary.json`, and a verified paired report under each sanitized model directory. + +Compatibility pilot: + +```sh +npm run eval:cor:matrix -- \ + --models gpt-5.6-sol,claude-sonnet-5 \ + --scenarios api-ts-functions-minimal,crud-react-functions-postgres \ + --attempts 1 \ + --through scaffold \ + --concurrency 2 \ + --seed compatibility-pilot-v1 \ + --output evals/results/compatibility-pilot +``` + +Representative scaffold matrix: + +```sh +npm run eval:cor:matrix -- \ + --models gpt-5.6-sol,claude-sonnet-5,gpt-5.4-mini \ + --scenarios api-ts-functions-minimal,crud-react-functions-postgres,multiservice-react-functions-worker-postgres-queue,worker-python-functions-blob \ + --attempts 2 \ + --through scaffold \ + --concurrency 3 \ + --seed representative-matrix-v1 \ + --output evals/results/representative-matrix +``` + +**Cost warning:** real matrix runs make two full evaluator runs per model, scenario, and attempt. +The representative command above launches 48 paid, potentially long-running model evaluations. +Run the same command with `--dry-run` first to inspect its deterministic manifest without making +model calls. Real and dry runs both require explicit, non-empty model and scenario lists; repeated +`--models` and `--scenarios` flags are also accepted. + +Legacy adapter release recommendations use only explicit VS Code parity and live-deployment result +inputs. A deployment passes the gate only when its result passed and cleanup was verified. +Live deployment also requires `--source-result /run-result.json`; it deploys only the +archived workspace beside that result, restores local `azd` state, and verifies that the dedicated +subscription returns to its pre-run resource inventory. + +## Legacy offline summary adapter + +`npm run eval:cor:vally` adapts archived native summaries into Vally-shaped records and a legacy +offline release-policy report. It is distinct from the Vally-native `vally experiment run` records +and reports produced by the ACA backend. ACA build/runtime, browser, accessibility, persistence, +worker-event, VS Code debugger, and deployment validators remain its authoritative evidence +producers. Adapter report generation never executes generated code, invokes an agent/model, or +calls an LLM judge. + +The adapter uses Vally's trajectory taxonomy, built-in `custom-metrics` grader, suites/oracles, +multi-trial pass@k and pass^k statistics, flakiness, and portable JSONL records. Its trajectories +contain only evidence actually retained in the summary: the evaluator stimulus, stage summaries, +tool-call summaries, errors, and aggregate token/nano-AIU events. They contain no +`assistant_message` or reasoning events, `output` is empty, and metadata explicitly declares +`source: copilot-on-rails-summary-adapter` and `transcriptFidelity: summary-only`. A stage-level +token event is an aggregate, not an individual API response. + +Generate the legacy adapter report: + +```sh +npm run eval:cor:vally -- \ + --input evals/results/treatment/summary.json \ + --baseline evals/results/baseline/summary.json \ + --vscode-parity evals/results/vscode-parity-result.json \ + --deployment evals/results/live-deployment.json \ + --output evals/results/vally-report +``` + +`--input` and `--baseline` are repeatable. Inputs must have unique run IDs, known scenarios, +declared arms, pinned/observed model provenance, and (for the controlled baseline) evidence that +Rails assets and custom tools were not injected. Treatment and baseline matching is exact on +`model + scenario + attempt + endpoint`, with exact modern evaluation-definition parity. Models, +endpoints, definitions, and arms are never pooled. + +The output contains: + +- `treatment/results.jsonl` and, when supplied, `baseline/results.jsonl`: Vally + `trial-result` records with trajectories and grades; +- `*/attempts//custom_metrics.json`: the authoritative per-attempt metrics artifact; +- per-attempt `trajectory.json` and `grade.json`; +- `regrade-eval.yaml`: a generated, no-LLM custom-metrics spec matching the emitted stimuli; +- `report.json` and `report.md`: the legacy offline same-model comparison, per-model/scenario/arm + pass rate, unbiased pass@k, pass^k reliability, flakiness, hard-gate-normalized score, release + thresholds, and recommendation; +- `authoritative-evidence.json`: low-level native aggregation consumed by the legacy adapter report; + and +- `comparison-manifest.json`: exact pairs and unmatched records. + +An inapplicable gate has `*_applicable: false`, `*_status: "not-applicable"`, and a null result. +An applicable gate with no archived evidence has `*_status: "missing-evidence"` and fails. The +adapter builds per-attempt `custom-metrics` assertions dynamically, requiring final product +success plus only that scenario/endpoint's applicable browser, accessibility, persistence, +worker, debugger, or present deployment gates. Explicit `--vscode-parity` and `--deployment` +artifacts are matched back to their exact source run and incorporated into that attempt's custom +metrics as well as aggregate release gates. `gradeTrajectory` runs the built-in grader. +Because its fractional assertion score is not a safe release score, any failed hard gate +normalizes the reported aggregate to zero. + +The JSONL retains each trajectory's strict per-attempt artifact directory. While the output +remains at that location, it can be deterministically re-graded without model calls: + +```sh +vally grade \ + --eval-spec evals/results/vally-report/regrade-eval.yaml \ + --output jsonl \ + < evals/results/vally-report/treatment/results.jsonl +``` + +Validate the checked-in Vally-native contract without model calls: + +```sh +npm run eval:cor:vally:native:check +npm run eval:cor:vally:native:lint +npm run eval:cor:vally:native:oracle +npm run eval:cor:vally:native:test +``` + +The pinned `@microsoft/vally@0.12.0` and `@microsoft/vally-cli@0.12.0` public APIs used here are +`Trajectory`, `gradeTrajectory`, `computeMetrics`, `computeStimulusScore`, `computeSkillScore`, +`passAtK`, and `passToTheK`. Both packages declare Node.js `>=22.0.0` and npm `>=11.11.1`; +use those versions for supported local and CI execution. + +### Supplemental qualitative comparison + +`vally compare` uses a qualitative judge and position-swap debiasing. It is supplemental, costs +model calls, and is never run by the adapter. **Do not run it on these summary-only records**: +they omit assistant responses and reasoning, so they are invalid qualitative evidence. If a +future archive contains faithful full transcripts for both exactly matched arms, use a matching +rubric-bearing eval spec and pin the judge: + +```sh +vally compare \ + --baseline \ + --treatment \ + --eval-spec \ + --judge-model gpt-5.6-sol \ + --judge-reasoning-effort medium \ + --output +``` diff --git a/evals/grader-certification/manifest.json b/evals/grader-certification/manifest.json new file mode 100644 index 000000000..0d849d2d4 --- /dev/null +++ b/evals/grader-certification/manifest.json @@ -0,0 +1,224 @@ +{ + "schemaVersion": 1, + "fixture": { + "id": "reference-node-fullstack", + "path": "evals/grader-certification/reference-node-fullstack", + "description": "Dependency-free Node.js full-stack reference project with build, test, lint, HTTP, browser, accessibility, persistence, debugger, and deployment artifacts.", + "offlineValidators": [ + "requirements", + "project-plan", + "plan-gate", + "preview", + "integration-plan", + "integration-output", + "local-debug", + "deployment", + "target-discovery" + ], + "acaValidators": [ + "project-build", + "local-runtime", + "browser", + "accessibility", + "persistence", + "debugger-readiness", + "security", + "cleanup" + ] + }, + "deployFixture": { + "id": "reference-deployable", + "path": "evals/grader-certification/reference-deployable", + "description": "Known-good deployable project used to certify the deployment gate. Packaging runs through azd with a remote container build, so the tier needs no Azure subscription, no azd login, and no local container runtime.", + "deployValidators": [ + "deployment-readiness" + ] + }, + "mutations": [ + { + "id": "requirements-schema-version", + "tier": "offline", + "validator": "requirements", + "file": ".azure/requirements.json", + "operation": "replace", + "search": "\"schemaVersion\": \"2\"", + "replacement": "\"schemaVersion\": \"1\"", + "expectedCode": "schemaVersion" + }, + { + "id": "project-plan-numbering", + "tier": "offline", + "validator": "project-plan", + "file": ".azure/project-plan.md", + "operation": "replace", + "search": "## 3. Prerequisites", + "replacement": "## 4. Prerequisites", + "expectedCode": "nonSequentialHeading" + }, + { + "id": "preview-not-ready", + "tier": "offline", + "validator": "preview", + "file": ".azure/.preview-temp/manifest.json", + "operation": "replace", + "search": "\"previewStatus\": \"ready\"", + "replacement": "\"previewStatus\": \"draft\"", + "expectedCode": "previewNotReady" + }, + { + "id": "integration-seed-policy", + "tier": "offline", + "validator": "integration-plan", + "file": ".azure/integration-plan.md", + "operation": "replace", + "search": "NO seed data.", + "replacement": "Use sample data when convenient.", + "expectedCode": "missingNoSeedRule" + }, + { + "id": "frontend-mock-import", + "tier": "offline", + "validator": "integration-output", + "file": "public/app.js", + "operation": "append", + "replacement": "\nimport('./mockClient.js');\n", + "expectedCode": "frontendMockStillImported" + }, + { + "id": "debug-task-graph", + "tier": "offline", + "validator": "local-debug", + "file": ".vscode/launch.json", + "operation": "replace", + "search": "\"preLaunchTask\": \"prepare\"", + "replacement": "\"preLaunchTask\": \"missing task\"", + "expectedCode": "invalidPreLaunchTask" + }, + { + "id": "deployment-hook-shell", + "tier": "offline", + "validator": "deployment", + "file": "azure.yaml", + "operation": "replace", + "search": "shell: sh", + "replacement": "shell: cmd", + "expectedCode": "invalidAzdHookShell" + }, + { + "id": "deployment-secret", + "tier": "offline", + "validator": "deployment", + "file": "azure.yaml", + "operation": "append", + "replacement": "\nmetadata:\n password: super-secret-value\n", + "expectedCode": "hardcodedSecret" + }, + { + "id": "no-build-target", + "tier": "offline", + "validator": "target-discovery", + "file": "package.json", + "operation": "delete", + "expectedCode": "noBuildTargets" + }, + { + "id": "build-syntax-error", + "tier": "aca", + "validator": "project-build", + "file": "src/server.js", + "operation": "append", + "replacement": "\nconst broken = ;\n", + "expectedCode": "sandboxCommandFailed", + "expectedCommand": "npm run build" + }, + { + "id": "generated-test-failure", + "tier": "aca", + "validator": "project-build", + "file": "test/server.test.js", + "operation": "replace", + "search": "assert.equal(response.status, 200);", + "replacement": "assert.equal(response.status, 500);", + "expectedCode": "sandboxCommandFailed", + "expectedCommand": "npm test" + }, + { + "id": "runtime-status-mismatch", + "tier": "aca", + "validator": "local-runtime", + "operation": "scenario-status", + "expectedCode": "localProbeFailed" + }, + { + "id": "accessibility-missing-button-name", + "tier": "aca", + "validator": "local-runtime", + "file": "public/index.html", + "operation": "replace", + "search": " ", + "replacement": " \n ", + "expectedCode": "localBrowserFailed" + }, + { + "id": "persistence-not-durable", + "tier": "aca", + "validator": "local-runtime", + "file": "src/server.js", + "operation": "replace", + "search": "path.join(__dirname, '..', 'data', 'items.json')", + "replacement": "path.join('/tmp', 'items-' + process.pid + '.json')", + "expectedCode": "localPersistenceFailed" + }, + { + "id": "debug-port-mismatch", + "tier": "aca", + "validator": "local-runtime", + "file": ".vscode/launch.json", + "operation": "replace", + "search": "\"--inspect=9229\"", + "replacement": "\"--inspect=9229\",\n \"--inspect-port=9331\"", + "comment": "The app still serves, but the inspector moves off the declared port. Debug-surface failures currently surface through the probe gate because probes carry debugPort; the dedicated debugger gate is certified by evals/test/debuggerPrerequisite.test.ts.", + "expectedCode": "localProbeFailed" + }, + { + "id": "security-auth-bypass", + "tier": "aca", + "validator": "local-runtime", + "file": "src/server.js", + "operation": "replace", + "search": "if (!isAuthorizedAdmin(request.headers.authorization, adminToken)) {", + "replacement": "if (false) {", + "comment": "Serves the protected admin path to an anonymous caller while public paths stay healthy, which is the exact shape the security gate exists to catch. The local-runtime task graph is install -> build -> prepare and never runs npm test, so the unit tests that also cover this logic do not pre-empt the gate.", + "expectedCode": "localSecurityFailed" + }, + { + "id": "deployment-plan-deleted", + "tier": "deploy", + "validator": "deployment-readiness", + "file": ".azure/deployment-plan.md", + "operation": "delete", + "expectedCode": "deploymentPlanMissing" + }, + { + "id": "deployment-services-removed", + "tier": "deploy", + "validator": "deployment-readiness", + "file": "azure.yaml", + "operation": "replace", + "search": "services:", + "replacement": "disabledServices:", + "expectedCode": "deploymentArtifactsInvalid" + }, + { + "id": "deployment-unsupported-host", + "tier": "deploy", + "validator": "deployment-readiness", + "file": "azure.yaml", + "operation": "replace", + "search": "host: containerapp", + "replacement": "host: nowhere", + "comment": "Static artifact validation only checks that a host field exists, so this reaches azd itself and proves the packaging command actually ran rather than being reported from cached artifact checks.", + "expectedCode": "azdPackageFailed" + } + ] +} diff --git a/evals/grader-certification/reference-deployable/.azure/.gitignore b/evals/grader-certification/reference-deployable/.azure/.gitignore new file mode 100644 index 000000000..3a5e24c35 --- /dev/null +++ b/evals/grader-certification/reference-deployable/.azure/.gitignore @@ -0,0 +1,9 @@ +# azd writes environment state here (config.json and per-environment directories +# holding .env files). That state is machine-local and must never be committed. +* + +# deployment-plan.md is not azd state. It is a checked-in part of this fixture: +# the deployment gate fails with `deploymentPlanMissing` without it, so a fresh +# checkout that lacks it would fail the golden case for the wrong reason. +!.gitignore +!deployment-plan.md diff --git a/evals/grader-certification/reference-deployable/.azure/deployment-plan.md b/evals/grader-certification/reference-deployable/.azure/deployment-plan.md new file mode 100644 index 000000000..251ffa019 --- /dev/null +++ b/evals/grader-certification/reference-deployable/.azure/deployment-plan.md @@ -0,0 +1,79 @@ +# Azure Deployment Plan + +> **Status:** Ready for Validation + +Generated: 2026-08-14 + +## 1. Project Overview + +Deploy the known-good deployable reference as a Node.js Azure Container App using Azure Developer +CLI and Bicep. This fixture exists to certify the deployment gate: a failure here indicates a +harness or environment defect rather than a Copilot on Rails defect. + +## 2. Requirements + +| Attribute | Value | +|---|---| +| Classification | Development | +| Scale | Small | +| Budget | Cost-Optimized | +| Subscription | Dedicated evaluation subscription | +| Location | East US 2 | + +## 3. Components Detected + +| Component | Type | Technology | Path | +|---|---|---|---| +| app | Web and API | Node.js 22 | `.` | + +## 4. Recipe Selection + +**Selected:** AZD with Bicep. + +## 5. Architecture + +The application runs in Azure Container Apps. The image is built remotely in Azure Container +Registry so packaging does not depend on a local container runtime, and no secrets are embedded in +the image or the infrastructure templates. + +### Azure Resources + +| Component | Azure Service | SKU / Tier | +|---|---|---| +| app | Azure Container Apps | Consumption | +| registry | Azure Container Registry | Basic | +| logs | Log Analytics | Pay-as-you-go | + +```text +Internet -> Azure Container Apps -> Node.js application +``` + +## 6. Execution Checklist + +- [x] Analyze workspace +- [x] Select recipe +- [x] Generate `azure.yaml` +- [x] Generate Bicep infrastructure +- [ ] Run validation +- [ ] Deploy after explicit authorization + +## 7. Validation Proof + +| Check | Command Run | Result | Timestamp | +|---|---|---|---| +| Static artifacts | grader certification | Passed | 2026-08-14 | +| Packaging | `azd package` | Passed | 2026-08-14 | + +## 8. Files to Generate + +| File | Purpose | Status | +|---|---|---| +| `.azure/deployment-plan.md` | Deployment plan | Complete | +| `azure.yaml` | Azure Developer CLI manifest | Complete | +| `infra/main.bicep` | Infrastructure | Complete | +| `Dockerfile` | Container image definition | Complete | + +## 9. Next Steps + +1. Run `azd package`. +2. Deploy only in the dedicated evaluation subscription after explicit authorization. diff --git a/evals/grader-certification/reference-deployable/Dockerfile b/evals/grader-certification/reference-deployable/Dockerfile new file mode 100644 index 000000000..9f4b2fdbb --- /dev/null +++ b/evals/grader-certification/reference-deployable/Dockerfile @@ -0,0 +1,6 @@ +FROM mcr.microsoft.com/devcontainers/javascript-node:20 +WORKDIR /app +COPY package*.json ./ +COPY src ./src +EXPOSE 3000 +CMD ["node", "src/server.js"] diff --git a/evals/grader-certification/reference-deployable/azure.yaml b/evals/grader-certification/reference-deployable/azure.yaml new file mode 100644 index 000000000..8fb6c5c13 --- /dev/null +++ b/evals/grader-certification/reference-deployable/azure.yaml @@ -0,0 +1,14 @@ +# Known-good deployable reference. A tier-2 failure here indicates a harness defect, +# not a Copilot on Rails defect. +name: cor-eval-deployable +metadata: + template: cor-eval-deployable@1.0.0 +services: + app: + project: . + host: containerapp + language: js + docker: + path: ./Dockerfile + # Build in ACR so a deploy does not require a local container runtime. + remoteBuild: true diff --git a/evals/grader-certification/reference-deployable/infra/main.bicep b/evals/grader-certification/reference-deployable/infra/main.bicep new file mode 100644 index 000000000..b3f8a561a --- /dev/null +++ b/evals/grader-certification/reference-deployable/infra/main.bicep @@ -0,0 +1,30 @@ +targetScope = 'subscription' + +@minLength(1) +param environmentName string + +@minLength(1) +param location string + +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) +var tags = { 'azd-env-name': environmentName, 'cor-eval': 'true' } + +resource rg 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: 'rg-${environmentName}' + location: location + tags: tags +} + +module resources 'resources.bicep' = { + name: 'resources' + scope: rg + params: { + location: location + resourceToken: resourceToken + tags: tags + } +} + +output AZURE_CONTAINER_REGISTRY_ENDPOINT string = resources.outputs.AZURE_CONTAINER_REGISTRY_ENDPOINT +output AZURE_CONTAINER_APPS_ENVIRONMENT_ID string = resources.outputs.AZURE_CONTAINER_APPS_ENVIRONMENT_ID +output SERVICE_APP_URI string = resources.outputs.SERVICE_APP_URI diff --git a/evals/grader-certification/reference-deployable/infra/main.parameters.json b/evals/grader-certification/reference-deployable/infra/main.parameters.json new file mode 100644 index 000000000..579c3be9a --- /dev/null +++ b/evals/grader-certification/reference-deployable/infra/main.parameters.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { "value": "${AZURE_ENV_NAME}" }, + "location": { "value": "${AZURE_LOCATION}" } + } +} diff --git a/evals/grader-certification/reference-deployable/infra/resources.bicep b/evals/grader-certification/reference-deployable/infra/resources.bicep new file mode 100644 index 000000000..10d17c6ab --- /dev/null +++ b/evals/grader-certification/reference-deployable/infra/resources.bicep @@ -0,0 +1,96 @@ +param location string +param resourceToken string +param tags object + +resource logs 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { + name: 'log-${resourceToken}' + location: location + tags: tags + properties: { + sku: { name: 'PerGB2018' } + retentionInDays: 30 + } +} + +resource registry 'Microsoft.ContainerRegistry/registries@2023-07-01' = { + name: 'acr${resourceToken}' + location: location + tags: tags + sku: { name: 'Basic' } + properties: { adminUserEnabled: true } +} + +resource identity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: 'id-${resourceToken}' + location: location + tags: tags +} + +// AcrPull, so the container app pulls with a managed identity instead of a stored secret. +resource acrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: registry + name: guid(registry.id, identity.id, '7f951dda-4ed3-4680-a7ca-43fe172d538d') + properties: { + principalId: identity.properties.principalId + principalType: 'ServicePrincipal' + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') + } +} + +resource env 'Microsoft.App/managedEnvironments@2024-03-01' = { + name: 'cae-${resourceToken}' + location: location + tags: tags + properties: { + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: logs.properties.customerId + sharedKey: logs.listKeys().primarySharedKey + } + } + } +} + +resource app 'Microsoft.App/containerApps@2024-03-01' = { + name: 'ca-${resourceToken}' + location: location + // azd matches this tag to the service key in azure.yaml. + tags: union(tags, { 'azd-service-name': 'app' }) + identity: { + type: 'UserAssigned' + userAssignedIdentities: { '${identity.id}': {} } + } + dependsOn: [acrPull] + properties: { + managedEnvironmentId: env.id + configuration: { + activeRevisionsMode: 'Single' + ingress: { + external: true + targetPort: 3000 + transport: 'auto' + } + registries: [ + { + server: registry.properties.loginServer + identity: identity.id + } + ] + } + template: { + containers: [ + { + name: 'app' + image: 'mcr.microsoft.com/k8se/quickstart:latest' + resources: { cpu: json('0.25'), memory: '0.5Gi' } + } + ] + scale: { minReplicas: 1, maxReplicas: 1 } + } + } +} + +output AZURE_CONTAINER_REGISTRY_ENDPOINT string = registry.properties.loginServer +output AZURE_CONTAINER_APPS_ENVIRONMENT_ID string = env.id +output SERVICE_APP_URI string = 'https://${app.properties.configuration.ingress.fqdn}' diff --git a/evals/grader-certification/reference-deployable/package.json b/evals/grader-certification/reference-deployable/package.json new file mode 100644 index 000000000..abb280406 --- /dev/null +++ b/evals/grader-certification/reference-deployable/package.json @@ -0,0 +1,7 @@ +{ + "name": "cor-eval-deployable", + "version": "1.0.0", + "private": true, + "main": "src/server.js", + "scripts": { "start": "node src/server.js" } +} diff --git a/evals/grader-certification/reference-deployable/src/server.js b/evals/grader-certification/reference-deployable/src/server.js new file mode 100644 index 000000000..7f5faa1b3 --- /dev/null +++ b/evals/grader-certification/reference-deployable/src/server.js @@ -0,0 +1,13 @@ +const http = require('http'); + +const port = process.env.PORT || 3000; + +http.createServer((req, res) => { + if (req.url === '/health') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'healthy' })); + return; + } + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('cor-eval-deployable is serving traffic'); +}).listen(port); diff --git a/evals/grader-certification/reference-node-fullstack/.azure/.preview-temp/manifest.json b/evals/grader-certification/reference-node-fullstack/.azure/.preview-temp/manifest.json new file mode 100644 index 000000000..f98c8bc51 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.azure/.preview-temp/manifest.json @@ -0,0 +1,9 @@ +{ + "previewStatus": "ready", + "pages": [ + { + "slug": "project-tracker", + "title": "Golden Project Tracker" + } + ] +} diff --git a/evals/grader-certification/reference-node-fullstack/.azure/.preview-temp/project-tracker.html b/evals/grader-certification/reference-node-fullstack/.azure/.preview-temp/project-tracker.html new file mode 100644 index 000000000..57246dd37 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.azure/.preview-temp/project-tracker.html @@ -0,0 +1,5 @@ + + +Golden Project Tracker Preview +

Golden Project Tracker

Create and track projects.

+ diff --git a/evals/grader-certification/reference-node-fullstack/.azure/.preview-temp/theme.css b/evals/grader-certification/reference-node-fullstack/.azure/.preview-temp/theme.css new file mode 100644 index 000000000..d97d562f1 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.azure/.preview-temp/theme.css @@ -0,0 +1,4 @@ +:root { + color: #1b1b1b; + background: #ffffff; +} diff --git a/evals/grader-certification/reference-node-fullstack/.azure/deployment-plan.md b/evals/grader-certification/reference-node-fullstack/.azure/deployment-plan.md new file mode 100644 index 000000000..2b355d94e --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.azure/deployment-plan.md @@ -0,0 +1,72 @@ +# Azure Deployment Plan + +> **Status:** Ready for Validation + +Generated: 2026-08-07 + +## 1. Project Overview + +Deploy the Golden Project Tracker as a Node.js Azure Container App using Azure Developer CLI and Bicep. + +## 2. Requirements + +| Attribute | Value | +|---|---| +| Classification | Development | +| Scale | Small | +| Budget | Cost-Optimized | +| Subscription | Dedicated evaluation subscription | +| Location | East US 2 | + +## 3. Components Detected + +| Component | Type | Technology | Path | +|---|---|---|---| +| app | Web and API | Node.js 22 | `.` | + +## 4. Recipe Selection + +**Selected:** AZD with Bicep. + +## 5. Architecture + +The application runs in Azure Container Apps with managed logging and no embedded secrets. + +### Azure Resources + +| Component | Azure Service | SKU / Tier | +|---|---|---| +| app | Azure Container Apps | Consumption | +| logs | Log Analytics | Pay-as-you-go | + +```text +Internet -> Azure Container Apps -> Node.js application +``` + +## 6. Execution Checklist + +- [x] Analyze workspace +- [x] Select recipe +- [x] Generate `azure.yaml` +- [x] Generate Bicep infrastructure +- [ ] Run validation +- [ ] Deploy after explicit authorization + +## 7. Validation Proof + +| Check | Command Run | Result | Timestamp | +|---|---|---|---| +| Static artifacts | grader certification | Passed | 2026-08-07 | + +## 8. Files to Generate + +| File | Purpose | Status | +|---|---|---| +| `.azure/deployment-plan.md` | Deployment plan | Complete | +| `azure.yaml` | Azure Developer CLI manifest | Complete | +| `infra/main.bicep` | Infrastructure | Complete | + +## 9. Next Steps + +1. Run `azd package`. +2. Deploy only in the dedicated evaluation subscription after explicit authorization. diff --git a/evals/grader-certification/reference-node-fullstack/.azure/integration-plan.md b/evals/grader-certification/reference-node-fullstack/.azure/integration-plan.md new file mode 100644 index 000000000..a1f01ab4a --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.azure/integration-plan.md @@ -0,0 +1,33 @@ +# Integration Plan + +## 1. Overview + +Integrate the browser client and Node.js API through same-origin HTTP routes. The browser calls the live backend and never imports preview or mock data modules. + +## 2. Frontend + +The frontend in `public/` uses `GET /api/items` to render persisted projects and `POST /api/items` to create a project. It uses semantic labels, a form, a button, an `aria-live` project list, and high-contrast styling. + +## 3. Backend + +The Node.js service in `src/server.js` exposes health, list, and create routes. It validates project names and returns explicit HTTP statuses. + +## 4. Database + +The file-backed repository persists records in `data/items.json`. Production migration would replace the file store with managed storage. NO seed data. + +## 5. Services + +The service also serves static HTML, JavaScript, and CSS from `public/`. Browser and API traffic use the same origin and port. + +## 6. API Routes + +| Method | Route | Purpose | +|---|---|---| +| GET | `/api/health` | Report readiness | +| GET | `/api/items` | List projects | +| POST | `/api/items` | Create a project | + +## 7. Validation + +Run build, generated tests, lint, browser actions, accessibility checks, persistence restart, and debugger readiness. diff --git a/evals/grader-certification/reference-node-fullstack/.azure/project-plan.md b/evals/grader-certification/reference-node-fullstack/.azure/project-plan.md new file mode 100644 index 000000000..10bf9d32b --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.azure/project-plan.md @@ -0,0 +1,68 @@ +# Project Plan + +**Status**: Integrated +**Created**: 2026-08-07 +**Mode**: New Project + +## 1. Project Overview + +**App Type**: Web Application + +Build a dependency-free full-stack Node.js project tracker with an accessible browser interface and durable file-backed storage. + +### User Flow + +A user opens the tracker, enters a project name, submits it, and sees the project after the application restarts. + +### Architecture + +- A Node.js HTTP service serves the API and static browser assets. +- A JSON file stores projects across service restarts. +- Browser code calls the API and updates an accessible list. + +## 2. Services Required + +| Name | Responsibility | +|---|---| +| Node server | Serve health, item, and static asset routes | +| Browser client | Create and render projects | +| File store | Persist project records | + +## 3. Prerequisites + +- Node.js 22 +- npm +- A browser for acceptance testing + +## 4. Project Structure + +```text +src/server.js +public/index.html +public/app.js +public/styles.css +test/server.test.js +``` + +## 5. Route Definitions + +| Method | Route | Purpose | +|---|---|---| +| GET | `/api/health` | Report service health | +| GET | `/api/items` | List projects | +| POST | `/api/items` | Create a project | + +## 6. Design System + +**Component Library**: Native semantic HTML + +The interface uses a high-contrast blue action, white card, dark text, and a restrained neutral background. + +Native semantic HTML controls are used so the fixture has no runtime dependencies. + +## 7. Next Steps + +1. Scaffold the server, browser assets, and tests. +2. Integrate browser API calls. +3. Generate debug artifacts. +4. Run build, test, lint, browser, accessibility, persistence, and debugger checks. diff --git a/evals/grader-certification/reference-node-fullstack/.azure/requirements.json b/evals/grader-certification/reference-node-fullstack/.azure/requirements.json new file mode 100644 index 000000000..263d4a01d --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.azure/requirements.json @@ -0,0 +1,129 @@ +{ + "schemaVersion": "2", + "generatedAt": "2026-08-07T00:00:00.000Z", + "mode": "NEW", + "summary": "A small full-stack project tracker with a browser UI, Node.js API, and durable file-backed storage.", + "workspaceSignals": { + "decision": "NEW", + "decisionReason": "Certification reference project.", + "detectedFiles": [] + }, + "services": [ + { + "id": "golden-app", + "label": "Golden Project Tracker API", + "role": "backend", + "root": "." + }, + { + "id": "golden-web", + "label": "Golden Project Tracker UI", + "role": "frontend", + "root": "." + } + ], + "questions": [ + { + "id": "golden-app:language", + "category": "service", + "question": "Which language should the project use?", + "header": "Language", + "answer": "JavaScript", + "status": "confirmed", + "options": [ + { + "label": "JavaScript", + "description": "Node.js with browser JavaScript" + } + ], + "recommendedChoice": "JavaScript", + "multiSelect": false, + "allowFreeformInput": false, + "serviceId": "golden-app" + }, + { + "id": "golden-web:language", + "category": "service", + "question": "Which language should the browser UI use?", + "header": "Language", + "answer": "JavaScript", + "status": "confirmed", + "options": [ + { + "label": "JavaScript", + "description": "Browser JavaScript" + } + ], + "recommendedChoice": "JavaScript", + "multiSelect": false, + "allowFreeformInput": false, + "serviceId": "golden-web" + }, + { + "id": "golden-web:features", + "category": "service", + "question": "What should the browser UI do?", + "header": "Features", + "answer": "Create and list projects through accessible controls.", + "status": "confirmed", + "recommendedChoice": "Create and list projects through accessible controls.", + "multiSelect": false, + "serviceId": "golden-web" + }, + { + "id": "golden-app:features", + "category": "service", + "question": "What should the project tracker do?", + "header": "Features", + "answer": "Create and list projects through an accessible browser UI.", + "status": "confirmed", + "recommendedChoice": "Create and list projects through an accessible browser UI.", + "multiSelect": false, + "serviceId": "golden-app" + }, + { + "id": "dataStores", + "category": "data", + "question": "Which data stores does the app need?", + "header": "Data Stores", + "answer": [ + "File storage" + ], + "status": "confirmed", + "options": [ + { + "label": "No datastore required", + "description": "No persistent data", + "exclusive": true + }, + { + "label": "File storage", + "description": "Durable local JSON file" + } + ], + "recommendedChoice": [ + "File storage" + ], + "multiSelect": true, + "allowFreeformInput": false + }, + { + "id": "auth", + "category": "auth", + "question": "Does the app need authentication?", + "header": "Authentication", + "answer": "No auth", + "status": "confirmed", + "options": [ + { + "label": "No auth", + "description": "Public local application" + } + ], + "recommendedChoice": "No auth", + "multiSelect": false, + "allowFreeformInput": false + } + ], + "executionMode": "guided" +} diff --git a/evals/grader-certification/reference-node-fullstack/.azure/vscode-debug-plan.md b/evals/grader-certification/reference-node-fullstack/.azure/vscode-debug-plan.md new file mode 100644 index 000000000..1695f8d16 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.azure/vscode-debug-plan.md @@ -0,0 +1,48 @@ +# Azure Debug Plan + +> **Status:** Implemented +> **Execution Mode:** Auto +> **Last Updated:** 2026-08-07T00:00:00.000Z + +## Prerequisites + +| Tool | Installed | +|---|---| +| Node.js 22 | Yes | +| npm | Yes | + +## Debug Configurations + +| Generate | Debug Config Name | Service Root | Project Type | Runtime | Notes | +|---|---|---|---|---|---| +| [x] | Golden App (debug) | `.` | Backend | Node.js | Launch with CDP inspector | + +## Orchestrator + +| Orchestrator | Selected | Notes | +|---|---|---| +| VS Code task and launch configuration | Yes | Build before launch | + +## Architecture + +```text +Browser -> Node.js API and static server -> JSON file + | + +-> CDP inspector on port 9229 +``` + +## Emulators + +No emulators are required. + +## API Test Collections + +No generated API test collection is required. + +## Debug Configuration Checklist + +Debug Configuration Checklist: + +✅ Golden App (debug) — HTTP health returned 200 and CDP inspector metadata was available. +✅ Golden Project Tracker — browser interaction and accessibility checks passed. +✅ Persistence — created project remained visible after service restart. diff --git a/evals/grader-certification/reference-node-fullstack/.vscode/extensions.json b/evals/grader-certification/reference-node-fullstack/.vscode/extensions.json new file mode 100644 index 000000000..ea3b1d810 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "ms-vscode.js-debug" + ] +} diff --git a/evals/grader-certification/reference-node-fullstack/.vscode/launch.json b/evals/grader-certification/reference-node-fullstack/.vscode/launch.json new file mode 100644 index 000000000..1c959118d --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Golden App (debug)", + "type": "pwa-node", + "request": "launch", + "program": "${workspaceFolder}/src/server.js", + "cwd": "${workspaceFolder}", + "runtimeArgs": [ + "--inspect=9229" + ], + "env": { + "PORT": "7071" + }, + "preLaunchTask": "prepare", + "skipFiles": [ + "/**" + ] + } + ] +} diff --git a/evals/grader-certification/reference-node-fullstack/.vscode/settings.json b/evals/grader-certification/reference-node-fullstack/.vscode/settings.json new file mode 100644 index 000000000..3d082906a --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "debug.javascript.autoAttachFilter": "disabled" +} diff --git a/evals/grader-certification/reference-node-fullstack/.vscode/tasks.json b/evals/grader-certification/reference-node-fullstack/.vscode/tasks.json new file mode 100644 index 000000000..c5ec241eb --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/.vscode/tasks.json @@ -0,0 +1,46 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "install", + "type": "shell", + "command": "npm ci --ignore-scripts", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [], + "runOptions": { + "instanceLimit": 1, + "instancePolicy": "silent" + } + }, + { + "label": "build", + "type": "shell", + "command": "npm run build", + "dependsOn": "install", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [], + "runOptions": { + "instanceLimit": 1, + "instancePolicy": "silent" + } + }, + { + "label": "prepare", + "type": "shell", + "command": "node -e \"process.exit(0)\"", + "dependsOn": "build", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [], + "runOptions": { + "instanceLimit": 1, + "instancePolicy": "silent" + } + } + ] +} diff --git a/evals/grader-certification/reference-node-fullstack/azure.yaml b/evals/grader-certification/reference-node-fullstack/azure.yaml new file mode 100644 index 000000000..b8eadce13 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/azure.yaml @@ -0,0 +1,12 @@ +name: cor-grader-reference +metadata: + template: cor-grader-reference@1.0.0 +services: + app: + project: . + host: containerapp + language: js +hooks: + prepackage: + shell: sh + run: npm run build diff --git a/evals/grader-certification/reference-node-fullstack/infra/main.bicep b/evals/grader-certification/reference-node-fullstack/infra/main.bicep new file mode 100644 index 000000000..ffa127b7d --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/infra/main.bicep @@ -0,0 +1,12 @@ +targetScope = 'resourceGroup' + +param environmentName string +param location string = resourceGroup().location + +resource environment 'Microsoft.App/managedEnvironments@2024-03-01' = { + name: '${environmentName}-environment' + location: location + properties: {} +} + +output AZURE_CONTAINER_APPS_ENVIRONMENT_ID string = environment.id diff --git a/evals/grader-certification/reference-node-fullstack/package-lock.json b/evals/grader-certification/reference-node-fullstack/package-lock.json new file mode 100644 index 000000000..15c9f909d --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/package-lock.json @@ -0,0 +1,15 @@ +{ + "name": "cor-grader-reference-node-fullstack", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cor-grader-reference-node-fullstack", + "version": "1.0.0", + "engines": { + "node": ">=22" + } + } + } +} diff --git a/evals/grader-certification/reference-node-fullstack/package.json b/evals/grader-certification/reference-node-fullstack/package.json new file mode 100644 index 000000000..cf6fc6bea --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/package.json @@ -0,0 +1,14 @@ +{ + "name": "cor-grader-reference-node-fullstack", + "version": "1.0.0", + "private": true, + "scripts": { + "build": "node --check src/server.js && node --check public/app.js", + "test": "node --test", + "lint": "node scripts/lint.js", + "start": "node src/server.js" + }, + "engines": { + "node": ">=22" + } +} diff --git a/evals/grader-certification/reference-node-fullstack/public/app.js b/evals/grader-certification/reference-node-fullstack/public/app.js new file mode 100644 index 000000000..39b2fc069 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/public/app.js @@ -0,0 +1,36 @@ +'use strict'; + +const form = document.querySelector('#project-form'); +const input = document.querySelector('#project-name'); +const list = document.querySelector('#projects'); +const emptyState = document.querySelector('#empty-state'); + +function render(items) { + list.replaceChildren(...items.map(item => { + const element = document.createElement('li'); + element.textContent = item.name; + return element; + })); + emptyState.hidden = items.length > 0; +} + +async function load() { + const response = await fetch('/api/items'); + render(await response.json()); +} + +form.addEventListener('submit', async event => { + event.preventDefault(); + const response = await fetch('/api/items', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: input.value }), + }); + if (!response.ok) { + throw new Error(`Create failed with ${response.status}`); + } + input.value = ''; + await load(); +}); + +void load(); diff --git a/evals/grader-certification/reference-node-fullstack/public/index.html b/evals/grader-certification/reference-node-fullstack/public/index.html new file mode 100644 index 000000000..4d80df77d --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/public/index.html @@ -0,0 +1,22 @@ + + + + + + Golden Project Tracker + + + +
+

Golden Project Tracker

+
+ + + +
+

No projects yet

+
    +
    + + + diff --git a/evals/grader-certification/reference-node-fullstack/public/styles.css b/evals/grader-certification/reference-node-fullstack/public/styles.css new file mode 100644 index 000000000..9b056d77d --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/public/styles.css @@ -0,0 +1,41 @@ +:root { + color: #1b1b1b; + background: #f7f9fc; + font-family: Arial, sans-serif; +} + +body { + margin: 0; +} + +main { + max-width: 42rem; + margin: 4rem auto; + padding: 2rem; + background: #ffffff; + border: 1px solid #c5ced8; + border-radius: 0.75rem; +} + +form { + display: grid; + gap: 0.75rem; +} + +input, +button { + min-height: 2.75rem; + font: inherit; +} + +button { + color: #ffffff; + background: #1479c9; + border: 0; + border-radius: 0.25rem; + cursor: pointer; +} + +li { + margin-block: 0.75rem; +} diff --git a/evals/grader-certification/reference-node-fullstack/scenario.json b/evals/grader-certification/reference-node-fullstack/scenario.json new file mode 100644 index 000000000..84b209760 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/scenario.json @@ -0,0 +1,107 @@ +{ + "schemaVersion": "1", + "id": "grader-reference-node-fullstack", + "prompt": "Build a small project tracker with a browser UI and a persistent Node.js API.", + "baselinePrompt": "Create a runnable project tracker with a browser UI and persistent Node.js API. Include build, test, lint, launch, and deployment files.", + "tags": { + "archetype": "crud", + "frontend": "html", + "backend": "node", + "database": "file", + "auth": "bearer-token", + "complexity": "small" + }, + "requirementsAnswers": { + "dataStores": [ + "File storage" + ], + "auth": "No auth" + }, + "validation": { + "profile": "advanced", + "build": true, + "test": true, + "lint": "required", + "timeoutMinutes": 5, + "maxAgentRetries": 0 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 60, + "probes": [ + { + "name": "Golden app health", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200, + "bodyIncludes": "\"status\":\"ok\"", + "debugPort": 9229, + "debugProtocol": "cdp" + }, + { + "name": "Golden project UI", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/", + "expectedStatus": 200, + "browser": { + "expectedText": "Golden Project Tracker", + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "actions": [ + { + "kind": "fill", + "selector": "Project name", + "selectorType": "label", + "value": "Certified project" + }, + { + "kind": "click", + "selector": "Add project", + "selectorType": "role", + "role": "button" + } + ], + "assertions": [ + { + "kind": "text", + "selector": "li", + "value": "Certified project" + } + ], + "persistence": { + "restartTargets": [ + "backend" + ], + "reload": "current-url", + "assertions": [ + { + "kind": "text", + "selector": "li", + "value": "Certified project" + } + ] + } + } + } + ], + "debugParity": { + "target": "backend", + "sourceGlob": "src/server.js", + "lineIncludes": "requestUrl.pathname === '/api/health'", + "triggerUrl": "http://127.0.0.1:7071/api/health", + "timeoutSeconds": 30 + }, + "security": { + "publicPaths": [ + "/api/health", + "/" + ], + "protectedPaths": [ + "/api/admin/stats" + ] + } + } + } +} diff --git a/evals/grader-certification/reference-node-fullstack/scripts/lint.js b/evals/grader-certification/reference-node-fullstack/scripts/lint.js new file mode 100644 index 000000000..e5ec6b0f3 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/scripts/lint.js @@ -0,0 +1,11 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); + +for (const relativePath of ['src/server.js', 'public/app.js', 'test/server.test.js']) { + const content = fs.readFileSync(path.join(__dirname, '..', relativePath), 'utf8'); + if (content.includes('\t') || / +$/m.test(content)) { + throw new Error(`${relativePath} contains tabs or trailing whitespace`); + } +} diff --git a/evals/grader-certification/reference-node-fullstack/src/server.js b/evals/grader-certification/reference-node-fullstack/src/server.js new file mode 100644 index 000000000..1ec37dcf2 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/src/server.js @@ -0,0 +1,118 @@ +'use strict'; + +const fs = require('node:fs/promises'); +const http = require('node:http'); +const path = require('node:path'); +const { randomBytes } = require('node:crypto'); + +const publicRoot = path.join(__dirname, '..', 'public'); + +async function readItems(dataFile) { + try { + return JSON.parse(await fs.readFile(dataFile, 'utf8')); + } catch (error) { + if (error.code === 'ENOENT') { + return []; + } + throw error; + } +} + +async function writeItems(dataFile, items) { + await fs.mkdir(path.dirname(dataFile), { recursive: true }); + await fs.writeFile(dataFile, `${JSON.stringify(items, null, 2)}\n`, 'utf8'); +} + +function send(response, status, contentType, body) { + response.writeHead(status, { + 'content-type': contentType, + 'cache-control': 'no-store', + }); + response.end(body); +} + +/** + * Compares the presented bearer token against the configured one. This is a full value + * comparison rather than a presence check on purpose: the security gate probes protected paths + * with a syntactically well-formed token that must not validate, so an app that merely looks for + * an Authorization header would serve protected data and has to be treated as unauthenticated. + */ +function isAuthorizedAdmin(header, expected) { + if (!expected) { + return false; + } + const match = /^Bearer (.+)$/u.exec(header ?? ''); + return match !== null && match[1] === expected; +} + +function readBody(request) { + return new Promise((resolve, reject) => { + let body = ''; + request.setEncoding('utf8'); + request.on('data', chunk => { + body += chunk; + }); + request.on('end', () => resolve(body)); + request.on('error', reject); + }); +} + +function createServer(options = {}) { + const dataFile = options.dataFile ?? path.join(__dirname, '..', 'data', 'items.json'); + // Falling back to an unguessable random secret rather than an empty string keeps the token + // comparison on the live path: with an empty expected value every request would short-circuit + // before the comparison ran, so the security gate would exercise less of this code than it + // appears to. No credential is committed anywhere as a result. + const adminToken = options.adminToken ?? process.env.ADMIN_TOKEN ?? randomBytes(32).toString('hex'); + return http.createServer(async (request, response) => { + const requestUrl = new URL(request.url, 'http://localhost'); + if (request.method === 'GET' && requestUrl.pathname === '/api/health') { + send(response, 200, 'application/json', JSON.stringify({ status: 'ok' })); + return; + } + if (request.method === 'GET' && requestUrl.pathname === '/api/items') { + send(response, 200, 'application/json', JSON.stringify(await readItems(dataFile))); + return; + } + if (request.method === 'POST' && requestUrl.pathname === '/api/items') { + const payload = JSON.parse(await readBody(request)); + if (typeof payload.name !== 'string' || !payload.name.trim()) { + send(response, 400, 'application/json', JSON.stringify({ error: 'name is required' })); + return; + } + const items = await readItems(dataFile); + const item = { id: items.length + 1, name: payload.name.trim() }; + await writeItems(dataFile, [...items, item]); + send(response, 201, 'application/json', JSON.stringify(item)); + return; + } + if (request.method === 'GET' && requestUrl.pathname === '/api/admin/stats') { + if (!isAuthorizedAdmin(request.headers.authorization, adminToken)) { + send(response, 401, 'application/json', JSON.stringify({ error: 'unauthorized' })); + return; + } + const items = await readItems(dataFile); + send(response, 200, 'application/json', JSON.stringify({ projectCount: items.length })); + return; + } + const asset = requestUrl.pathname === '/' ? 'index.html' : requestUrl.pathname.slice(1); + if (!['index.html', 'app.js', 'styles.css'].includes(asset)) { + send(response, 404, 'text/plain; charset=utf-8', 'Not found'); + return; + } + const contentTypes = { + 'index.html': 'text/html; charset=utf-8', + 'app.js': 'text/javascript; charset=utf-8', + 'styles.css': 'text/css; charset=utf-8', + }; + send(response, 200, contentTypes[asset], await fs.readFile(path.join(publicRoot, asset))); + }); +} + +if (require.main === module) { + const server = createServer(); + server.listen(Number(process.env.PORT ?? 7071), '0.0.0.0'); + process.on('SIGTERM', () => server.close()); +} + +module.exports = { createServer }; diff --git a/evals/grader-certification/reference-node-fullstack/test/server.test.js b/evals/grader-certification/reference-node-fullstack/test/server.test.js new file mode 100644 index 000000000..d103e0b22 --- /dev/null +++ b/evals/grader-certification/reference-node-fullstack/test/server.test.js @@ -0,0 +1,66 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { createServer } = require('../src/server'); + +test('health and persisted item workflow succeeds', async t => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'cor-grader-reference-')); + const server = createServer({ dataFile: path.join(directory, 'items.json') }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(async () => { + await new Promise(resolve => server.close(resolve)); + await fs.rm(directory, { recursive: true, force: true }); + }); + const address = server.address(); + const baseUrl = `http://127.0.0.1:${address.port}`; + const response = await fetch(`${baseUrl}/api/health`); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { status: 'ok' }); + const created = await fetch(`${baseUrl}/api/items`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: 'Certified project' }), + }); + assert.equal(created.status, 201); + const items = await fetch(`${baseUrl}/api/items`); + assert.deepEqual(await items.json(), [{ id: 1, name: 'Certified project' }]); +}); + +test('admin stats endpoint enforces bearer token authorization', async t => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'cor-grader-reference-auth-')); + const server = createServer({ + dataFile: path.join(directory, 'items.json'), + adminToken: 'certified-admin-token', + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(async () => { + await new Promise(resolve => server.close(resolve)); + await fs.rm(directory, { recursive: true, force: true }); + }); + const baseUrl = `http://127.0.0.1:${server.address().port}`; + + const anonymous = await fetch(`${baseUrl}/api/admin/stats`); + assert.equal(anonymous.status, 401); + + // A syntactically valid bearer token that cannot validate. This is the case that separates + // real verification from an app that only checks whether an Authorization header is present. + const malformed = await fetch(`${baseUrl}/api/admin/stats`, { + headers: { Authorization: 'Bearer not.a.valid.token' }, + }); + assert.equal(malformed.status, 401); + + const authorized = await fetch(`${baseUrl}/api/admin/stats`, { + headers: { Authorization: 'Bearer certified-admin-token' }, + }); + assert.equal(authorized.status, 200); + assert.deepEqual(await authorized.json(), { projectCount: 0 }); + + // Public paths must stay reachable, which is what proves the refusal above is selective + // rather than an application that is simply unavailable. + const health = await fetch(`${baseUrl}/api/health`); + assert.equal(health.status, 200); +}); diff --git a/evals/release-thresholds.v1.json b/evals/release-thresholds.v1.json new file mode 100644 index 000000000..d5d37f1eb --- /dev/null +++ b/evals/release-thresholds.v1.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": "1", + "thresholdSet": "copilot-on-rails-release-v1", + "evidence": { + "minimumScenarioCoverage": 1, + "minimumAttemptsPerModelScenario": 3 + }, + "criticalFailures": { + "maximumCount": 0, + "failureCodes": [ + "criticalSecurityViolation", + "destructiveOperationDetected", + "agentCleanupFailed", + "resultFinalizationFailed", + "sandboxCleanupFailed", + "localSandboxCleanupFailed", + "parityCleanupFailed" + ] + }, + "outcomes": { + "minimumFinalSuccessRate": 0.85, + "minimumFirstPassSuccessRate": 0.7 + }, + "ui": { + "minimumBrowserSuccessRate": 1, + "minimumAccessibilityScanCoverage": 1, + "maximumSeriousOrCriticalViolations": 0 + }, + "sideEffects": { + "minimumPersistenceSuccessRate": 1, + "minimumWorkerSuccessRate": 1 + }, + "debugger": { + "minimumRuns": 1, + "minimumSuccessRate": 1 + }, + "deployment": { + "minimumRuns": 1, + "minimumSuccessRate": 1, + "minimumCleanupCoverage": 1 + }, + "baseline": { + "required": true, + "minimumMatchedPairCoverage": 1, + "minimumPairsPerModelScenario": 3, + "minimumFinalSuccessRateDelta": -0.05, + "minimumFirstPassSuccessRateDelta": -0.1 + }, + "efficiency": { + "maximumLatencyMultiplier": 1.5, + "maximumCostMultiplier": 1.5, + "costMetric": "totalNanoAiu" + }, + "provenance": { + "requireDeclaredArms": true, + "requireAttemptCommitAndAssetHash": true, + "requireRequestedAndObservedModel": true, + "requireEvaluationDefinitionHash": true, + "requireCleanupEvidence": true + } +} diff --git a/evals/sandbox-dotnet.yaml b/evals/sandbox-dotnet.yaml new file mode 100644 index 000000000..d87e6b4b2 --- /dev/null +++ b/evals/sandbox-dotnet.yaml @@ -0,0 +1,45 @@ +# ACA Sandbox manifest for .NET generated-project validation. +diskId: 344c39f7-88b1-4c71-8d84-2cb26930d1cc +resources: + cpu: 4000m + memory: 8192Mi + disk: 81920Mi +labels: + workload: copilot-on-rails-eval + role: validator +lifecycle: + autoSuspendPolicy: + enabled: true + interval: 600 + mode: Memory + autoDeletePolicy: + enabled: true + deleteIntervalInSeconds: 3600 +egressPolicy: + defaultAction: Deny + trafficInspection: Full + hostRules: + - pattern: api.nuget.org + action: Allow + - pattern: cdn.functions.azure.com + action: Allow + - pattern: mcr.microsoft.com + action: Allow + - pattern: "*.data.mcr.microsoft.com" + action: Allow + - pattern: "*.docker.io" + action: Allow + - pattern: production.cloudflare.docker.com + action: Allow + - pattern: "*.r2.cloudflarestorage.com" + action: Allow + - pattern: update.code.visualstudio.com + action: Allow + - pattern: vscode.download.prss.microsoft.com + action: Allow + - pattern: ms-azuretools.gallerycdn.vsassets.io + action: Allow + - pattern: archive.ubuntu.com + action: Allow + - pattern: security.ubuntu.com + action: Allow diff --git a/evals/sandbox-python.yaml b/evals/sandbox-python.yaml new file mode 100644 index 000000000..7a043cb41 --- /dev/null +++ b/evals/sandbox-python.yaml @@ -0,0 +1,47 @@ +# ACA Sandbox manifest for Python generated-project validation. +diskId: 344c39f7-88b1-4c71-8d84-2cb26930d1cc +resources: + cpu: 4000m + memory: 8192Mi + disk: 81920Mi +labels: + workload: copilot-on-rails-eval + role: validator +lifecycle: + autoSuspendPolicy: + enabled: true + interval: 600 + mode: Memory + autoDeletePolicy: + enabled: true + deleteIntervalInSeconds: 3600 +egressPolicy: + defaultAction: Deny + trafficInspection: Full + hostRules: + - pattern: pypi.org + action: Allow + - pattern: files.pythonhosted.org + action: Allow + - pattern: cdn.functions.azure.com + action: Allow + - pattern: mcr.microsoft.com + action: Allow + - pattern: "*.data.mcr.microsoft.com" + action: Allow + - pattern: "*.docker.io" + action: Allow + - pattern: production.cloudflare.docker.com + action: Allow + - pattern: "*.r2.cloudflarestorage.com" + action: Allow + - pattern: update.code.visualstudio.com + action: Allow + - pattern: vscode.download.prss.microsoft.com + action: Allow + - pattern: ms-azuretools.gallerycdn.vsassets.io + action: Allow + - pattern: archive.ubuntu.com + action: Allow + - pattern: security.ubuntu.com + action: Allow diff --git a/evals/sandbox.yaml b/evals/sandbox.yaml new file mode 100644 index 000000000..67693a1a8 --- /dev/null +++ b/evals/sandbox.yaml @@ -0,0 +1,45 @@ +# ACA Sandbox manifest for generated-project validation. +diskId: 344c39f7-88b1-4c71-8d84-2cb26930d1cc +resources: + cpu: 4000m + memory: 8192Mi + disk: 81920Mi +labels: + workload: copilot-on-rails-eval + role: validator +lifecycle: + autoSuspendPolicy: + enabled: true + interval: 600 + mode: Memory + autoDeletePolicy: + enabled: true + deleteIntervalInSeconds: 3600 +egressPolicy: + defaultAction: Deny + trafficInspection: Full + hostRules: + - pattern: registry.npmjs.org + action: Allow + - pattern: cdn.functions.azure.com + action: Allow + - pattern: mcr.microsoft.com + action: Allow + - pattern: "*.data.mcr.microsoft.com" + action: Allow + - pattern: "*.docker.io" + action: Allow + - pattern: production.cloudflare.docker.com + action: Allow + - pattern: "*.r2.cloudflarestorage.com" + action: Allow + - pattern: update.code.visualstudio.com + action: Allow + - pattern: vscode.download.prss.microsoft.com + action: Allow + - pattern: ms-azuretools.gallerycdn.vsassets.io + action: Allow + - pattern: archive.ubuntu.com + action: Allow + - pattern: security.ubuntu.com + action: Allow diff --git a/evals/scenarios/api-csharp-functions-sql.json b/evals/scenarios/api-csharp-functions-sql.json new file mode 100644 index 000000000..cbc484e50 --- /dev/null +++ b/evals/scenarios/api-csharp-functions-sql.json @@ -0,0 +1,40 @@ +{ + "schemaVersion": "1", + "id": "api-csharp-functions-sql", + "prompt": "Build a backend-only C# Azure Functions API for managing warehouse inventory with CRUD endpoints, Azure SQL persistence, no authentication, and xUnit tests.", + "baselinePrompt": "Build a complete backend-only warehouse inventory project using a C# Azure Functions API with CRUD endpoints, Azure SQL persistence, no authentication, and xUnit tests. Implement all application code and configuration, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging the API, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "api", + "frontend": "none", + "backend": "csharp-functions", + "database": "azure-sql", + "auth": "none", + "complexity": "medium" + }, + "requirementsAnswers": { + "dataStores": ["Azure SQL"], + "auth": "No auth" + }, + "validation": { + "profile": "standard", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 10 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "dependency health endpoint", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200, + "bodyIncludes": "healthy" + } + ] + } + } +} diff --git a/evals/scenarios/api-node-express-redis.json b/evals/scenarios/api-node-express-redis.json new file mode 100644 index 000000000..db5ca616d --- /dev/null +++ b/evals/scenarios/api-node-express-redis.json @@ -0,0 +1,39 @@ +{ + "schemaVersion": "1", + "id": "api-node-express-redis", + "prompt": "Build a backend-only TypeScript Express API for a rate-limited URL shortener using Redis, with no authentication and automated API tests.", + "baselinePrompt": "Build a complete backend-only rate-limited URL shortener project using a TypeScript Express API and Redis, with no authentication and automated API tests. Implement all application code and configuration, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging the API, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "api", + "frontend": "none", + "backend": "typescript-express", + "database": "redis", + "auth": "none", + "complexity": "low" + }, + "requirementsAnswers": { + "dataStores": [ + "Redis" + ], + "auth": "No auth" + }, + "validation": { + "profile": "minimal", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 5 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "express api process", + "target": "backend", + "processPattern": "node .*(server|index|app|main)|npm run (dev|start)|nodemon|ts-node|tsx " + } + ] + } + } +} diff --git a/evals/scenarios/api-python-cosmos.json b/evals/scenarios/api-python-cosmos.json new file mode 100644 index 000000000..101ce386a --- /dev/null +++ b/evals/scenarios/api-python-cosmos.json @@ -0,0 +1,39 @@ +{ + "schemaVersion": "1", + "id": "api-python-cosmos", + "prompt": "Build a backend-only Python API for tracking equipment inspections. It should expose CRUD endpoints, store inspection records in Cosmos DB, require no authentication, and include automated API tests.", + "baselinePrompt": "Build a complete backend-only equipment inspection tracking project using a Python API with CRUD endpoints, Cosmos DB persistence for inspection records, no authentication, and automated API tests. Implement all application code and configuration, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging the API, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "api", + "frontend": "none", + "backend": "python", + "database": "cosmos", + "auth": "none", + "complexity": "low" + }, + "requirementsAnswers": { + "dataStores": [ + "CosmosDB" + ], + "auth": "No auth" + }, + "validation": { + "profile": "minimal", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 5 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "python api process", + "target": "backend", + "processPattern": "uvicorn|gunicorn|flask run|python .*(app|main|server)|func( host)? start|Microsoft.Azure.WebJobs.Script.WebHost" + } + ] + } + } +} diff --git a/evals/scenarios/api-python-flask-blob.json b/evals/scenarios/api-python-flask-blob.json new file mode 100644 index 000000000..abfb9dd30 --- /dev/null +++ b/evals/scenarios/api-python-flask-blob.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": "1", + "id": "api-python-flask-blob", + "prompt": "Build a backend-only Python Flask API for uploading and downloading compliance documents from Blob Storage, protected by mock authentication and covered by API tests.", + "baselinePrompt": "Build a complete backend-only compliance document project using a Python Flask API that uploads documents to and downloads documents from Azure Blob Storage, protected by mock authentication and covered by API tests. Implement all application code and configuration, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging the API, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "api", + "frontend": "none", + "backend": "python-flask", + "database": "blob", + "auth": "mock", + "complexity": "low" + }, + "requirementsAnswers": { + "dataStores": ["Blob Storage"], + "auth": "Mock auth middleware" + }, + "validation": { + "profile": "minimal", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 5 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 120, + "probes": [ + { + "name": "health endpoint", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:5000/api/health", + "expectedStatus": 200, + "bodyIncludes": "healthy", + "debugPort": 5678, + "debugProtocol": "tcp" + } + ] + } + } +} diff --git a/evals/scenarios/api-python-functions-queue.json b/evals/scenarios/api-python-functions-queue.json new file mode 100644 index 000000000..9740d18f0 --- /dev/null +++ b/evals/scenarios/api-python-functions-queue.json @@ -0,0 +1,39 @@ +{ + "schemaVersion": "1", + "id": "api-python-functions-queue", + "prompt": "Build a backend-only Python Azure Functions API that accepts image-processing jobs, writes them to Queue Storage, exposes job status endpoints, requires no authentication, and includes pytest coverage.", + "baselinePrompt": "Build a complete backend-only image-processing job project using a Python Azure Functions API that accepts jobs, writes them to Azure Queue Storage, exposes job status endpoints, requires no authentication, and includes pytest coverage. Implement all application code and configuration, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging the API, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "api", + "frontend": "none", + "backend": "python-functions", + "database": "queue", + "auth": "none", + "complexity": "low" + }, + "requirementsAnswers": { + "dataStores": [ + "Queue Storage" + ], + "auth": "No auth" + }, + "validation": { + "profile": "minimal", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 5 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "functions host process", + "target": "backend", + "processPattern": "func( host)? start|Microsoft.Azure.WebJobs.Script.WebHost" + } + ] + } + } +} diff --git a/evals/scenarios/api-ts-functions-minimal.json b/evals/scenarios/api-ts-functions-minimal.json new file mode 100644 index 000000000..35dcef13c --- /dev/null +++ b/evals/scenarios/api-ts-functions-minimal.json @@ -0,0 +1,41 @@ +{ + "schemaVersion": "1", + "id": "api-ts-functions-minimal", + "prompt": "Build a minimal backend-only TypeScript Azure Functions API with health, list, and create endpoints for ephemeral notes, no database, no authentication, and unit tests.", + "baselinePrompt": "Build a complete minimal backend-only ephemeral notes project using a TypeScript Azure Functions API with health, list, and create endpoints, no database, no authentication, and unit tests. Implement all application code and configuration, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging the API, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "api", + "frontend": "none", + "backend": "typescript-functions", + "database": "none", + "auth": "none", + "complexity": "low" + }, + "requirementsAnswers": { + "dataStores": ["No datastore required"], + "auth": "No auth" + }, + "validation": { + "profile": "minimal", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 5 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 90, + "probes": [ + { + "name": "health endpoint", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200, + "debugPort": 9229, + "debugProtocol": "cdp" + } + ] + } + } +} diff --git a/evals/scenarios/crud-angular-csharp-functions-sql-entra.json b/evals/scenarios/crud-angular-csharp-functions-sql-entra.json new file mode 100644 index 000000000..81b719281 --- /dev/null +++ b/evals/scenarios/crud-angular-csharp-functions-sql-entra.json @@ -0,0 +1,69 @@ +{ + "schemaVersion": "1", + "id": "crud-angular-csharp-functions-sql-entra", + "prompt": "Build an employee scheduling app with an Angular frontend, C# Azure Functions API, Azure SQL persistence, Microsoft Entra ID authentication with test doubles, and component plus API tests. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/shifts collection.", + "baselinePrompt": "Build a complete employee scheduling application using an Angular frontend, a C# Azure Functions API, Azure SQL persistence, and Microsoft Entra ID authentication with test doubles, including component and API tests. Implement all frontend, backend, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/shifts collection.", + "tags": { + "archetype": "crud", + "frontend": "angular", + "backend": "csharp-functions", + "database": "azure-sql", + "auth": "entra-id", + "complexity": "high" + }, + "requirementsAnswers": { + "dataStores": [ + "Azure SQL" + ], + "auth": "Microsoft Entra ID" + }, + "validation": { + "profile": "advanced", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 15 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "api health", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200 + }, + { + "name": "shifts collection requires auth", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/shifts", + "expectedStatus": 401 + }, + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:4200/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "advisory", + "viewport": { + "width": 1440, + "height": 900 + } + } + } + ], + "security": { + "publicPaths": [ + "http://127.0.0.1:7071/api/health" + ] + } + } + } +} diff --git a/evals/scenarios/crud-angular-ts-functions-cosmos-clerk.json b/evals/scenarios/crud-angular-ts-functions-cosmos-clerk.json new file mode 100644 index 000000000..f0a89ad98 --- /dev/null +++ b/evals/scenarios/crud-angular-ts-functions-cosmos-clerk.json @@ -0,0 +1,69 @@ +{ + "schemaVersion": "1", + "id": "crud-angular-ts-functions-cosmos-clerk", + "prompt": "Build a volunteer coordination app with an Angular frontend, TypeScript Azure Functions API, Cosmos DB, Clerk authentication isolated behind a mockable adapter, and CRUD plus authorization tests. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/items collection.", + "baselinePrompt": "Build a complete volunteer coordination application using an Angular frontend, a TypeScript Azure Functions API, Cosmos DB, and Clerk authentication isolated behind a mockable adapter, including CRUD and authorization tests. Implement all frontend, backend, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/items collection.", + "tags": { + "archetype": "crud", + "frontend": "angular", + "backend": "typescript-functions", + "database": "cosmos", + "auth": "clerk", + "complexity": "high" + }, + "requirementsAnswers": { + "dataStores": [ + "CosmosDB" + ], + "auth": "Clerk" + }, + "validation": { + "profile": "advanced", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 15 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "api health", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200 + }, + { + "name": "items collection requires auth", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/items", + "expectedStatus": 401 + }, + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:4200/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "advisory", + "viewport": { + "width": 1440, + "height": 900 + } + } + } + ], + "security": { + "publicPaths": [ + "http://127.0.0.1:7071/api/health" + ] + } + } + } +} diff --git a/evals/scenarios/crud-react-fastapi-postgres-auth0.json b/evals/scenarios/crud-react-fastapi-postgres-auth0.json new file mode 100644 index 000000000..e05e109d6 --- /dev/null +++ b/evals/scenarios/crud-react-fastapi-postgres-auth0.json @@ -0,0 +1,69 @@ +{ + "schemaVersion": "1", + "id": "crud-react-fastapi-postgres-auth0", + "prompt": "Build a subscription dashboard with a React and Vite frontend, Python FastAPI backend, PostgreSQL, Auth0 behind a configuration-driven adapter that is mocked in tests, and full CRUD coverage. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/subscriptions collection.", + "baselinePrompt": "Build a complete subscription dashboard application using a React and Vite frontend, a Python FastAPI backend, PostgreSQL, and Auth0 behind a configuration-driven adapter that is mocked in tests, with full CRUD coverage. Implement all frontend, backend, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/subscriptions collection.", + "tags": { + "archetype": "crud", + "frontend": "react", + "backend": "python-fastapi", + "database": "postgres", + "auth": "auth0", + "complexity": "high" + }, + "requirementsAnswers": { + "dataStores": [ + "PostgreSQL" + ], + "auth": "Auth0" + }, + "validation": { + "profile": "advanced", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 15 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "api health", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:8000/api/health", + "expectedStatus": 200 + }, + { + "name": "subscriptions collection requires auth", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:8000/api/subscriptions", + "expectedStatus": 401 + }, + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "advisory", + "viewport": { + "width": 1440, + "height": 900 + } + } + } + ], + "security": { + "publicPaths": [ + "http://127.0.0.1:8000/api/health" + ] + } + } + } +} diff --git a/evals/scenarios/crud-react-functions-cosmos-entra.json b/evals/scenarios/crud-react-functions-cosmos-entra.json new file mode 100644 index 000000000..43b437f39 --- /dev/null +++ b/evals/scenarios/crud-react-functions-cosmos-entra.json @@ -0,0 +1,69 @@ +{ + "schemaVersion": "1", + "id": "crud-react-functions-cosmos-entra", + "prompt": "Build a document review app with a React and Vite frontend, TypeScript Azure Functions API, Cosmos DB persistence, Microsoft Entra ID authentication with mocked token validation in tests, and review workflow coverage. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/reviews collection.", + "baselinePrompt": "Build a complete document review application using a React and Vite frontend, a TypeScript Azure Functions API, Cosmos DB persistence, and Microsoft Entra ID authentication with mocked token validation in tests, including review workflow coverage. Implement all frontend, backend, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/reviews collection.", + "tags": { + "archetype": "crud", + "frontend": "react", + "backend": "typescript-functions", + "database": "cosmos", + "auth": "entra-id", + "complexity": "medium" + }, + "requirementsAnswers": { + "dataStores": [ + "CosmosDB" + ], + "auth": "Microsoft Entra ID" + }, + "validation": { + "profile": "standard", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 10 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "api health", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200 + }, + { + "name": "reviews collection requires auth", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/reviews", + "expectedStatus": 401 + }, + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "advisory", + "viewport": { + "width": 1440, + "height": 900 + } + } + } + ], + "security": { + "publicPaths": [ + "http://127.0.0.1:7071/api/health" + ] + } + } + } +} diff --git a/evals/scenarios/crud-react-functions-postgres-external-id.json b/evals/scenarios/crud-react-functions-postgres-external-id.json new file mode 100644 index 000000000..00a5177ee --- /dev/null +++ b/evals/scenarios/crud-react-functions-postgres-external-id.json @@ -0,0 +1,69 @@ +{ + "schemaVersion": "1", + "id": "crud-react-functions-postgres-external-id", + "prompt": "Build a customer membership portal with a React and Vite frontend, TypeScript Azure Functions API, PostgreSQL, Microsoft Entra External ID behind a mockable identity adapter, and membership lifecycle tests. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/orders collection.", + "baselinePrompt": "Build a complete customer membership portal using a React and Vite frontend, a TypeScript Azure Functions API, PostgreSQL, and Microsoft Entra External ID behind a mockable identity adapter, including membership lifecycle tests. Implement all frontend, backend, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/orders collection.", + "tags": { + "archetype": "crud", + "frontend": "react", + "backend": "typescript-functions", + "database": "postgres", + "auth": "entra-external-id", + "complexity": "high" + }, + "requirementsAnswers": { + "dataStores": [ + "PostgreSQL" + ], + "auth": "Microsoft Entra External ID" + }, + "validation": { + "profile": "advanced", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 15 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "api health", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200 + }, + { + "name": "orders collection requires auth", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/orders", + "expectedStatus": 401 + }, + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "advisory", + "viewport": { + "width": 1440, + "height": 900 + } + } + } + ], + "security": { + "publicPaths": [ + "http://127.0.0.1:7071/api/health" + ] + } + } + } +} diff --git a/evals/scenarios/crud-react-functions-postgres.json b/evals/scenarios/crud-react-functions-postgres.json new file mode 100644 index 000000000..5ea50c2ba --- /dev/null +++ b/evals/scenarios/crud-react-functions-postgres.json @@ -0,0 +1,110 @@ +{ + "schemaVersion": "1", + "id": "crud-react-functions-postgres", + "prompt": "Build a customer support ticket application with a React frontend, a TypeScript Azure Functions API, PostgreSQL persistence, and mock authentication. Users should create, assign, comment on, and close tickets.", + "baselinePrompt": "Build a complete customer support ticket application using a React frontend, a TypeScript Azure Functions API, PostgreSQL persistence, and mock authentication. Users must be able to create, assign, comment on, and close tickets. Implement all frontend, backend, and configuration code, comprehensive automated tests for the ticket workflows, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "crud", + "frontend": "react", + "backend": "functions", + "database": "postgres", + "auth": "mock", + "complexity": "medium" + }, + "requirementsAnswers": { + "dataStores": [ + "PostgreSQL" + ], + "auth": "Mock auth middleware" + }, + "validation": { + "profile": "standard", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 10, + "maxAgentRetries": 6 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "compound": true, + "debugParity": { + "target": "backend", + "sourceGlob": "**/src/utils/http.ts", + "lineIncludes": "const start = Date.now()", + "triggerUrl": "http://127.0.0.1:7071/api/health", + "timeoutSeconds": 180 + }, + "probes": [ + { + "name": "ticket API health", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200 + }, + { + "name": "ticket application browser", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "bodyIncludes": "root", + "browser": { + "expectedText": "ticket", + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "required", + "viewport": { + "width": 1440, + "height": 900 + }, + "actions": [ + { + "kind": "click", + "selector": "Create ticket", + "selectorType": "role", + "role": "button" + }, + { + "kind": "fillForm", + "values": { + "Subject": "Browser acceptance ticket", + "Description": "Created by the evaluator to verify the full-stack ticket journey." + } + }, + { + "kind": "click", + "selector": "Create ticket", + "selectorType": "role", + "role": "button" + } + ], + "assertions": [ + { + "kind": "text", + "selector": "body", + "value": "Browser acceptance ticket" + } + ], + "persistence": { + "restartTargets": [ + "backend", + "frontend" + ], + "reload": "current-url", + "assertions": [ + { + "kind": "text", + "selector": "body", + "value": "Browser acceptance ticket" + } + ] + } + } + } + ] + } + } +} diff --git a/evals/scenarios/crud-svelte-js-functions-sql-auth0.json b/evals/scenarios/crud-svelte-js-functions-sql-auth0.json new file mode 100644 index 000000000..407b7f66a --- /dev/null +++ b/evals/scenarios/crud-svelte-js-functions-sql-auth0.json @@ -0,0 +1,69 @@ +{ + "schemaVersion": "1", + "id": "crud-svelte-js-functions-sql-auth0", + "prompt": "Build an event registration app with a Svelte frontend, JavaScript Azure Functions API, Azure SQL, Auth0 behind a mockable adapter, and registration plus authorization tests. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/expenses collection.", + "baselinePrompt": "Build a complete event registration application using a Svelte frontend, a JavaScript Azure Functions API, Azure SQL, and Auth0 behind a mockable adapter, including registration and authorization tests. Implement all frontend, backend, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/expenses collection.", + "tags": { + "archetype": "crud", + "frontend": "svelte", + "backend": "javascript-functions", + "database": "azure-sql", + "auth": "auth0", + "complexity": "medium" + }, + "requirementsAnswers": { + "dataStores": [ + "Azure SQL" + ], + "auth": "Auth0" + }, + "validation": { + "profile": "standard", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 10 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "api health", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200 + }, + { + "name": "expenses collection requires auth", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/expenses", + "expectedStatus": 401 + }, + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "advisory", + "viewport": { + "width": 1440, + "height": 900 + } + } + } + ], + "security": { + "publicPaths": [ + "http://127.0.0.1:7071/api/health" + ] + } + } + } +} diff --git a/evals/scenarios/crud-svelte-python-functions-cosmos.json b/evals/scenarios/crud-svelte-python-functions-cosmos.json new file mode 100644 index 000000000..4c42f8d67 --- /dev/null +++ b/evals/scenarios/crud-svelte-python-functions-cosmos.json @@ -0,0 +1,93 @@ +{ + "schemaVersion": "1", + "id": "crud-svelte-python-functions-cosmos", + "prompt": "Build a personal reading tracker with a Svelte frontend, Python Azure Functions API, Cosmos DB, mock authentication, and UI plus API tests for reading-list workflows.", + "baselinePrompt": "Build a complete personal reading tracker application using a Svelte frontend, a Python Azure Functions API, Cosmos DB, and mock authentication, including UI and API tests for reading-list workflows. Implement all frontend, backend, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "crud", + "frontend": "svelte", + "backend": "python-functions", + "database": "cosmos", + "auth": "mock", + "complexity": "medium" + }, + "requirementsAnswers": { + "dataStores": [ + "CosmosDB" + ], + "auth": "Mock auth middleware" + }, + "validation": { + "profile": "standard", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 10 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "required", + "viewport": { + "width": 1440, + "height": 900 + }, + "actions": [ + { + "kind": "click", + "selector": "Add book", + "selectorType": "role", + "role": "button", + "optional": true + }, + { + "kind": "fillForm", + "values": { + "title": "Browser acceptance book", + "name": "Browser acceptance book", + "book": "Browser acceptance book" + } + }, + { + "kind": "click", + "selector": "Add book", + "selectorType": "role", + "role": "button" + } + ], + "assertions": [ + { + "kind": "text", + "selector": "body", + "value": "Browser acceptance book" + } + ], + "persistence": { + "restartTargets": [ + "frontend" + ], + "reload": "current-url", + "assertions": [ + { + "kind": "text", + "selector": "body", + "value": "Browser acceptance book" + } + ] + } + } + } + ] + } + } +} diff --git a/evals/scenarios/crud-vue-functions-postgres.json b/evals/scenarios/crud-vue-functions-postgres.json new file mode 100644 index 000000000..574194123 --- /dev/null +++ b/evals/scenarios/crud-vue-functions-postgres.json @@ -0,0 +1,93 @@ +{ + "schemaVersion": "1", + "id": "crud-vue-functions-postgres", + "prompt": "Build a project milestone tracker with a Vue and Vite frontend, TypeScript Azure Functions API, PostgreSQL persistence, no authentication, and component plus API tests.", + "baselinePrompt": "Build a complete project milestone tracker application using a Vue and Vite frontend, a TypeScript Azure Functions API, PostgreSQL persistence, and no authentication, including component and API tests. Implement all frontend, backend, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "crud", + "frontend": "vue", + "backend": "typescript-functions", + "database": "postgres", + "auth": "none", + "complexity": "medium" + }, + "requirementsAnswers": { + "dataStores": [ + "PostgreSQL" + ], + "auth": "No auth" + }, + "validation": { + "profile": "standard", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 10 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "required", + "viewport": { + "width": 1440, + "height": 900 + }, + "actions": [ + { + "kind": "click", + "selector": "Add milestone", + "selectorType": "role", + "role": "button", + "optional": true + }, + { + "kind": "fillForm", + "values": { + "name": "Browser acceptance milestone", + "title": "Browser acceptance milestone", + "milestone": "Browser acceptance milestone" + } + }, + { + "kind": "click", + "selector": "Add milestone", + "selectorType": "role", + "role": "button" + } + ], + "assertions": [ + { + "kind": "text", + "selector": "body", + "value": "Browser acceptance milestone" + } + ], + "persistence": { + "restartTargets": [ + "frontend" + ], + "reload": "current-url", + "assertions": [ + { + "kind": "text", + "selector": "body", + "value": "Browser acceptance milestone" + } + ] + } + } + } + ] + } + } +} diff --git a/evals/scenarios/multiservice-react-functions-worker-postgres-queue.json b/evals/scenarios/multiservice-react-functions-worker-postgres-queue.json new file mode 100644 index 000000000..fe89673e3 --- /dev/null +++ b/evals/scenarios/multiservice-react-functions-worker-postgres-queue.json @@ -0,0 +1,95 @@ +{ + "schemaVersion": "1", + "id": "multiservice-react-functions-worker-postgres-queue", + "prompt": "Build an order processing system with a React and Vite frontend, TypeScript Azure Functions CRUD API backed by PostgreSQL, a Python Azure Functions queue worker for fulfillment events, mock authentication, and cross-service contract tests.", + "baselinePrompt": "Build a complete order processing system using a React and Vite frontend, a TypeScript Azure Functions CRUD API backed by PostgreSQL, a Python Azure Functions queue worker using Azure Queue Storage for fulfillment events, and mock authentication, including cross-service contract tests. Implement all frontend, API, worker, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "multiservice", + "frontend": "react", + "backend": "typescript-python-functions", + "database": "postgres-queue", + "auth": "mock", + "complexity": "high" + }, + "requirementsAnswers": { + "dataStores": [ + "Queue Storage", + "PostgreSQL" + ], + "auth": "Mock auth middleware" + }, + "validation": { + "profile": "advanced", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 15 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "required", + "viewport": { + "width": 1440, + "height": 900 + }, + "actions": [ + { + "kind": "click", + "selector": "Create order", + "selectorType": "role", + "role": "button", + "optional": true + }, + { + "kind": "fillForm", + "values": { + "order": "Browser acceptance order", + "customer": "Browser acceptance order", + "name": "Browser acceptance order", + "item": "Browser acceptance order" + } + }, + { + "kind": "click", + "selector": "Create order", + "selectorType": "role", + "role": "button" + } + ], + "assertions": [ + { + "kind": "text", + "selector": "body", + "value": "Browser acceptance order" + } + ], + "persistence": { + "restartTargets": [ + "frontend" + ], + "reload": "current-url", + "assertions": [ + { + "kind": "text", + "selector": "body", + "value": "Browser acceptance order" + } + ] + } + } + } + ] + } + } +} diff --git a/evals/scenarios/multiservice-vue-functions-worker-cosmos-queue.json b/evals/scenarios/multiservice-vue-functions-worker-cosmos-queue.json new file mode 100644 index 000000000..9b0193177 --- /dev/null +++ b/evals/scenarios/multiservice-vue-functions-worker-cosmos-queue.json @@ -0,0 +1,70 @@ +{ + "schemaVersion": "1", + "id": "multiservice-vue-functions-worker-cosmos-queue", + "prompt": "Build an incident response system with a Vue and Vite frontend, C# Azure Functions API using Cosmos DB, a TypeScript Azure Functions queue worker for notifications, Microsoft Entra ID with mocked token validation, and cross-service tests. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/jobs collection.", + "baselinePrompt": "Build a complete incident response system using a Vue and Vite frontend, a C# Azure Functions API using Cosmos DB, a TypeScript Azure Functions queue worker using Azure Queue Storage for notifications, and Microsoft Entra ID with mocked token validation, including cross-service tests. Implement all frontend, API, worker, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified. Expose an unauthenticated GET /api/health endpoint and require a valid bearer token for the GET /api/jobs collection.", + "tags": { + "archetype": "multiservice", + "frontend": "vue", + "backend": "csharp-typescript-functions", + "database": "cosmos-queue", + "auth": "entra-id", + "complexity": "high" + }, + "requirementsAnswers": { + "dataStores": [ + "Queue Storage", + "CosmosDB" + ], + "auth": "Microsoft Entra ID" + }, + "validation": { + "profile": "advanced", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 15 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "api health", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/health", + "expectedStatus": 200 + }, + { + "name": "jobs collection requires auth", + "target": "backend", + "method": "GET", + "url": "http://127.0.0.1:7071/api/jobs", + "expectedStatus": 401 + }, + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "advisory", + "viewport": { + "width": 1440, + "height": 900 + } + } + } + ], + "security": { + "publicPaths": [ + "http://127.0.0.1:7071/api/health" + ] + } + } + } +} diff --git a/evals/scenarios/static-html-functions-blob.json b/evals/scenarios/static-html-functions-blob.json new file mode 100644 index 000000000..334b680c3 --- /dev/null +++ b/evals/scenarios/static-html-functions-blob.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": "1", + "id": "static-html-functions-blob", + "prompt": "Build a static TypeScript and HTML photo metadata browser with Pico.css, a TypeScript Azure Functions API, Blob Storage for images, no authentication, and frontend plus API tests.", + "baselinePrompt": "Build a complete photo metadata browser using a static TypeScript and HTML frontend with Pico.css, a TypeScript Azure Functions API, Azure Blob Storage for images, and no authentication, including frontend and API tests. Implement all frontend, backend, and configuration code, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging every application service, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "static-api", + "frontend": "html", + "backend": "typescript-functions", + "database": "blob", + "auth": "none", + "complexity": "medium" + }, + "requirementsAnswers": { + "dataStores": [ + "Blob Storage" + ], + "auth": "No auth" + }, + "validation": { + "profile": "standard", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 10 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 180, + "probes": [ + { + "name": "application frontend", + "target": "frontend", + "method": "GET", + "url": "http://127.0.0.1:5173/", + "expectedStatus": 200, + "browser": { + "requireInteractiveElements": true, + "maxSeriousAccessibilityViolations": 0, + "journeySeverity": "advisory", + "viewport": { + "width": 1440, + "height": 900 + } + } + } + ] + } + } +} diff --git a/evals/scenarios/worker-python-functions-blob.json b/evals/scenarios/worker-python-functions-blob.json new file mode 100644 index 000000000..507bac6fa --- /dev/null +++ b/evals/scenarios/worker-python-functions-blob.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": "1", + "id": "worker-python-functions-blob", + "prompt": "Build a backend-only Python Azure Functions timer worker that archives expired Blob Storage documents, has no authentication, and includes deterministic worker tests. Locally, read documents from an active-documents container, treat an expiresAt blob metadata value earlier than the current time as expired, move expired blobs to an archived-documents container with the same blob name and content, and delete the source blob.", + "baselinePrompt": "Build a complete backend-only document archival project using a Python Azure Functions timer worker that archives expired documents in Azure Blob Storage, has no authentication, and includes deterministic worker tests. Locally, read documents from an active-documents container, treat an expiresAt blob metadata value earlier than the current time as expired, move expired blobs to an archived-documents container with the same blob name and content, and delete the source blob. Implement all worker code and configuration, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging the worker, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "worker", + "frontend": "none", + "backend": "python-functions", + "database": "blob", + "auth": "none", + "complexity": "low" + }, + "requirementsAnswers": { + "dataStores": ["Blob Storage"], + "auth": "No auth" + }, + "validation": { + "profile": "minimal", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 5 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 120, + "probes": [ + { + "name": "timer worker process", + "target": "worker", + "processPattern": "func( host)? start|Microsoft.Azure.WebJobs.Script.WebHost" + } + ], + "storageEvents": [ + { + "name": "expired document is durably archived", + "kind": "blob", + "sourceContainer": "active-documents", + "destinationContainer": "archived-documents", + "blobName": "eval/expired-document.txt", + "content": "copilot-on-rails durable blob acceptance", + "metadata": { + "expiresAt": "2000-01-01T00:00:00Z" + }, + "sourceMustBeDeleted": true, + "timeoutSeconds": 120 + } + ] + } + } +} diff --git a/evals/scenarios/worker-ts-functions-queue.json b/evals/scenarios/worker-ts-functions-queue.json new file mode 100644 index 000000000..58d5ee841 --- /dev/null +++ b/evals/scenarios/worker-ts-functions-queue.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": "1", + "id": "worker-ts-functions-queue", + "prompt": "Build a backend-only TypeScript Azure Functions queue worker that reads JSON billing events from the billing-events-in Queue Storage queue, validates them, and writes routed JSON events to billing-events-out. For the deterministic input {\"eventId\":\"eval-billing-001\",\"accountId\":\"acct-eval-001\",\"amount\":1250,\"currency\":\"USD\"}, the output must preserve eventId and accountId and include route:\"standard\". Use AzureWebJobsStorage so this works with Azurite, has no authentication, and includes unit tests for success and poison-message paths.", + "baselinePrompt": "Build a complete backend-only billing event processing project using a TypeScript Azure Functions queue worker that reads JSON events from the billing-events-in Azure Queue Storage queue and writes routed JSON events to billing-events-out. The deterministic input {\"eventId\":\"eval-billing-001\",\"accountId\":\"acct-eval-001\",\"amount\":1250,\"currency\":\"USD\"} must produce output preserving eventId and accountId and containing route:\"standard\". Use AzureWebJobsStorage so local execution works with Azurite, has no authentication, and includes unit tests for success and poison-message paths. Implement all worker code and configuration, comprehensive automated tests, local VS Code tasks and launch configuration for running and debugging the worker and Azurite, and deployment-ready Azure infrastructure-as-code and application configuration. The repository must be installable, buildable, testable, runnable locally, debuggable in VS Code, and deployable to Azure. Work autonomously to completion without asking questions or requiring user interaction; choose reasonable defaults where unspecified.", + "tags": { + "archetype": "worker", + "frontend": "none", + "backend": "typescript-functions", + "database": "queue", + "auth": "none", + "complexity": "low" + }, + "requirementsAnswers": { + "dataStores": ["Queue Storage"], + "auth": "No auth" + }, + "validation": { + "profile": "minimal", + "build": true, + "test": true, + "lint": "if-present", + "timeoutMinutes": 5 + }, + "acceptance": { + "local": { + "startupTimeoutSeconds": 120, + "storageEvents": [ + { + "name": "billing event is routed", + "kind": "queue", + "inputQueue": "billing-events-in", + "outputQueue": "billing-events-out", + "message": { + "eventId": "eval-billing-001", + "accountId": "acct-eval-001", + "amount": 1250, + "currency": "USD" + }, + "expectedMessageIncludes": { + "eventId": "eval-billing-001", + "accountId": "acct-eval-001", + "route": "standard" + }, + "timeoutSeconds": 90 + } + ], + "probes": [ + { + "name": "queue worker process", + "target": "worker", + "processPattern": "func( host)? start|Microsoft.Azure.WebJobs.Script.WebHost" + } + ] + } + } +} diff --git a/evals/src/BaselineCopilotSdkExecutor.ts b/evals/src/BaselineCopilotSdkExecutor.ts new file mode 100644 index 000000000..ffe59f3c0 --- /dev/null +++ b/evals/src/BaselineCopilotSdkExecutor.ts @@ -0,0 +1,189 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { + CopilotClientOptions, + SessionConfig, + SessionEvent, +} from '@github/copilot-sdk-eval'; +import * as path from 'path'; +import type { + CorAgentRunResult, +} from '../../src/utils/copilotOnRails/agentExecution/CorAgentExecutor'; +import { + appendCorAgentCaptureError, + createCorAgentEventCapture, + reduceCorAgentEvent, +} from '../../src/utils/copilotOnRails/agentExecution/CorAgentExecutor'; + +export const baselineWorkspaceFileTools = [ + 'apply_patch', + 'create', + 'edit', + 'glob', + 'grep', + 'rg', + 'view', +] as const; + +export const baselineAvailableTools = baselineWorkspaceFileTools.map(tool => `builtin:${tool}`); + +export const baselineSystemMessage = [ + 'Complete the user request directly in the current workspace.', + 'Use only the provided workspace file tools.', + 'Do not use shell, network, MCP, delegation, custom agents, skills, or custom instructions.', + 'Do not ask for user input. Stop when the workspace implementation is complete.', +].join(' '); + +export interface BaselineAgentRunRequest { + prompt: string; + workingDirectory: string; + model: string; + timeoutMs: number; +} + +export interface BaselineAgentExecutor { + run(request: BaselineAgentRunRequest): Promise; +} + +export function getBaselineStateDirectory(request: Pick): string { + return path.join( + path.dirname(request.workingDirectory), + '.copilot-baseline-state', + path.basename(request.workingDirectory), + ); +} + +export function isBaselineFilePermissionAllowed( + request: { kind: string; requestSandboxBypass?: boolean; path?: string; fileName?: string }, + workingDirectory: string, +): boolean { + if (request.requestSandboxBypass || (request.kind !== 'read' && request.kind !== 'write')) { + return false; + } + const requestedPath = request.kind === 'read' ? request.path : request.fileName; + if (!requestedPath) { + return false; + } + const root = path.resolve(workingDirectory); + const resolved = path.resolve(root, requestedPath); + const relative = path.relative(root, resolved); + return relative === '' || (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)); +} + +export function createBaselineClientOptions(request: BaselineAgentRunRequest): CopilotClientOptions { + return { + workingDirectory: request.workingDirectory, + baseDirectory: getBaselineStateDirectory(request), + logLevel: 'warning', + mode: 'empty', + useLoggedInUser: true, + }; +} + +export function createBaselineSessionConfig(request: BaselineAgentRunRequest): SessionConfig { + return { + clientName: 'vscode-azureresourcegroups-baseline-evaluation', + model: request.model, + availableTools: baselineAvailableTools, + customAgents: [], + enableConfigDiscovery: false, + enableMcpApps: false, + mcpServers: {}, + pluginDirectories: [], + requestCanvasRenderer: false, + requestExtensions: false, + skillDirectories: [], + skipCustomInstructions: true, + systemMessage: { + mode: 'append', + content: baselineSystemMessage, + }, + onPermissionRequest: permission => { + if (!isBaselineFilePermissionAllowed(permission, request.workingDirectory)) { + return { + kind: 'reject', + feedback: `The controlled baseline does not permit this ${permission.kind} operation.`, + }; + } + return { kind: 'approve-once' }; + }, + workingDirectory: request.workingDirectory, + }; +} + +/** + * Runs the SDK's generic coding agent without any Copilot-on-Rails prompt, asset, custom tool, + * or delegation surface. + */ +export class BaselineCopilotSdkExecutor implements BaselineAgentExecutor { + public async run(request: BaselineAgentRunRequest): Promise { + const started = Date.now(); + let capture = createCorAgentEventCapture(); + const { CopilotClient } = await import('@github/copilot-sdk-eval'); + const client = new CopilotClient(createBaselineClientOptions(request)); + let sessionId: string | undefined; + let finalMessage: string | undefined; + let outcome: CorAgentRunResult['outcome'] = 'failed'; + + try { + await client.start(); + const session = await client.createSession(createBaselineSessionConfig(request)); + sessionId = session.sessionId; + const unsubscribe = session.on((event: SessionEvent) => { + capture = reduceCorAgentEvent(capture, event); + }); + try { + const response = await session.sendAndWait({ prompt: request.prompt }, request.timeoutMs); + finalMessage = response?.data.content; + outcome = 'completed'; + } catch (error) { + const message = getErrorMessage(error); + capture = appendCorAgentCaptureError(capture, message); + outcome = /timed?\s*out|timeout/i.test(message) ? 'timedOut' : 'failed'; + } finally { + unsubscribe(); + try { + await session.disconnect(); + } catch (error) { + capture = appendCorAgentCaptureError(capture, `SDK cleanup: ${getErrorMessage(error)}`); + outcome = 'failed'; + } + } + } catch (error) { + capture = appendCorAgentCaptureError(capture, getErrorMessage(error)); + } + + try { + const stopErrors = await client.stop(); + for (const error of stopErrors) { + capture = appendCorAgentCaptureError(capture, `SDK cleanup: ${error.message}`); + } + if (stopErrors.length) { + outcome = 'failed'; + } + } catch (error) { + capture = appendCorAgentCaptureError(capture, `SDK cleanup: ${getErrorMessage(error)}`); + outcome = 'failed'; + } + const completed = Date.now(); + return { + outcome, + sessionId, + finalMessage, + startedAt: new Date(started).toISOString(), + completedAt: new Date(completed).toISOString(), + durationMs: completed - started, + usage: capture.usage, + toolCalls: capture.toolCalls, + errors: capture.errors, + eventTimeline: capture.eventTimeline, + }; + } +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/evals/src/CopilotSdkAgentExecutor.ts b/evals/src/CopilotSdkAgentExecutor.ts new file mode 100644 index 000000000..80c4004c2 --- /dev/null +++ b/evals/src/CopilotSdkAgentExecutor.ts @@ -0,0 +1,332 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { + SessionEvent, + Tool, +} from '@github/copilot-sdk-eval'; +import { realpathSync } from 'fs'; +import * as path from 'path'; +import { + CorAgentExecutor, + CorAgentRunRequest, + CorAgentRunResult, + appendCorAgentCaptureError, + createCorAgentEventCapture, + reduceCorAgentEvent, +} from '../../src/utils/copilotOnRails/agentExecution/CorAgentExecutor'; +import { loadAgentSystemPrompt } from './agentAssets'; + +const defaultTimeoutMs = 5 * 60 * 1000; +const defaultStallTimeoutMs = 90 * 1000; +const sdkLogLevels = ['error', 'warning', 'info', 'debug'] as const; + +/** + * The stall threshold is coupled to how many agent sessions run at once: concurrency slows + * individual turns, so a budget tuned for one worker starts reporting healthy-but-slow turns as + * stalls at eight. Configurable so raising workers does not require a code change. + */ +export function stallTimeoutMs(): number { + const configured = Number(process.env.COR_EVAL_STALL_TIMEOUT_MS); + return Number.isFinite(configured) && configured > 0 ? configured : defaultStallTimeoutMs; +} + +type SdkLogLevel = typeof sdkLogLevels[number]; + +/** + * Diagnosing a stalled agent turn requires the SDK's own transport logs, which are + * suppressed at the default level. Raising it is opt-in so normal runs stay quiet. + */ +function sdkLogLevel(): SdkLogLevel { + const configured = process.env.COR_EVAL_SDK_LOG_LEVEL; + return sdkLogLevels.find(level => level === configured) ?? 'warning'; +} + +export const agentStallMessagePrefix = 'Agent produced no session events for'; + +/** + * An upstream turn can start and never return: the SDK emits `assistant.turn_start`, then + * nothing at all until the overall timeout expires. Waiting out the full budget wastes + * minutes per attempt and, when the upstream incident is broad, loses an entire matrix to + * dead time. Silence is measured directly so a stall is caught in seconds and can be + * retried, and so it stays distinguishable from an agent that is genuinely working. + */ +export function createStallWatchdog(timeoutMs: number): { + stalled: Promise; + recordActivity: () => void; + dispose: () => void; + describe: () => string; +} { + let lastActivity = Date.now(); + let timer: NodeJS.Timeout | undefined; + let settled = false; + let signalStalled: (() => void) | undefined; + const stalled = new Promise(resolve => { + signalStalled = resolve; + }); + const check = (): void => { + if (settled) { + return; + } + const idleMs = Date.now() - lastActivity; + if (idleMs >= timeoutMs) { + settled = true; + signalStalled?.(); + return; + } + timer = setTimeout(check, timeoutMs - idleMs).unref(); + }; + timer = setTimeout(check, timeoutMs).unref(); + return { + stalled, + recordActivity: () => { + lastActivity = Date.now(); + }, + dispose: () => { + settled = true; + if (timer) { + clearTimeout(timer); + } + }, + describe: () => `${agentStallMessagePrefix} ${timeoutMs}ms.`, + }; +} + +const evaluationBuiltInTools = [ + 'apply_patch', + 'create', + 'edit', + 'glob', + 'grep', + 'rg', + 'view', +] as const; + +export class CopilotSdkAgentExecutor implements CorAgentExecutor { + public constructor(private readonly repoRoot: string) { + } + + public async run(request: CorAgentRunRequest): Promise { + const started = Date.now(); + const startedAt = new Date(started).toISOString(); + let capture = createCorAgentEventCapture(); + const { CopilotClient, ToolSet } = await import('@github/copilot-sdk-eval'); + const client = new CopilotClient({ + workingDirectory: request.workingDirectory, + logLevel: sdkLogLevel(), + useLoggedInUser: true, + }); + let sessionId: string | undefined; + let finalMessage: string | undefined; + let outcome: CorAgentRunResult['outcome']; + let resolveCompletion: ((toolName: string) => void) | undefined; + const completion = new Promise(resolve => { + resolveCompletion = resolve; + }); + const completionToolNames = new Set(request.completionToolNames ?? []); + + try { + await client.start(); + const session = await client.createSession({ + clientName: 'vscode-azureresourcegroups-evaluation', + model: request.model, + tools: request.tools?.map(tool => toSdkTool( + tool, + completionToolNames.has(tool.name) ? () => resolveCompletion?.(tool.name) : undefined, + )), + availableTools: new ToolSet() + .addBuiltIn(request.builtInTools ?? evaluationBuiltInTools) + .addCustom('*'), + systemMessage: { + mode: 'append', + content: await loadAgentSystemPrompt(this.repoRoot, request.agentName, request.additionalSystemMessage), + }, + onPermissionRequest: permission => { + if (!isTreatmentPermissionAllowed(permission, request.workingDirectory)) { + return { + kind: 'reject', + feedback: `The Phase 0 evaluation does not permit this ${permission.kind} operation.`, + }; + } + return { kind: 'approve-once' }; + }, + }); + sessionId = session.sessionId; + const stall = createStallWatchdog(request.stallTimeoutMs ?? stallTimeoutMs()); + const unsubscribe = session.on((event: SessionEvent) => { + stall.recordActivity(); + capture = reduceCorAgentEvent(capture, event); + }); + try { + const responsePromise = session.sendAndWait( + { prompt: request.prompt }, + request.timeoutMs ?? defaultTimeoutMs, + ); + const settled = await Promise.race([ + responsePromise.then( + response => ({ kind: 'idle' as const, response }), + error => ({ kind: 'error' as const, error }), + ), + completionToolNames.size + ? completion.then(toolName => ({ kind: 'completion' as const, toolName })) + : new Promise(() => undefined), + stall.stalled.then(() => ({ kind: 'stalled' as const })), + ]); + if (settled.kind === 'completion') { + await session.abort(); + await responsePromise.catch(() => undefined); + outcome = 'completed'; + } else if (settled.kind === 'stalled') { + await session.abort().catch(() => undefined); + await responsePromise.catch(() => undefined); + throw new Error(stall.describe()); + } else if (settled.kind === 'error') { + throw settled.error; + } else { + finalMessage = settled.response?.data.content; + outcome = 'completed'; + } + } catch (error) { + const message = getErrorMessage(error); + capture = appendCorAgentCaptureError(capture, message); + outcome = /timed?\s*out|timeout/i.test(message) ? 'timedOut' : 'failed'; + } finally { + stall.dispose(); + unsubscribe(); + try { + await session.disconnect(); + } catch (error) { + capture = appendCorAgentCaptureError(capture, `SDK cleanup: ${getErrorMessage(error)}`); + outcome = 'failed'; + } + } + } catch (error) { + capture = appendCorAgentCaptureError(capture, getErrorMessage(error)); + outcome = 'failed'; + } + + try { + const stopErrors = await client.stop(); + for (const error of stopErrors) { + capture = appendCorAgentCaptureError(capture, `SDK cleanup: ${error.message}`); + } + if (stopErrors.length) { + outcome = 'failed'; + } + } catch (error) { + capture = appendCorAgentCaptureError(capture, `SDK cleanup: ${getErrorMessage(error)}`); + outcome = 'failed'; + } + const completed = Date.now(); + return { + outcome, + sessionId, + finalMessage, + startedAt, + completedAt: new Date(completed).toISOString(), + durationMs: completed - started, + usage: capture.usage, + toolCalls: capture.toolCalls, + errors: capture.errors, + eventTimeline: capture.eventTimeline, + }; + } +} + +export function isTreatmentPermissionAllowed( + permission: unknown, + workingDirectory: string, +): boolean { + if (!permission || typeof permission !== 'object') { + return false; + } + const request = permission as Record; + if ( + request.kind === 'shell' + || request.kind === 'url' + || request.kind === 'mcp' + || request.requestSandboxBypass === true + ) { + return false; + } + if (request.kind === 'custom-tool') { + return true; + } + if (request.kind !== 'read' && request.kind !== 'write') { + return false; + } + const requestedPaths: string[] = []; + for (const field of ['path', 'fileName', 'directory']) { + const value = request[field]; + if (value === undefined) { + continue; + } + if (typeof value !== 'string' || !value) { + return false; + } + requestedPaths.push(value); + } + if (!requestedPaths.length) { + return false; + } + const root = canonicalizePath(workingDirectory); + if (!root) { + return false; + } + for (const requestedPath of requestedPaths) { + const resolved = path.isAbsolute(requestedPath) + ? path.resolve(requestedPath) + : path.resolve(root, requestedPath); + const canonicalRequestedPath = canonicalizePath(resolved); + if (!canonicalRequestedPath) { + return false; + } + const relative = path.relative(root, canonicalRequestedPath); + if (relative.startsWith(`..${path.sep}`) || relative === '..' || path.isAbsolute(relative)) { + return false; + } + } + return true; +} + +function canonicalizePath(value: string): string | undefined { + let existingAncestor = path.resolve(value); + const missingSegments: string[] = []; + while (true) { + try { + return path.join(realpathSync(existingAncestor), ...missingSegments); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + return undefined; + } + const parent = path.dirname(existingAncestor); + if (parent === existingAncestor) { + return undefined; + } + missingSegments.unshift(path.basename(existingAncestor)); + existingAncestor = parent; + } + } +} + +function toSdkTool( + tool: NonNullable[number], + onCompleted?: () => void, +): Tool { + return { + name: tool.name, + description: tool.description, + parameters: tool.parameters, + handler: async args => { + const result = await tool.handler(args); + onCompleted?.(); + return result; + }, + }; +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/evals/src/SandboxLocalRuntimeValidator.ts b/evals/src/SandboxLocalRuntimeValidator.ts new file mode 100644 index 000000000..8d8dc1750 --- /dev/null +++ b/evals/src/SandboxLocalRuntimeValidator.ts @@ -0,0 +1,2839 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { execFile } from 'child_process'; +import { createHmac, randomUUID } from 'crypto'; +import { promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { promisify } from 'util'; +import { parse } from 'jsonc-parser'; +import { + findColumnIndex, + findSection, + findTable, + isChecked, + parseLocalDebugPlanMarkdown, +} from '../../src/webviews/copilotOnRails/views/utils/parseLocalDebugPlanMarkdown'; +import { + AcaCommandRunner, + createSandboxManifest, + createWorkspaceArchive, + readSandboxId, + readSandboxIds, + ValidationEcosystem, +} from './SandboxProjectValidator'; +import { + CorEvaluationScenario, + LocalAcceptanceProbe, + SecurityContract, + StorageBlobEventContract, + StorageEventContract, + StorageQueueEventContract, +} from './scenario'; +import { + SecurityCheckPlan, + isSecurityPlanConclusive, + planSecurityChecks, +} from './securityChecks'; + +const execFileAsync = promisify(execFile); +const maxLogLength = 20_000; +export const debugpyEvaluationPort = 5678; +const debuggerPrerequisiteAttempts = 30; +const debuggerPrerequisiteTimeoutMs = 75 * 1000; + +export interface PlannedDebugConfiguration { + name: string; + serviceRoot: string; + projectType: string; + runtime: string; +} + +interface ProbeConfigurationGroup { + configuration: PlannedDebugConfiguration; + probes: LocalAcceptanceProbe[]; +} + +interface LaunchConfiguration { + name?: unknown; + type?: unknown; + request?: unknown; + preLaunchTask?: unknown; + module?: unknown; + program?: unknown; + python?: unknown; + runtimeExecutable?: unknown; + runtimeArgs?: unknown; + cwd?: unknown; + args?: unknown; + env?: unknown; + processName?: unknown; + port?: unknown; + attachSimplePort?: unknown; + url?: unknown; + webRoot?: unknown; +} + +interface VsCodeTask { + type?: unknown; + label?: unknown; + command?: unknown; + script?: unknown; + args?: unknown; + dependsOn?: unknown; + isBackground?: unknown; + options?: { + cwd?: unknown; + env?: unknown; + }; +} + +export interface LocalRuntimeCommandResult { + kind: 'setup' | 'task' | 'probe' | 'debugger' | 'diagnostic' | 'restart' | 'storage-event' | 'security'; + name: string; + command: string; + success: boolean; + durationMs: number; + stdout: string; + stderr: string; +} + +export interface LocalRuntimeProbeResult { + name: string; + target: LocalAcceptanceProbe['target']; + method?: string; + url?: string; + expectedStatus?: number; + processPattern?: string; + success: boolean; + durationMs: number; + response?: string; + responseStatus?: number; + responseHeaders?: string; + responseBody?: string; + error?: string; +} + +export interface LocalRuntimeBrowserResult { + name: string; + url: string; + success: boolean; + durationMs: number; + title?: string; + bodyTextLength?: number; + interactiveElements?: number; + seriousAccessibilityViolations?: string[]; + accessibilityScanned?: boolean; + accessibilityScanError?: string; + consoleErrors?: string[]; + actionsCompleted?: number; + actionsExpected?: number; + /** + * Actions that observably changed the page. An action can complete without doing anything — + * clicking a mis-resolved target, or filling a form that exposed no fields — so this is the + * count the journey score is built from. + */ + actionsEffective?: number; + actionLedger?: Array<{ action: string; effective: boolean; reason?: string }>; + assertionsCompleted?: number; + assertionsExpected?: number; + /** The app served the page, rendered content and exposed interactive elements. */ + loadPassed?: boolean; + journeyStatus?: 'passed' | 'failed' | 'not-attempted'; + journeySeverity?: 'required' | 'advisory'; + journeyError?: string; + viewport?: { width: number; height: number }; + currentUrl?: string; + bodyTextExcerpt?: string; + error?: string; +} + +export interface LocalRuntimePersistenceResult { + name: string; + restartTargets: Array<'backend' | 'frontend'>; + processIdsBefore: number[]; + processIdsAfter: number[]; + preservedProcessIds?: number[]; + readinessProbes: LocalRuntimeProbeResult[]; + postRestartBrowser: LocalRuntimeBrowserResult; + success: boolean; + /** + * Persistence re-asserts the record the journey created, so it has nothing to verify when the + * journey never completed. Skipping is reported distinctly from failing: treating an + * unattempted check as a failure would relocate the journey's false negative one gate + * downstream instead of removing it. + */ + skipped?: boolean; + skipReason?: string; + durationMs: number; + error?: string; +} + +export interface LocalRuntimeStorageEventResult { + name: string; + kind: 'queue' | 'blob'; + inputQueue?: string; + outputQueue?: string; + sourceContainer?: string; + destinationContainer?: string; + blobName?: string; + stimulus: unknown; + expectedMessageIncludes?: unknown; + observedMessage?: unknown; + observedContent?: string; + sourceDeleted?: boolean; + pollAttempts?: number; + success: boolean; + durationMs: number; + error?: string; +} + +export interface LocalRuntimeSecurityResult { + name: string; + url: string; + /** + * `public` proves the server is alive and enforcing selectively. Without it a crashed or + * blanket-deny app would satisfy every negative check and pass the gate for the wrong reason. + */ + kind: 'unauthenticated' | 'malformed-token' | 'public'; + expectedStatuses: number[]; + responseStatus?: number; + /** + * Retained only for failures, where the body is the evidence that protected data leaked. + */ + responseExcerpt?: string; + success: boolean; + durationMs: number; + error?: string; +} + +export interface SandboxLocalRuntimeValidationResult { + outcome: 'passed' | 'failed'; + failureCode?: + | 'acceptanceSpecMissing' + | 'acceptanceTargetMissing' + | 'debugTaskGraphInvalid' + | 'localRuntimeUnsupported' + | 'localSandboxCreateFailed' + | 'localSandboxSetupFailed' + | 'localToolchainUnavailable' + | 'localContainerRegistryUnavailable' + | 'localTaskFailed' + | 'localProbeFailed' + | 'localBrowserFailed' + | 'localPersistenceFailed' + | 'localStorageEventFailed' + | 'localSecurityFailed' + | 'localDebuggerUnavailable' + | 'localSandboxCleanupFailed'; + error?: string; + commands: LocalRuntimeCommandResult[]; + probes: LocalRuntimeProbeResult[]; + browserChecks?: LocalRuntimeBrowserResult[]; + persistenceChecks?: LocalRuntimePersistenceResult[]; + workerEvents?: LocalRuntimeStorageEventResult[]; + securityChecks?: LocalRuntimeSecurityResult[]; +} + +interface LaunchedProcess { + pid: number; + label: string; + target: LocalAcceptanceProbe['target']; + task: VsCodeTask; + serviceRoot: string; + restartable: boolean; +} + +export function isLocalRuntimeInfrastructureFailureCode(code: string | undefined): boolean { + return [ + 'localSandboxCreateFailed', + 'localSandboxSetupFailed', + 'localToolchainUnavailable', + 'localContainerRegistryUnavailable', + 'localSandboxCleanupFailed', + ].includes(code ?? ''); +} + +/** + * A registry that refuses or rate-limits an image pull says nothing about the generated project. + * Counting it as a product failure understates the real success rate. + */ +export function isContainerRegistryFailure(output: string | undefined): boolean { + if (!output) { + return false; + } + return /error pulling image configuration|denied: requested access to the resource is denied/u.test(output) + || /toomanyrequests|rate limit|pull rate limit/iu.test(output) + || /failed to (?:resolve|pull) (?:reference|image)/u.test(output) + || /(?:dial tcp|TLS handshake|i\/o) timeout.*registry|registry.*(?:dial tcp|TLS handshake|i\/o) timeout/u.test(output); +} + +export class SandboxLocalRuntimeValidator { + public constructor( + private readonly repoRoot: string, + private readonly aca: AcaCommandRunner = new DefaultAcaCommandRunner(), + ) { + } + + public async validate( + workspace: string, + scenario: CorEvaluationScenario, + planContent: string, + ): Promise { + const contract = scenario.acceptance?.local; + if (!contract?.probes.length) { + return { + outcome: 'failed', + failureCode: 'acceptanceSpecMissing', + error: `Scenario "${scenario.id}" has no evaluator-owned local acceptance probes.`, + commands: [], + probes: [], + browserChecks: [], + persistenceChecks: [], + workerEvents: [], + }; + } + + const configurations = parsePlannedConfigurations(planContent); + const groups = groupProbesByConfiguration(contract.probes, configurations); + if ('error' in groups) { + return { + outcome: 'failed', + failureCode: 'acceptanceTargetMissing', + error: groups.error, + commands: [], + probes: [], + browserChecks: [], + persistenceChecks: [], + workerEvents: [], + }; + } + const archivePath = path.join(os.tmpdir(), `cor-local-${randomUUID()}.tar.gz`); + const commands: LocalRuntimeCommandResult[] = []; + const probes: LocalRuntimeProbeResult[] = []; + const browserChecks: LocalRuntimeBrowserResult[] = []; + const persistenceChecks: LocalRuntimePersistenceResult[] = []; + const workerEvents: LocalRuntimeStorageEventResult[] = []; + const securityChecks: LocalRuntimeSecurityResult[] = []; + try { + await createWorkspaceArchive(workspace, archivePath); + if (contract.compound) { + const result = await this.validateCompoundConfigurations( + workspace, + archivePath, + [...groups.values()], + contract.startupTimeoutSeconds ?? 90, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + contract.storageEvents ?? [], + ); + return result ?? { outcome: 'passed', commands, probes, browserChecks, persistenceChecks, workerEvents }; + } + for (const group of groups.values()) { + const result = await this.validateConfiguration( + workspace, + archivePath, + group.configuration, + group.probes, + contract.startupTimeoutSeconds ?? 90, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + group.probes.some(probe => probe.target === 'worker') ? (contract.storageEvents ?? []) : [], + contract.security, + securityChecks, + ); + if (result) { + return result; + } + } + return { outcome: 'passed', commands, probes, browserChecks, persistenceChecks, workerEvents, securityChecks }; + } finally { + await fs.rm(archivePath, { force: true }); + } + } + + private async validateCompoundConfigurations( + workspace: string, + archivePath: string, + groups: ProbeConfigurationGroup[], + startupTimeoutSeconds: number, + commands: LocalRuntimeCommandResult[], + probes: LocalRuntimeProbeResult[], + browserChecks: LocalRuntimeBrowserResult[], + persistenceChecks: LocalRuntimePersistenceResult[], + workerEvents: LocalRuntimeStorageEventResult[], + storageEvents: StorageEventContract[], + ): Promise { + const ecosystems = new Set(groups.map(group => runtimeToEcosystem(group.configuration.runtime))); + if (ecosystems.has(undefined) || ecosystems.size !== 1) { + return failure( + 'localRuntimeUnsupported', + 'Compound local acceptance currently requires every service to use one supported runtime ecosystem.', + commands, + probes, + ); + } + const ecosystem = [...ecosystems][0] as ValidationEcosystem; + const debugArtifacts = await readDebugArtifacts(workspace); + if ('error' in debugArtifacts) { + return failure('debugTaskGraphInvalid', debugArtifacts.error, commands, probes); + } + const prepared: Array<{ + group: ProbeConfigurationGroup; + launch: LaunchConfiguration; + tasks: VsCodeTask[]; + processLogName: string; + }> = []; + for (const group of groups) { + const launch = debugArtifacts.launchConfigurations.find(value => value.name === group.configuration.name); + if (!launch) { + return failure('debugTaskGraphInvalid', `Launch configuration "${group.configuration.name}" does not exist.`, commands, probes); + } + const taskChain = typeof launch.preLaunchTask === 'string' + ? resolveTaskChain(launch.preLaunchTask, debugArtifacts.tasks) + : { tasks: [] }; + if ('error' in taskChain) { + return failure('debugTaskGraphInvalid', taskChain.error, commands, probes); + } + prepared.push({ + group, + launch, + tasks: taskChain.tasks, + processLogName: typeof launch.preLaunchTask === 'string' + ? launch.preLaunchTask + : String(launch.name), + }); + } + + const created = await this.createSandbox(ecosystem); + if ('error' in created) { + return failure('localSandboxCreateFailed', created.error, commands, probes); + } + const sandboxId = created.sandboxId; + const launchedProcesses: LaunchedProcess[] = []; + let validationFailure: SandboxLocalRuntimeValidationResult | undefined; + let cleanupError: string | undefined; + try { + validationFailure = await this.setupWorkspace(sandboxId, archivePath, ecosystem, commands); + if (!validationFailure && prepared.some(item => item.tasks.some(task => task.type === 'func'))) { + validationFailure = await this.ensureFunctionsCoreTools(sandboxId, ecosystem, commands); + } + const startedTasks = new Set(); + for (const item of prepared) { + if (validationFailure) { + break; + } + for (const task of item.tasks) { + const label = typeof task.label === 'string' ? task.label : 'unnamed task'; + if (startedTasks.has(label)) { + continue; + } + startedTasks.add(label); + const taskResult = await this.runTask( + sandboxId, + task, + item.group.configuration.serviceRoot, + launchedProcesses, + { + target: item.group.probes[0].target, + restartable: label === item.launch.preLaunchTask + && ['backend', 'frontend'].includes(item.group.probes[0].target), + }, + ); + commands.push(taskResult); + if (!taskResult.success) { + const registryFailure = isContainerRegistryFailure( + `${taskResult.stderr ?? ''}\n${taskResult.stdout ?? ''}`); + validationFailure = failure( + registryFailure ? 'localContainerRegistryUnavailable' : 'localTaskFailed', + registryFailure + ? `Debug task "${label}" could not pull its container images.` + : `Debug task "${label}" failed.`, + commands, + probes); + break; + } + } + if (validationFailure) { + break; + } + const launchTask = resolveLaunchTask(item.launch, item.group.configuration.serviceRoot); + if (launchTask && 'error' in launchTask) { + validationFailure = failure('localRuntimeUnsupported', launchTask.error, commands, probes); + } else if (launchTask) { + const launchResult = await this.runTask( + sandboxId, + launchTask, + item.group.configuration.serviceRoot, + launchedProcesses, + { + target: item.group.probes[0].target, + restartable: ['backend', 'frontend'].includes(item.group.probes[0].target), + }, + ); + commands.push(launchResult); + item.processLogName = String(launchTask.label); + if (!launchResult.success) { + validationFailure = failure('localTaskFailed', `Debug launch configuration "${item.group.configuration.name}" failed.`, commands, probes); + } + } + } + for (const item of prepared) { + if (validationFailure) { + break; + } + for (const probe of item.group.probes) { + const probeResult = await this.runProbe( + sandboxId, + probe, + startupTimeoutSeconds, + String(item.launch.preLaunchTask), + ); + probes.push(probeResult.probe); + commands.push(probeResult.command); + if (!probeResult.probe.success) { + commands.push(await this.readProcessLogs(sandboxId, item.processLogName)); + validationFailure = failure( + 'localProbeFailed', + `Local acceptance probe "${probe.name}" failed. ${probeResult.probe.error ?? ''}`.trim(), + commands, + probes, + ); + break; + } + if (probe.browser) { + const browserResult = await this.runBrowserProbe(sandboxId, probe); + browserChecks.push(browserResult.browser); + commands.push(browserResult.command); + if (!browserResult.browser.success) { + validationFailure = failure( + 'localBrowserFailed', + `Browser acceptance probe "${probe.name}" failed. ${browserResult.browser.error ?? ''}`.trim(), + commands, + probes, + browserChecks, + ); + break; + } + } + } + } + for (const storageEvent of storageEvents) { + if (validationFailure) { + break; + } + const eventResult = await this.runStorageEvent(sandboxId, storageEvent); + commands.push(eventResult.command); + workerEvents.push(eventResult.event); + if (!eventResult.event.success) { + validationFailure = failure( + 'localStorageEventFailed', + `Storage event "${storageEvent.name}" failed. ${eventResult.event.error ?? ''}`.trim(), + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + } + } + for (const item of prepared) { + if (validationFailure) { + break; + } + for (const probe of item.group.probes.filter(value => value.browser?.persistence)) { + const persistenceResult = await this.runPersistenceCheck( + sandboxId, + probe, + groups.flatMap(group => group.probes), + launchedProcesses, + startupTimeoutSeconds, + commands, + [...browserChecks].reverse().find(check => check.name === probe.name), + ); + persistenceChecks.push(persistenceResult); + if (!persistenceResult.success && !persistenceResult.skipped) { + validationFailure = failure( + 'localPersistenceFailed', + `Persistence acceptance "${probe.name}" failed. ${persistenceResult.error ?? ''}`.trim(), + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + break; + } + } + } + for (const item of prepared) { + if (validationFailure) { + break; + } + validationFailure = await this.runDebuggerPrerequisites( + sandboxId, + item.launch, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + } + } finally { + try { + await this.aca.run(['sandbox', 'delete', '--id', sandboxId, '--yes'], 5 * 60 * 1000); + } catch (error) { + cleanupError = getErrorMessage(error); + } + } + if (cleanupError) { + return failure( + 'localSandboxCleanupFailed', + cleanupError, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + } + return validationFailure; + } + + private async validateConfiguration( + workspace: string, + archivePath: string, + configuration: PlannedDebugConfiguration, + acceptanceProbes: LocalAcceptanceProbe[], + startupTimeoutSeconds: number, + commands: LocalRuntimeCommandResult[], + probes: LocalRuntimeProbeResult[], + browserChecks: LocalRuntimeBrowserResult[], + persistenceChecks: LocalRuntimePersistenceResult[], + workerEvents: LocalRuntimeStorageEventResult[], + storageEvents: StorageEventContract[], + securityContract: SecurityContract | undefined, + securityChecks: LocalRuntimeSecurityResult[], + ): Promise { + const ecosystem = runtimeToEcosystem(configuration.runtime); + if (!ecosystem) { + return failure( + 'localRuntimeUnsupported', + `Runtime "${configuration.runtime}" is not supported by the isolated local-runtime validator.`, + commands, + probes, + ); + } + const debugArtifacts = await readDebugArtifacts(workspace); + if ('error' in debugArtifacts) { + return failure('debugTaskGraphInvalid', debugArtifacts.error, commands, probes); + } + const launchConfiguration = debugArtifacts.launchConfigurations + .find(value => value.name === configuration.name); + if (!launchConfiguration) { + return failure( + 'debugTaskGraphInvalid', + `Launch configuration "${configuration.name}" does not exist.`, + commands, + probes, + ); + } + const taskChain = typeof launchConfiguration.preLaunchTask === 'string' + ? resolveTaskChain(launchConfiguration.preLaunchTask, debugArtifacts.tasks) + : { tasks: [] }; + if ('error' in taskChain) { + return failure('debugTaskGraphInvalid', taskChain.error, commands, probes); + } + + const created = await this.createSandbox(ecosystem); + if ('error' in created) { + return failure('localSandboxCreateFailed', created.error, commands, probes); + } + const sandboxId = created.sandboxId; + const launchedProcesses: LaunchedProcess[] = []; + let validationFailure: SandboxLocalRuntimeValidationResult | undefined; + let cleanupError: string | undefined; + let processLogName = typeof launchConfiguration.preLaunchTask === 'string' + ? launchConfiguration.preLaunchTask + : String(launchConfiguration.name); + try { + validationFailure = await this.setupWorkspace(sandboxId, archivePath, ecosystem, commands); + if (!validationFailure && taskChain.tasks.some(task => task.type === 'func')) { + validationFailure = await this.ensureFunctionsCoreTools(sandboxId, ecosystem, commands); + } + if (!validationFailure) { + for (const task of taskChain.tasks) { + const target = acceptanceProbes[0].target; + const taskResult = await this.runTask( + sandboxId, + task, + configuration.serviceRoot, + launchedProcesses, + { + target, + restartable: task.label === launchConfiguration.preLaunchTask + && ['backend', 'frontend'].includes(target), + }, + ); + commands.push(taskResult); + if (!taskResult.success) { + const output = `${taskResult.stdout}\n${taskResult.stderr}`; + const unavailable = /(?:command not found|not recognized|no such file or directory)/i.test(output); + // The compound path already distinguished these; without the same check here a + // blocked image pull is charged to the generated project. + const registryFailure = isContainerRegistryFailure(output); + validationFailure = failure( + registryFailure + ? 'localContainerRegistryUnavailable' + : unavailable ? 'localToolchainUnavailable' : 'localTaskFailed', + registryFailure + ? `Debug task "${String(task.label)}" could not pull its container images.` + : `Debug task "${String(task.label)}" failed.`, + commands, + probes, + ); + break; + } + } + } + if (!validationFailure) { + const launchTask = resolveLaunchTask(launchConfiguration, configuration.serviceRoot); + if (launchTask && 'error' in launchTask) { + validationFailure = failure('localRuntimeUnsupported', launchTask.error, commands, probes); + } else if (launchTask) { + const target = acceptanceProbes[0].target; + const launchResult = await this.runTask( + sandboxId, + launchTask, + configuration.serviceRoot, + launchedProcesses, + { target, restartable: ['backend', 'frontend'].includes(target) }, + ); + commands.push(launchResult); + processLogName = String(launchTask.label); + if (!launchResult.success) { + const registryFailure = isContainerRegistryFailure( + `${launchResult.stderr ?? ''}\n${launchResult.stdout ?? ''}`); + validationFailure = failure( + registryFailure ? 'localContainerRegistryUnavailable' : 'localTaskFailed', + registryFailure + ? `Debug launch configuration "${configuration.name}" could not pull its container images.` + : `Debug launch configuration "${configuration.name}" failed.`, + commands, + probes, + ); + } + } + } + if (!validationFailure) { + for (const probe of acceptanceProbes) { + const probeResult = await this.runProbe( + sandboxId, + probe, + startupTimeoutSeconds, + processLogName, + ); + probes.push(probeResult.probe); + commands.push(probeResult.command); + if (!probeResult.probe.success) { + const logs = await this.readProcessLogs(sandboxId, processLogName); + commands.push(logs); + validationFailure = failure( + 'localProbeFailed', + `Local acceptance probe "${probe.name}" failed. ${probeResult.probe.error ?? ''}`.trim(), + commands, + probes, + ); + break; + } + if (probe.browser) { + const browserResult = await this.runBrowserProbe(sandboxId, probe); + browserChecks.push(browserResult.browser); + commands.push(browserResult.command); + if (!browserResult.browser.success) { + validationFailure = failure( + 'localBrowserFailed', + `Browser acceptance probe "${probe.name}" failed. ${browserResult.browser.error ?? ''}`.trim(), + commands, + probes, + browserChecks, + ); + break; + } + } + } + } + if (!validationFailure) { + for (const storageEvent of storageEvents) { + const eventResult = await this.runStorageEvent(sandboxId, storageEvent); + commands.push(eventResult.command); + workerEvents.push(eventResult.event); + if (!eventResult.event.success) { + validationFailure = failure( + 'localStorageEventFailed', + `Storage event "${storageEvent.name}" failed. ${eventResult.event.error ?? ''}`.trim(), + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + break; + } + } + } + if (!validationFailure) { + for (const probe of acceptanceProbes.filter(value => value.browser?.persistence)) { + const persistenceResult = await this.runPersistenceCheck( + sandboxId, + probe, + acceptanceProbes, + launchedProcesses, + startupTimeoutSeconds, + commands, + [...browserChecks].reverse().find(check => check.name === probe.name), + ); + persistenceChecks.push(persistenceResult); + if (!persistenceResult.success) { + validationFailure = failure( + 'localPersistenceFailed', + `Persistence acceptance "${probe.name}" failed. ${persistenceResult.error ?? ''}`.trim(), + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + break; + } + } + } + if (!validationFailure) { + validationFailure = await this.runSecurityChecks( + sandboxId, + securityContract, + acceptanceProbes, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + securityChecks, + ); + } + if (!validationFailure) { + validationFailure = await this.runDebuggerPrerequisites( + sandboxId, + launchConfiguration, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + } + } finally { + try { + await this.aca.run(['sandbox', 'delete', '--id', sandboxId, '--yes'], 5 * 60 * 1000); + } catch (error) { + cleanupError = getErrorMessage(error); + } + } + if (cleanupError) { + return failure( + 'localSandboxCleanupFailed', + cleanupError, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + } + return validationFailure; + } + + private async createSandbox( + ecosystem: ValidationEcosystem, + ): Promise<{ sandboxId: string } | { error: string }> { + const runLabel = randomUUID(); + const manifestPath = path.join(os.tmpdir(), `cor-local-${runLabel}-${ecosystem}.yaml`); + try { + await createSandboxManifest(this.getManifestPath(ecosystem), manifestPath, runLabel); + await this.aca.run(['sandbox', 'validate', '--file', manifestPath], 60 * 1000); + const created = await this.aca.run([ + 'sandbox', 'apply', + '--file', manifestPath, + '--wait-timeout', '300', + '-o', 'json', + ], 6 * 60 * 1000); + return { sandboxId: readSandboxId(created.stdout) }; + } catch (error) { + const cleanupError = await this.cleanupAfterCreateFailure(error, runLabel); + return { error: [getErrorMessage(error), cleanupError].filter(Boolean).join(' ') }; + } finally { + await fs.rm(manifestPath, { force: true }); + } + } + + private async setupWorkspace( + sandboxId: string, + archivePath: string, + ecosystem: ValidationEcosystem, + commands: LocalRuntimeCommandResult[], + ): Promise { + const toolchain = `${getToolchainCheckCommand(ecosystem)} && command -v curl && command -v setsid`; + const toolchainResult = await this.runCommand( + sandboxId, + 'setup', + `${ecosystem} toolchain`, + toolchain, + '/tmp', + 60 * 1000, + ); + commands.push(toolchainResult); + if (!toolchainResult.success) { + return failure('localToolchainUnavailable', `${ecosystem} local-runtime toolchain is unavailable.`, commands, []); + } + try { + await this.aca.run([ + 'sandbox', 'fs', 'write', + '--id', sandboxId, + '--path', '/tmp/workspace.tar.gz', + '--file', archivePath, + ], 5 * 60 * 1000); + const extractResult = await this.runCommand( + sandboxId, + 'setup', + 'extract workspace', + 'mkdir -p /workspace && tar -xzf /tmp/workspace.tar.gz -C /workspace', + '/tmp', + 5 * 60 * 1000, + ); + commands.push(extractResult); + if (!extractResult.success) { + return failure('localSandboxSetupFailed', 'Could not extract the generated workspace.', commands, []); + } + } catch (error) { + return failure('localSandboxSetupFailed', getErrorMessage(error), commands, []); + } + return undefined; + } + + private async ensureFunctionsCoreTools( + sandboxId: string, + ecosystem: ValidationEcosystem, + commands: LocalRuntimeCommandResult[], + ): Promise { + const check = await this.runCommand( + sandboxId, + 'setup', + 'Azure Functions Core Tools', + 'func --version', + '/workspace', + 30 * 1000, + ); + commands.push(check); + if (check.success) { + return undefined; + } + if (ecosystem !== 'node') { + return failure( + 'localToolchainUnavailable', + `Azure Functions Core Tools are not available in the ${ecosystem} validation disk.`, + commands, + [], + ); + } + const install = await this.runCommand( + sandboxId, + 'setup', + 'install Azure Functions Core Tools', + 'npm install --global azure-functions-core-tools@4 --unsafe-perm true', + '/workspace', + 5 * 60 * 1000, + ); + commands.push(install); + if (!install.success) { + return failure('localToolchainUnavailable', 'Could not install Azure Functions Core Tools.', commands, []); + } + return undefined; + } + + private async runTask( + sandboxId: string, + task: VsCodeTask, + serviceRoot: string, + launchedProcesses?: LaunchedProcess[], + process?: { target: LocalAcceptanceProbe['target']; restartable: boolean }, + ): Promise { + const label = typeof task.label === 'string' ? task.label : 'unnamed task'; + const resolved = resolveTaskCommand(task, serviceRoot); + if (!resolved.command) { + return { + kind: 'task', + name: label, + command: '', + success: true, + durationMs: 0, + stdout: 'Dependency-only task.', + stderr: '', + }; + } + if (task.isBackground === true) { + const logPath = processLogPath(label); + const backgroundCommand = `mkdir -p /workspace/.cor-eval || exit 1; nohup setsid sh -lc ${shellQuote(resolved.command)} >${shellQuote(logPath)} 2>&1 & pid=$!; printf '%s\\n' "$pid"`; + const result = await this.runCommand( + sandboxId, + 'task', + label, + backgroundCommand, + resolved.cwd, + 60 * 1000, + ); + if (result.success && launchedProcesses && process) { + try { + launchedProcesses.push({ + pid: parseLaunchedProcessId(result.stdout), + label, + target: process.target, + task, + serviceRoot, + restartable: process.restartable, + }); + } catch (error) { + result.success = false; + result.stderr = getErrorMessage(error); + } + } + return result; + } + return await this.runCommand( + sandboxId, + 'task', + label, + resolved.command, + resolved.cwd, + 10 * 60 * 1000, + ); + } + + private async runProbe( + sandboxId: string, + probe: LocalAcceptanceProbe, + startupTimeoutSeconds: number, + launchTask: string, + ): Promise<{ probe: LocalRuntimeProbeResult; command: LocalRuntimeCommandResult }> { + const started = Date.now(); + if (probe.processPattern && !probe.url) { + const attempts = Math.max(1, startupTimeoutSeconds); + const command = [ + `for i in $(seq 1 ${attempts}); do`, + `if pgrep -f -- ${shellQuote(probe.processPattern)} >/dev/null; then exit 0; fi;`, + 'sleep 1;', + 'done;', + `printf '%s\\n' ${shellQuote(`Timed out waiting for process pattern ${probe.processPattern}; launch task: ${launchTask}`)} >&2;`, + 'exit 1', + ].join(' '); + const commandResult = await this.runCommand( + sandboxId, + 'probe', + probe.name, + command, + '/workspace', + (startupTimeoutSeconds + 15) * 1000, + ); + return { + command: commandResult, + probe: { + name: probe.name, + target: probe.target, + processPattern: probe.processPattern, + success: commandResult.success, + durationMs: Date.now() - started, + error: commandResult.success ? undefined : commandResult.stderr, + }, + }; + } + if (!probe.url || !probe.method || probe.expectedStatus === undefined) { + throw new Error(`Probe "${probe.name}" has no complete HTTP or process acceptance contract.`); + } + const command = createHttpProbeCommand(probe, startupTimeoutSeconds, launchTask); + const commandResult = await this.runCommand( + sandboxId, + 'probe', + probe.name, + command, + '/workspace', + (startupTimeoutSeconds + 15) * 1000, + ); + const response = parseHttpProbeEvidence(commandResult.stdout); + return { + command: commandResult, + probe: { + name: probe.name, + target: probe.target, + method: probe.method, + url: probe.url, + expectedStatus: probe.expectedStatus, + success: commandResult.success, + durationMs: Date.now() - started, + response: commandResult.stdout || undefined, + responseStatus: response.status, + responseHeaders: response.headers, + responseBody: response.body, + error: commandResult.success ? undefined : commandResult.stderr, + }, + }; + } + + private async runBrowserProbe( + sandboxId: string, + probe: LocalAcceptanceProbe, + browserOverride?: NonNullable, + urlOverride?: string, + nameOverride?: string, + ): Promise<{ browser: LocalRuntimeBrowserResult; command: LocalRuntimeCommandResult }> { + const started = Date.now(); + const browser = browserOverride ?? probe.browser; + if (!browser) { + throw new Error('Browser acceptance configuration is required.'); + } + const url = urlOverride ?? probe.url; + if (!url) { + throw new Error('Browser acceptance requires a probe URL.'); + } + const script = createBrowserProbeScript(url, browser); + const name = nameOverride ?? probe.name; + const commandResult = await this.runCommand( + sandboxId, + 'probe', + `${name} browser`, + `node -e ${shellQuote(script)}`, + '/workspace', + 2 * 60 * 1000, + ); + let evidence: Partial = {}; + if (commandResult.stdout.trim()) { + try { + evidence = JSON.parse(commandResult.stdout.trim()) as Partial; + } catch (error) { + if (commandResult.success) { + commandResult.success = false; + commandResult.stderr = `Browser evidence was not valid JSON: ${getErrorMessage(error)}`; + } + } + } + return { + command: commandResult, + browser: { + name, + url, + success: commandResult.success, + durationMs: Date.now() - started, + ...evidence, + error: commandResult.success ? undefined : commandResult.stderr, + }, + }; + } + + /** + * Both the compound and single-configuration paths need identical debugger evidence, and the + * gate reads only commands recorded with kind 'debugger'. Sharing one implementation keeps a + * configuration shape from being verified on one path and silently skipped on the other. + */ + private async runDebuggerPrerequisites( + sandboxId: string, + launch: LaunchConfiguration, + commands: LocalRuntimeCommandResult[], + probes: LocalRuntimeProbeResult[], + browserChecks: LocalRuntimeBrowserResult[], + persistenceChecks: LocalRuntimePersistenceResult[], + workerEvents: LocalRuntimeStorageEventResult[], + ): Promise { + const resolved = resolveDebuggerPrerequisite(launch); + if ('error' in resolved) { + return failure( + 'debugTaskGraphInvalid', + resolved.error, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + } + for (const check of resolved.checks) { + const result = await this.runCommand( + sandboxId, + 'debugger', + check.name, + check.command, + '/workspace', + debuggerPrerequisiteTimeoutMs, + ); + commands.push(result); + if (!result.success) { + return failure( + 'localDebuggerUnavailable', + check.errorMessage, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + ); + } + } + return undefined; + } + + private async runPersistenceCheck( + sandboxId: string, + probe: LocalAcceptanceProbe, + acceptanceProbes: LocalAcceptanceProbe[], + launchedProcesses: LaunchedProcess[], + startupTimeoutSeconds: number, + commands: LocalRuntimeCommandResult[], + initialBrowser: LocalRuntimeBrowserResult | undefined, + ): Promise { + const started = Date.now(); + const contract = probe.browser?.persistence; + if (!contract) { + throw new Error('Persistence acceptance configuration is required.'); + } + if (initialBrowser?.journeyStatus === 'failed' || initialBrowser?.journeyStatus === 'not-attempted') { + const reason = `The browser journey did not complete (${initialBrowser.journeyError ?? initialBrowser.journeyStatus}), ` + + 'so there is no created record whose survival a restart could verify.'; + return { + name: probe.name, + restartTargets: contract.restartTargets, + processIdsBefore: [], + processIdsAfter: [], + readinessProbes: [], + postRestartBrowser: { + name: `${probe.name} after restart`, + url: probe.url ?? '', + success: false, + durationMs: 0, + }, + success: false, + skipped: true, + skipReason: reason, + durationMs: Date.now() - started, + error: reason, + }; + } + const selected = launchedProcesses.filter(process => + process.restartable && contract.restartTargets.includes(process.target as 'backend' | 'frontend')); + const preserved = launchedProcesses.filter(process => !selected.includes(process)); + const missingTarget = contract.restartTargets.find(target => + !selected.some(process => process.target === target)); + const emptyBrowser: LocalRuntimeBrowserResult = { + name: `${probe.name} after restart`, + url: probe.url ?? '', + success: false, + durationMs: 0, + error: missingTarget ? `No evaluator-launched process was recorded for ${missingTarget}.` : undefined, + }; + if (missingTarget) { + return { + name: probe.name, + restartTargets: contract.restartTargets, + processIdsBefore: selected.map(process => process.pid), + processIdsAfter: [], + preservedProcessIds: preserved.map(process => process.pid), + readinessProbes: [], + postRestartBrowser: emptyBrowser, + success: false, + durationMs: Date.now() - started, + error: emptyBrowser.error, + }; + } + + const processIdsBefore = selected.map(process => process.pid); + for (const process of selected) { + const termination = await this.runCommand( + sandboxId, + 'restart', + `stop ${process.label}`, + createProcessGroupTerminationCommand(process.pid), + '/workspace', + 30 * 1000, + ); + commands.push(termination); + if (!termination.success) { + return { + name: probe.name, + restartTargets: contract.restartTargets, + processIdsBefore, + processIdsAfter: [], + readinessProbes: [], + postRestartBrowser: emptyBrowser, + success: false, + durationMs: Date.now() - started, + error: `Could not stop evaluator-launched process group ${process.pid}: ${termination.stderr}`, + }; + } + } + for (const process of preserved) { + const liveness = await this.runCommand( + sandboxId, + 'restart', + `verify preserved ${process.label}`, + `/bin/kill -0 -- -${process.pid}`, + '/workspace', + 10 * 1000, + ); + commands.push(liveness); + if (!liveness.success) { + return { + name: probe.name, + restartTargets: contract.restartTargets, + processIdsBefore, + processIdsAfter: [], + preservedProcessIds: preserved.map(value => value.pid), + readinessProbes: [], + postRestartBrowser: emptyBrowser, + success: false, + durationMs: Date.now() - started, + error: `Evaluator-launched dependency process group ${process.pid} did not survive the application restart.`, + }; + } + } + for (const process of selected) { + const index = launchedProcesses.indexOf(process); + if (index >= 0) { + launchedProcesses.splice(index, 1); + } + } + + const restarted: LaunchedProcess[] = []; + for (const process of selected) { + const restart = await this.runTask( + sandboxId, + process.task, + process.serviceRoot, + restarted, + { target: process.target, restartable: true }, + ); + restart.kind = 'restart'; + restart.name = `restart ${process.label}`; + commands.push(restart); + if (!restart.success) { + return { + name: probe.name, + restartTargets: contract.restartTargets, + processIdsBefore, + processIdsAfter: restarted.map(value => value.pid), + preservedProcessIds: preserved.map(value => value.pid), + readinessProbes: [], + postRestartBrowser: emptyBrowser, + success: false, + durationMs: Date.now() - started, + error: `Could not restart "${process.label}": ${restart.stderr}`, + }; + } + } + launchedProcesses.push(...restarted); + + const readinessProbes: LocalRuntimeProbeResult[] = []; + for (const readinessProbe of acceptanceProbes.filter(value => + contract.restartTargets.includes(value.target as 'backend' | 'frontend'))) { + const result = await this.runProbe( + sandboxId, + readinessProbe, + startupTimeoutSeconds, + 'post-restart readiness', + ); + result.command.kind = 'restart'; + commands.push(result.command); + readinessProbes.push(result.probe); + if (!result.probe.success) { + return { + name: probe.name, + restartTargets: contract.restartTargets, + processIdsBefore, + processIdsAfter: restarted.map(value => value.pid), + preservedProcessIds: preserved.map(value => value.pid), + readinessProbes, + postRestartBrowser: emptyBrowser, + success: false, + durationMs: Date.now() - started, + error: `Post-restart readiness probe "${readinessProbe.name}" failed.`, + }; + } + } + + const reloadUrl = contract.reload === 'current-url' ? initialBrowser?.currentUrl : contract.reload; + if (!reloadUrl) { + return { + name: probe.name, + restartTargets: contract.restartTargets, + processIdsBefore, + processIdsAfter: restarted.map(value => value.pid), + preservedProcessIds: preserved.map(value => value.pid), + readinessProbes, + postRestartBrowser: emptyBrowser, + success: false, + durationMs: Date.now() - started, + error: 'The initial browser journey did not record a current URL for persistence reload.', + }; + } + const postRestart = await this.runBrowserProbe( + sandboxId, + probe, + { + ...probe.browser, + actions: [], + assertions: contract.assertions, + persistence: undefined, + }, + reloadUrl, + `${probe.name} after restart`, + ); + postRestart.command.kind = 'restart'; + commands.push(postRestart.command); + return { + name: probe.name, + restartTargets: contract.restartTargets, + processIdsBefore, + processIdsAfter: restarted.map(value => value.pid), + preservedProcessIds: preserved.map(value => value.pid), + readinessProbes, + postRestartBrowser: postRestart.browser, + success: postRestart.browser.success, + durationMs: Date.now() - started, + error: postRestart.browser.error, + }; + } + + /** + * The gate is a hard release blocker, so a contract that cannot produce conclusive evidence + * fails rather than passing on a technicality. + */ + private async runSecurityChecks( + sandboxId: string, + securityContract: SecurityContract | undefined, + acceptanceProbes: LocalAcceptanceProbe[], + commands: LocalRuntimeCommandResult[], + probes: LocalRuntimeProbeResult[], + browserChecks: LocalRuntimeBrowserResult[], + persistenceChecks: LocalRuntimePersistenceResult[], + workerEvents: LocalRuntimeStorageEventResult[], + securityChecks: LocalRuntimeSecurityResult[], + ): Promise { + if (!securityContract) { + return undefined; + } + const plan = planSecurityChecks(securityContract, acceptanceProbes); + const fail = (error: string): SandboxLocalRuntimeValidationResult => failure( + 'localSecurityFailed', + error, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + securityChecks, + ); + if (!isSecurityPlanConclusive(plan)) { + return fail( + 'The security contract produced no conclusive checks: it needs at least one public ' + + 'path and one protected path.', + ); + } + for (const check of plan) { + const { result, command } = await this.runSecurityCheck(sandboxId, check); + commands.push(command); + securityChecks.push(result); + if (!result.success) { + return fail(`Security check "${check.name}" failed. ${result.error ?? ''}`.trim()); + } + } + return undefined; + } + + /** + * Runs one negative-authorization request. The app is already known to be up by this point, + * so a single attempt is enough and a non-answer is itself a failure. + */ + private async runSecurityCheck( + sandboxId: string, + check: SecurityCheckPlan, + ): Promise<{ result: LocalRuntimeSecurityResult; command: LocalRuntimeCommandResult }> { + const started = Date.now(); + const bodyPath = '/tmp/cor-security-body'; + const headerArguments = Object.entries(check.headers ?? {}) + .map(([name, value]) => `--header ${shellQuote(`${name}: ${value}`)}`) + .join(' '); + const script = [ + `code=$(curl --silent --show-error --output ${shellQuote(bodyPath)}`, + `--write-out '%{http_code}' --max-time 20 ${headerArguments} ${shellQuote(check.url)} || true);`, + `printf 'COR_STATUS:%s\\nCOR_HEADERS_BEGIN\\n\\nCOR_BODY_BEGIN\\n' "$code";`, + `cat ${shellQuote(bodyPath)} 2>/dev/null || true`, + ].join(' '); + const commandResult = await this.runCommand( + sandboxId, + 'security', + check.name, + script, + '/workspace', + 45000, + ); + const evidence = parseHttpProbeEvidence(commandResult.stdout); + const success = evidence.status !== undefined + && check.expectedStatuses.includes(evidence.status); + // A refusal body is uninteresting; a leak is the whole finding, so only failures keep it. + const responseExcerpt = success ? undefined : evidence.body?.slice(0, 500); + return { + command: { ...commandResult, success }, + result: { + name: check.name, + url: check.url, + kind: check.kind, + expectedStatuses: check.expectedStatuses, + responseStatus: evidence.status, + responseExcerpt, + success, + durationMs: Date.now() - started, + error: success + ? undefined + : evidence.status === undefined + ? `No HTTP response from ${check.url}.` + : `Expected ${check.expectedStatuses.join(' or ')} but got ${evidence.status}.`, + }, + }; + } + + private async runStorageEvent( + sandboxId: string, + contract: StorageEventContract, + ): Promise<{ event: LocalRuntimeStorageEventResult; command: LocalRuntimeCommandResult }> { + const started = Date.now(); + const script = contract.kind === 'queue' + ? createQueueStorageEventScript(contract) + : createBlobStorageEventScript(contract); + const commandResult = await this.runCommand( + sandboxId, + 'storage-event', + contract.name, + `node -e ${shellQuote(script)}`, + '/workspace', + ((contract.timeoutSeconds ?? 90) + 15) * 1000, + ); + let evidence: { + observedMessage?: unknown; + observedContent?: string; + sourceDeleted?: boolean; + pollAttempts?: number; + error?: string; + } = {}; + if (commandResult.stdout.trim()) { + try { + evidence = JSON.parse(commandResult.stdout.trim()) as typeof evidence; + } catch (error) { + commandResult.success = false; + commandResult.stderr = `Storage event evidence was not valid JSON: ${getErrorMessage(error)}`; + } + } + return { + command: commandResult, + event: { + name: contract.name, + kind: contract.kind, + inputQueue: contract.kind === 'queue' ? contract.inputQueue : undefined, + outputQueue: contract.kind === 'queue' ? contract.outputQueue : undefined, + sourceContainer: contract.kind === 'blob' ? contract.sourceContainer : undefined, + destinationContainer: contract.kind === 'blob' ? contract.destinationContainer : undefined, + blobName: contract.kind === 'blob' ? contract.blobName : undefined, + stimulus: contract.kind === 'queue' + ? contract.message + : { content: contract.content, metadata: contract.metadata }, + expectedMessageIncludes: contract.kind === 'queue' ? contract.expectedMessageIncludes : undefined, + observedMessage: evidence.observedMessage, + observedContent: evidence.observedContent, + sourceDeleted: evidence.sourceDeleted, + pollAttempts: evidence.pollAttempts, + success: commandResult.success, + durationMs: Date.now() - started, + error: commandResult.success ? undefined : evidence.error ?? commandResult.stderr, + }, + }; + } + + private async readProcessLogs(sandboxId: string, launchTask: string): Promise { + return await this.runCommand( + sandboxId, + 'diagnostic', + `${launchTask} logs`, + `cat ${shellQuote(processLogPath(launchTask))} 2>/dev/null || true`, + '/workspace', + 30 * 1000, + ); + } + + private async runCommand( + sandboxId: string, + kind: LocalRuntimeCommandResult['kind'], + name: string, + command: string, + workingDirectory: string, + timeoutMs: number, + ): Promise { + const started = Date.now(); + try { + const value = await this.aca.run([ + 'sandbox', 'exec', + '--id', sandboxId, + '--working-directory', workingDirectory, + '-c', command, + ], timeoutMs); + return { + kind, + name, + command, + success: true, + durationMs: Date.now() - started, + stdout: truncate(value.stdout), + stderr: truncate(value.stderr), + }; + } catch (error) { + const commandError = error as Error & { stdout?: string; stderr?: string }; + return { + kind, + name, + command, + success: false, + durationMs: Date.now() - started, + stdout: truncate(commandError.stdout ?? ''), + stderr: truncate(commandError.stderr ?? getErrorMessage(error)), + }; + } + } + + private async cleanupAfterCreateFailure(error: unknown, runLabel: string): Promise { + const commandError = error as Error & { stdout?: string; stderr?: string }; + const sandboxIds = new Set(); + const output = [commandError.stdout, commandError.stderr].filter(Boolean).join('\n'); + try { + sandboxIds.add(readSandboxId(output)); + } catch { + // The apply command may fail before returning an id. + } + try { + const listed = await this.aca.run([ + 'sandbox', 'list', + '-l', `run-id=${runLabel}`, + '-o', 'json', + ], 60 * 1000); + for (const id of readSandboxIds(listed.stdout)) { + sandboxIds.add(id); + } + } catch (listError) { + if (!sandboxIds.size) { + return `Could not recover a created sandbox by label: ${getErrorMessage(listError)}`; + } + } + const errors: string[] = []; + for (const id of sandboxIds) { + try { + await this.aca.run(['sandbox', 'delete', '--id', id, '--yes'], 5 * 60 * 1000); + } catch (deleteError) { + errors.push(`${id}: ${getErrorMessage(deleteError)}`); + } + } + return errors.length ? `Sandbox cleanup failed: ${errors.join('; ')}` : undefined; + } + + private getManifestPath(ecosystem: ValidationEcosystem): string { + switch (ecosystem) { + case 'node': + return path.join(this.repoRoot, 'evals', 'sandbox.yaml'); + case 'python': + return path.join(this.repoRoot, 'evals', 'sandbox-python.yaml'); + case 'dotnet': + return path.join(this.repoRoot, 'evals', 'sandbox-dotnet.yaml'); + } + } +} + +export function parsePlannedConfigurations(content: string): PlannedDebugConfiguration[] { + const plan = parseLocalDebugPlanMarkdown(content); + const section = findSection(plan, 'Debug Configurations'); + const table = section && findTable(section, [ + 'Generate', + 'Debug Config Name', + 'Service Root', + 'Project Type', + 'Runtime', + ]); + if (!table) { + return []; + } + const generateIndex = findColumnIndex(table.headers, 'Generate'); + const nameIndex = findColumnIndex(table.headers, 'Debug Config Name'); + const serviceRootIndex = findColumnIndex(table.headers, 'Service Root'); + const projectTypeIndex = findColumnIndex(table.headers, 'Project Type'); + const runtimeIndex = findColumnIndex(table.headers, 'Runtime'); + return table.rows + .filter(row => isChecked(row[generateIndex] ?? '') && !row.some(cell => /compound\s+config/i.test(cell))) + .flatMap(row => { + const name = row[nameIndex]?.trim(); + const projectType = row[projectTypeIndex]?.trim(); + const runtime = row[runtimeIndex]?.trim(); + if (!name || !projectType || !runtime) { + return []; + } + return [{ + name, + serviceRoot: normalizeServiceRoot(row[serviceRootIndex] ?? '.'), + projectType, + runtime, + }]; + }); +} + +function groupProbesByConfiguration( + probes: LocalAcceptanceProbe[], + configurations: PlannedDebugConfiguration[], +): Map | { error: string } { + const groups = new Map(); + for (const probe of probes) { + const matches = configurations.filter(configuration => targetMatches(probe.target, configuration)); + if (matches.length !== 1) { + return { + error: `Probe "${probe.name}" target "${probe.target}" matched ${matches.length} debug configurations; exactly one is required.`, + }; + } + const configuration = matches[0]; + const group = groups.get(configuration.name) ?? { configuration, probes: [] }; + group.probes.push(probe); + groups.set(configuration.name, group); + } + return groups; +} + +export function targetMatches(target: LocalAcceptanceProbe['target'], configuration: PlannedDebugConfiguration): boolean { + const normalized = `${configuration.name} ${configuration.projectType}`.toLowerCase(); + switch (target) { + case 'frontend': + return normalized.includes('frontend'); + case 'worker': + return normalized.includes('worker') || normalized.includes('background'); + case 'backend': + return !normalized.includes('frontend') && !normalized.includes('worker') && !normalized.includes('background'); + } +} + +async function readDebugArtifacts( + workspace: string, +): Promise<{ launchConfigurations: LaunchConfiguration[]; tasks: VsCodeTask[] } | { error: string }> { + try { + const [launchContent, tasksContent] = await Promise.all([ + fs.readFile(path.join(workspace, '.vscode', 'launch.json'), 'utf8'), + fs.readFile(path.join(workspace, '.vscode', 'tasks.json'), 'utf8'), + ]); + const launch = parse(launchContent) as { configurations?: unknown }; + const tasks = parse(tasksContent) as { tasks?: unknown }; + if (!Array.isArray(launch?.configurations) || !Array.isArray(tasks?.tasks)) { + return { error: 'launch.json and tasks.json must contain configuration and task arrays.' }; + } + return { + launchConfigurations: launch.configurations as LaunchConfiguration[], + tasks: tasks.tasks as VsCodeTask[], + }; + } catch (error) { + return { error: `Could not read generated debug artifacts: ${getErrorMessage(error)}` }; + } +} + +function getDebugCheck(probe: LocalAcceptanceProbe): string { + if (!probe.debugPort) { + return ''; + } + if (probe.debugProtocol === 'cdp') { + return ` && curl --silent --show-error --fail http://127.0.0.1:${probe.debugPort}/json/list >/dev/null`; + } + return ` && bash -c 'exec 3<>/dev/tcp/127.0.0.1/${probe.debugPort}'`; +} + +export function createHttpProbeCommand( + probe: LocalAcceptanceProbe, + startupTimeoutSeconds: number, + launchTask: string, + ): string { + if (!probe.url || !probe.method || probe.expectedStatus === undefined) { + throw new Error(`Probe "${probe.name}" has no complete HTTP acceptance contract.`); + } + const id = randomUUID(); + const directory = '/workspace/.cor-eval'; + const responseBodyPath = `${directory}/http-${id}.body`; + const responseHeadersPath = `${directory}/http-${id}.headers`; + const requestBodyPath = `${directory}/http-${id}.request`; + const serializedBody = probe.body === undefined + ? undefined + : typeof probe.body === 'string' ? probe.body : JSON.stringify(probe.body); + const bodySetup = serializedBody === undefined + ? '' + : `printf %s ${shellQuote(Buffer.from(serializedBody, 'utf8').toString('base64'))} | base64 --decode > ${shellQuote(requestBodyPath)};`; + const headerArguments = Object.entries(probe.headers ?? {}) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, value]) => `--header ${shellQuote(`${name}: ${value}`)}`) + .join(' '); + const requestBodyArgument = serializedBody === undefined ? '' : `--data-binary @${shellQuote(requestBodyPath)}`; + const bodyCheck = probe.bodyIncludes + ? `grep -F -- ${shellQuote(probe.bodyIncludes)} ${shellQuote(responseBodyPath)} >/dev/null || exit 1;` + : ''; + const debugCheck = getDebugCheck(probe).replace(/^ && /, ''); + const evidence = [ + `printf 'COR_STATUS:%s\\nCOR_HEADERS_BEGIN\\n' "$code";`, + `cat ${shellQuote(responseHeadersPath)} 2>/dev/null || true;`, + `printf '\\nCOR_BODY_BEGIN\\n';`, + `cat ${shellQuote(responseBodyPath)} 2>/dev/null || true;`, + ].join(' '); + const attempts = Math.max(1, startupTimeoutSeconds); + return [ + `mkdir -p ${shellQuote(directory)};`, + bodySetup, + `for i in $(seq 1 ${attempts}); do`, + `code=$(curl --silent --show-error --output ${shellQuote(responseBodyPath)} --dump-header ${shellQuote(responseHeadersPath)} --write-out '%{http_code}' --request ${probe.method} ${headerArguments} ${requestBodyArgument} ${shellQuote(probe.url)} || true);`, + `if [ "$code" = "${probe.expectedStatus}" ]; then ${evidence} ${bodyCheck} ${debugCheck ? `${debugCheck} || exit 1;` : ''} exit 0; fi;`, + 'sleep 1;', + 'done;', + evidence, + `printf '%s\\n' ${shellQuote(`Timed out waiting for ${probe.url}; launch task: ${launchTask}`)} >&2;`, + 'exit 1', + ].filter(Boolean).join(' '); + } + + function parseHttpProbeEvidence(output: string): { status?: number; headers?: string; body?: string } { + const match = /^COR_STATUS:(\d+)\nCOR_HEADERS_BEGIN\n([\s\S]*?)\nCOR_BODY_BEGIN\n([\s\S]*)$/.exec(output); + if (!match) { + return {}; + } + return { + status: Number(match[1]), + headers: match[2], + body: match[3], + }; + } + + export function parseLaunchedProcessId(output: string): number { + const trimmed = output.trim(); + if (!/^[1-9]\d*$/.test(trimmed)) { + throw new Error(`Background launch did not return one process id: ${trimmed || ''}`); + } + const pid = Number(trimmed); + if (!Number.isSafeInteger(pid) || pid <= 1) { + throw new Error(`Background launch returned an invalid process id: ${trimmed}`); + } + return pid; + } + + export function createProcessGroupTerminationCommand(pid: number): string { + if (!Number.isSafeInteger(pid) || pid <= 1) { + throw new Error(`Refusing to terminate invalid process group id: ${pid}`); + } + return [ + `/bin/kill -TERM -- -${pid} || exit 1`, + `for i in $(seq 1 20); do if ! /bin/kill -0 -- -${pid} 2>/dev/null; then exit 0; fi; sleep 0.25; done`, + `/bin/kill -KILL -- -${pid}`, + ].join('; '); + } + + const azuriteDevelopmentAccount = 'devstoreaccount1'; + // Azurite's well-known development account key. This is a published constant, not a secret: + // https://learn.microsoft.com/azure/storage/common/storage-use-azurite#well-known-storage-account-and-key + // It must match byte-for-byte or Azurite rejects every request with 403 AuthenticationFailed, + // which silently fails the storage-event probe no matter how correct the generated app is. + // Pinned by azuriteDevelopmentKey.test.ts. + const azuriteDevelopmentKey = + 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='; + const storageApiVersion = '2023-11-03'; + + export function createAzureStorageSharedKeyAuthorization( + method: string, + pathname: string, + query: Record, + headers: Record, + ): string { + const normalizedHeaders = Object.fromEntries( + Object.entries(headers).map(([name, value]) => [name.toLowerCase(), value.trim()]), + ); + const canonicalHeaders = Object.entries(normalizedHeaders) + .filter(([name]) => name.startsWith('x-ms-')) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, value]) => `${name}:${value}\n`) + .join(''); + const canonicalQuery = Object.entries(query) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, value]) => `\n${name.toLowerCase()}:${value}`) + .join(''); + const contentLength = normalizedHeaders['content-length']; + const stringToSign = [ + method, + normalizedHeaders['content-encoding'] ?? '', + normalizedHeaders['content-language'] ?? '', + contentLength === '0' ? '' : contentLength ?? '', + normalizedHeaders['content-md5'] ?? '', + normalizedHeaders['content-type'] ?? '', + '', + normalizedHeaders['if-modified-since'] ?? '', + normalizedHeaders['if-match'] ?? '', + normalizedHeaders['if-none-match'] ?? '', + normalizedHeaders['if-unmodified-since'] ?? '', + normalizedHeaders.range ?? '', + `${canonicalHeaders}/${azuriteDevelopmentAccount}${pathname}${canonicalQuery}`, + ].join('\n'); + const signature = createHmac('sha256', Buffer.from(azuriteDevelopmentKey, 'base64')) + .update(stringToSign, 'utf8') + .digest('base64'); + return `SharedKey ${azuriteDevelopmentAccount}:${signature}`; + } + + /* eslint-disable no-template-curly-in-string -- Generated JavaScript contains its own template literals. */ + export function createQueueStorageEventScript(contract: StorageQueueEventContract): string { + const timeoutMs = (contract.timeoutSeconds ?? 90) * 1000; + return [ + "const http = require('http');", + "const crypto = require('crypto');", + `const account = ${JSON.stringify(azuriteDevelopmentAccount)};`, + `const accountKey = ${JSON.stringify(azuriteDevelopmentKey)};`, + `const version = ${JSON.stringify(storageApiVersion)};`, + `const inputQueue = ${JSON.stringify(contract.inputQueue)};`, + `const outputQueue = ${JSON.stringify(contract.outputQueue)};`, + `const stimulus = ${JSON.stringify(contract.message)};`, + `const expected = ${JSON.stringify(contract.expectedMessageIncludes)};`, + `const deadline = Date.now() + ${timeoutMs};`, + 'let pollAttempts = 0;', + 'const subset = (actual, wanted) => wanted !== null && typeof wanted === "object" && !Array.isArray(wanted)', + ' ? actual !== null && typeof actual === "object" && !Array.isArray(actual) && Object.entries(wanted).every(([key, value]) => subset(actual[key], value))', + ' : Array.isArray(wanted) ? Array.isArray(actual) && wanted.length === actual.length && wanted.every((value, index) => subset(actual[index], value)) : Object.is(actual, wanted);', + 'const decodeXml = value => value.replace(/"/g, \'"\').replace(/'/g, "\'").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&");', + 'const parseMessage = value => {', + ' const decoded = decodeXml(value);', + ' const candidates = [decoded];', + ' try { candidates.push(Buffer.from(decoded, "base64").toString("utf8")); } catch {}', + ' for (const candidate of candidates) { try { return JSON.parse(candidate); } catch {} }', + ' return decoded;', + '};', + 'const request = (method, queue, query = {}, body = "") => new Promise((resolve, reject) => {', + ' const search = new URLSearchParams(query).toString();', + ' const pathname = `/${account}/${queue}${method === "GET" && query.messageid ? `/messages/${query.messageid}` : ""}`;', + ' const date = new Date().toUTCString();', + ' const headers = { "x-ms-date": date, "x-ms-version": version, "content-length": String(Buffer.byteLength(body)) };', + ' if (body) headers["content-type"] = "application/xml";', + ' const canonicalHeaders = Object.entries(headers).filter(([name]) => name.startsWith("x-ms-")).sort().map(([name, value]) => `${name}:${value}\\n`).join("");', + ' const canonicalQuery = Object.entries(query).filter(([name]) => name !== "messageid").sort().map(([name, value]) => `\\n${name.toLowerCase()}:${value}`).join("");', + ' const stringToSign = [method, "", "", body.length ? headers["content-length"] : "", "", headers["content-type"] || "", "", "", "", "", "", "", `${canonicalHeaders}/${account}${pathname}${canonicalQuery}`].join("\\n");', + ' headers.Authorization = `SharedKey ${account}:` + crypto.createHmac("sha256", Buffer.from(accountKey, "base64")).update(stringToSign).digest("base64");', + ' const req = http.request({ hostname: "127.0.0.1", port: 10001, method, path: pathname + (search ? `?${search}` : ""), headers }, response => {', + ' const chunks = []; response.on("data", chunk => chunks.push(chunk)); response.on("end", () => resolve({ status: response.statusCode, body: Buffer.concat(chunks).toString("utf8") }));', + ' });', + ' req.on("error", reject); if (body) req.write(body); req.end();', + '});', + '(async () => {', + ' for (const queue of [inputQueue, outputQueue]) {', + ' const created = await request("PUT", queue, { restype: "queue" });', + ' if (![201, 204, 409].includes(created.status)) throw new Error(`Could not create queue ${queue}: ${created.status} ${created.body}`);', + ' }', + ' const encoded = Buffer.from(JSON.stringify(stimulus)).toString("base64");', + ' const sent = await request("POST", `${inputQueue}/messages`, {}, `${encoded}`);', + ' if (sent.status !== 201) throw new Error(`Could not enqueue stimulus: ${sent.status} ${sent.body}`);', + ' while (Date.now() < deadline) {', + ' pollAttempts++;', + ' const result = await request("GET", `${outputQueue}/messages`, { numofmessages: "1", visibilitytimeout: "1" });', + ' if (result.status !== 200) throw new Error(`Could not poll output queue: ${result.status} ${result.body}`);', + ' const match = /([\\s\\S]*?)<\\/MessageText>/.exec(result.body);', + ' if (match) { const observedMessage = parseMessage(match[1]); if (subset(observedMessage, expected)) { process.stdout.write(JSON.stringify({ observedMessage, pollAttempts })); return; } }', + ' await new Promise(resolve => setTimeout(resolve, 1000));', + ' }', + ' throw new Error(`Timed out waiting for matching output after ${pollAttempts} polls.`);', + '})().catch(error => { const message = error instanceof Error ? error.message : String(error); process.stdout.write(JSON.stringify({ pollAttempts, error: message })); console.error(message); process.exit(1); });', + ].join('\n'); + } + + export function createBlobStorageEventScript(contract: StorageBlobEventContract): string { + const timeoutMs = (contract.timeoutSeconds ?? 120) * 1000; + return [ + "const http = require('http');", + "const crypto = require('crypto');", + `const account = ${JSON.stringify(azuriteDevelopmentAccount)};`, + `const accountKey = ${JSON.stringify(azuriteDevelopmentKey)};`, + `const version = ${JSON.stringify(storageApiVersion)};`, + `const sourceContainer = ${JSON.stringify(contract.sourceContainer)};`, + `const destinationContainer = ${JSON.stringify(contract.destinationContainer)};`, + `const blobName = ${JSON.stringify(contract.blobName)};`, + `const content = ${JSON.stringify(contract.content)};`, + `const metadata = ${JSON.stringify(contract.metadata)};`, + `const sourceMustBeDeleted = ${JSON.stringify(contract.sourceMustBeDeleted ?? true)};`, + `const deadline = Date.now() + ${timeoutMs};`, + 'let pollAttempts = 0;', + 'const request = (method, pathname, query = {}, body = "", extraHeaders = {}) => new Promise((resolve, reject) => {', + ' const search = new URLSearchParams(query).toString();', + ' const date = new Date().toUTCString();', + ' const headers = { "x-ms-date": date, "x-ms-version": version, ...extraHeaders };', + ' if (body) headers["content-length"] = String(Buffer.byteLength(body));', + ' const normalized = Object.fromEntries(Object.entries(headers).map(([name, value]) => [name.toLowerCase(), String(value).trim()]));', + ' const canonicalHeaders = Object.entries(normalized).filter(([name]) => name.startsWith("x-ms-")).sort(([left], [right]) => left.localeCompare(right)).map(([name, value]) => `${name}:${value}\\n`).join("");', + ' const canonicalQuery = Object.entries(query).sort(([left], [right]) => left.localeCompare(right)).map(([name, value]) => `\\n${name.toLowerCase()}:${value}`).join("");', + ' const contentLength = normalized["content-length"];', + ' const stringToSign = [method, "", "", contentLength === "0" ? "" : contentLength || "", "", normalized["content-type"] || "", "", "", "", "", "", normalized.range || "", `${canonicalHeaders}/${account}${pathname}${canonicalQuery}`].join("\\n");', + ' headers.Authorization = `SharedKey ${account}:` + crypto.createHmac("sha256", Buffer.from(accountKey, "base64")).update(stringToSign).digest("base64");', + ' const req = http.request({ hostname: "127.0.0.1", port: 10000, method, path: pathname + (search ? `?${search}` : ""), headers }, response => {', + ' const chunks = []; response.on("data", chunk => chunks.push(chunk)); response.on("end", () => resolve({ status: response.statusCode, body: Buffer.concat(chunks).toString("utf8") }));', + ' });', + ' req.on("error", reject); if (body) req.write(body); req.end();', + '});', + '(async () => {', + ' for (const container of [sourceContainer, destinationContainer]) {', + ' const created = await request("PUT", `/${account}/${container}`, { restype: "container" });', + ' if (![201, 409].includes(created.status)) throw new Error(`Could not create container ${container}: ${created.status} ${created.body}`);', + ' }', + ' const metadataHeaders = Object.fromEntries(Object.entries(metadata).map(([name, value]) => [`x-ms-meta-${name}`, value]));', + ' const sourcePath = `/${account}/${sourceContainer}/${blobName.split("/").map(encodeURIComponent).join("/")}`;', + ' const destinationPath = `/${account}/${destinationContainer}/${blobName.split("/").map(encodeURIComponent).join("/")}`;', + ' const uploaded = await request("PUT", sourcePath, {}, content, { "content-type": "application/octet-stream", "x-ms-blob-type": "BlockBlob", ...metadataHeaders });', + ' if (uploaded.status !== 201) throw new Error(`Could not upload stimulus blob: ${uploaded.status} ${uploaded.body}`);', + ' while (Date.now() < deadline) {', + ' pollAttempts++;', + ' const destination = await request("GET", destinationPath);', + ' const source = await request("HEAD", sourcePath);', + ' const sourceDeleted = source.status === 404;', + ' if (destination.status === 200 && destination.body === content && (!sourceMustBeDeleted || sourceDeleted)) {', + ' process.stdout.write(JSON.stringify({ observedContent: destination.body, sourceDeleted, pollAttempts }));', + ' return;', + ' }', + ' if (![200, 404].includes(destination.status)) throw new Error(`Could not poll destination blob: ${destination.status} ${destination.body}`);', + ' if (![200, 404].includes(source.status)) throw new Error(`Could not inspect source blob: ${source.status} ${source.body}`);', + ' await new Promise(resolve => setTimeout(resolve, 1000));', + ' }', + ' throw new Error(`Timed out waiting for archived blob after ${pollAttempts} polls.`);', + '})().catch(error => { const message = error instanceof Error ? error.message : String(error); process.stdout.write(JSON.stringify({ pollAttempts, error: message })); console.error(message); process.exit(1); });', + ].join('\n'); + } + /* eslint-enable no-template-curly-in-string */ + + export function createBrowserProbeScript( + url: string, + contract: NonNullable, +): string { + const expectedText = contract.expectedText?.toLowerCase(); + const requireInteractive = contract.requireInteractiveElements ?? true; + const maxViolations = contract.maxSeriousAccessibilityViolations === null + ? null + : contract.maxSeriousAccessibilityViolations ?? 0; + const viewport = contract.viewport ?? { width: 1440, height: 900 }; + const journeySeverity = contract.journeySeverity ?? 'required'; + const actions = contract.actions ?? []; + const assertions = contract.assertions ?? []; + return [ + "const { chromium } = require('/home/vscode/.cor-browser/node_modules/playwright');", + "const axe = require('/home/vscode/.cor-browser/node_modules/axe-core');", + '(async () => {', + 'const consoleErrors = [];', + 'let actionsCompleted = 0;', + 'let actionsEffective = 0;', + 'const actionLedger = [];', + 'let assertionsCompleted = 0;', + 'const actionsSkipped = [];', + 'const adaptedTargets = [];', + 'const ambiguousTargets = [];', + // A role+name matching several elements is a legitimate UI (a header CTA and a form submit + // button can share a label). Playwright strict mode rejects it, which failed real, working + // apps. Disambiguate to the first visible+enabled match and record it as evidence; the + // assertions that follow stay strict, so this cannot mask a broken flow. + // A generated app is free to label an empty-state CTA differently from the populated-state + // one ("Create first ticket" vs "Create ticket"), and the evaluator always boots against an + // empty database because seed data is forbidden. Demanding the populated-state label makes + // every CRUD app fail a working create flow, so an absent target falls back to the + // intent-equivalent control instead of waiting out the click timeout. + `const findIntentEquivalent = async (role, name) => { + const stop = new Set(['a', 'an', 'the', 'new', 'first', 'my', 'this', 'to', 'add']); + const words = String(name || '').toLowerCase().split(/[^a-z0-9]+/) + .filter(word => word && !stop.has(word)); + if (!words.length) { return null; } + // The prompt never says whether a create affordance is a button or a link, and a router + // link renders as an anchor. Searching only the requested role made a working app fail + // at the first action, so the search widens across the activatable roles. The requested + // role is tried first, so an exact match always wins over a widened one. + const requested = role || 'button'; + const roles = [requested, ...['button', 'link', 'menuitem', 'tab'].filter(value => value !== requested)]; + let best = null; + let bestScore = 0; + let bestName = ''; + let bestRole = requested; + for (const candidateRole of roles) { + const candidates = page.getByRole(candidateRole); + const count = await candidates.count().catch(() => 0); + for (let index = 0; index < count; index++) { + const candidate = candidates.nth(index); + const usable = await candidate.isVisible().catch(() => false) + && await candidate.isEnabled().catch(() => false); + if (!usable) { continue; } + const text = ((await candidate.textContent().catch(() => '')) || '').toLowerCase(); + const matched = words.filter(word => text.includes(word)).length; + // Require every meaningful word so "Create ticket" never resolves to "Delete ticket". + if (matched !== words.length) { continue; } + // Prefer the tightest label, so "Create ticket" beats "Create ticket from template". + const score = 1000 - text.trim().length; + if (score > bestScore) { bestScore = score; best = candidate; bestName = text.trim(); bestRole = candidateRole; } + } + // A widened role is a fallback, not a preference: stop as soon as one role matched. + if (best) { break; } + } + if (best) { adaptedTargets.push({ requested: name, requestedRole: requested, resolved: bestName, resolvedRole: bestRole }); } + return best; + };`, + // Once a form has been filled, a "click create/submit" action means submit *that* form. The + // app is free to label its submit differently from the CTA that opened it ("Create support + // ticket" vs "Create ticket"), so an exact match on a nav control elsewhere on the page is + // not the intended target. + `const preferFilledFormSubmit = async (locator) => { + if (formFieldsFilled.length === 0) { return null; } + const inForm = await locator.first().evaluate(el => Boolean(el.closest('form'))).catch(() => false); + if (inForm) { return null; } + const submit = page.locator('form button[type="submit"], form input[type="submit"]'); + const count = await submit.count().catch(() => 0); + for (let index = 0; index < count; index++) { + const candidate = submit.nth(index); + const usable = await candidate.isVisible().catch(() => false) + && await candidate.isEnabled().catch(() => false); + if (!usable) { continue; } + const text = ((await candidate.textContent().catch(() => '')) || '').trim(); + adaptedTargets.push({ requested: 'submit of filled form', resolved: text }); + return candidate; + } + return null; + };`, + `const resolveClickTarget = async (locator, label, role, name) => { + const count = await locator.count().catch(() => 1); + if (count === 1) { + const preferred = await preferFilledFormSubmit(locator); + if (preferred) { return preferred; } + } + if (count === 0) { + const adapted = await findIntentEquivalent(role, name); + if (adapted) { return adapted; } + return locator; + } + if (count <= 1) { return locator; } + ambiguousTargets.push({ target: label, matches: count }); + let best = null; + let bestScore = -1; + for (let index = 0; index < count; index++) { + const candidate = locator.nth(index); + const usable = await candidate.isVisible().catch(() => false) + && await candidate.isEnabled().catch(() => false); + if (!usable) { continue; } + // Once a form has been filled, the intended target is that form's own submit + // control, not a same-labelled CTA elsewhere on the page (a header "Create ticket" + // button next to the form's "Create ticket" submit is a legitimate design). + const inForm = await candidate.evaluate(el => Boolean(el.closest('form'))).catch(() => false); + const isSubmit = await candidate.evaluate( + el => el.getAttribute('type') === 'submit').catch(() => false); + const score = (formFieldsFilled.length > 0 && inForm ? 4 : 0) + (isSubmit ? 2 : 0) + 1; + if (score > bestScore) { bestScore = score; best = candidate; } + } + return best ?? locator.first(); + };`, + 'const formFieldsFilled = [];', + 'const formFieldsUnsatisfiable = [];', + 'let invalidFields = [];', + // Discover the form at runtime instead of hard-coding a field list. A generated app is free + // to invent required fields the prompt never specified, and that must not read as a defect. + 'const discoverFields = async (scope) => await page.evaluate((scopeSelector) => {', + 'const root = scopeSelector ? document.querySelector(scopeSelector) : document;', + 'if (!root) return [];', + "const controls = Array.from(root.querySelectorAll('input, textarea, select'));", + 'const labelFor = (el) => {', + "let text = el.getAttribute('aria-label') || '';", + "if (!text && el.getAttribute('aria-labelledby')) { const owner = document.getElementById(el.getAttribute('aria-labelledby')); if (owner) text = owner.textContent || ''; }", + "if (!text && el.id) { const tag = document.querySelector('label[for=\"' + CSS.escape(el.id) + '\"]'); if (tag) text = tag.textContent || ''; }", + "if (!text) { const wrapper = el.closest('label'); if (wrapper) text = wrapper.textContent || ''; }", + // Fluent UI and similar wrappers render the label as a sibling rather than a `for` target. + "if (!text) { const field = el.closest('.fui-Field, [class*=\"Field\"]'); if (field) { const tag = field.querySelector('label'); if (tag) text = tag.textContent || ''; } }", + "if (!text) text = el.getAttribute('placeholder') || el.name || '';", + "return text.replace(/\\s+/g, ' ').trim();", + '};', + 'return controls.map((el, index) => {', + "el.setAttribute('data-cor-probe-field', String(index));", + 'const style = globalThis.getComputedStyle(el);', + 'return {', + 'key: index,', + 'label: labelFor(el),', + 'tag: el.tagName.toLowerCase(),', + "type: (el.getAttribute('type') || '').toLowerCase(),", + 'required: el.required || el.getAttribute(\'aria-required\') === \'true\',', + 'disabled: el.disabled,', + 'readOnly: Boolean(el.readOnly),', + "hidden: style.display === 'none' || style.visibility === 'hidden' || el.type === 'hidden',", + "hasValue: Boolean(el.value),", + "pattern: el.getAttribute('pattern') || '',", + "placeholder: el.getAttribute('placeholder') || '',", + "options: el.tagName.toLowerCase() === 'select' ? Array.from(el.options).map(option => option.value).filter(Boolean) : [],", + '};', + '});', + '}, scope ?? null);', + // Synthesize a value the control will actually accept, so discovery does not stall on + // formats the scenario never mentioned. + 'const synthesizeValue = (field) => {', + 'if (field.options.length) return field.options[0];', + 'const hint = (field.label + \' \' + field.placeholder).toLowerCase();', + "if (field.type === 'email' || /e-?mail/.test(hint)) return 'evaluation.user@example.com';", + "if (field.type === 'number' || field.type === 'range') return '1';", + "if (field.type === 'tel' || /phone/.test(hint)) return '5555550123';", + "if (field.type === 'url' || /url|website/.test(hint)) return 'https://example.com';", + "if (field.type === 'date') return '2026-01-15';", + "if (field.type === 'datetime-local') return '2026-01-15T09:00';", + "if (field.type === 'time') return '09:00';", + "if (field.type === 'password') return 'Evaluation!1';", + // A raw identifier cannot be invented so that it matches a real row; record it and try + // anyway, so the report names the reason instead of timing out on an assertion. + "if (/uuid|guid/.test(hint)) return '00000000-0000-4000-8000-000000000000';", + "return 'Evaluation ' + (field.label || 'value');", + '};', + // A generated field may constrain its format. Try to satisfy it before reporting, so only a + // genuinely uncompletable control is recorded against the project. + 'const applyPattern = (field, value) => {', + 'if (!field.pattern) return value;', + "let expression; try { expression = new RegExp('^(?:' + field.pattern + ')$'); } catch { return value; }", + 'if (expression.test(value)) return value;', + "for (const candidate of ['00000000-0000-4000-8000-000000000000', '12345', '1', '2026-01-15', 'evaluation.user@example.com', 'Evaluation']) { if (expression.test(candidate)) return candidate; }", + "for (let length = 1; length <= 16; length++) { const digits = '1'.repeat(length); if (expression.test(digits)) return digits; }", + "formFieldsUnsatisfiable.push(field.label + ' (no value satisfies pattern ' + field.pattern + ')');", + 'return value;', + '};', + 'const fillDiscoveredForm = async (values, scope) => {', + // A click that changes route renders the form asynchronously. Discovering once, immediately, + // finds nothing, fills nothing, and still reports the action as completed - which is how a + // working create flow gets scored as a product failure. Wait for the form to actually exist. + "const fillable = (field) => !field.disabled && !field.hidden", + " && !['checkbox', 'radio', 'file', 'submit', 'button', 'image', 'reset'].includes(field.type);", + 'let fields = [];', + 'const formDeadline = Date.now() + 15000;', + 'while (Date.now() < formDeadline) {', + 'fields = await discoverFields(scope);', + 'if (fields.some(fillable)) break;', + 'await page.waitForTimeout(250);', + '}', + "if (!fields.some(fillable)) { formFieldsUnsatisfiable.push('(no fillable form field appeared within 15s" + + " of the form action)'); }", + 'for (const field of fields) {', + 'if (field.disabled || field.hidden) continue;', + "if (['checkbox', 'radio', 'file', 'submit', 'button', 'image', 'reset'].includes(field.type)) continue;", + 'const match = Object.keys(values).find(key => field.label.toLowerCase().includes(key.toLowerCase()));', + // Fill what the scenario asked for, plus whatever else the app decided to require. + 'if (match === undefined && !field.required) continue;', + 'if (field.readOnly) { if (!field.hasValue) formFieldsUnsatisfiable.push(field.label + \' (read-only)\'); continue; }', + 'const value = applyPattern(field, match === undefined ? synthesizeValue(field) : values[match]);', + "if (/uuid|guid/.test((field.label + ' ' + field.placeholder).toLowerCase()) && match === undefined) formFieldsUnsatisfiable.push(field.label + ' (expects an opaque identifier with no picker)');", + 'const locator = page.locator(\'[data-cor-probe-field="\' + field.key + \'"]\');', + 'try {', + "if (field.tag === 'select') await locator.selectOption(value, { timeout: 5000 });", + 'else await locator.fill(value, { timeout: 5000 });', + "formFieldsFilled.push(field.label + '=' + value);", + "} catch (error) { formFieldsUnsatisfiable.push(field.label + ' (' + (error instanceof Error ? error.message.split('\\n')[0] : String(error)) + ')'); }", + '}', + '};', + // Naming the controls the browser itself rejected turns a generic assertion timeout into a + // precise statement about which field blocked submission. + 'const collectInvalidFields = async () => await page.evaluate(() => Array.from(document.querySelectorAll(\':invalid\')).slice(0, 10).map(el => {', + "const label = el.getAttribute('aria-label') || el.getAttribute('name') || el.getAttribute('placeholder') || el.tagName.toLowerCase();", + "return label + (el.validationMessage ? ': ' + el.validationMessage : '');", + '})).catch(() => []);', + 'let page;', + // An action that leaves the URL and the rendered text untouched did not do what the + // contract asked, even when Playwright reported no error. Clicking a mis-resolved target + // is silent, so effect has to be observed rather than assumed. + "const pageSignature = async () => { try { return page.url() + '|' + (await page.locator('body').innerText()).length; } catch { return 'unavailable'; } };", + 'const recordEffect = async (action, before) => {', + 'const after = await pageSignature();', + "const effective = before === 'unavailable' || after !== before;", + 'if (effective) actionsEffective++;', + "actionLedger.push(effective ? { action, effective: true } : { action, effective: false, reason: 'the page did not change' });", + '};', + 'let accessibilityScanned = false;', + 'let accessibilityScanError;', + 'const scanAccessibility = async () => {', + 'if (!page) return [];', + 'try {', + 'await page.addScriptTag({ content: axe.source });', + // Fluent UI's focus manager (tabster) injects `` + // sentinels that axe reports as `aria-hidden-focus`. They are library internals the + // generated app neither writes nor can remove, so scoring them would fail every Fluent UI + // frontend for a defect it cannot fix. App-authored rules stay at zero tolerance. + "const accessibility = await page.evaluate(async () => await globalThis.axe.run({ exclude: [['[data-tabster-dummy]']] }, { runOnly: { type: 'tag', values: ['wcag2a', 'wcag2aa', 'wcag21aa'] } }));", + 'accessibilityScanned = true;', + "return accessibility.violations.filter(value => value.impact === 'serious' || value.impact === 'critical').map(value => value.id + ':' + value.impact + ' [' + value.nodes.slice(0, 3).flatMap(node => node.target).join(', ') + ']');", + '} catch (error) { accessibilityScanError = error instanceof Error ? error.message : String(error); return []; }', + '};', + 'const browser = await chromium.launch({ headless: true });', + 'let loadPassed = false;', + "let journeyStatus = 'not-attempted';", + 'let journeyError;', + 'try {', + `page = await browser.newPage({ viewport: ${JSON.stringify(viewport)} });`, + "page.on('console', message => { if (message.type() === 'error') consoleErrors.push(message.text()); });", + "page.on('pageerror', error => consoleErrors.push(error.message));", + `const response = await page.goto(${JSON.stringify(url)}, { waitUntil: 'domcontentloaded', timeout: 60000 });`, + "if (!response || !response.ok()) throw new Error('Browser navigation failed with status ' + (response?.status() ?? 'none') + '.');", + "await page.locator('body').waitFor({ state: 'visible', timeout: 15000 });", + // The load contract is evaluated before the journey drives the UI. These assertions are + // entirely app-controlled, so a failure here is always a real defect; running them first + // means a mis-resolved selector can never hide the fact that the app rendered correctly. + "const landingTitle = await page.title();", + "const landingBodyText = (await page.locator('body').innerText()).trim();", + "if (!landingBodyText) throw new Error('Rendered page body is empty.');", + expectedText + ? `if (!landingBodyText.toLowerCase().includes(${JSON.stringify(expectedText)})) throw new Error(${JSON.stringify(`Rendered page does not include expected text "${contract.expectedText}".`)});` + : '', + "const interactiveElements = await page.locator('a[href], button, input, select, textarea').count();", + requireInteractive + ? "if (interactiveElements === 0) throw new Error('Rendered page has no interactive elements.');" + : '', + 'loadPassed = true;', + // Everything below drives a UI whose labels the prompt never specified. Failures are + // recorded as journey evidence; whether they fail the probe is the contract's decision. + 'try {', + ...actions.map(action => { + const label = JSON.stringify(action.kind === 'fillForm' + ? `fillForm ${Object.keys(action.values ?? {}).join(', ')}` + : `${action.kind} ${action.selector ?? ''}`); + if (action.kind === 'fillForm') { + // A form fill that filled nothing is not a completed action. Counting it was how a + // probe reported "3/3 actions completed" with an empty formFieldsFilled list, and + // it pushed the real diagnosis downstream into an opaque assertion timeout. + return `{ const before = formFieldsFilled.length; ` + + `await fillDiscoveredForm(${JSON.stringify(action.values ?? {})}, ` + + `${action.scope ? JSON.stringify(action.scope) : 'null'}); ` + + `if (formFieldsFilled.length === before) { ` + + `actionLedger.push({ action: ${label}, effective: false, reason: 'no form field was filled' }); ` + + `throw new Error('Form fill completed without filling any field.'); } ` + + `actionsCompleted++; actionsEffective++; actionLedger.push({ action: ${label}, effective: true }); }`; + } + const locator = browserLocatorExpression({ ...action, selector: action.selector ?? '' }); + let statement: string; + switch (action.kind) { + case 'click': + statement = `await (await resolveClickTarget(${locator}, ${label}, ${JSON.stringify(action.selectorType === 'role' ? (action.role ?? 'button') : '')}, ${JSON.stringify(action.selectorType === 'role' ? (action.selector ?? '') : '')})).click();`; + break; + case 'fill': + statement = `await ${locator}.fill(${JSON.stringify(action.value ?? '')});`; + break; + case 'select': + statement = `await ${locator}.selectOption(${JSON.stringify(action.value ?? '')});`; + break; + } + const tracked = `{ const before = await pageSignature(); ${statement} ` + + `actionsCompleted++; await recordEffect(${label}, before); }`; + if (!action.optional) { + return tracked; + } + // The prompt leaves this control's shape open, so absence or read-only state is a valid + // design decision rather than a defect. Record the skip instead of failing the probe. + const probe = action.kind === 'click' ? 'isEnabled' : 'isEditable'; + return `if (await ${locator}.${probe}({ timeout: 2000 }).catch(() => false)) { ${tracked} } ` + + `else { actionsSkipped.push(${label}); }`; + }), + actions.some(action => action.kind === 'fillForm') + ? 'invalidFields = await collectInvalidFields();' + : '', + ...assertions.map(assertion => { + const locator = browserLocatorExpression(assertion); + switch (assertion.kind) { + case 'visible': + return `if (!await ${locator}.isVisible()) throw new Error(${JSON.stringify(`Expected ${assertion.selector} to be visible.`)}); assertionsCompleted++;`; + case 'hidden': + return `if (await ${locator}.isVisible()) throw new Error(${JSON.stringify(`Expected ${assertion.selector} to be hidden.`)}); assertionsCompleted++;`; + case 'text': + return `await ${locator}.filter({ hasText: ${JSON.stringify(assertion.value ?? '')} }).waitFor({ state: 'visible', timeout: 15000 }); assertionsCompleted++;`; + case 'value': + return `if ((await ${locator}.inputValue()) !== ${JSON.stringify(assertion.value ?? '')}) throw new Error(${JSON.stringify(`Expected ${assertion.selector} value to equal "${assertion.value ?? ''}".`)}); assertionsCompleted++;`; + } + }), + "journeyStatus = 'passed';", + '} catch (error) {', + "journeyStatus = 'failed';", + "journeyError = error instanceof Error ? error.message.split('\\n')[0] : String(error);", + journeySeverity === 'advisory' + ? "console.error('Browser journey did not complete (advisory): ' + journeyError);" + : 'throw error;', + '}', + // Re-read after the journey so the evidence reflects where the app ended up, while the + // gating decision above stayed pinned to the landing page. + 'const title = await page.title().catch(() => landingTitle);', + "const bodyText = await page.locator('body').innerText().then(value => value.trim()).catch(() => landingBodyText);", + 'const seriousAccessibilityViolations = await scanAccessibility();', + `process.stdout.write(JSON.stringify({ title, currentUrl: page.url(), bodyTextLength: bodyText.length, bodyTextExcerpt: bodyText.slice(0, 2000), interactiveElements, seriousAccessibilityViolations, accessibilityScanned, accessibilityScanError, consoleErrors: consoleErrors.slice(0, 20), loadPassed, journeyStatus, journeyError, journeySeverity: ${JSON.stringify(journeySeverity)}, actionsCompleted, actionsEffective, actionLedger, actionsSkipped, formFieldsFilled, formFieldsUnsatisfiable, invalidFields, ambiguousTargets, adaptedTargets, actionsExpected: ${actions.length}, assertionsCompleted, assertionsExpected: ${assertions.length}, viewport: ${JSON.stringify(viewport)} }));`, + maxViolations === null + ? '' + : `if (seriousAccessibilityViolations.length > ${maxViolations}) { console.error('Accessibility violations exceeded ${maxViolations}: ' + seriousAccessibilityViolations.join(', ')); process.exitCode = 1; }`, + '} catch (error) {', + "const bodyText = page ? await page.locator('body').innerText().catch(() => '') : '';", + "const title = page ? await page.title().catch(() => '') : '';", + 'const seriousAccessibilityViolations = await scanAccessibility();', + `process.stdout.write(JSON.stringify({ title, currentUrl: page?.url(), bodyTextLength: bodyText.length, bodyTextExcerpt: bodyText.slice(0, 2000), seriousAccessibilityViolations, accessibilityScanned, accessibilityScanError, consoleErrors: consoleErrors.slice(0, 20), loadPassed, journeyStatus, journeyError, journeySeverity: ${JSON.stringify(journeySeverity)}, actionsCompleted, actionsEffective, actionLedger, actionsSkipped, formFieldsFilled, formFieldsUnsatisfiable, invalidFields, ambiguousTargets, adaptedTargets, actionsExpected: ${actions.length}, assertionsCompleted, assertionsExpected: ${assertions.length}, viewport: ${JSON.stringify(viewport)} }));`, + "console.error(error instanceof Error ? error.stack : String(error));", + 'process.exitCode = 1;', + '} finally { await browser.close(); }', + "})().catch(error => { console.error(error instanceof Error ? error.stack : String(error)); process.exit(1); });", + ].filter(Boolean).join('\n'); +} + +function browserLocatorExpression( + step: { selector: string; selectorType?: 'css' | 'label' | 'role'; role?: string }, +): string { + if (step.selectorType === 'label') { + return `page.getByLabel(${JSON.stringify(step.selector)})`; + } + if (step.selectorType === 'role') { + return `page.getByRole(${JSON.stringify(step.role)}, { name: ${JSON.stringify(step.selector)}, exact: true })`; + } + return `page.locator(${JSON.stringify(step.selector)})`; +} + +export function resolveLaunchTask( + configuration: LaunchConfiguration, + serviceRoot: string, +): VsCodeTask | { error: string } | undefined { + if (configuration.request !== 'launch') { + return undefined; + } + if (['pwa-chrome', 'chrome', 'msedge'].includes(String(configuration.type))) { + return undefined; + } + const args = Array.isArray(configuration.args) + ? configuration.args.filter((value): value is string => typeof value === 'string') + : []; + let command: string; + if (configuration.type === 'debugpy') { + const interpreter = typeof configuration.python === 'string' ? configuration.python : 'python'; + if (typeof configuration.module === 'string') { + command = `${shellQuote(interpreter)} -m debugpy --listen 127.0.0.1:${debugpyEvaluationPort} -m ${shellQuote(configuration.module)}`; + } else if (typeof configuration.program === 'string') { + command = `${shellQuote(interpreter)} -m debugpy --listen 127.0.0.1:${debugpyEvaluationPort} ${shellQuote(configuration.program)}`; + } else { + return { error: 'A debugpy launch configuration requires a module or program.' }; + } + } else if (['node', 'pwa-node'].includes(String(configuration.type))) { + const executable = typeof configuration.runtimeExecutable === 'string' + ? configuration.runtimeExecutable + : 'node'; + const runtimeArgs = Array.isArray(configuration.runtimeArgs) + ? configuration.runtimeArgs.filter((value): value is string => typeof value === 'string') + : []; + if (executable === 'node' && typeof configuration.program !== 'string') { + return { error: 'A Node.js launch configuration requires a program or runtimeExecutable.' }; + } + command = [ + executable, + ...runtimeArgs, + ...(typeof configuration.program === 'string' ? [configuration.program] : []), + ].map(shellQuote).join(' '); + } else if (configuration.type === 'coreclr') { + if (typeof configuration.program !== 'string') { + return { error: 'A CoreCLR launch configuration requires a program.' }; + } + const executable = typeof configuration.runtimeExecutable === 'string' + ? configuration.runtimeExecutable + : 'dotnet'; + command = [executable, configuration.program].map(shellQuote).join(' '); + } else { + return { error: `Launch debugger type "${String(configuration.type)}" is not supported by isolated validation.` }; + } + if (args.length) { + command += ` ${args.map(shellQuote).join(' ')}`; + } + const workspaceFolder = '$' + '{workspaceFolder}'; + return { + type: 'shell', + label: `launch: ${String(configuration.name)}`, + command, + isBackground: true, + options: { + cwd: typeof configuration.cwd === 'string' + ? configuration.cwd + : serviceRoot === '.' + ? workspaceFolder + : `${workspaceFolder}/${serviceRoot}`, + env: configuration.env, + }, + }; +} + +export interface DebuggerPrerequisiteCheck { + name: string; + command: string; + errorMessage: string; +} + +const nodeDebuggerTypes = new Set(['node', 'pwa-node', 'node-terminal']); +const defaultNodeInspectorPort = 9229; +const browserDebuggerTypes = new Set(['chrome', 'pwa-chrome', 'msedge', 'pwa-msedge']); + +/** + * A debug configuration is only evidence that F5 works if the thing it attaches to is actually + * attachable. Each supported configuration shape is reduced to shell checks that observe the + * live debug surface, so the `debugger` gate measures the generated project rather than the + * evaluator's assumptions about it. Shapes we cannot observe return no checks instead of a + * synthetic pass, which keeps the gate honest by reporting missing evidence. + */ +export function resolveDebuggerPrerequisite( + configuration: LaunchConfiguration, +): { checks: DebuggerPrerequisiteCheck[] } | { error: string } { + const type = typeof configuration.type === 'string' ? configuration.type.toLowerCase() : ''; + if (configuration.request === 'attach' && type === 'coreclr') { + return resolveCoreClrPrerequisite(configuration); + } + if (configuration.request === 'attach' && nodeDebuggerTypes.has(type)) { + return resolveNodeAttachPrerequisite(configuration); + } + if (configuration.request === 'launch' && nodeDebuggerTypes.has(type)) { + return resolveNodeLaunchPrerequisite(configuration); + } + if (configuration.request === 'launch' && browserDebuggerTypes.has(type)) { + return resolveBrowserLaunchPrerequisite(configuration); + } + return { checks: [] }; +} + +function resolveCoreClrPrerequisite( + configuration: LaunchConfiguration, +): { checks: DebuggerPrerequisiteCheck[] } | { error: string } { + if (typeof configuration.processName !== 'string' || !configuration.processName.trim()) { + return { error: 'A CoreCLR attach configuration requires a literal processName.' }; + } + const processName = configuration.processName.replace(/\.exe$/i, ''); + const processPattern = `(^|[/ ])${escapeExtendedRegex(processName)}(\\.dll)?( |$)`; + return { + checks: [{ + command: `pgrep -f -- ${shellQuote(processPattern)} >/dev/null`, + name: `CoreCLR process ${processName}`, + errorMessage: `CoreCLR attach target process "${processName}" is not running.`, + }], + }; +} + +/** + * VS Code attaches to a Node target by reading the inspector's own HTTP endpoint, so asking that + * endpoint for an attachable target reproduces what F5 does. A listening socket is not enough: + * the port can accept a connection before the inspector publishes a debug target. + */ +/** + * Node `launch` configurations expose their debug surface through an `--inspect` runtime argument + * rather than a `port` property. Without this the gate resolved zero checks for the most common + * Node shape, which reported "no failure" and was indistinguishable from a verified debugger. + */ +function resolveNodeLaunchPrerequisite( + configuration: LaunchConfiguration, +): { checks: DebuggerPrerequisiteCheck[] } | { error: string } { + const port = readInspectPort(configuration); + if (port === undefined) { + return { checks: [] }; + } + return { + checks: [{ + command: createDebuggerRetryCommand( + `curl --silent --show-error --fail --max-time 5 http://127.0.0.1:${port}/json/list` + + ' 2>/dev/null | grep -q webSocketDebuggerUrl', + `Node inspector on port ${port} never published an attachable debug target.`, + ), + name: `Node inspector port ${port}`, + errorMessage: `Node launch target on port ${port} is not accepting a debugger.`, + }], + }; +} + +/** + * Accepts `--inspect`, `--inspect-brk`, and `--inspect=[host:]port` forms. A bare flag uses Node's + * default inspector port. + */ +function readInspectPort(configuration: LaunchConfiguration): number | undefined { + const runtimeArgs = Array.isArray(configuration.runtimeArgs) + ? configuration.runtimeArgs.filter((value): value is string => typeof value === 'string') + : []; + for (const argument of runtimeArgs) { + const match = /^--inspect(?:-brk)?(?:=(?:(?:\[[^\]]+\]|[^:]+):)?(\d+))?$/u.exec(argument.trim()); + if (!match) { + continue; + } + if (match[1] === undefined) { + return defaultNodeInspectorPort; + } + const parsed = Number.parseInt(match[1], 10); + if (parsed > 0 && parsed < 65536) { + return parsed; + } + } + return undefined; +} + +function resolveNodeAttachPrerequisite( + configuration: LaunchConfiguration, +): { checks: DebuggerPrerequisiteCheck[] } | { error: string } { + const port = readDebugPort(configuration); + if (port === undefined) { + return { + error: 'A Node attach configuration requires a literal port so the debug target can be verified.', + }; + } + return { + checks: [{ + command: createDebuggerRetryCommand( + `curl --silent --show-error --fail --max-time 5 http://127.0.0.1:${port}/json/list` + + ' 2>/dev/null | grep -q webSocketDebuggerUrl', + `Node inspector on port ${port} never published an attachable debug target.`, + ), + name: `Node inspector port ${port}`, + errorMessage: `Node attach target on port ${port} is not accepting a debugger.`, + }], + }; +} + +/** + * A browser launch configuration has no attach target to probe, but it still fails for users in + * two deterministic ways: the URL does not serve, or webRoot points somewhere that does not + * exist, in which case source maps never resolve and no breakpoint ever binds. + */ +function resolveBrowserLaunchPrerequisite( + configuration: LaunchConfiguration, +): { checks: DebuggerPrerequisiteCheck[] } | { error: string } { + const checks: DebuggerPrerequisiteCheck[] = []; + const url = typeof configuration.url === 'string' ? configuration.url.trim() : ''; + if (!url) { + return { error: 'A browser launch configuration requires a literal url.' }; + } + checks.push({ + command: createDebuggerRetryCommand( + `curl --silent --show-error --fail --max-time 5 --output /dev/null ${shellQuote(url)}`, + `Browser debug target ${url} never served a response.`, + ), + name: `Browser debug target ${url}`, + errorMessage: `Browser debug target "${url}" is not serving, so F5 would open a dead page.`, + }); + const webRoot = typeof configuration.webRoot === 'string' + ? replaceWorkspaceFolder(configuration.webRoot) + : undefined; + if (webRoot && !webRoot.includes('${')) { + checks.push({ + command: `test -d ${shellQuote(webRoot)}`, + name: `Browser webRoot ${webRoot}`, + errorMessage: `Browser debug configuration webRoot "${webRoot}" does not exist, so breakpoints cannot bind.`, + }); + } + return { checks }; +} + +function readDebugPort(configuration: LaunchConfiguration): number | undefined { + for (const candidate of [configuration.port, configuration.attachSimplePort]) { + if (typeof candidate === 'number' && Number.isInteger(candidate) && candidate > 0 && candidate < 65536) { + return candidate; + } + if (typeof candidate === 'string' && /^\d+$/u.test(candidate.trim())) { + const parsed = Number.parseInt(candidate.trim(), 10); + if (parsed > 0 && parsed < 65536) { + return parsed; + } + } + } + return undefined; +} + +/** + * Language workers frequently start lazily, so the debug surface can appear seconds after the + * probes that preceded this check. Retrying keeps a slow start from being reported as a project + * that cannot be debugged at all. + */ +function createDebuggerRetryCommand(attempt: string, timeoutMessage: string): string { + return [ + `for i in $(seq 1 ${debuggerPrerequisiteAttempts}); do`, + `if ${attempt}; then exit 0; fi;`, + 'sleep 1;', + 'done;', + `printf '%s\\n' ${shellQuote(timeoutMessage)} >&2;`, + 'exit 1', + ].join(' '); +} + +function escapeExtendedRegex(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function resolveTaskChain( + rootLabel: string, + tasks: VsCodeTask[], +): { tasks: VsCodeTask[] } | { error: string } { + const byLabel = new Map(); + for (const task of tasks) { + if (typeof task.label === 'string') { + byLabel.set(task.label, task); + } + } + const ordered: VsCodeTask[] = []; + const visited = new Set(); + const active = new Set(); + const visit = (label: string): string | undefined => { + if (active.has(label)) { + return `Task dependency cycle includes "${label}".`; + } + if (visited.has(label)) { + return undefined; + } + const task = byLabel.get(label); + if (!task) { + return `Task "${label}" does not exist.`; + } + active.add(label); + for (const dependency of normalizeStringList(task.dependsOn)) { + const error = visit(dependency); + if (error) { + return error; + } + } + active.delete(label); + visited.add(label); + ordered.push(task); + return undefined; + }; + const error = visit(rootLabel); + return error ? { error } : { tasks: ordered }; +} + +function resolveTaskCommand( + task: VsCodeTask, + _serviceRoot: string, +): { command?: string; cwd: string } { + const taskType = typeof task.type === 'string' ? task.type : 'shell'; + const rawCommand = typeof task.command === 'string' ? task.command : undefined; + let command: string | undefined; + if (taskType === 'npm') { + const script = typeof task.script === 'string' ? task.script : rawCommand; + command = script ? `npm run ${shellQuote(script)}` : undefined; + } else if (taskType === 'func') { + command = rawCommand ? `func ${rawCommand}` : undefined; + } else { + command = rawCommand; + } + const args = Array.isArray(task.args) + ? task.args.filter((value): value is string => typeof value === 'string') + : []; + if (command && args.length) { + command += ` ${args.map(shellQuote).join(' ')}`; + } + const env = task.options?.env && typeof task.options.env === 'object' && !Array.isArray(task.options.env) + ? Object.entries(task.options.env as Record) + .filter((entry): entry is [string, string] => typeof entry[1] === 'string') + : []; + if (command && env.length) { + command = `env ${env.map(([key, value]) => `${key}=${shellQuote(value)}`).join(' ')} ${command}`; + } + const cwd = typeof task.options?.cwd === 'string' + ? replaceWorkspaceFolder(task.options.cwd) + : '/workspace'; + return { + command: command && replaceWorkspaceFolder(command), + cwd, + }; +} + +function runtimeToEcosystem(runtime: string): ValidationEcosystem | undefined { + const normalized = runtime.toLowerCase(); + if (normalized.includes('node')) { + return 'node'; + } + if (normalized.includes('python')) { + return 'python'; + } + if (normalized.includes('dotnet') || normalized.includes('.net')) { + return 'dotnet'; + } + return undefined; +} + +function normalizeServiceRoot(value: string): string { + const normalized = value.replaceAll('\\', '/').replace(/^\.\//, '').replace(/\/$/, ''); + return normalized || '.'; +} + +function replaceWorkspaceFolder(value: string): string { + return value.replaceAll('$' + '{workspaceFolder}', '/workspace'); +} + +function normalizeStringList(value: unknown): string[] { + if (typeof value === 'string') { + return [value]; + } + return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : []; +} + +function getToolchainCheckCommand(ecosystem: ValidationEcosystem): string { + switch (ecosystem) { + case 'node': + return 'node --version && npm --version'; + case 'python': + return 'python --version && python -m pip --version'; + case 'dotnet': + return 'dotnet --info && test -n "$(dotnet --list-sdks)"'; + } +} + +function processLogPath(label: string): string { + return `/workspace/.cor-eval/cor-${label.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')}.log`; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} + +function failure( + failureCode: NonNullable, + error: string, + commands: LocalRuntimeCommandResult[], + probes: LocalRuntimeProbeResult[], + browserChecks?: LocalRuntimeBrowserResult[], + persistenceChecks?: LocalRuntimePersistenceResult[], + workerEvents?: LocalRuntimeStorageEventResult[], + securityChecks?: LocalRuntimeSecurityResult[], +): SandboxLocalRuntimeValidationResult { + return { + outcome: 'failed', + failureCode, + error, + commands, + probes, + browserChecks, + persistenceChecks, + workerEvents, + securityChecks, + }; +} + +function truncate(value: string): string { + return value.length <= maxLogLength ? value : `${value.slice(0, maxLogLength)}\n[truncated]`; +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +class DefaultAcaCommandRunner implements AcaCommandRunner { + public async run(args: string[], timeoutMs: number): Promise<{ stdout: string; stderr: string }> { + return await execFileAsync('aca', args, { + timeout: timeoutMs, + maxBuffer: 20 * 1024 * 1024, + encoding: 'utf8', + }); + } +} diff --git a/evals/src/SandboxProjectValidator.ts b/evals/src/SandboxProjectValidator.ts new file mode 100644 index 000000000..dc9680217 --- /dev/null +++ b/evals/src/SandboxProjectValidator.ts @@ -0,0 +1,865 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { execFile } from 'child_process'; +import { randomUUID } from 'crypto'; +import { promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { promisify } from 'util'; +import { CorEvaluationScenario } from './scenario'; + +const execFileAsync = promisify(execFile); +const ignoredDirectories = new Set(['.git', '.azure', 'dist', 'node_modules', 'out']); +const maxLogLength = 20_000; + +export type ValidationEcosystem = 'node' | 'python' | 'dotnet'; +const pythonValidationDirectory = '.cor-eval-venv'; +const pythonValidationInterpreter = `${pythonValidationDirectory}/bin/python`; +const pythonValidationPip = `${pythonValidationDirectory}/bin/pip`; + +export interface ProjectValidationTarget { + ecosystem: ValidationEcosystem; + relativeDirectory: string; + commands: string[]; +} + +interface NodePackageInfo { + directory: string; + relativeDirectory: string; + scripts: Record; + workspacePatterns: string[]; + hasPackageLock: boolean; +} + +export interface SandboxValidationCommandResult { + ecosystem: ValidationEcosystem; + relativeDirectory: string; + command: string; + success: boolean; + failureKind?: 'commandExit' | 'runnerError'; + durationMs: number; + stdout: string; + stderr: string; +} + +export type SandboxValidationCommandKind = 'build' | 'test' | 'lint'; + +export interface SandboxProjectValidationResult { + outcome: 'passed' | 'failed'; + failureCode?: 'noBuildTargets' | 'sandboxCreateFailed' | 'sandboxSetupFailed' | 'sandboxCommandFailed' | 'sandboxCleanupFailed'; + error?: string; + commands: SandboxValidationCommandResult[]; +} + +export interface AcaCommandResult { + stdout: string; + stderr: string; +} + +export interface AcaCommandRunner { + run(args: string[], timeoutMs: number): Promise; +} + +export function isSandboxInfrastructureFailureCode(code: string | undefined): boolean { + return [ + 'sandboxCreateFailed', + 'sandboxSetupFailed', + 'sandboxCleanupFailed', + 'agentRunTimedOut', + 'agentRunStalled', + ].includes(code ?? ''); +} + +export function classifySandboxValidationCommand( + command: Pick, +): SandboxValidationCommandKind | undefined { + const value = command.command.trim(); + switch (command.ecosystem) { + case 'node': + if (/Missing required npm script: build/u.test(value)) { + return 'build'; + } + if (/Missing required npm script: test/u.test(value)) { + return 'test'; + } + if (/Missing required npm script: lint/u.test(value)) { + return 'lint'; + } + if (/^npm\s+run\s+build(?:\s|$)/u.test(value)) { + return 'build'; + } + if (/^npm\s+(?:run\s+)?test(?:\s|$)/u.test(value)) { + return 'test'; + } + if (/^npm\s+run\s+lint(?:\s|$)/u.test(value)) { + return 'lint'; + } + return undefined; + case 'python': + if (/Missing required Python lint configuration/u.test(value)) { + return 'lint'; + } + if (/\s-m\s+compileall(?:\s|$)/u.test(value)) { + return 'build'; + } + if (/\s-m\s+pytest(?:\s|$)/u.test(value)) { + return 'test'; + } + if (/\s-m\s+ruff\s+check(?:\s|$)/u.test(value)) { + return 'lint'; + } + return undefined; + case 'dotnet': + if (/^dotnet\s+build(?:\s|$)/u.test(value)) { + return 'build'; + } + if (/^dotnet\s+test(?:\s|$)/u.test(value)) { + return 'test'; + } + if (/^dotnet\s+format\b.*\s--verify-no-changes(?:\s|$)/u.test(value)) { + return 'lint'; + } + return undefined; + } +} + +export function canContinueAfterProjectValidationFailure( + validation: SandboxProjectValidationResult, +): boolean { + if (validation.outcome !== 'failed' || validation.failureCode !== 'sandboxCommandFailed') { + return false; + } + const failedCommands = validation.commands.filter(command => !command.success); + return failedCommands.length > 0 && failedCommands.every(command => { + const kind = classifySandboxValidationCommand(command); + return command.failureKind === 'commandExit' && (kind === 'test' || kind === 'lint'); + }); +} + +export class SandboxProjectValidator { + public constructor( + private readonly repoRoot: string, + private readonly aca: AcaCommandRunner = new DefaultAcaCommandRunner(), + ) { + } + + public async validate(workspace: string, scenario: CorEvaluationScenario): Promise { + const targets = await discoverProjectValidationTargets(workspace, scenario); + if (!targets.length) { + return { + outcome: 'failed', + failureCode: 'noBuildTargets', + error: 'The scaffold did not produce a supported Node, Python, or .NET build target.', + commands: [], + }; + } + + const archivePath = path.join(os.tmpdir(), `cor-eval-${randomUUID()}.tar.gz`); + const commands: SandboxValidationCommandResult[] = []; + let qualityFailure: SandboxProjectValidationResult | undefined; + try { + await createWorkspaceArchive(workspace, archivePath); + for (const [ecosystem, ecosystemTargets] of groupTargets(targets)) { + const result = await this.validateEcosystem(ecosystem, ecosystemTargets, archivePath, scenario, commands); + if (result) { + if (canContinueAfterProjectValidationFailure(result)) { + qualityFailure ??= result; + } else { + return result; + } + } + } + return qualityFailure ?? { outcome: 'passed', commands }; + } finally { + await fs.rm(archivePath, { force: true }); + } + } + + private async validateEcosystem( + ecosystem: ValidationEcosystem, + targets: ProjectValidationTarget[], + archivePath: string, + scenario: CorEvaluationScenario, + commands: SandboxValidationCommandResult[], + ): Promise { + const runLabel = randomUUID(); + const manifestPath = path.join(os.tmpdir(), `cor-eval-${runLabel}-${ecosystem}.yaml`); + let sandboxId: string; + try { + await createSandboxManifest(this.getManifestPath(ecosystem), manifestPath, runLabel); + await this.aca.run([ + 'sandbox', + 'validate', + '--file', + manifestPath, + ], 60 * 1000); + const created = await this.aca.run([ + 'sandbox', + 'apply', + '--file', + manifestPath, + '--wait-timeout', + '300', + '-o', + 'json', + ], 6 * 60 * 1000); + sandboxId = readSandboxId(created.stdout); + } catch (error) { + const cleanupError = await this.cleanupAfterCreateFailure(error, runLabel); + return { + outcome: 'failed', + failureCode: 'sandboxCreateFailed', + error: [getErrorMessage(error), cleanupError].filter(Boolean).join(' '), + commands, + }; + } finally { + await fs.rm(manifestPath, { force: true }); + } + + let validationFailure: SandboxProjectValidationResult | undefined; + try { + await this.aca.run([ + 'sandbox', 'exec', + '--id', sandboxId, + '-c', getToolchainCheckCommand(ecosystem), + ], 60 * 1000); + await this.aca.run([ + 'sandbox', 'fs', 'write', + '--id', sandboxId, + '--path', '/tmp/workspace.tar.gz', + '--file', archivePath, + ], 5 * 60 * 1000); + await this.aca.run([ + 'sandbox', 'exec', + '--id', sandboxId, + '--working-directory', '/tmp', + '-c', 'mkdir -p /workspace && tar -xzf /tmp/workspace.tar.gz -C /workspace', + ], 5 * 60 * 1000); + } catch (error) { + validationFailure = { + outcome: 'failed', + failureCode: 'sandboxSetupFailed', + error: getErrorMessage(error), + commands, + }; + } + if (!validationFailure) { + try { + validation: for (const target of targets) { + for (const command of target.commands) { + const commandResult = await this.runValidationCommand( + sandboxId, + target, + command, + scenario.validation.timeoutMinutes * 60 * 1000, + ); + commands.push(commandResult); + if (!commandResult.success) { + const failure: SandboxProjectValidationResult = { + outcome: 'failed', + failureCode: 'sandboxCommandFailed', + error: `${target.relativeDirectory}: "${command}" failed.`, + commands, + }; + if (canContinueAfterProjectValidationFailure(failure)) { + validationFailure ??= failure; + continue; + } + validationFailure = failure; + break validation; + } + } + } + } catch (error) { + validationFailure = { + outcome: 'failed', + failureCode: 'sandboxCommandFailed', + error: getErrorMessage(error), + commands, + }; + } + } + + try { + await this.aca.run(['sandbox', 'delete', '--id', sandboxId, '--yes'], 5 * 60 * 1000); + } catch (error) { + return { + outcome: 'failed', + failureCode: 'sandboxCleanupFailed', + error: getErrorMessage(error), + commands, + }; + } + return validationFailure; + } + + private async cleanupAfterCreateFailure(error: unknown, runLabel: string): Promise { + const commandError = error as Error & { stdout?: string; stderr?: string }; + const sandboxIds = new Set(); + const explicitId = tryReadSandboxId([commandError.stdout, commandError.stderr].filter(Boolean).join('\n')); + if (explicitId) { + sandboxIds.add(explicitId); + } + + let recoveryError: string | undefined; + try { + const listed = await this.aca.run([ + 'sandbox', 'list', + '-l', `run-id=${runLabel}`, + '-o', 'json', + ], 60 * 1000); + for (const id of readSandboxIds(listed.stdout)) { + sandboxIds.add(id); + } + } catch (listError) { + if (!sandboxIds.size) { + recoveryError = `Could not recover a created sandbox by label: ${getErrorMessage(listError)}`; + } + } + + const deleteErrors: string[] = []; + for (const id of sandboxIds) { + try { + await this.aca.run(['sandbox', 'delete', '--id', id, '--yes'], 5 * 60 * 1000); + } catch (deleteError) { + deleteErrors.push(`${id}: ${getErrorMessage(deleteError)}`); + } + } + if (deleteErrors.length) { + return `Sandbox cleanup failed: ${deleteErrors.join('; ')}`; + } + return recoveryError; + } + + private async runValidationCommand( + sandboxId: string, + target: ProjectValidationTarget, + command: string, + timeoutMs: number, + ): Promise { + const started = Date.now(); + const remoteExitMarker = `__COR_EVAL_REMOTE_EXIT_${randomUUID()}__=`; + const wrappedCommand = `( ${command} ); __cor_eval_status=$?; ` + + `printf '\\n${remoteExitMarker}%s\\n' "$__cor_eval_status" >&2; ` + + 'exit "$__cor_eval_status"'; + try { + const result = await this.aca.run([ + 'sandbox', 'exec', + '--id', sandboxId, + '--working-directory', toSandboxPath(target.relativeDirectory), + '-c', wrappedCommand, + ], timeoutMs); + const stderr = removeRemoteExitMarker(result.stderr, remoteExitMarker); + return { + ecosystem: target.ecosystem, + relativeDirectory: target.relativeDirectory, + command, + success: true, + durationMs: Date.now() - started, + stdout: truncate(result.stdout), + stderr: truncate(stderr.output), + }; + } catch (error) { + const commandError = error as Error & { stdout?: string; stderr?: string }; + const stderr = removeRemoteExitMarker(commandError.stderr ?? '', remoteExitMarker); + return { + ecosystem: target.ecosystem, + relativeDirectory: target.relativeDirectory, + command, + success: false, + failureKind: stderr.exitCode !== undefined && stderr.exitCode !== 0 + ? 'commandExit' + : 'runnerError', + durationMs: Date.now() - started, + stdout: truncate(commandError.stdout ?? ''), + stderr: truncate(stderr.output || getErrorMessage(error)), + }; + } + } + + private getManifestPath(ecosystem: ValidationEcosystem): string { + switch (ecosystem) { + case 'node': + return path.join(this.repoRoot, 'evals', 'sandbox.yaml'); + case 'python': + return path.join(this.repoRoot, 'evals', 'sandbox-python.yaml'); + case 'dotnet': + return path.join(this.repoRoot, 'evals', 'sandbox-dotnet.yaml'); + } + } +} + +export async function discoverProjectValidationTargets( + workspace: string, + scenario: CorEvaluationScenario, +): Promise { + const files = await listProjectFiles(workspace); + const targets: ProjectValidationTarget[] = []; + + const nodePackages = await Promise.all( + files + .filter(file => path.basename(file) === 'package.json') + .map(file => readNodePackageInfo(workspace, file)), + ); + nodePackages.sort((left, right) => + pathDepth(left.relativeDirectory) - pathDepth(right.relativeDirectory) + || left.relativeDirectory.localeCompare(right.relativeDirectory)); + const workspaceRoots = new Map(); + for (const nodePackage of nodePackages) { + workspaceRoots.set(nodePackage, findWorkspaceRoot(nodePackage, nodePackages)); + } + for (const nodePackage of nodePackages) { + const workspaceRoot = workspaceRoots.get(nodePackage); + const workspaceMembers = nodePackages.filter(candidate => workspaceRoots.get(candidate) === nodePackage); + const commands: string[] = []; + if (!workspaceRoot) { + commands.push(nodePackage.hasPackageLock ? 'npm ci --ignore-scripts' : 'npm install --ignore-scripts'); + } + for (const validation of getRequestedNodeValidations(scenario)) { + if (workspaceRoot?.scripts[validation.script]) { + continue; + } + const script = nodePackage.scripts[validation.script]; + if (script) { + commands.push(validation.command); + } else if (validation.required && workspaceMembers.length === 0) { + commands.push(missingScriptCommand(validation.script)); + } + } + if (commands.length) { + targets.push(createTarget('node', workspace, nodePackage.directory, commands)); + } + } + + const pythonDirectories = [...new Set( + files + .filter(file => ['requirements.txt', 'pyproject.toml'].includes(path.basename(file))) + .map(file => path.dirname(file)), + )].sort(); + for (const directory of pythonDirectories) { + const commands = [ + `python -m venv ${pythonValidationDirectory}`, + ...await createPythonInstallCommands(directory, files), + ]; + if (scenario.validation.build) { + commands.push(`${pythonValidationInterpreter} -m compileall -q .`); + } + if (scenario.validation.test) { + commands.push(`${pythonValidationInterpreter} -m pytest`); + } + const pythonLintCommand = await detectPythonLintCommand(directory); + if (pythonLintCommand && scenario.validation.lint !== 'skip') { + commands.push(pythonLintCommand); + } else if (!pythonLintCommand && scenario.validation.lint === 'required') { + commands.push(missingPythonLintCommand()); + } + targets.push(createTarget('python', workspace, directory, commands)); + } + + const solutions = files.filter(file => file.endsWith('.sln')); + const dotnetProjects = solutions.length ? solutions : files.filter(file => file.endsWith('.csproj')); + for (const file of dotnetProjects) { + const commands = ['dotnet restore']; + if (scenario.validation.build) { + commands.push('dotnet build --no-restore'); + } + if (scenario.validation.test) { + commands.push('dotnet test --no-build'); + } + if (scenario.validation.lint === 'required' + || (scenario.validation.lint === 'if-present' && await hasDotnetLintConfiguration(path.dirname(file)))) { + commands.push('dotnet format --verify-no-changes --no-restore'); + } + targets.push(createTarget('dotnet', workspace, path.dirname(file), commands)); + } + + return targets; +} + +async function createPythonInstallCommands(directory: string, files: string[]): Promise { + const requirementFiles = files + .filter(file => path.dirname(file) === directory + && /^requirements(?:[-_.](?:dev|test|tests|lint))?\.txt$/i.test(path.basename(file))) + .sort((left, right) => { + const leftBase = path.basename(left).toLowerCase(); + const rightBase = path.basename(right).toLowerCase(); + if (leftBase === 'requirements.txt') { + return -1; + } + if (rightBase === 'requirements.txt') { + return 1; + } + return leftBase.localeCompare(rightBase); + }); + const commands = requirementFiles.map(file => `${pythonValidationPip} install -r ${shellQuote(path.basename(file))}`); + const pyprojectPath = path.join(directory, 'pyproject.toml'); + const pyproject = await readOptionalFile(pyprojectPath); + if (pyproject !== undefined && isPythonPackageProject(pyproject)) { + const extras = findPythonValidationExtras(pyproject); + commands.push(extras.length + ? `${pythonValidationPip} install ${shellQuote(`.[${extras.join(',')}]`)}` + : `${pythonValidationPip} install .`); + } + return commands; +} + +function isPythonPackageProject(pyproject: string): boolean { + return /^\s*\[(?:build-system|project|tool\.(?:hatch|pdm|poetry|setuptools))(?:\.|\])[^]*$/im.test(pyproject); +} + +function findPythonValidationExtras(pyproject: string): string[] { + const header = /^\s*\[project\.optional-dependencies\]\s*$/im.exec(pyproject); + if (!header) { + return []; + } + const remainder = pyproject.slice(header.index + header[0].length); + const nextSectionIndex = /^\s*\[/m.exec(remainder)?.index ?? remainder.length; + const section = remainder.slice(0, nextSectionIndex); + const preferredGroups = new Set(['dev', 'test', 'tests', 'lint']); + return [...section.matchAll(/^\s*([A-Za-z0-9_-]+)\s*=/gm)] + .map(match => match[1]) + .filter(group => preferredGroups.has(group.toLowerCase())) + .sort(); +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} + +async function readNodePackageInfo(workspace: string, file: string): Promise { + const packageJson = JSON.parse(await fs.readFile(file, 'utf8')) as { + scripts?: Record; + workspaces?: string[] | { packages?: string[] }; + }; + const directory = path.dirname(file); + return { + directory, + relativeDirectory: path.relative(workspace, directory) || '.', + scripts: packageJson.scripts ?? {}, + workspacePatterns: Array.isArray(packageJson.workspaces) + ? packageJson.workspaces + : packageJson.workspaces?.packages ?? [], + hasPackageLock: await hasSibling(file, 'package-lock.json'), + }; +} + +function findWorkspaceRoot( + nodePackage: NodePackageInfo, + candidates: NodePackageInfo[], +): NodePackageInfo | undefined { + return candidates + .filter(candidate => candidate !== nodePackage + && candidate.workspacePatterns.length > 0 + && isWithinDirectory(nodePackage.directory, candidate.directory) + && candidate.workspacePatterns.some(pattern => + workspacePatternMatches(path.relative(candidate.directory, nodePackage.directory), pattern))) + .sort((left, right) => pathDepth(right.relativeDirectory) - pathDepth(left.relativeDirectory))[0]; +} + +function workspacePatternMatches(relativeDirectory: string, pattern: string): boolean { + const candidate = relativeDirectory.split(path.sep).join('/'); + const normalizedPattern = pattern.replaceAll('\\', '/').replace(/^\.\//, '').replace(/\/$/, ''); + let expression = '^'; + for (let index = 0; index < normalizedPattern.length; index++) { + const character = normalizedPattern[index]; + if (character === '*' && normalizedPattern[index + 1] === '*') { + expression += '.*'; + index++; + } else if (character === '*') { + expression += '[^/]*'; + } else if (character === '?') { + expression += '[^/]'; + } else { + expression += character.replace(/[|\\{}()[\]^$+?.]/g, '\\$&'); + } + } + return new RegExp(`${expression}$`).test(candidate); +} + +function getRequestedNodeValidations( + scenario: CorEvaluationScenario, +): { script: 'build' | 'test' | 'lint'; command: string; required: boolean }[] { + const validations: { script: 'build' | 'test' | 'lint'; command: string; required: boolean }[] = []; + if (scenario.validation.build) { + validations.push({ script: 'build', command: 'npm run build', required: true }); + } + if (scenario.validation.test) { + validations.push({ script: 'test', command: 'npm test', required: true }); + } + if (scenario.validation.lint !== 'skip') { + validations.push({ + script: 'lint', + command: 'npm run lint', + required: scenario.validation.lint === 'required', + }); + } + return validations; +} + +function createTarget( + ecosystem: ValidationEcosystem, + workspace: string, + directory: string, + commands: string[], +): ProjectValidationTarget { + return { + ecosystem, + relativeDirectory: path.relative(workspace, directory) || '.', + commands, + }; +} + +async function listProjectFiles(directory: string): Promise { + const files: string[] = []; + for (const entry of await fs.readdir(directory, { withFileTypes: true })) { + if (entry.isDirectory() && ignoredDirectories.has(entry.name)) { + continue; + } + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + files.push(...await listProjectFiles(entryPath)); + } else if (entry.isFile()) { + files.push(entryPath); + } + } + return files; +} + +async function hasSibling(file: string, siblingName: string): Promise { + try { + await fs.access(path.join(path.dirname(file), siblingName)); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return false; + } + throw error; + } +} + +function missingScriptCommand(script: string): string { + return `node -e "console.error('Missing required npm script: ${script}'); process.exit(1)"`; +} + +function missingPythonLintCommand(): string { + return `${pythonValidationInterpreter} -c "import sys; print('Missing required Python lint configuration', file=sys.stderr); sys.exit(1)"`; +} + +async function detectPythonLintCommand(directory: string): Promise { + const ruffFiles = ['ruff.toml', '.ruff.toml']; + if (await anyFileExists(directory, ruffFiles)) { + return `${pythonValidationInterpreter} -m ruff check .`; + } + const pyproject = await readOptionalFile(path.join(directory, 'pyproject.toml')); + const requirementNames = (await fs.readdir(directory)) + .filter(name => /^requirements(?:[-_.](?:dev|test|tests|lint))?\.txt$/i.test(name)); + const requirements = (await Promise.all( + requirementNames.map(name => fs.readFile(path.join(directory, name), 'utf8')), + )).join('\n'); + if (/\[tool\.ruff(?:\.|\])/.test(pyproject ?? '') || /^\s*ruff(?:[<=>~!].*)?$/im.test(requirements ?? '')) { + return `${pythonValidationInterpreter} -m ruff check .`; + } + const setupConfig = await readOptionalFile(path.join(directory, 'setup.cfg')); + if (await anyFileExists(directory, ['.flake8']) + || /^\s*\[flake8\]\s*$/im.test(setupConfig ?? '') + || /^\s*flake8(?:[<=>~!].*)?$/im.test(requirements ?? '')) { + return `${pythonValidationInterpreter} -m flake8 .`; + } + return undefined; +} + +async function hasDotnetLintConfiguration(directory: string): Promise { + return await anyFileExists(directory, [ + '.editorconfig', + path.join('.config', 'dotnet-tools.json'), + ]); +} + +async function anyFileExists(directory: string, names: string[]): Promise { + for (const name of names) { + try { + await fs.access(path.join(directory, name)); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + } + return false; +} + +async function readOptionalFile(file: string): Promise { + try { + return await fs.readFile(file, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return undefined; + } + throw error; + } +} + +function isWithinDirectory(candidate: string, parent: string): boolean { + const relative = path.relative(parent, candidate); + return relative !== '' && !relative.startsWith('..') && !path.isAbsolute(relative); +} + +function pathDepth(relativeDirectory: string): number { + return relativeDirectory === '.' ? 0 : relativeDirectory.split(path.sep).length; +} + +function groupTargets( + targets: ProjectValidationTarget[], +): Map { + const groups = new Map(); + for (const target of targets) { + const group = groups.get(target.ecosystem) ?? []; + group.push(target); + groups.set(target.ecosystem, group); + } + return groups; +} + +function getToolchainCheckCommand(ecosystem: ValidationEcosystem): string { + switch (ecosystem) { + case 'node': + return 'node --version && npm --version'; + case 'python': + return 'python --version && python -m pip --version'; + case 'dotnet': + return 'dotnet --info && test -n "$(dotnet --list-sdks)"'; + } +} + +export async function createWorkspaceArchive(workspace: string, archivePath: string): Promise { + await execFileAsync('tar', [ + '-czf', archivePath, + '--exclude=.git', + '--exclude=.DS_Store', + '--exclude=._*', + '--exclude=*/._*', + '--exclude=node_modules', + '--exclude=dist', + '--exclude=out', + '-C', workspace, + '.', + ], { + env: { + ...process.env, + ['COPYFILE_DISABLE']: '1', + }, + maxBuffer: 10 * 1024 * 1024, + }); +} + +export async function createSandboxManifest( + sourcePath: string, + destinationPath: string, + runLabel: string, +): Promise { + const source = await fs.readFile(sourcePath, 'utf8'); + if (!/^labels:\s*$/m.test(source)) { + throw new Error(`Sandbox manifest does not define labels: ${sourcePath}`); + } + const ownerLabel = process.env.COR_EVAL_OWNER_ID; + if (ownerLabel !== undefined && !/^[a-z0-9][a-z0-9-]{0,62}$/.test(ownerLabel)) { + throw new Error('COR_EVAL_OWNER_ID must be a lowercase alphanumeric/hyphen ACA label value.'); + } + const labels = [ + ` run-id: ${runLabel}`, + ...(ownerLabel ? [` owner-id: ${ownerLabel}`] : []), + ].join('\n'); + const content = source.replace(/^labels:\s*$/m, `labels:\n${labels}`); + await fs.writeFile(destinationPath, content); +} + +export function readSandboxId(stdout: string): string { + const jsonStart = stdout.indexOf('{'); + const jsonEnd = stdout.lastIndexOf('}'); + if (jsonStart >= 0 && jsonEnd > jsonStart) { + try { + const parsed = JSON.parse(stdout.slice(jsonStart, jsonEnd + 1)) as { id?: unknown }; + if (typeof parsed.id === 'string' && parsed.id) { + return parsed.id; + } + } catch { + // Fall through to the CLI's human-readable "Created sandbox " output. + } + } + const uuidPattern = '[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}'; + const id = new RegExp(`\\bCreated sandbox\\b[^\\r\\n]*?\\b(${uuidPattern})\\b`, 'i').exec(stdout)?.[1] + ?? new RegExp(`^\\s*(${uuidPattern})\\s*$`, 'i').exec(stdout)?.[1]; + if (!id) { + throw new Error('ACA sandbox apply did not return an id.'); + } + return id; +} + +export function readSandboxIds(stdout: string): string[] { + const jsonStart = stdout.indexOf('['); + const jsonEnd = stdout.lastIndexOf(']'); + if (jsonStart < 0 || jsonEnd <= jsonStart) { + throw new Error('ACA sandbox list did not return a JSON array.'); + } + const parsed: unknown = JSON.parse(stdout.slice(jsonStart, jsonEnd + 1)); + if (!Array.isArray(parsed)) { + throw new Error('ACA sandbox list did not return a JSON array.'); + } + return parsed.flatMap(value => { + if (!value || typeof value !== 'object') { + return []; + } + const id = (value as { id?: unknown }).id; + return typeof id === 'string' && id ? [id] : []; + }); +} + +function tryReadSandboxId(stdout: string): string | undefined { + try { + return readSandboxId(stdout); + } catch { + return undefined; + } +} + +function toSandboxPath(relativeDirectory: string): string { + const normalized = relativeDirectory.split(path.sep).join('/'); + return normalized === '.' ? '/workspace' : `/workspace/${normalized}`; +} + +function truncate(value: string): string { + return value.length <= maxLogLength ? value : `${value.slice(0, maxLogLength)}\n[truncated]`; +} + +function removeRemoteExitMarker(value: string, marker: string): { output: string; exitCode?: number } { + let exitCode: number | undefined; + const output = value + .split(/\r?\n/u) + .filter(line => { + if (line.startsWith(marker) && /^\d+$/u.test(line.slice(marker.length))) { + exitCode = Number.parseInt(line.slice(marker.length), 10); + return false; + } + return true; + }) + .join('\n'); + return { output, exitCode }; +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export class DefaultAcaCommandRunner implements AcaCommandRunner { + public async run(args: string[], timeoutMs: number): Promise { + return await execFileAsync('aca', args, { + timeout: timeoutMs, + maxBuffer: 20 * 1024 * 1024, + encoding: 'utf8', + }); + } +} diff --git a/evals/src/SandboxVsCodeParityValidator.ts b/evals/src/SandboxVsCodeParityValidator.ts new file mode 100644 index 000000000..664f2fa34 --- /dev/null +++ b/evals/src/SandboxVsCodeParityValidator.ts @@ -0,0 +1,332 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { randomUUID } from 'crypto'; +import { promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + AcaCommandRunner, + createSandboxManifest, + createWorkspaceArchive, + DefaultAcaCommandRunner, + readSandboxId, + readSandboxIds, +} from './SandboxProjectValidator'; +import { + parsePlannedConfigurations, + targetMatches, +} from './SandboxLocalRuntimeValidator'; +import { CorEvaluationScenario, DebugParityContract } from './scenario'; +import { EvaluationDefinitionProvenance } from './evaluationDefinition'; + +const resultPrefix = 'COR_VSCODE_PARITY_RESULT='; + +export interface VsCodeParityEvidence { + outcome: 'passed'; + configurationName: string; + source: string; + line: number; + column: number; + sessions: { id: string; name: string; type: string }[]; + stoppedReason?: string; + hitBreakpointIds: number[]; +} + +export interface SandboxVsCodeParityResult { + outcome: 'passed' | 'failed' | 'skipped'; + failureCode?: 'paritySpecMissing' | 'parityTargetMissing' | 'paritySandboxCreateFailed' | 'paritySetupFailed' | 'parityExecutionFailed' | 'parityEvidenceInvalid' | 'parityCleanupFailed'; + error?: string; + codeVersion?: string; + evidence?: VsCodeParityEvidence; + sourceProvenance?: { + evaluationArm: 'rails'; + through: 'local'; + runId: string; + scenarioId: string; + attempt: number; + candidateCommit: string; + agentAssetsHash: string; + evaluationDefinition?: EvaluationDefinitionProvenance; + requestedModel: string; + observedModels: string[]; + debugParity: DebugParityContract; + }; +} + +export class SandboxVsCodeParityValidator { + public constructor( + private readonly repoRoot: string, + private readonly aca: AcaCommandRunner = new DefaultAcaCommandRunner(), + ) { + } + + public async validate( + workspace: string, + scenario: CorEvaluationScenario, + debugPlanContent: string, + ): Promise { + const contract = scenario.acceptance?.local?.debugParity; + if (!contract) { + return { outcome: 'skipped', failureCode: 'paritySpecMissing', error: 'Scenario has no VS Code debug parity contract.' }; + } + const matches = parsePlannedConfigurations(debugPlanContent) + .filter(configuration => targetMatches(contract.target, configuration)); + if (matches.length !== 1) { + return { + outcome: 'failed', + failureCode: 'parityTargetMissing', + error: `Debug parity target "${contract.target}" matched ${matches.length} configurations; exactly one is required.`, + }; + } + + const runLabel = randomUUID(); + const manifestPath = path.join(os.tmpdir(), `cor-vscode-parity-${runLabel}.yaml`); + const workspaceArchive = path.join(os.tmpdir(), `cor-vscode-parity-workspace-${runLabel}.tar.gz`); + const parityArchive = path.join(os.tmpdir(), `cor-vscode-parity-extension-${runLabel}.tar.gz`); + let sandboxId: string | undefined; + let result: SandboxVsCodeParityResult; + try { + await Promise.all([ + createWorkspaceArchive(workspace, workspaceArchive), + createWorkspaceArchive(path.join(this.repoRoot, 'evals', 'vscode-parity'), parityArchive), + createSandboxManifest(path.join(this.repoRoot, 'evals', 'sandbox.yaml'), manifestPath, runLabel), + ]); + try { + await this.aca.run(['sandbox', 'validate', '--file', manifestPath], 60_000); + const created = await this.aca.run([ + 'sandbox', 'apply', + '--file', manifestPath, + '--wait-timeout', '300', + '-o', 'json', + ], 6 * 60_000); + sandboxId = readSandboxId(created.stdout); + } catch (error) { + const cleanupError = await this.cleanupCreatedSandbox(error, runLabel); + return failure( + 'paritySandboxCreateFailed', + cleanupError + ? new Error(`${getErrorMessage(error)} Cleanup failed: ${cleanupError}`) + : error, + ); + } + + try { + await this.aca.run([ + 'sandbox', 'exec', '--id', sandboxId, '-c', + 'if ! ldconfig -p | grep -q libgtk-3.so.0; then sudo sed -i "s|http://archive.ubuntu.com|https://archive.ubuntu.com|g; s|http://security.ubuntu.com|https://security.ubuntu.com|g" /etc/apt/sources.list.d/ubuntu.sources && sudo apt-get update -qq -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources -o Dir::Etc::sourceparts=- -o APT::Get::List-Cleanup=0 && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources -o Dir::Etc::sourceparts=- libgtk-3-0 libnss3 libasound2t64 libxss1 libgbm1; fi', + ], 5 * 60_000); + await this.aca.run([ + 'sandbox', 'exec', '--id', sandboxId, '-c', + 'if [ ! -x /home/vscode/.cor-vscode/VSCode-linux-x64/bin/code ]; then mkdir -p /home/vscode/.cor-vscode && curl -fsSL https://update.code.visualstudio.com/1.106.3/linux-x64/stable -o /tmp/cor-vscode.tar.gz && tar -xzf /tmp/cor-vscode.tar.gz -C /home/vscode/.cor-vscode; fi', + ], 5 * 60_000); + const codeVersion = (await this.aca.run([ + 'sandbox', 'exec', '--id', sandboxId, '-c', + 'node -p "require(\'/home/vscode/.cor-vscode/VSCode-linux-x64/resources/app/package.json\').version"', + ], 60_000)).stdout.trim(); + await this.aca.run([ + 'sandbox', 'exec', '--id', sandboxId, '-c', + 'mkdir -p /tmp/cor-vscode-extensions /tmp/cor-vscode-vsix/resource-groups /tmp/cor-vscode-vsix/functions && curl -fsSL https://ms-azuretools.gallerycdn.vsassets.io/extensions/ms-azuretools/vscode-azureresourcegroups/0.12.7/1781209805226/Microsoft.VisualStudio.Services.VSIXPackage -o /tmp/vscode-azureresourcegroups.vsix && curl -fsSL https://ms-azuretools.gallerycdn.vsassets.io/extensions/ms-azuretools/vscode-azurefunctions/1.22.0/1780435477995/Microsoft.VisualStudio.Services.VSIXPackage -o /tmp/vscode-azurefunctions.vsix && unzip -q /tmp/vscode-azureresourcegroups.vsix "extension/*" -d /tmp/cor-vscode-vsix/resource-groups && unzip -q /tmp/vscode-azurefunctions.vsix "extension/*" -d /tmp/cor-vscode-vsix/functions && mv /tmp/cor-vscode-vsix/resource-groups/extension /tmp/cor-vscode-extensions/ms-azuretools.vscode-azureresourcegroups-0.12.7 && mv /tmp/cor-vscode-vsix/functions/extension /tmp/cor-vscode-extensions/ms-azuretools.vscode-azurefunctions-1.22.0', + ], 5 * 60_000); + await Promise.all([ + this.aca.run([ + 'sandbox', 'fs', 'write', '--id', sandboxId, + '--path', '/tmp/workspace.tar.gz', '--file', workspaceArchive, + ], 5 * 60_000), + this.aca.run([ + 'sandbox', 'fs', 'write', '--id', sandboxId, + '--path', '/tmp/parity.tar.gz', '--file', parityArchive, + ], 5 * 60_000), + ]); + await this.aca.run([ + 'sandbox', 'exec', '--id', sandboxId, '-c', + 'mkdir -p /workspace /home/vscode/cor-vscode-parity && tar -xzf /tmp/workspace.tar.gz -C /workspace && tar -xzf /tmp/parity.tar.gz -C /home/vscode/cor-vscode-parity && cd /workspace && npm install', + ], 5 * 60_000); + result = { outcome: 'passed', codeVersion }; + } catch (error) { + result = failure('paritySetupFailed', error); + } + + if (result.outcome === 'passed') { + const command = createParityCommand({ + configurationName: matches[0].name, + sourceGlob: contract.sourceGlob, + lineIncludes: contract.lineIncludes, + triggerUrl: contract.triggerUrl, + timeoutMs: (contract.timeoutSeconds ?? 120) * 1000, + }); + try { + const executed = await this.executeParityCommand( + sandboxId, + command, + (contract.timeoutSeconds ?? 120) * 2000 + 60_000, + ); + try { + const evidence = parseParityEvidence(executed.output); + result = { ...result, evidence }; + } catch (error) { + result = { + outcome: 'failed', + failureCode: executed.exitCode === 0 ? 'parityEvidenceInvalid' : 'parityExecutionFailed', + error: `${getErrorMessage(error)}\n${executed.output}`.trim(), + codeVersion: result.codeVersion, + }; + } + } catch (error) { + const output = getCommandOutput(error); + try { + const evidence = parseParityEvidence(output); + result = evidence.outcome === 'passed' + ? { ...result, evidence } + : failure('parityExecutionFailed', error); + } catch { + result = failure('parityExecutionFailed', error); + } + } + } + } finally { + await Promise.all([ + fs.rm(manifestPath, { force: true }), + fs.rm(workspaceArchive, { force: true }), + fs.rm(parityArchive, { force: true }), + ]); + } + + if (sandboxId) { + try { + await this.aca.run(['sandbox', 'delete', '--id', sandboxId, '--yes'], 3 * 60_000); + } catch (error) { + return failure('parityCleanupFailed', error); + } + } + return result; + } + + private async executeParityCommand( + sandboxId: string, + command: string, + timeoutMs: number, + ): Promise<{ output: string; exitCode: number }> { + const logPath = '/tmp/cor-vscode-parity.log'; + const exitPath = '/tmp/cor-vscode-parity.exit'; + const resultPath = '/tmp/cor-vscode-parity-result.json'; + const wrapped = `${command}; parity_status=$?; printf '%s\\n' "$parity_status" > ${exitPath}; exit "$parity_status"`; + await this.aca.run([ + 'sandbox', 'exec', '--id', sandboxId, + '--working-directory', '/workspace', + '-c', `rm -f ${logPath} ${exitPath} ${resultPath}; nohup sh -c ${shellQuote(wrapped)} > ${logPath} 2>&1 < /dev/null & echo $!`, + ], 60_000); + + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const status = await this.aca.run([ + 'sandbox', 'exec', '--id', sandboxId, '-c', + `cat ${exitPath} 2>/dev/null || true`, + ], 60_000); + const rawExitCode = status.stdout.trim(); + const exitCode = Number(rawExitCode); + if (rawExitCode && Number.isInteger(exitCode)) { + const output = await this.aca.run([ + 'sandbox', 'exec', '--id', sandboxId, '-c', + `cat ${logPath} 2>/dev/null || true; if test -f ${resultPath}; then printf '\\n${resultPrefix}'; cat ${resultPath}; fi`, + ], 60_000); + return { output: `${output.stdout}\n${output.stderr}`.trim(), exitCode }; + } + await new Promise(resolve => setTimeout(resolve, 2_000)); + } + + const output = await this.aca.run([ + 'sandbox', 'exec', '--id', sandboxId, '-c', + `cat ${logPath} 2>/dev/null || true`, + ], 60_000); + const error = new Error(`VS Code parity timed out after ${timeoutMs}ms.`) as Error & { stdout: string }; + error.stdout = `${output.stdout}\n${output.stderr}`.trim(); + throw error; + } + + private async cleanupCreatedSandbox(error: unknown, runLabel: string): Promise { + const ids = new Set(); + try { + ids.add(readSandboxId(getCommandOutput(error))); + } catch { + // Recover by the unique run label below. + } + let recoveryError: string | undefined; + try { + const listed = await this.aca.run([ + 'sandbox', 'list', '-l', `run-id=${runLabel}`, '-o', 'json', + ], 60_000); + readSandboxIds(listed.stdout).forEach(id => ids.add(id)); + } catch (listError) { + recoveryError = getErrorMessage(listError); + } + const deletionErrors = (await Promise.all([...ids].map(async id => { + try { + await this.aca.run(['sandbox', 'delete', '--id', id, '--yes'], 3 * 60_000); + return undefined; + } catch (deleteError) { + return `${id}: ${getErrorMessage(deleteError)}`; + } + }))).filter((value): value is string => value !== undefined); + return [recoveryError, ...deletionErrors].filter(Boolean).join('; ') || undefined; + } +} + +export function createParityCommand(input: { + configurationName: string; + sourceGlob: string; + lineIncludes: string; + triggerUrl: string; + timeoutMs: number; +}): string { + const environment: [string, string][] = [ + ['COR_PARITY_CONFIGURATION', input.configurationName], + ['COR_PARITY_SOURCE_GLOB', input.sourceGlob], + ['COR_PARITY_LINE_INCLUDES', input.lineIncludes], + ['COR_PARITY_TRIGGER_URL', input.triggerUrl], + ['COR_PARITY_TIMEOUT_MS', String(input.timeoutMs)], + ['COR_PARITY_RESULT_PATH', '/tmp/cor-vscode-parity-result.json'], + ]; + const variables = environment + .map(([name, value]) => `${name}=${shellQuote(value)}`) + .join(' '); + return `${variables} xvfb-run -a /home/vscode/.cor-vscode/VSCode-linux-x64/code --no-sandbox --disable-gpu --disable-updates --disable-workspace-trust --skip-welcome --user-data-dir /tmp/cor-vscode-user --extensions-dir /tmp/cor-vscode-extensions --extensionDevelopmentPath=/home/vscode/cor-vscode-parity --extensionTestsPath=/home/vscode/cor-vscode-parity/test.js /workspace`; +} + +export function parseParityEvidence(output: string): VsCodeParityEvidence { + const line = output.split(/\r?\n/).find(value => value.includes(resultPrefix)); + if (!line) { + throw new Error('VS Code parity output did not contain structured evidence.'); + } + const parsed = JSON.parse(line.slice(line.indexOf(resultPrefix) + resultPrefix.length)) as VsCodeParityEvidence; + if ( + parsed.outcome !== 'passed' + || parsed.stoppedReason !== 'breakpoint' + || !parsed.configurationName + || !parsed.source + || !Number.isInteger(parsed.line) + || !Number.isInteger(parsed.column) + ) { + throw new Error('VS Code parity evidence did not prove a breakpoint stop.'); + } + return parsed; +} + +function failure(code: NonNullable, error: unknown): SandboxVsCodeParityResult { + return { outcome: 'failed', failureCode: code, error: getCommandOutput(error) }; +} + +function getCommandOutput(error: unknown): string { + const value = error as { stdout?: string; stderr?: string }; + return [value.stdout, value.stderr, getErrorMessage(error)].filter(Boolean).join('\n'); +} + +function getErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} diff --git a/evals/src/agentAssets.ts b/evals/src/agentAssets.ts new file mode 100644 index 000000000..202d0e918 --- /dev/null +++ b/evals/src/agentAssets.ts @@ -0,0 +1,92 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createHash } from 'crypto'; +import { promises as fs } from 'fs'; +import * as path from 'path'; + +const instructionFolders = [ + 'azure-debug-generate', + 'azure-debug-plan', + 'azure-project-plan', + 'azure-project-scaffold', + 'azure-project-integrate', + 'shared-references', +] as const; + +export async function prepareAgentWorkspace(repoRoot: string, workspace: string): Promise { + const sourceRoot = path.join(repoRoot, 'resources', 'agents'); + const destinationRoot = path.join(workspace, '.github', 'agents'); + await fs.rm(destinationRoot, { recursive: true, force: true }); + await Promise.all([ + fs.mkdir(destinationRoot, { recursive: true }), + // The production create flow writes .azure/.pending-create before launching + // the planning agent, and VS Code's create_file creates nested preview + // parents automatically. The SDK evaluation create tool does neither. + fs.mkdir(path.join(workspace, '.azure', '.preview-temp'), { recursive: true }), + ]); + for (const folder of instructionFolders) { + await fs.cp(path.join(sourceRoot, folder), path.join(destinationRoot, folder), { + recursive: true, + force: true, + }); + } +} + +export async function loadAgentSystemPrompt(repoRoot: string, agentName: string, additionalSystemMessage?: string): Promise { + const filePath = path.join(repoRoot, 'resources', 'agents', `${agentName}.agent.md`); + const body = stripFrontmatter(await fs.readFile(filePath, 'utf8')); + const runtimePreamble = [ + `You are the "${agentName}" agent for the Azure Copilot-on-Rails workflow.`, + 'This is an automated evaluation in an isolated workspace.', + 'The production webview gates are represented by tools with the same names.', + 'When an instruction says to call one of those tools and stop, call it and end the turn.', + ...additionalSystemMessage + ? [] + : ['Do not delegate to sub-agents; this evaluation provides only the workspace file tools needed for the current phase.'], + `Read detailed instructions under \`.github/agents/${agentName}/\` exactly as the production agent does.`, + '', + ].join('\n'); + const runtimeOverride = additionalSystemMessage + ? `\n\n## Evaluation runtime constraints\n\n${additionalSystemMessage.trim()}` + : ''; + return `${runtimePreamble}\n${body}${runtimeOverride}`.trim(); +} + +export async function computeAgentAssetsHash(repoRoot: string): Promise { + const root = path.join(repoRoot, 'resources', 'agents'); + const files = await listFiles(root); + const hash = createHash('sha256'); + for (const file of files) { + hash.update(path.relative(root, file)); + hash.update('\0'); + hash.update(await fs.readFile(file)); + hash.update('\0'); + } + return hash.digest('hex'); +} + +function stripFrontmatter(markdown: string): string { + const normalized = markdown.replace(/^\uFEFF/, ''); + if (!normalized.startsWith('---\n')) { + return normalized.trim(); + } + const end = normalized.indexOf('\n---\n', 4); + return end === -1 ? normalized.trim() : normalized.slice(end + 5).trim(); +} + +async function listFiles(directory: string): Promise { + const entries = await fs.readdir(directory, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + files.push(...await listFiles(entryPath)); + } else if (entry.isFile()) { + files.push(entryPath); + } + } + return files.sort(); +} diff --git a/evals/src/artifacts/deployment.ts b/evals/src/artifacts/deployment.ts new file mode 100644 index 000000000..be0e10110 --- /dev/null +++ b/evals/src/artifacts/deployment.ts @@ -0,0 +1,223 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { promises as fs } from 'fs'; +import * as path from 'path'; +import { + getDeploymentPlanRenderIssue, + parseDeploymentPlanMarkdown, +} from '../../../src/webviews/copilotOnRails/views/utils/parseDeploymentPlanMarkdown'; +import { + ArtifactValidationIssue, + ArtifactValidationResult, + createValidationResult, +} from './validationTypes'; + +export interface DeploymentArtifactValidationResult extends ArtifactValidationResult { + packageCommand: 'azd package'; + serviceNames: string[]; + infrastructure: 'bicep' | 'terraform' | 'missing'; +} + +export async function validateDeploymentArtifacts( + workspace: string, + planContent: string, +): Promise { + const issues: ArtifactValidationIssue[] = []; + const plan = parseDeploymentPlanMarkdown(planContent); + const renderIssue = getDeploymentPlanRenderIssue(planContent, plan); + if (renderIssue) { + addIssue(issues, renderIssue, '$.deploymentPlan', 'Deployment plan is empty or lacks structured architecture and resource sections.'); + } + if (!plan.location || !plan.resources.rows.length || !plan.workspaceScan.rows.length) { + addIssue(issues, 'incompleteDeploymentPlan', '$.deploymentPlan', 'Deployment plan requires a location, component inventory, and Azure resource mapping.'); + } + + const azureYamlPath = path.join(workspace, 'azure.yaml'); + const azureYaml = await readRequiredFile(azureYamlPath, '$.azureYaml', issues); + const serviceNames = azureYaml ? parseServiceNames(azureYaml) : []; + if (azureYaml && !/^\s*name\s*:\s*\S+/m.test(azureYaml)) { + addIssue(issues, 'missingAzdName', '$.azureYaml.name', 'azure.yaml requires a project name.'); + } + if (!serviceNames.length) { + addIssue(issues, 'missingAzdServices', '$.azureYaml.services', 'azure.yaml requires at least one service.'); + } + validateHooks(azureYaml, issues); + await validateServicePaths(workspace, azureYaml, serviceNames, issues); + + const infrastructure = await detectInfrastructure(workspace); + if (infrastructure === 'missing') { + addIssue(issues, 'missingInfrastructure', '$.infra', 'Deployment artifacts require Bicep or Terraform infrastructure.'); + } + await validateSecretHygiene(workspace, issues); + + return { + ...createValidationResult(issues), + packageCommand: 'azd package', + serviceNames, + infrastructure, + }; +} + +function parseServiceNames(content: string): string[] { + const services = content.match(/(?:^|\n)services\s*:\s*\n((?:[ \t]+.*(?:\n|$))*)/m)?.[1] ?? ''; + const minimumIndent = services + .split('\n') + .filter(line => line.trim()) + .map(line => line.match(/^\s*/)?.[0].length ?? 0) + .reduce((minimum, value) => Math.min(minimum, value), Number.POSITIVE_INFINITY); + if (!Number.isFinite(minimumIndent)) { + return []; + } + return services + .split('\n') + .flatMap(line => { + const match = line.match(new RegExp(`^\\s{${minimumIndent}}([A-Za-z0-9_-]+)\\s*:\\s*$`)); + return match ? [match[1]] : []; + }); +} + +function validateHooks(content: string, issues: ArtifactValidationIssue[]): void { + for (const match of content.matchAll(/^\s*shell\s*:\s*(\S+)\s*$/gm)) { + if (!['sh', 'pwsh'].includes(match[1])) { + addIssue(issues, 'invalidAzdHookShell', '$.azureYaml.hooks', `azd hook shell "${match[1]}" is unsupported; use sh or pwsh.`); + } + } +} + +async function validateServicePaths( + workspace: string, + content: string, + serviceNames: string[], + issues: ArtifactValidationIssue[], +): Promise { + const workspaceRoot = await fs.realpath(workspace); + for (const serviceName of serviceNames) { + const block = content.match(new RegExp(`^\\s+${escapeRegex(serviceName)}\\s*:\\s*\\n((?:\\s{4,}.*(?:\\n|$))*)`, 'm'))?.[1] ?? ''; + const project = block.match(/^\s+project\s*:\s*["']?([^"'#\n]+)["']?\s*$/m)?.[1].trim(); + const host = block.match(/^\s+host\s*:\s*(\S+)\s*$/m)?.[1]; + if (!project || !host) { + addIssue(issues, 'incompleteAzdService', `$.azureYaml.services.${serviceName}`, 'Each azd service requires project and host fields.'); + continue; + } + try { + const projectPath = await fs.realpath(path.resolve(workspace, project)); + const relative = path.relative(workspaceRoot, projectPath); + if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + addIssue( + issues, + 'azdServicePathOutsideWorkspace', + `$.azureYaml.services.${serviceName}.project`, + `Service project path "${project}" must stay within the evaluated workspace.`, + ); + } + } catch { + addIssue(issues, 'missingAzdServicePath', `$.azureYaml.services.${serviceName}.project`, `Service project path "${project}" does not exist.`); + } + } +} + +async function detectInfrastructure(workspace: string): Promise { + if (await exists(path.join(workspace, 'infra', 'main.bicep'))) { + return 'bicep'; + } + const terraformFiles = await listFiles(path.join(workspace, 'infra'), file => file.endsWith('.tf')); + return terraformFiles.length ? 'terraform' : 'missing'; +} + +async function validateSecretHygiene(workspace: string, issues: ArtifactValidationIssue[]): Promise { + const candidates = [ + path.join(workspace, 'azure.yaml'), + ...await listFiles(path.join(workspace, 'infra'), file => /\.(?:bicep|tf|json|ya?ml)$/i.test(file)), + ]; + const secretPattern = /\b(?:password|clientSecret|accountKey)\b\s*[:=]\s*["']?(?!\$\{|parameters?\(|getSecret\(|@secure\(|<)[A-Za-z0-9+/=_-]{8,}/i; + for (const file of candidates) { + const content = await readOptionalFile(file); + if (content !== undefined && secretPattern.test(content)) { + addIssue(issues, 'hardcodedSecret', path.relative(workspace, file), 'Deployment artifacts must not contain hard-coded credentials.'); + } + } +} + +async function readRequiredFile(file: string, issuePath: string, issues: ArtifactValidationIssue[]): Promise { + try { + return await fs.readFile(file, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + addIssue(issues, 'missingDeploymentArtifact', issuePath, `${path.basename(file)} is required.`); + return ''; + } + throw error; + } +} + +async function readOptionalFile(file: string): Promise { + try { + return await fs.readFile(file, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return undefined; + } + throw error; + } +} + +async function listFiles(directory: string, include: (file: string) => boolean): Promise { + try { + const entries = await fs.readdir(directory, { withFileTypes: true }); + const nested = await Promise.all(entries.map(async entry => { + const file = path.join(directory, entry.name); + return entry.isDirectory() ? await listFiles(file, include) : include(file) ? [file] : []; + })); + return nested.flat(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return []; + } + throw error; + } +} + +async function exists(file: string): Promise { + try { + await fs.access(file); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return false; + } + throw error; + } +} + +function escapeRegex(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function addIssue(issues: ArtifactValidationIssue[], code: string, issuePath: string, message: string): void { + issues.push({ code, path: issuePath, message }); +} + +async function main(): Promise { + const args = process.argv.slice(2); + const value = (name: string): string | undefined => { + const index = args.indexOf(name); + return index >= 0 ? args[index + 1] : undefined; + }; + const workspace = path.resolve(value('--workspace') ?? process.cwd()); + const planPath = path.resolve(value('--plan') ?? path.join(workspace, '.azure', 'deployment-plan.md')); + const result = await validateDeploymentArtifacts(workspace, await fs.readFile(planPath, 'utf8')); + process.stdout.write(JSON.stringify(result, null, 2) + '\n'); + if (!result.valid) { + process.exitCode = 1; + } +} + +if (require.main === module) { + void main().catch(error => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/evals/src/artifacts/integrationOutput.ts b/evals/src/artifacts/integrationOutput.ts new file mode 100644 index 000000000..b5bf49613 --- /dev/null +++ b/evals/src/artifacts/integrationOutput.ts @@ -0,0 +1,115 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { promises as fs } from 'fs'; +import * as path from 'path'; +import { + ArtifactValidationIssue, + ArtifactValidationResult, + createValidationResult, +} from './validationTypes'; + +const ignoredDirectories = new Set(['.git', '.azure', 'dist', 'node_modules', 'out']); +const sourceExtension = /\.(?:[cm]?[jt]sx?|html|svelte|vue)$/i; +const testFilePattern = /(?:^|\/)(?:[^/]+\.(?:test|spec)\.[^/]+|tests?\/|__tests__\/)/i; +const mockImportPattern = /\b(?:from\s+|import\s+|import\s*\()(['"`])[^'"`]*(?:mockClient|previewState|PreviewStateSwitcher|(?:^|\/)mocks?(?:\/|(?=['"`])))[^'"`]*\1/i; +const mockIdentityPattern = /(?:x-mock-user-id|mockUserId|MOCK_USER_ID)/i; +const uuidPattern = /\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b/gi; +const persistenceSetupPattern = /(?:^|\/)(?:migrations?|seeds?)(?:\/|$)|(?:^|\/)[^/]*seed[^/]*\.[^/]+$/i; + +export async function validateIntegrationOutput( + workspace: string, + options: { hasFrontend: boolean }, +): Promise { + if (!options.hasFrontend) { + return createValidationResult([]); + } + + const issues: ArtifactValidationIssue[] = []; + const files = await listSourceFiles(workspace); + const frontendRoots = inferFrontendRoots(files); + if (!frontendRoots.length) { + issues.push(issue('frontendSourceMissing', '$', 'Integrated project does not contain frontend source files.')); + } + const frontendMockIdentities = new Map(); + for (const relativePath of files.filter(file => + frontendRoots.some(root => root === '.' || file === root || file.startsWith(`${root}/`)))) { + if (testFilePattern.test(relativePath)) { + continue; + } + const content = await fs.readFile(path.join(workspace, relativePath), 'utf8'); + if (mockImportPattern.test(content)) { + issues.push(issue( + 'frontendMockStillImported', + relativePath, + 'Integrated frontend source must not import mock data or preview-state modules.', + )); + } + if (mockIdentityPattern.test(content)) { + for (const identity of content.match(uuidPattern) ?? []) { + frontendMockIdentities.set(identity.toLowerCase(), relativePath); + } + } + } + + if (frontendMockIdentities.size) { + const persistenceContents = await Promise.all(files + .filter(file => persistenceSetupPattern.test(file)) + .filter(file => !testFilePattern.test(file)) + .map(file => fs.readFile(path.join(workspace, file), 'utf8'))); + for (const [identity, relativePath] of frontendMockIdentities) { + if (!persistenceContents.some(content => content.toLowerCase().includes(identity))) { + issues.push(issue( + 'mockAuthIdentityUnseeded', + relativePath, + `Frontend mock identity ${identity} must be provisioned by backend migration or seed source.`, + )); + } + } + } + + return createValidationResult(issues); +} + +function inferFrontendRoots(files: string[]): string[] { + const roots = new Set(); + for (const file of files) { + const parts = file.split('/'); + const namedRoot = parts.findIndex(part => ['client', 'frontend', 'ui', 'web'].includes(part.toLowerCase())); + if (namedRoot >= 0) { + roots.add(parts.slice(0, namedRoot + 1).join('/')); + continue; + } + if (/\.(?:jsx|tsx|svelte|vue)$/i.test(file) || /(?:^|\/)index\.html$/i.test(file)) { + const sourceIndex = parts.lastIndexOf('src'); + roots.add(sourceIndex > 0 ? parts.slice(0, sourceIndex).join('/') : '.'); + } + } + return [...roots].sort(); +} + +async function listSourceFiles(root: string): Promise { + const files: string[] = []; + async function visit(directory: string): Promise { + const entries = await fs.readdir(directory, { withFileTypes: true }); + await Promise.all(entries.map(async entry => { + if (entry.isDirectory() && ignoredDirectories.has(entry.name)) { + return; + } + const absolutePath = path.join(directory, entry.name); + if (entry.isDirectory()) { + await visit(absolutePath); + } else if (entry.isFile() && sourceExtension.test(entry.name)) { + files.push(path.relative(root, absolutePath).split(path.sep).join('/')); + } + })); + } + await visit(root); + return files.sort(); +} + +function issue(code: string, issuePath: string, message: string): ArtifactValidationIssue { + return { code, path: issuePath, message }; +} diff --git a/evals/src/artifacts/integrationPlan.ts b/evals/src/artifacts/integrationPlan.ts new file mode 100644 index 000000000..a21c56a3f --- /dev/null +++ b/evals/src/artifacts/integrationPlan.ts @@ -0,0 +1,41 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { + ArtifactValidationIssue, + ArtifactValidationResult, + createValidationResult, +} from './validationTypes'; + +const requiredContent = [ + { code: 'missingBackend', pattern: /\bbackend\b/i, message: 'Integration plan must describe the backend project and commands.' }, + { code: 'missingRoutes', pattern: /\b(api routes?|endpoints?)\b/i, message: 'Integration plan must inventory API routes.' }, + { code: 'missingDatabase', pattern: /\b(database|data store|cosmos|blob storage|queue storage|redis)\b/i, message: 'Integration plan must describe persistence or explicitly state none.' }, + { code: 'missingServices', pattern: /\bservices?\b/i, message: 'Integration plan must list planned services.' }, + { code: 'missingNoSeedRule', pattern: /\bno seed data\b/i, message: 'Integration plan must explicitly prohibit seed data.' }, +] as const; + +export function validateIntegrationPlanArtifact( + content: string, + options: { hasFrontend: boolean }, +): ArtifactValidationResult { + const issues: ArtifactValidationIssue[] = []; + if (content.trim().length < 200) { + issues.push(issue('artifactTooShort', '$', 'Integration plan is too short to hand off the scaffold.')); + } + for (const requirement of requiredContent) { + if (!requirement.pattern.test(content)) { + issues.push(issue(requirement.code, '$', requirement.message)); + } + } + if (options.hasFrontend && !/\bfrontend\b/i.test(content)) { + issues.push(issue('missingFrontend', '$', 'Integration plan must describe the frontend commands and API seam.')); + } + return createValidationResult(issues); +} + +function issue(code: string, path: string, message: string): ArtifactValidationIssue { + return { code, path, message }; +} diff --git a/evals/src/artifacts/localDebug.ts b/evals/src/artifacts/localDebug.ts new file mode 100644 index 000000000..5c5da91f5 --- /dev/null +++ b/evals/src/artifacts/localDebug.ts @@ -0,0 +1,1544 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { promises as fs } from 'fs'; +import * as path from 'path'; +import { parse, ParseError, printParseErrorCode } from 'jsonc-parser'; +import { + findColumnIndex, + findSection, + findTable, + flattenContent, + isChecked, + parseLocalDebugPlanMarkdown, +} from '../../../src/webviews/copilotOnRails/views/utils/parseLocalDebugPlanMarkdown'; +import { ArtifactValidationIssue, ArtifactValidationResult } from './validationTypes'; +import { SandboxLocalRuntimeValidationResult } from '../SandboxLocalRuntimeValidator'; + +export interface LocalDebugPlanValidationOptions { + expectedStatus?: 'Planning' | 'Approved' | 'Executing' | 'Implemented'; + requireAutoMode?: boolean; + requireSuccessfulChecklist?: boolean; +} + +interface DebugConfiguration { + name?: unknown; + preLaunchTask?: unknown; +} + +interface DebugCompound { + name?: unknown; + configurations?: unknown; +} + +interface DebugTask { + label?: unknown; + command?: unknown; + args?: unknown; + dependsOn?: unknown; + isBackground?: unknown; + problemMatcher?: unknown; + options?: { + cwd?: unknown; + }; + runOptions?: { + instanceLimit?: unknown; + instancePolicy?: unknown; + }; +} + +export function validateLocalDebugPlanArtifact( + content: string, + options: LocalDebugPlanValidationOptions = {}, +): ArtifactValidationResult { + const issues: ArtifactValidationIssue[] = []; + const plan = parseLocalDebugPlanMarkdown(content); + if (!content.trim()) { + addIssue(issues, 'emptyArtifact', '$', 'Local debug plan must not be empty.'); + return result(issues); + } + if (plan.title !== 'Azure Debug Plan') { + addIssue(issues, 'invalidTitle', '$.title', 'Local debug plan title must be "Azure Debug Plan".'); + } + if (!['Planning', 'Approved', 'Executing', 'Implemented'].includes(plan.status)) { + addIssue(issues, 'invalidStatus', '$.status', `Unsupported local debug plan status "${plan.status}".`); + } else if (options.expectedStatus && plan.status !== options.expectedStatus) { + addIssue( + issues, + 'unexpectedStatus', + '$.status', + `Expected local debug plan status "${options.expectedStatus}", found "${plan.status}".`, + ); + } + if (!['Auto', 'Guided'].includes(plan.executionMode)) { + addIssue(issues, 'invalidExecutionMode', '$.executionMode', 'Execution Mode must be Auto or Guided.'); + } else if (options.requireAutoMode && plan.executionMode !== 'Auto') { + addIssue(issues, 'unexpectedExecutionMode', '$.executionMode', 'Headless evaluation requires Execution Mode Auto.'); + } + + const prerequisites = findSection(plan, 'Prerequisites'); + const prerequisiteTable = prerequisites && findTable(prerequisites, ['Tool', 'Installed']); + if (!prerequisiteTable?.rows.length) { + addIssue(issues, 'missingPrerequisites', '$.sections.prerequisites', 'Prerequisites must contain a populated tool inventory.'); + } + + const configurationsSection = findSection(plan, 'Debug Configurations'); + const configurations = configurationsSection + && findTable(configurationsSection, ['Generate', 'Debug Config Name', 'Service Root', 'Project Type', 'Runtime']); + if (!configurations) { + addIssue( + issues, + 'missingDebugConfigurations', + '$.sections.debugConfigurations', + 'Debug Configurations must contain the canonical service table.', + ); + } else { + const generateIndex = findColumnIndex(configurations.headers, 'Generate'); + const nameIndex = findColumnIndex(configurations.headers, 'Debug Config Name'); + const serviceRootIndex = findColumnIndex(configurations.headers, 'Service Root'); + const projectTypeIndex = findColumnIndex(configurations.headers, 'Project Type'); + const runtimeIndex = findColumnIndex(configurations.headers, 'Runtime'); + const checked = configurations.rows.filter(row => isChecked(row[generateIndex] ?? '')); + if (!checked.length) { + addIssue(issues, 'noGeneratedConfigurations', '$.sections.debugConfigurations', 'At least one debug configuration must be selected.'); + } + checked.forEach((row, index) => { + if (!row[nameIndex]?.trim()) { + addIssue(issues, 'missingDebugConfigName', `$.sections.debugConfigurations.rows[${index}]`, 'Selected configuration must have a name.'); + } + const compound = row.some(cell => /compound\s+config/i.test(cell)); + if (!compound && (!row[serviceRootIndex]?.trim() || !row[projectTypeIndex]?.trim() || !row[runtimeIndex]?.trim())) { + addIssue( + issues, + 'incompleteDebugConfiguration', + `$.sections.debugConfigurations.rows[${index}]`, + 'Selected service configurations require Service Root, Project Type, and Runtime.', + ); + } + }); + } + + const orchestrator = findSection(plan, 'Orchestrator'); + if (!orchestrator || !findTable(orchestrator, ['Orchestrator'])?.rows.length) { + addIssue(issues, 'missingOrchestrator', '$.sections.orchestrator', 'Orchestrator must contain a selected local orchestration strategy.'); + } + const architecture = findSection(plan, 'Architecture'); + if (!architecture || !flattenContent(architecture.content).some(value => value.type === 'codeBlock')) { + addIssue(issues, 'missingArchitectureDiagram', '$.sections.architecture', 'Architecture Diagram must contain a code block.'); + } + + const requireSuccessfulChecklist = options.requireSuccessfulChecklist ?? plan.status === 'Implemented'; + if (requireSuccessfulChecklist) { + const checklist = findSection(plan, 'Debug Configuration Checklist'); + const checklistText = checklist + ? flattenContent(checklist.content) + .flatMap(value => { + switch (value.type) { + case 'paragraph': + case 'blockquote': + return [value.text]; + case 'bulletList': + return value.items; + default: + return []; + } + }) + .join('\n') + : ''; + if (!checklistText.includes('✅')) { + addIssue(issues, 'missingSuccessfulChecklist', '$.sections.debugConfigurationChecklist', 'Implemented plans require real successful validation evidence.'); + } + if (/❌||placeholder|not run/i.test(checklistText)) { + addIssue(issues, 'unsuccessfulChecklist', '$.sections.debugConfigurationChecklist', 'Implemented plans must not contain failed or placeholder checklist entries.'); + } + } + return result(issues); +} + +export async function validateLocalDebugArtifacts( + workspace: string, + planContent: string, + options: { requireSuccessfulChecklist?: boolean } = {}, +): Promise { + const planResult = validateLocalDebugPlanArtifact(planContent, { + requireAutoMode: true, + requireSuccessfulChecklist: options.requireSuccessfulChecklist, + }); + const issues = [...planResult.issues]; + const selected = getSelectedConfigurations(planContent); + const launch = await readJsoncFile(path.join(workspace, '.vscode', 'launch.json'), '$.launch', issues); + const tasks = await readJsoncFile(path.join(workspace, '.vscode', 'tasks.json'), '$.tasks', issues); + const extensions = await readJsoncFile(path.join(workspace, '.vscode', 'extensions.json'), '$.extensions', issues); + await readJsoncFile(path.join(workspace, '.vscode', 'settings.json'), '$.settings', issues); + + const launchConfigurations = Array.isArray(launch?.configurations) + ? launch.configurations as DebugConfiguration[] + : []; + const launchCompounds = Array.isArray(launch?.compounds) + ? launch.compounds as DebugCompound[] + : []; + if (!launchConfigurations.length) { + addIssue(issues, 'missingLaunchConfigurations', '$.launch.configurations', 'launch.json must contain at least one configuration.'); + } + const configurationNames = new Set( + launchConfigurations.flatMap(value => typeof value.name === 'string' ? [value.name] : []), + ); + const compoundNames = new Set( + launchCompounds.flatMap(value => typeof value.name === 'string' ? [value.name] : []), + ); + for (const configuration of selected.services) { + if (!configurationNames.has(configuration)) { + addIssue(issues, 'missingLaunchConfiguration', '$.launch.configurations', `Missing selected launch configuration "${configuration}".`); + } + } + for (const compound of selected.compounds) { + if (!compoundNames.has(compound)) { + addIssue(issues, 'missingCompoundConfiguration', '$.launch.compounds', `Missing selected compound configuration "${compound}".`); + } + } + for (const [index, compound] of launchCompounds.entries()) { + const members = Array.isArray(compound.configurations) ? compound.configurations : []; + if (!members.length || members.some(value => typeof value !== 'string' || !configurationNames.has(value))) { + addIssue( + issues, + 'invalidCompoundMembers', + `$.launch.compounds[${index}].configurations`, + 'Every compound member must reference an existing launch configuration.', + ); + } + } + + const taskValues = Array.isArray(tasks?.tasks) ? tasks.tasks as DebugTask[] : []; + const taskLabels = taskValues.flatMap(task => typeof task.label === 'string' ? [task.label] : []); + const uniqueTaskLabels = new Set(taskLabels); + if (!taskValues.length) { + addIssue(issues, 'missingTasks', '$.tasks.tasks', 'tasks.json must contain at least one task.'); + } + if (uniqueTaskLabels.size !== taskLabels.length) { + addIssue(issues, 'duplicateTaskLabels', '$.tasks.tasks', 'Task labels must be unique.'); + } + taskValues.forEach((task, index) => { + if (task.runOptions?.instanceLimit !== 1 || task.runOptions?.instancePolicy !== 'silent') { + addIssue( + issues, + 'invalidTaskRunOptions', + `$.tasks.tasks[${index}].runOptions`, + 'Every generated task requires instanceLimit 1 and instancePolicy "silent".', + ); + } + if (task.isBackground === true && isEmptyProblemMatcher(task.problemMatcher)) { + addIssue( + issues, + 'missingBackgroundProblemMatcher', + `$.tasks.tasks[${index}].problemMatcher`, + 'Background tasks require a non-empty problem matcher.', + ); + } + for (const dependency of normalizeStringList(task.dependsOn)) { + if (!uniqueTaskLabels.has(dependency)) { + addIssue( + issues, + 'missingTaskDependency', + `$.tasks.tasks[${index}].dependsOn`, + `Task dependency "${dependency}" does not exist.`, + ); + } + } + }); + for (const [index, configuration] of launchConfigurations.entries()) { + if (typeof configuration.preLaunchTask !== 'string' || !uniqueTaskLabels.has(configuration.preLaunchTask)) { + addIssue( + issues, + 'invalidPreLaunchTask', + `$.launch.configurations[${index}].preLaunchTask`, + 'Every launch configuration must reference an existing preLaunchTask.', + ); + } + } + detectTaskCycles(taskValues, issues); + const rootManifest = await readPackageManifest(path.resolve(workspace)); + validateNodeServiceInstallTasks(workspace, taskValues, issues, rootManifest?.declaresWorkspaces === true); + await validateNpmWorkspaceTaskTooling(workspace, taskValues, issues, rootManifest); + await validateWorkspaceDependencyBuildOrder(workspace, taskValues, issues, rootManifest); + await validateRedactedSecretPlaceholders(workspace, issues); + await validateComposeInterpolationSource(workspace, issues); + await validateTaskEnvironmentAvailability(workspace, taskValues, issues, rootManifest); + await validateDeclaredTestScripts(workspace, issues, rootManifest); + await validateFluentUiTestInterop(workspace, issues, rootManifest); + await validateKnexCheckConstraints(workspace, issues); + await validateGeneratedConfigSyntax(workspace, issues); + + const recommendations = Array.isArray(extensions?.recommendations) ? extensions.recommendations : []; + if (!recommendations.length || recommendations.some(value => typeof value !== 'string' || !value.includes('.'))) { + addIssue( + issues, + 'missingExtensionRecommendations', + '$.extensions.recommendations', + 'extensions.json must contain valid VS Code extension recommendations.', + ); + } + + if (planSectionHasRows(planContent, 'Emulators')) { + const composePath = await findFirstExisting(workspace, ['docker-compose.yml', 'docker-compose.yaml', 'compose.yml', 'compose.yaml']); + if (!composePath) { + addIssue(issues, 'missingComposeFile', '$.dockerCompose', 'Plans with emulators require a Docker Compose file.'); + } else if (/\bazurite\b/i.test(planContent)) { + const composeContent = await fs.readFile(path.join(workspace, composePath), 'utf8'); + if (!/skipApiVersionCheck/i.test(composeContent)) { + addIssue( + issues, + 'azuriteApiVersionCheckEnabled', + '$.dockerCompose.services.azurite.command', + 'Azurite must start with --skipApiVersionCheck so newer Azure Storage SDK API versions work locally.', + ); + } + } + if (!uniqueTaskLabels.has('Start Emulators')) { + addIssue(issues, 'missingEmulatorTask', '$.tasks.tasks', 'Plans with emulators require a "Start Emulators" task.'); + } + } + if (planSectionHasCheckedRows(planContent, 'API Test Collections')) { + const apiTests = await listFilesIfPresent(path.join(workspace, 'api-test-collections')); + if (!apiTests.some(file => /invoke\.(?:sh|ps1)$/i.test(file))) { + addIssue( + issues, + 'missingApiTestCollections', + '$.apiTestCollections', + 'Selected API test collections require generated invoke.sh or invoke.ps1 scripts.', + ); + } + } + + return result(issues); +} + +export function applyLocalRuntimeEvidence( + content: string, + validation: SandboxLocalRuntimeValidationResult, + now: string = new Date().toISOString(), +): string { + if (validation.outcome !== 'passed' || !validation.probes.length) { + throw new Error('Successful local runtime evidence is required before implementing the debug plan.'); + } + const checklist = [ + '## Debug Configuration Checklist', + '', + 'Debug Configuration Checklist:', + ...validation.probes.map(probe => probe.processPattern + ? `✅ ${probe.name} — process matching \`${probe.processPattern}\` remained live.` + : `✅ ${probe.name} — ${probe.method} ${probe.url} returned ${probe.expectedStatus}${probe.response ? `; response captured (${probe.response.length} characters)` : ''}.`), + '', + ].join('\n'); + let updated = content + .replace( + /^(\s*>\s*\*\*Status:?\*\*:?\s*)(Planning|Approved|Executing|Implemented)\s*$/im, + '$1Implemented', + ) + .replace( + /^(\s*>\s*\*\*Last Updated:?\*\*:?\s*).+$/im, + `$1${now}`, + ); + if (/^##\s+Debug Configuration Checklist\s*$/im.test(updated)) { + updated = updated.replace(/^##\s+Debug Configuration Checklist\s*$[\s\S]*$/im, checklist); + } else { + updated = `${updated.trimEnd()}\n\n---\n\n${checklist}`; + } + return `${updated.trimEnd()}\n`; +} + +function getSelectedConfigurations(content: string): { services: string[]; compounds: string[] } { + const plan = parseLocalDebugPlanMarkdown(content); + const section = findSection(plan, 'Debug Configurations'); + const table = section && findTable(section, ['Generate', 'Debug Config Name']); + if (!table) { + return { services: [], compounds: [] }; + } + const generateIndex = findColumnIndex(table.headers, 'Generate'); + const nameIndex = findColumnIndex(table.headers, 'Debug Config Name'); + const services: string[] = []; + const compounds: string[] = []; + for (const row of table.rows.filter(value => isChecked(value[generateIndex] ?? ''))) { + const name = row[nameIndex]?.trim(); + if (!name) { + continue; + } + (row.some(cell => /compound\s+config/i.test(cell)) ? compounds : services).push(name); + } + return { services, compounds }; +} + +async function readJsoncFile( + filePath: string, + issuePath: string, + issues: ArtifactValidationIssue[], +): Promise | undefined> { + let content: string; + try { + content = await fs.readFile(filePath, 'utf8'); + } catch { + addIssue(issues, 'missingDebugArtifact', issuePath, `Required debug artifact is missing: ${filePath}.`); + return undefined; + } + const errors: ParseError[] = []; + const value: unknown = parse(content, errors, { allowTrailingComma: true, disallowComments: false }); + if (errors.length || !value || typeof value !== 'object' || Array.isArray(value)) { + const details = errors.map(error => printParseErrorCode(error.error)).join(', '); + addIssue(issues, 'invalidDebugArtifactJson', issuePath, `Debug artifact must be valid JSONC${details ? `: ${details}` : ''}.`); + return undefined; + } + return value as Record; +} + +function detectTaskCycles(tasks: DebugTask[], issues: ArtifactValidationIssue[]): void { + const graph = new Map(); + for (const task of tasks) { + if (typeof task.label === 'string') { + graph.set(task.label, normalizeStringList(task.dependsOn)); + } + } + const visited = new Set(); + const active = new Set(); + const visit = (label: string): boolean => { + if (active.has(label)) { + return true; + } + if (visited.has(label)) { + return false; + } + visited.add(label); + active.add(label); + const cyclic = (graph.get(label) ?? []).some(visit); + active.delete(label); + return cyclic; + }; + if ([...graph.keys()].some(visit)) { + addIssue(issues, 'cyclicTaskDependency', '$.tasks.tasks', 'Task dependency graph must not contain cycles.'); + } +} + +function validateNodeServiceInstallTasks( + workspace: string, + tasks: DebugTask[], + issues: ArtifactValidationIssue[], + isWorkspacesMonorepo: boolean, +): void { + const byLabel = new Map(tasks.flatMap(task => + typeof task.label === 'string' ? [[task.label, task] as const] : [])); + const root = path.resolve(workspace); + const installTasksByCwd = new Map(); + const rootInstallLabels: string[] = []; + for (const task of tasks) { + const command = typeof task.command === 'string' ? task.command : ''; + const label = typeof task.label === 'string' ? task.label : ''; + const cwd = typeof task.options?.cwd === 'string' ? task.options.cwd : ''; + if (!label || !/\bnpm\s+(?:install|ci)\b/i.test(command)) { + continue; + } + if (cwd) { + installTasksByCwd.set(cwd, [...(installTasksByCwd.get(cwd) ?? []), label]); + } + if (resolveTaskDirectory(workspace, task.options?.cwd) === root) { + rootInstallLabels.push(label); + } + } + + for (const [index, task] of tasks.entries()) { + const command = typeof task.command === 'string' ? task.command : ''; + const cwd = typeof task.options?.cwd === 'string' ? task.options.cwd : ''; + if (!cwd || !/\bnpm\s+run\s+(?:dev|start|watch)\b/i.test(command) || /\bnpm\s+(?:install|ci)\b/i.test(command)) { + continue; + } + // In a workspaces monorepo the correct install runs at the workspace root, so a shared + // root install task satisfies a service whose own cwd is a workspace member. + const installTasks = [ + ...(installTasksByCwd.get(cwd) ?? []), + ...(isWorkspacesMonorepo ? rootInstallLabels : []), + ]; + if (!installTasks.length) { + addIssue( + issues, + 'missingServiceInstallTask', + `$.tasks.tasks[${index}]`, + `Node service task "${String(task.label)}" requires an npm install or npm ci task with the same cwd.`, + ); + continue; + } + const reachable = collectTaskDependencies(task, byLabel); + if (!installTasks.some(label => reachable.has(label))) { + addIssue( + issues, + 'unreachableServiceInstallTask', + `$.tasks.tasks[${index}].dependsOn`, + `Node service task "${String(task.label)}" must depend on its same-cwd install task.`, + ); + } + } +} + +/** + * npm workspaces install scoping: running `npm install` inside a member directory installs only + * that member's dependency graph and skips the root package's own devDependencies. A tool declared + * only at the root is therefore never materialized, and the member script fails with exit code 127. + */ +async function validateNpmWorkspaceTaskTooling( + workspace: string, + tasks: DebugTask[], + issues: ArtifactValidationIssue[], + rootManifest: PackageManifestSummary | undefined, +): Promise { + const root = path.resolve(workspace); + if (!rootManifest || !rootManifest.declaresWorkspaces) { + return; + } + const byLabel = new Map(tasks.flatMap(task => + typeof task.label === 'string' ? [[task.label, task] as const] : [])); + const rootInstallLabels = new Set(tasks.flatMap(task => { + const command = typeof task.command === 'string' ? task.command : ''; + const label = typeof task.label === 'string' ? task.label : ''; + if (!label || !/\bnpm\s+(?:install|ci)\b/i.test(command)) { + return []; + } + return resolveTaskDirectory(workspace, task.options?.cwd) === root ? [label] : []; + })); + + for (const [index, task] of tasks.entries()) { + const command = typeof task.command === 'string' ? task.command : ''; + const script = /\bnpm\s+run\s+([\w:.-]+)/i.exec(command)?.[1]; + const directory = resolveTaskDirectory(workspace, task.options?.cwd); + if (!script || directory === root) { + continue; + } + const reachable = collectTaskDependencies(task, byLabel); + if (![...rootInstallLabels].some(label => reachable.has(label))) { + addIssue( + issues, + 'missingWorkspaceRootInstallTask', + `$.tasks.tasks[${index}].dependsOn`, + `Task "${String(task.label)}" runs an npm script inside workspace member "${path.relative(root, directory) || '.'}" but never depends on a workspace-root install task. A member-scoped npm install does not install root dependencies.`, + ); + } + const manifest = await readPackageManifest(directory); + const body = manifest?.scripts[script]; + if (!manifest || !body) { + continue; + } + for (const executable of invokedExecutables(body)) { + const owner = BIN_PACKAGE_OWNERS[executable] ?? executable; + if (manifest.dependencyNames.has(owner) || !rootManifest.dependencyNames.has(owner)) { + continue; + } + addIssue( + issues, + 'undeclaredWorkspaceToolDependency', + `$.tasks.tasks[${index}].command`, + `Script "${script}" in "${path.relative(root, directory)}/package.json" invokes "${executable}", which is declared only in the workspace root package.json. Declare "${owner}" in the package that invokes it.`, + ); + } + } +} + +interface PackageManifestSummary { + declaresWorkspaces: boolean; + dependencyNames: Set; + scripts: Record; + name?: string; + workspacePatterns: string[]; + buildOutputEntries: string[]; +} + +const BIN_PACKAGE_OWNERS: Record = { + 'cross-env': 'cross-env', + eslint: 'eslint', + jest: 'jest', + nodemon: 'nodemon', + prettier: 'prettier', + rimraf: 'rimraf', + 'ts-node': 'ts-node', + tsc: 'typescript', + tsx: 'tsx', + vite: 'vite', + vitest: 'vitest', +}; + +const SHELL_KEYWORDS = new Set([ + 'cd', 'do', 'done', 'echo', 'else', 'exit', 'export', 'false', 'fi', 'for', 'if', 'set', 'then', 'true', 'while', +]); + +function resolveTaskDirectory(workspace: string, cwd: unknown): string { + const root = path.resolve(workspace); + if (typeof cwd !== 'string' || !cwd.trim()) { + return root; + } + const expanded = cwd.replace(/\$\{workspaceFolder\}/g, root).replace(/\\/g, '/'); + return path.resolve(path.isAbsolute(expanded) ? expanded : path.join(root, expanded)); +} + +async function readPackageManifest(directory: string): Promise { + let raw: Record; + try { + raw = JSON.parse(await fs.readFile(path.join(directory, 'package.json'), 'utf8')) as Record; + } catch { + return undefined; + } + const dependencyNames = new Set(); + for (const field of ['dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies']) { + const value = raw[field]; + if (value && typeof value === 'object' && !Array.isArray(value)) { + Object.keys(value).forEach(name => dependencyNames.add(name)); + } + } + const scripts: Record = {}; + const rawScripts = raw.scripts; + if (rawScripts && typeof rawScripts === 'object' && !Array.isArray(rawScripts)) { + for (const [name, body] of Object.entries(rawScripts)) { + if (typeof body === 'string') { + scripts[name] = body; + } + } + } + const workspaces = raw.workspaces; + const workspacePatterns = Array.isArray(workspaces) + ? workspaces.filter((value): value is string => typeof value === 'string') + : (workspaces && typeof workspaces === 'object' && Array.isArray((workspaces as { packages?: unknown }).packages) + ? ((workspaces as { packages: unknown[] }).packages).filter((value): value is string => typeof value === 'string') + : []); + const declaresWorkspaces = Array.isArray(workspaces) + || (!!workspaces && typeof workspaces === 'object' && Array.isArray((workspaces as { packages?: unknown }).packages)); + const buildOutputEntries = ['main', 'types', 'typings', 'module'] + .flatMap(field => typeof raw[field] === 'string' ? [raw[field] as string] : []); + return { + declaresWorkspaces, + dependencyNames, + scripts, + name: typeof raw.name === 'string' ? raw.name : undefined, + workspacePatterns, + buildOutputEntries, + }; +} + +function invokedExecutables(script: string): string[] { + return script + .split(/&&|\|\||[;|]/) + .flatMap(segment => { + const token = segment + .trim() + .split(/\s+/) + .find(candidate => candidate && !/^[A-Za-z_][A-Za-z0-9_]*=/.test(candidate)); + const normalized = token?.replace(/^["']|["']$/g, '') ?? ''; + if (!normalized || normalized.startsWith('-') || normalized.includes('/') || SHELL_KEYWORDS.has(normalized)) { + return []; + } + return [normalized]; + }); +} + +function collectTaskDependencies(task: DebugTask, byLabel: Map): Set { + const found = new Set(); + const visit = (label: string): void => { + if (found.has(label)) { + return; + } + found.add(label); + const dependency = byLabel.get(label); + if (dependency) { + normalizeStringList(dependency.dependsOn).forEach(visit); + } + }; + normalizeStringList(task.dependsOn).forEach(visit); + return found; +} + +function normalizeStringList(value: unknown): string[] { + if (typeof value === 'string') { + return [value]; + } + return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : []; +} + +function isEmptyProblemMatcher(value: unknown): boolean { + return value === undefined || value === null || value === '' || (Array.isArray(value) && value.length === 0); +} + +function planSectionHasRows(content: string, sectionName: string): boolean { + const section = findSection(parseLocalDebugPlanMarkdown(content), sectionName); + return !!section && flattenContent(section.content).some(value => value.type === 'table' && value.rows.length > 0); +} + +function planSectionHasCheckedRows(content: string, sectionName: string): boolean { + const section = findSection(parseLocalDebugPlanMarkdown(content), sectionName); + if (!section) { + return false; + } + return flattenContent(section.content).some(value => + value.type === 'table' && value.rows.some(row => row.some(isChecked))); +} + +async function findFirstExisting(directory: string, names: string[]): Promise { + for (const name of names) { + try { + await fs.access(path.join(directory, name)); + return name; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + } + return undefined; +} + +async function listFilesIfPresent(directory: string): Promise { + let entries: import('fs').Dirent[]; + try { + entries = await fs.readdir(directory, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return []; + } + throw error; + } + return (await Promise.all(entries.map(async entry => { + const entryPath = path.join(directory, entry.name); + return entry.isDirectory() ? await listFilesIfPresent(entryPath) : [entryPath]; + }))).flat(); +} + +function addIssue( + issues: ArtifactValidationIssue[], + code: string, + issuePath: string, + message: string, +): void { + issues.push({ code, path: issuePath, message }); +} + +/** + * In an npm workspaces monorepo a member that imports another member resolves it through the + * dependency's published entry points (`main` / `types`), which point at compiled output. Nothing + * emits that output until the dependency's own build runs, so a service task that builds or watches + * without first building its workspace dependencies compiles against missing type declarations. + * + * This fails silently: `tsc --watch` reports the errors but never exits, so the task stays "running" + * and downstream hosts start against output that was never emitted. + */ +async function validateWorkspaceDependencyBuildOrder( + workspace: string, + tasks: DebugTask[], + issues: ArtifactValidationIssue[], + rootManifest: PackageManifestSummary | undefined, +): Promise { + if (!rootManifest?.declaresWorkspaces) { + return; + } + const root = path.resolve(workspace); + const members = await readWorkspaceMembers(root, rootManifest); + if (members.size < 2) { + return; + } + const byDirectory = new Map([...members.values()].map(member => [member.directory, member] as const)); + const byLabel = new Map(tasks.flatMap(task => + typeof task.label === 'string' ? [[task.label, task] as const] : [])); + + for (const [index, task] of tasks.entries()) { + const command = typeof task.command === 'string' ? task.command : ''; + if (!/\bnpm\s+run\s+(?:build|watch|dev|start)\b|\btsc\b/i.test(command)) { + continue; + } + const member = byDirectory.get(resolveTaskDirectory(workspace, task.options?.cwd) ?? ''); + if (!member) { + continue; + } + const internalDependencies = [...member.manifest.dependencyNames] + .flatMap(name => { + const dependency = members.get(name); + return dependency && dependency.directory !== member.directory && dependency.producesBuildOutput + ? [dependency] + : []; + }); + if (!internalDependencies.length) { + continue; + } + const reachable = collectTaskDependencies(task, byLabel); + const reachableTasks = tasks.filter(candidate => + typeof candidate.label === 'string' && reachable.has(candidate.label)); + + for (const dependency of internalDependencies) { + if (await hasProjectReferenceTo(member.directory, dependency.directory)) { + continue; + } + const built = reachableTasks.some(candidate => buildsWorkspacePackage( + workspace, + candidate, + dependency, + root, + rootManifest, + )); + if (!built) { + addIssue( + issues, + 'missingWorkspaceDependencyBuild', + `$.tasks.tasks[${index}].dependsOn`, + `Task "${String(task.label)}" builds workspace package ${member.name ?? member.directory}, which depends on ` + + `${dependency.name}, but nothing builds ${dependency.name} first. ` + + `${dependency.name} resolves through compiled output (${dependency.manifest.buildOutputEntries.join(', ')}), ` + + 'so the compile sees missing type declarations. Add a build task for the dependency and depend on it, ' + + 'or use TypeScript project references so `tsc -b` builds it automatically.', + ); + } + } + } +} + +interface WorkspaceMember { + name?: string; + directory: string; + manifest: PackageManifestSummary; + producesBuildOutput: boolean; +} + +async function readWorkspaceMembers( + root: string, + rootManifest: PackageManifestSummary, +): Promise> { + const directories = new Set(); + for (const pattern of rootManifest.workspacePatterns) { + const normalized = pattern.replace(/\\/g, '/'); + if (!normalized.includes('*')) { + directories.add(path.resolve(root, normalized)); + continue; + } + // Support the common `dir/*` shape without pulling in a glob dependency. + const base = normalized.slice(0, normalized.indexOf('*')).replace(/\/$/, ''); + let entries: string[]; + try { + entries = (await fs.readdir(path.resolve(root, base), { withFileTypes: true })) + .filter(entry => entry.isDirectory()) + .map(entry => path.resolve(root, base, entry.name)); + } catch { + continue; + } + entries.forEach(entry => directories.add(entry)); + } + + const members = new Map(); + for (const directory of directories) { + const manifest = await readPackageManifest(directory); + if (!manifest?.name) { + continue; + } + const producesBuildOutput = typeof manifest.scripts.build === 'string' + && manifest.buildOutputEntries.some(entry => /^(?:\.\/)?(?:dist|build|lib|out|es[m5]?)\b/i.test(entry)); + members.set(manifest.name, { name: manifest.name, directory, manifest, producesBuildOutput }); + } + return members; +} + +function buildsWorkspacePackage( + workspace: string, + task: DebugTask, + dependency: WorkspaceMember, + root: string, + rootManifest: PackageManifestSummary, +): boolean { + const command = typeof task.command === 'string' ? task.command : ''; + const args = normalizeStringList(task.args).join(' '); + const full = `${command} ${args}`.trim(); + if (!/\b(?:npm\s+run\s+build|npm\s+run\s+watch|tsc)\b/i.test(full)) { + return false; + } + const directory = resolveTaskDirectory(workspace, task.options?.cwd); + // A task running inside the dependency's own directory builds it directly. + if (directory === dependency.directory) { + return true; + } + // `npm run build -w ` / `--workspace ` targets it explicitly. + const name = dependency.name ?? ''; + if (name && new RegExp(`(?:-w|--workspace)[\\s=]+${escapeRegExp(name)}(?:\\s|$)`, 'i').test(full)) { + return true; + } + // A root aggregate build counts when the root build script targets the dependency. + if (directory === root && /\bnpm\s+run\s+build\b/i.test(full)) { + const rootBuild = rootManifest.scripts.build ?? ''; + if (name && rootBuild.includes(name)) { + return true; + } + } + return false; +} + +async function hasProjectReferenceTo(memberDirectory: string, dependencyDirectory: string): Promise { + const visited = new Set(); + const visit = async (tsconfigPath: string, depth: number): Promise => { + if (depth > 3 || visited.has(tsconfigPath)) { + return false; + } + visited.add(tsconfigPath); + let content: string; + try { + content = await fs.readFile(tsconfigPath, 'utf8'); + } catch { + return false; + } + const parsed = parse(content, [], { allowTrailingComma: true, disallowComments: false }) as + { references?: { path?: unknown }[] } | undefined; + const references = Array.isArray(parsed?.references) ? parsed.references : []; + for (const reference of references) { + if (typeof reference?.path !== 'string') { + continue; + } + const resolved = path.resolve(path.dirname(tsconfigPath), reference.path); + const directory = /\.json$/i.test(resolved) ? path.dirname(resolved) : resolved; + if (directory === dependencyDirectory || directory.startsWith(`${dependencyDirectory}${path.sep}`)) { + return true; + } + const nested = /\.json$/i.test(resolved) ? resolved : path.join(resolved, 'tsconfig.json'); + if (await visit(nested, depth + 1)) { + return true; + } + } + return false; + }; + return await visit(path.join(memberDirectory, 'tsconfig.json'), 0); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +const CONFIG_FILE_PATTERN = /(^|[/\\])(docker-compose\.ya?ml|compose\.ya?ml|local\.settings\.json|\.env(\..+)?)$/i;const REDACTION_MASK_PATTERN = /\*{3,}/; + +/** + * Secret-redaction filters rewrite concrete `scheme://user:password@host` literals to a run of + * asterisks. When that masked text reaches a generated config file it is silently corrupt, and in + * YAML it is fatal: a leading `*` is an alias indicator, so the whole compose file fails to parse. + */ +async function validateRedactedSecretPlaceholders( + workspace: string, + issues: ArtifactValidationIssue[], +): Promise { + const candidates = (await listFilesIfPresent(workspace)).filter(filePath => + !filePath.includes(`${path.sep}node_modules${path.sep}`) + && !filePath.includes(`${path.sep}.git${path.sep}`) + && CONFIG_FILE_PATTERN.test(filePath)); + for (const filePath of candidates) { + let content: string; + try { + content = await fs.readFile(filePath, 'utf8'); + } catch { + continue; + } + const relative = path.relative(workspace, filePath) || path.basename(filePath); + content.split('\n').forEach((line, index) => { + const match = REDACTION_MASK_PATTERN.exec(line); + // Skip glob patterns such as `**/*`, which legitimately contain adjacent asterisks. + if (!match || line.includes('*/') || line.includes('/*')) { + return; + } + addIssue( + issues, + 'redactedSecretPlaceholder', + `$.generatedConfig["${relative}"].line[${index + 1}]`, + `${relative} line ${index + 1} contains a redaction mask ("${match[0]}") where a value is expected. ` + + 'A concrete credential literal was masked by a secret-redaction filter before it was written. ' + + 'Build connection strings from discrete variables instead of inlining user:password@host.', + ); + }); + } +} + +/** + * `${VAR}` in a Compose file interpolates from `.env` or the shell, never from a service's own + * `environment:` block. Without a `.env` the value resolves to an empty string and the database + * fails to authenticate at runtime rather than failing fast. + */ +async function validateComposeInterpolationSource( + workspace: string, + issues: ArtifactValidationIssue[], +): Promise { + const composePath = await findFirstExisting(workspace, ['docker-compose.yml', 'docker-compose.yaml', 'compose.yml', 'compose.yaml']); + if (!composePath) { + return; + } + let composeContent: string; + try { + composeContent = await fs.readFile(path.join(workspace, composePath), 'utf8'); + } catch { + return; + } + const referenced = new Set(); + for (const match of composeContent.matchAll(/\$\{([A-Za-z_][A-Za-z0-9_]*)(?::?-[^}]*)?\}/g)) { + // A default value (`${VAR:-fallback}`) makes the reference safe on its own. + if (!match[0].includes('-')) { + referenced.add(match[1]); + } + } + if (!referenced.size) { + return; + } + const envPath = await findFirstExisting(workspace, ['.env']); + let declared = new Set(); + if (envPath) { + const envContent = await fs.readFile(path.join(workspace, envPath), 'utf8'); + declared = new Set(envContent + .split('\n') + .flatMap(line => { + const parsed = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line); + return parsed ? [parsed[1]] : []; + })); + } + const missing = [...referenced].filter(name => !declared.has(name)).sort(); + if (missing.length) { + addIssue( + issues, + 'missingComposeEnvValues', + '$.dockerCompose.interpolation', + `${composePath} interpolates ${missing.map(name => `\${${name}}`).join(', ')} but ${envPath ? `.env does not declare ${missing.length === 1 ? 'it' : 'them'}` : 'no .env file exists'}. ` + + 'Compose resolves undeclared variables to an empty string, so services start with blank credentials instead of failing fast.', + ); + } +} + +/** Tasks whose process inherits environment from a runtime-owned settings file rather than the shell. */ +const FUNCTIONS_HOST_COMMAND = /\bfunc\b[^\n]*\b(?:host\s+start|start)\b/; + +/** `require('dotenv')`, `-r dotenv/config`, `dotenv -e .env --`, and the dotenvx equivalents. */ +const DOTENV_LOADER_PATTERN = /\bdotenv(?:x|-cli)?\b/; + +/** Reads a value straight out of the environment with no inline fallback of any kind. */ +function readsEnvWithoutFallback(content: string, name: string): boolean { + const pattern = new RegExp( + `process\\.env\\.${escapeRegExp(name)}\\b|process\\.env\\[\\s*['"\`]${escapeRegExp(name)}['"\`]\\s*\\]`, + 'g', + ); + for (const match of content.matchAll(pattern)) { + const rest = content.slice(match.index + match[0].length, match.index + match[0].length + 40); + // `?? 'x'`, `|| 'x'`, and `: 'x'` all supply a value when the variable is absent. + if (/^\s*[!]?\s*(?:\?\?|\|\||\?\.|:)/.test(rest)) { + continue; + } + return true; + } + return false; +} + +function parseEnvKeys(content: string): Set { + return new Set(content.split('\n').flatMap(line => { + const parsed = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line); + return parsed ? [parsed[1]] : []; + })); +} + +/** + * A debug task that runs outside a runtime host gets its environment from the shell, so a `.env` + * file only reaches it when something in the command chain actually loads dotenv. When the generated + * project documents a value in `.env.example`, omits it from `.env`, and then reads it without a + * fallback, the task starts with the variable undefined. + * + * This is how a database migration fails with a connection error that looks like an unready server: + * the client never attempts a connection because it was handed `undefined` instead of a URL. + */ +async function validateTaskEnvironmentAvailability( + workspace: string, + tasks: DebugTask[], + issues: ArtifactValidationIssue[], + rootManifest: PackageManifestSummary | undefined, +): Promise { + const examplePath = await findFirstExisting(workspace, ['.env.example', '.env.sample', '.env.template']); + if (!examplePath) { + return; + } + let documented: Set; + try { + documented = parseEnvKeys(await fs.readFile(path.join(workspace, examplePath), 'utf8')); + } catch { + return; + } + const envPath = await findFirstExisting(workspace, ['.env']); + let declared = new Set(); + if (envPath) { + try { + declared = parseEnvKeys(await fs.readFile(path.join(workspace, envPath), 'utf8')); + } catch { + return; + } + } + const undeclared = [...documented].filter(name => !declared.has(name)); + if (!undeclared.length) { + return; + } + + const members = rootManifest ? await readWorkspaceMembers(path.resolve(workspace), rootManifest) : new Map(); + const byLabel = new Map(); + tasks.forEach(task => { + if (typeof task.label === 'string') { + byLabel.set(task.label, task); + } + }); + + for (const task of tasks) { + const label = typeof task.label === 'string' ? task.label : ''; + const commandText = [task.command, ...(Array.isArray(task.args) ? task.args : [])] + .filter((value): value is string => typeof value === 'string') + .join(' ') + .trim(); + if (!commandText) { + continue; + } + const taskDirectory = resolveTaskDirectory(workspace, task.options?.cwd); + const segments = await resolveScriptChain(commandText, taskDirectory, workspace, members); + const chainText = [commandText, ...segments.map(segment => segment.body)].join('\n'); + // A runtime host injects its own settings file, so the shell environment is not the source. + if (FUNCTIONS_HOST_COMMAND.test(chainText) || DOTENV_LOADER_PATTERN.test(chainText)) { + continue; + } + // Compose supplies `environment:` to the container it starts. + if (/\bdocker\s+compose\b/.test(chainText)) { + continue; + } + // Resolve config paths against the directory each script segment actually runs in. + const searchRoots = [taskDirectory, ...segments.map(segment => segment.directory), path.resolve(workspace)]; + const entryFiles = await collectCommandEntryFiles(chainText, searchRoots, workspace); + if (!entryFiles.length) { + continue; + } + const required = new Set(); + for (const file of entryFiles) { + let content: string; + try { + content = await fs.readFile(file, 'utf8'); + } catch { + continue; + } + for (const name of undeclared) { + if (readsEnvWithoutFallback(content, name)) { + required.add(name); + } + } + } + const missing = [...required].sort(); + if (missing.length) { + addIssue( + issues, + 'missingTaskEnvValue', + `$.tasks["${label}"].environment`, + `Task "${label}" runs \`${commandText}\`, which reads ${missing.join(', ')} from the environment with no fallback, ` + + `but ${envPath ? `.env does not declare ${missing.length === 1 ? 'it' : 'them'}` : 'no .env file exists'} ` + + `(${examplePath} does). The task starts with the value undefined instead of failing fast. ` + + 'Declare every value the debug tasks need in .env, or run the work through the docker compose service that already defines it.', + ); + } + } +} + +/** Follows `npm run