diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 07f8cac942..6d70082180 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -56,7 +56,7 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - inventory_path="tools/e2e-scenarios/free-standing-jobs.env" + inventory_output="$(npx tsx tools/e2e-scenarios/free-standing-workflow-inventory.mts --shell)" allowed_jobs="" free_standing_scenarios_csv="" free_standing_scenario_jobs_csv="" @@ -68,13 +68,13 @@ jobs: continue fi if [[ ! "${line}" =~ ^(allowed_jobs|free_standing_scenarios_csv|free_standing_scenario_jobs_csv)=([A-Za-z0-9_:-]+(,[A-Za-z0-9_:-]+)*)$ ]]; then - echo "::error::free-standing jobs inventory must be data-only key=value" >&2 + echo "::error::free-standing workflow inventory must be data-only key=value" >&2 exit 1 fi inventory_key="${BASH_REMATCH[1]}" inventory_value="${BASH_REMATCH[2]}" if [[ -n "${seen_inventory_keys[${inventory_key}]:-}" ]]; then - echo "::error::free-standing jobs inventory must not redefine ${inventory_key}" >&2 + echo "::error::free-standing workflow inventory must not redefine ${inventory_key}" >&2 exit 1 fi seen_inventory_keys["${inventory_key}"]=1 @@ -83,10 +83,10 @@ jobs: free_standing_scenarios_csv) free_standing_scenarios_csv="${inventory_value}" ;; free_standing_scenario_jobs_csv) free_standing_scenario_jobs_csv="${inventory_value}" ;; esac - done < "${inventory_path}" + done <<< "${inventory_output}" for required_inventory_key in allowed_jobs free_standing_scenarios_csv free_standing_scenario_jobs_csv; do if [[ -z "${!required_inventory_key:-}" ]]; then - echo "::error::free-standing jobs inventory missing ${required_inventory_key}" >&2 + echo "::error::free-standing workflow inventory missing ${required_inventory_key}" >&2 exit 1 fi done @@ -94,11 +94,11 @@ jobs: IFS=',' read -r -a allowed_job_entries <<< "${allowed_jobs}" for job in "${allowed_job_entries[@]}"; do if [[ ! "${job}" =~ ^[A-Za-z0-9_-]+$ ]]; then - echo "::error::free-standing jobs inventory contains invalid job id" >&2 + echo "::error::free-standing workflow inventory contains invalid job id" >&2 exit 1 fi if [[ -n "${seen_allowed_jobs[${job}]:-}" ]]; then - echo "::error::free-standing jobs inventory repeats job id" >&2 + echo "::error::free-standing workflow inventory repeats job id" >&2 exit 1 fi seen_allowed_jobs["${job}"]=1 @@ -107,11 +107,11 @@ jobs: IFS=',' read -r -a free_standing_scenario_entries <<< "${free_standing_scenarios_csv}" for scenario in "${free_standing_scenario_entries[@]}"; do if [[ ! "${scenario}" =~ ^[A-Za-z0-9_-]+$ ]]; then - echo "::error::free-standing jobs inventory contains invalid scenario id" >&2 + echo "::error::free-standing workflow inventory contains invalid scenario id" >&2 exit 1 fi if [[ -n "${seen_free_standing_scenarios[${scenario}]:-}" ]]; then - echo "::error::free-standing jobs inventory repeats scenario id" >&2 + echo "::error::free-standing workflow inventory repeats scenario id" >&2 exit 1 fi seen_free_standing_scenarios["${scenario}"]=1 @@ -127,7 +127,7 @@ jobs: scenario="${entry%%:*}" job="${entry#*:}" if [[ -n "${seen_scenario_mappings[${scenario}]:-}" ]]; then - echo "::error::free-standing jobs inventory repeats scenario mapping" >&2 + echo "::error::free-standing workflow inventory repeats scenario mapping" >&2 exit 1 fi seen_scenario_mappings["${scenario}"]=1 @@ -323,6 +323,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "openshell-version-pin" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/openshell-version-pin NEMOCLAW_RUN_E2E_SCENARIOS: "1" steps: @@ -364,6 +366,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "onboard-negative-paths" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/onboard-negative-paths NEMOCLAW_RUN_E2E_SCENARIOS: "1" steps: @@ -412,6 +416,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "skill-agent" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/skill-agent NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -511,6 +517,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "hermes-root-entrypoint-smoke" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-root-entrypoint-smoke NEMOCLAW_RUN_E2E_SCENARIOS: "1" steps: @@ -553,6 +561,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "inference-routing" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/inference-routing NEMOCLAW_RUN_E2E_SCENARIOS: "1" steps: @@ -601,6 +611,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "issue-4434-tui-unreachable-inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/issue-4434-tui-unreachable-inference NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -699,6 +711,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 65 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "credential-sanitization" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/credential-sanitization NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -783,6 +797,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 50 env: + FREE_STANDING_VITEST_JOB: "1" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/credential-migration NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -861,6 +876,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "runtime-overrides" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/runtime-overrides NEMOCLAW_RUN_E2E_SCENARIOS: "1" steps: @@ -902,6 +919,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 75 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "hermes-e2e" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-e2e NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -954,6 +973,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 90 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "network-policy" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/network-policy NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1016,6 +1037,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "shields-config" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/shields-config NEMOCLAW_RUN_E2E_SCENARIOS: "1" NEMOCLAW_NON_INTERACTIVE: "1" @@ -1092,6 +1115,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 130 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "rebuild-openclaw" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/rebuild-openclaw NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1184,6 +1209,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 90 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "sandbox-rebuild" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/sandbox-rebuild NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1277,6 +1304,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 90 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "double-onboard" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/double-onboard NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1364,6 +1393,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "token-rotation" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/token-rotation NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1447,6 +1478,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 90 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "messaging-providers" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/messaging-providers NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1534,6 +1567,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 env: + FREE_STANDING_VITEST_JOB: "1" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/launchable-smoke NEMOCLAW_RUN_E2E_SCENARIOS: "1" NEMOCLAW_SANDBOX_NAME: "e2e-launchable" @@ -1608,6 +1642,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "model-router-provider-routed-inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/model-router-provider-routed-inference NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1697,6 +1733,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "sandbox-survival" DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-sandbox-survival E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/sandbox-survival NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1788,6 +1826,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 75 env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "openclaw-tui-chat-correlation" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/openclaw-tui-chat-correlation NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1868,10 +1908,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: + FREE_STANDING_VITEST_JOB: "1" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/gateway-guard-recovery NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" # nemoclaw onboard registers the gateway under the canonical name @@ -1910,6 +1950,8 @@ jobs: run: bash scripts/install-openshell.sh - name: Run Vitest gateway-guard-recovery scenario + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} run: | set -euo pipefail # OpenShell installs to /usr/local/bin on GitHub-hosted runners diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 51c035c1ef..535e97cac4 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -12,7 +12,7 @@ import { evaluateE2eVitestWorkflowDispatchSelectors, readFreeStandingJobsInventory, validateE2eVitestScenariosWorkflowBoundary, - validateFreeStandingJobsEnvContent, + validateFreeStandingWorkflowInventory, } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; function readWorkflow(): Record { @@ -257,106 +257,102 @@ describe("e2e-vitest-scenarios workflow boundary", () => { }); }); - it("keeps the free-standing inventory internally consistent and data-only", () => { - const inventory = fs.readFileSync( - path.join(process.cwd(), "tools/e2e-scenarios/free-standing-jobs.env"), - "utf-8", - ); - expect(validateFreeStandingJobsEnvContent(inventory)).toEqual([]); - expect( - validateFreeStandingJobsEnvContent( - [ - "allowed_jobs=openshell-version-pin-vitest", - "free_standing_scenarios_csv=openshell-version-pin", - "free_standing_scenario_jobs_csv=openshell-version-pin:missing-vitest", - ].join("\n"), - ), - ).toContain("free-standing inventory maps openshell-version-pin to unknown job missing-vitest"); - expect( - validateFreeStandingJobsEnvContent( - [ - "allowed_jobs=openshell-version-pin-vitest", - "free_standing_scenarios_csv=openshell-version-pin,extra-scenario", - "free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest", - ].join("\n"), - ), - ).toContain( - "free_standing_scenarios_csv must exactly match free_standing_scenario_jobs_csv keys", + it("derives the free-standing inventory from workflow job metadata", () => { + const inventory = readFreeStandingJobsInventory(); + expect(validateFreeStandingWorkflowInventory()).toEqual([]); + expect(inventory.allowedJobs).toContain("openshell-version-pin-vitest"); + expect(inventory.allowedJobs).toContain("gateway-guard-recovery"); + expect(inventory.scenarioToJob.get("openshell-version-pin")).toBe( + "openshell-version-pin-vitest", ); + expect(inventory.scenarioToJob.get("credential-migration")).toBeUndefined(); expect( - validateFreeStandingJobsEnvContent( - [ - "allowed_jobs=openshell-version-pin-vitest", - "free_standing_scenarios_csv=openshell-version-pin", - "free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest", - "export injected=true", - "allowed_jobs=$(touch /tmp/nope)", - ].join("\n"), + inventory.allowedJobs.every((job) => + Object.keys((readWorkflow().jobs as Record) ?? {}).includes(job), ), - ).toEqual( - expect.arrayContaining([ - "free-standing jobs inventory line 4 must be data-only key=value", - "free-standing jobs inventory line 5 must be data-only key=value", - ]), - ); + ).toBe(true); }); - it("rejects malformed inventory in the workflow runtime before matrix generation", () => { - const malformedInventories = [ - [ - "allowed_jobs=openshell-version-pin-vitest", - "allowed_jobs=onboard-negative-paths-vitest", - "free_standing_scenarios_csv=openshell-version-pin", - "free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest", - ], - [ - "allowed_jobs=openshell-version-pin-vitest,openshell-version-pin-vitest", - "free_standing_scenarios_csv=openshell-version-pin", - "free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest", - ], - [ - "allowed_jobs=bad:job", - "free_standing_scenarios_csv=openshell-version-pin", - "free_standing_scenario_jobs_csv=openshell-version-pin:bad:job", - ], - [ - "allowed_jobs=openshell-version-pin-vitest", - "free_standing_scenarios_csv=openshell-version-pin", - "free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest,openshell-version-pin:openshell-version-pin-vitest", - ], + it("rejects malformed free-standing workflow metadata before matrix generation", () => { + const malformedWorkflows = [ + { + body: ` +jobs: + openshell-version-pin-vitest: + env: + FREE_STANDING_VITEST_JOB: "yes" + FREE_STANDING_SCENARIO_ID: openshell-version-pin +`, + error: 'openshell-version-pin-vitest job FREE_STANDING_VITEST_JOB must be "1"', + }, + { + body: ` +jobs: + openshell-version-pin-vitest: + env: + FREE_STANDING_SCENARIO_ID: openshell-version-pin +`, + error: + "openshell-version-pin-vitest job FREE_STANDING_SCENARIO_ID requires FREE_STANDING_VITEST_JOB", + }, + { + body: ` +jobs: + openshell-version-pin-vitest: + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "bad:scenario" +`, + error: "openshell-version-pin-vitest job FREE_STANDING_SCENARIO_ID must be a selector id", + }, + { + body: ` +jobs: + first-vitest: + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: duplicate-scenario + second-vitest: + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: duplicate-scenario +`, + error: "free-standing workflow metadata repeats scenario id: duplicate-scenario", + }, ]; - for (const inventory of malformedInventories) { - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-bad-inventory-")); + for (const { body, error } of malformedWorkflows) { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-bad-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); try { - fs.mkdirSync(path.join(tmp, "tools", "e2e-scenarios"), { recursive: true }); - fs.writeFileSync( - path.join(tmp, "tools", "e2e-scenarios", "free-standing-jobs.env"), - `${inventory.join("\n")}\n`, - ); - const result = spawnSync("bash", ["-c", generateMatrixScript()], { - cwd: tmp, - encoding: "utf-8", - timeout: 10_000, - killSignal: "SIGKILL", - env: { - ...process.env, - GITHUB_OUTPUT: path.join(tmp, "github-output"), - GITHUB_STEP_SUMMARY: path.join(tmp, "github-summary"), - JOBS: "", - SCENARIOS: "", + fs.writeFileSync(workflowPath, body); + expect(validateFreeStandingWorkflowInventory(workflowPath)).toContain(error); + const result = spawnSync( + "npx", + [ + "tsx", + "tools/e2e-scenarios/free-standing-workflow-inventory.mts", + "--shell", + "--workflow", + workflowPath, + ], + { + cwd: process.cwd(), + encoding: "utf-8", + timeout: 30_000, + killSignal: "SIGKILL", }, - }); + ); expect(result.signal).toBeNull(); expect(result.status).not.toBe(0); - expect(result.stderr).toContain("::error::"); + expect(result.stderr).toContain(`::error::${error}`); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } } }); - it("keeps each free-standing scenario out of the registry matrix", { timeout: 15_000 }, () => { + it("keeps each free-standing scenario out of the registry matrix", { timeout: 60_000 }, () => { const inventory = readFreeStandingJobsInventory(); for (const job of inventory.allowedJobs) { expect(generateMatrixForDispatch({ JOBS: job, SCENARIOS: "" })).toMatchObject({ @@ -633,27 +629,70 @@ jobs: } }); - it("requires runtime-overrides workflow and report coverage", () => { + it("applies boundary checks to newly marked free-standing jobs", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record>; + }; + workflow.jobs["ad-hoc-derived-vitest"] = { + "runs-on": "ubuntu-latest", + needs: "live-scenarios", + if: "${{ inputs.scenarios != '' }}", + env: { + FREE_STANDING_VITEST_JOB: "1", + FREE_STANDING_SCENARIO_ID: "ad-hoc-derived", + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }, + steps: [ + { uses: "actions/checkout@v4" }, + { + name: "Run ad hoc", + run: "echo ${{ inputs.jobs }} && echo ${{ secrets.NVIDIA_API_KEY }}", + }, + ], + }; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "ad-hoc-derived-vitest job must depend on generate-matrix", + "ad-hoc-derived-vitest job must use the shared jobs selector condition", + "ad-hoc-derived-vitest job env must not include NVIDIA_API_KEY", + "ad-hoc-derived-vitest step 'actions/checkout@v4' action must be pinned to a full commit SHA", + "step 'Run ad hoc' run script must not interpolate dispatch inputs directly", + "ad-hoc-derived-vitest step 'Run ad hoc' run script must not interpolate secrets directly", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("requires runtime-overrides workflow and report coverage", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const renamedWorkflowPath = path.join(tmp, "renamed-workflow.yaml"); + const missingReportNeedPath = path.join(tmp, "missing-report-need.yaml"); const workflow = fs.readFileSync( path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"), "utf8", ); fs.writeFileSync( - workflowPath, - workflow - .replace(/runtime-overrides-vitest/g, "runtime-overrides-missing") - .replace(/runtime-overrides/g, "runtime-overrides-missing"), + renamedWorkflowPath, + workflow.replace(/^ runtime-overrides-vitest:$/m, " runtime-overrides-missing:"), + ); + fs.writeFileSync( + missingReportNeedPath, + workflow.replace(" runtime-overrides-vitest,\n", ""), ); try { - const errors = validateE2eVitestScenariosWorkflowBoundary(workflowPath); - expect(errors).toEqual( - expect.arrayContaining([ - "workflow missing runtime-overrides-vitest job", - "report-to-pr job must wait for runtime-overrides-vitest", - ]), + expect(validateE2eVitestScenariosWorkflowBoundary(renamedWorkflowPath)).toContain( + "workflow missing runtime-overrides-vitest job", + ); + expect(validateE2eVitestScenariosWorkflowBoundary(missingReportNeedPath)).toContain( + "report-to-pr job must wait for runtime-overrides-vitest", ); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/tools/e2e-scenarios/free-standing-jobs.env b/tools/e2e-scenarios/free-standing-jobs.env deleted file mode 100644 index 5467e6f2f4..0000000000 --- a/tools/e2e-scenarios/free-standing-jobs.env +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -allowed_jobs=openshell-version-pin-vitest,onboard-negative-paths-vitest,skill-agent-vitest,inference-routing-vitest,credential-migration-vitest,runtime-overrides-vitest,hermes-e2e-vitest,hermes-root-entrypoint-smoke-vitest,network-policy-vitest,shields-config-vitest,rebuild-openclaw-vitest,sandbox-rebuild-vitest,token-rotation-vitest,messaging-providers-vitest,launchable-smoke-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,double-onboard-vitest,issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference-vitest,credential-sanitization-vitest,sandbox-survival-vitest -free_standing_scenarios_csv=openshell-version-pin,onboard-negative-paths,skill-agent,inference-routing,runtime-overrides,hermes-e2e,hermes-root-entrypoint-smoke,network-policy,shields-config,rebuild-openclaw,sandbox-rebuild,token-rotation,messaging-providers,openclaw-tui-chat-correlation,double-onboard,issue-4434-tui-unreachable-inference,model-router-provider-routed-inference,credential-sanitization,sandbox-survival -free_standing_scenario_jobs_csv=openshell-version-pin:openshell-version-pin-vitest,onboard-negative-paths:onboard-negative-paths-vitest,skill-agent:skill-agent-vitest,inference-routing:inference-routing-vitest,runtime-overrides:runtime-overrides-vitest,hermes-e2e:hermes-e2e-vitest,hermes-root-entrypoint-smoke:hermes-root-entrypoint-smoke-vitest,network-policy:network-policy-vitest,shields-config:shields-config-vitest,rebuild-openclaw:rebuild-openclaw-vitest,sandbox-rebuild:sandbox-rebuild-vitest,token-rotation:token-rotation-vitest,messaging-providers:messaging-providers-vitest,openclaw-tui-chat-correlation:openclaw-tui-chat-correlation-vitest,double-onboard:double-onboard-vitest,issue-4434-tui-unreachable-inference:issue-4434-tui-unreachable-inference-vitest,model-router-provider-routed-inference:model-router-provider-routed-inference-vitest,credential-sanitization:credential-sanitization-vitest,sandbox-survival:sandbox-survival-vitest diff --git a/tools/e2e-scenarios/free-standing-workflow-inventory.mts b/tools/e2e-scenarios/free-standing-workflow-inventory.mts new file mode 100644 index 0000000000..a618c42575 --- /dev/null +++ b/tools/e2e-scenarios/free-standing-workflow-inventory.mts @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + formatFreeStandingJobsInventoryForShell, + readFreeStandingJobsInventory, +} from "./workflow-boundary.mts"; + +function usage(): string { + return [ + "Usage: npx tsx tools/e2e-scenarios/free-standing-workflow-inventory.mts [--shell] [--workflow PATH]", + "", + "Derives free-standing E2E Vitest selector mappings from workflow job metadata.", + ].join("\n"); +} + +function parseArgs(argv: readonly string[]): { shell: boolean; workflowPath?: string } { + const parsed: { shell: boolean; workflowPath?: string } = { shell: false }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--shell") { + parsed.shell = true; + continue; + } + if (arg === "--workflow") { + const workflowPath = argv[index + 1]; + if (!workflowPath) throw new Error("--workflow requires a path"); + parsed.workflowPath = workflowPath; + index += 1; + continue; + } + if (arg === "--help" || arg === "-h") { + console.log(usage()); + process.exit(0); + } + throw new Error(`Unknown argument: ${arg}`); + } + return parsed; +} + +try { + const options = parseArgs(process.argv.slice(2)); + const inventory = readFreeStandingJobsInventory(options.workflowPath); + if (options.shell) { + process.stdout.write(formatFreeStandingJobsInventoryForShell(inventory)); + } else { + process.stdout.write( + `${JSON.stringify( + { + allowedJobs: inventory.allowedJobs, + freeStandingScenarios: inventory.freeStandingScenarios, + scenarioJobs: Object.fromEntries(inventory.scenarioToJob), + }, + null, + 2, + )}\n`, + ); + } +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + for (const line of message.split("\n")) { + console.error(`::error::${line}`); + } + process.exitCode = 1; +} diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 9e55fc3ff3..ab774b1cc5 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -13,6 +13,8 @@ const DEFAULT_VITEST_WORKFLOW_PATH = join( "workflows", "e2e-vitest-scenarios.yaml", ); +export const FREE_STANDING_WORKFLOW_INVENTORY_SCRIPT = + "tools/e2e-scenarios/free-standing-workflow-inventory.mts"; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { name?: string; run?: string; uses?: string; with?: WorkflowRecord }; @@ -25,21 +27,23 @@ export interface FreeStandingJobsInventory { const SELECTOR_PATTERN = /^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$/; const SELECTOR_ID_PATTERN = /^[A-Za-z0-9_-]+$/; -const FREE_STANDING_JOBS_ENV_PATH = join(REPO_ROOT, "tools", "e2e-scenarios", "free-standing-jobs.env"); -const FREE_STANDING_JOBS_ENV_KEYS = [ - "allowed_jobs", - "free_standing_scenarios_csv", - "free_standing_scenario_jobs_csv", -] as const; -const FREE_STANDING_JOBS_ENV_LINE_PATTERN = - /^(allowed_jobs|free_standing_scenarios_csv|free_standing_scenario_jobs_csv)=[A-Za-z0-9_:-]+(,[A-Za-z0-9_:-]+)*$/; -const FREE_STANDING_SCENARIO_JOB_PATTERN = /^[A-Za-z0-9_-]+:[A-Za-z0-9_-]+$/; - -function splitEnvCsv(value: string | undefined): string[] { - return (value ?? "") - .split(",") - .map((part) => part.trim()) - .filter(Boolean); +const FREE_STANDING_JOB_MARKER = "FREE_STANDING_VITEST_JOB"; +const FREE_STANDING_SCENARIO_MARKER = "FREE_STANDING_SCENARIO_ID"; +const COMMON_SECRET_ENV_NAMES = [ + "NVIDIA_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", +]; +const FREE_STANDING_SELECTOR_SPECIAL_CASES = new Set([ + "hermes-e2e-vitest", + "hermes-root-entrypoint-smoke-vitest", +]); + +function asRecord(value: unknown): WorkflowRecord { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as WorkflowRecord) + : {}; } function findDuplicates(values: readonly string[]): string[] { @@ -52,104 +56,101 @@ function findDuplicates(values: readonly string[]): string[] { return [...duplicates].sort(); } -function parseFreeStandingJobsEnvContent(content: string): Record { - const env: Record = {}; - for (const rawLine of content.split("\n")) { - const line = rawLine.trim(); - if (!line || line.startsWith("#")) continue; - const [key, value] = line.split(/=(.*)/s).slice(0, 2).map((part) => part.trim()); - env[key] = value; - } - return env; -} - -export function validateFreeStandingJobsEnvContent(content: string): string[] { +function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { + errors: string[]; + inventory: FreeStandingJobsInventory; +} { const errors: string[] = []; - const env: Record = {}; - - content.split("\n").forEach((rawLine, index) => { - const line = rawLine.trim(); - if (!line || line.startsWith("#")) return; - if (!FREE_STANDING_JOBS_ENV_LINE_PATTERN.test(line)) { - errors.push(`free-standing jobs inventory line ${index + 1} must be data-only key=value`); - return; + const allowedJobs: string[] = []; + const freeStandingScenarios: string[] = []; + const scenarioToJob = new Map(); + + for (const [jobId, rawJob] of Object.entries(jobs)) { + const job = asRecord(rawJob); + const env = asRecord(job.env); + const hasJobMarker = Object.hasOwn(env, FREE_STANDING_JOB_MARKER); + const hasScenarioMarker = Object.hasOwn(env, FREE_STANDING_SCENARIO_MARKER); + if (!hasJobMarker && !hasScenarioMarker) continue; + + if (!SELECTOR_ID_PATTERN.test(jobId)) { + errors.push(`free-standing workflow metadata contains invalid job id: ${jobId}`); + } + if (!hasJobMarker) { + errors.push(`${jobId} job ${FREE_STANDING_SCENARIO_MARKER} requires ${FREE_STANDING_JOB_MARKER}`); + continue; } - const [key, value] = line.split(/=(.*)/s).slice(0, 2).map((part) => part.trim()); - if (Object.hasOwn(env, key)) { - errors.push(`free-standing jobs inventory must not redefine ${key}`); + if (env[FREE_STANDING_JOB_MARKER] !== "1") { + errors.push(`${jobId} job ${FREE_STANDING_JOB_MARKER} must be "1"`); + continue; } - env[key] = value; - }); - - for (const key of FREE_STANDING_JOBS_ENV_KEYS) { - if (!env[key]) errors.push(`free-standing jobs inventory must define non-empty ${key}`); - } - if (errors.length > 0) return errors; - const allowedJobs = splitEnvCsv(env.allowed_jobs); - const freeStandingScenarios = splitEnvCsv(env.free_standing_scenarios_csv); - const scenarioJobEntries = splitEnvCsv(env.free_standing_scenario_jobs_csv); - const scenarioJobPairs = scenarioJobEntries.map((entry) => entry.split(":", 2) as [string, string]); - const scenarioKeys = scenarioJobPairs.map(([scenario]) => scenario); + allowedJobs.push(jobId); + if (!hasScenarioMarker) continue; - for (const job of allowedJobs) { - if (!SELECTOR_ID_PATTERN.test(job)) { - errors.push(`free-standing jobs inventory contains invalid job id: ${job}`); - } - } - for (const scenario of freeStandingScenarios) { - if (!SELECTOR_ID_PATTERN.test(scenario)) { - errors.push(`free-standing jobs inventory contains invalid scenario id: ${scenario}`); - } - } - for (const entry of scenarioJobEntries) { - if (!FREE_STANDING_SCENARIO_JOB_PATTERN.test(entry)) { - errors.push(`free-standing jobs inventory contains invalid scenario job mapping: ${entry}`); + const scenario = env[FREE_STANDING_SCENARIO_MARKER]; + if (typeof scenario !== "string" || !SELECTOR_ID_PATTERN.test(scenario)) { + errors.push(`${jobId} job ${FREE_STANDING_SCENARIO_MARKER} must be a selector id`); + continue; } + freeStandingScenarios.push(scenario); + scenarioToJob.set(scenario, jobId); } + if (allowedJobs.length === 0) { + errors.push("free-standing workflow metadata must declare at least one job"); + } for (const duplicate of findDuplicates(allowedJobs)) { - errors.push(`free-standing jobs inventory repeats job id: ${duplicate}`); + errors.push(`free-standing workflow metadata repeats job id: ${duplicate}`); } for (const duplicate of findDuplicates(freeStandingScenarios)) { - errors.push(`free-standing jobs inventory repeats scenario id: ${duplicate}`); - } - for (const duplicate of findDuplicates(scenarioKeys)) { - errors.push(`free-standing jobs inventory repeats scenario mapping: ${duplicate}`); + errors.push(`free-standing workflow metadata repeats scenario id: ${duplicate}`); } - if (freeStandingScenarios.join(",") !== scenarioKeys.join(",")) { - errors.push("free_standing_scenarios_csv must exactly match free_standing_scenario_jobs_csv keys"); - } - for (const [scenario, job] of scenarioJobPairs) { - if (!allowedJobs.includes(job)) { - errors.push(`free-standing inventory maps ${scenario} to unknown job ${job}`); - } - } + return { + errors, + inventory: { + allowedJobs, + freeStandingScenarios, + scenarioToJob, + }, + }; +} - return errors; +function readWorkflowRecord(workflowPath: string): WorkflowRecord { + return asRecord(YAML.parse(readFileSync(workflowPath, "utf-8"))); +} + +export function validateFreeStandingWorkflowInventory( + workflowPath = DEFAULT_VITEST_WORKFLOW_PATH, +): string[] { + const workflow = readWorkflowRecord(workflowPath); + return deriveFreeStandingJobsInventoryFromJobs(asRecord(workflow.jobs)).errors; } -export function readFreeStandingJobsInventory(): FreeStandingJobsInventory { - const content = readFileSync(FREE_STANDING_JOBS_ENV_PATH, "utf-8"); - const errors = validateFreeStandingJobsEnvContent(content); +export function readFreeStandingJobsInventory( + workflowPath = DEFAULT_VITEST_WORKFLOW_PATH, +): FreeStandingJobsInventory { + const workflow = readWorkflowRecord(workflowPath); + const { errors, inventory } = deriveFreeStandingJobsInventoryFromJobs(asRecord(workflow.jobs)); if (errors.length > 0) { - throw new Error(`Invalid free-standing jobs inventory:\n${errors.join("\n")}`); + throw new Error(`Invalid free-standing workflow inventory:\n${errors.join("\n")}`); } - const env = parseFreeStandingJobsEnvContent(content); - const scenarioJobPairs = splitEnvCsv(env.free_standing_scenario_jobs_csv).map( - (entry) => entry.split(":", 2) as [string, string], - ); - return { - allowedJobs: splitEnvCsv(env.allowed_jobs), - freeStandingScenarios: splitEnvCsv(env.free_standing_scenarios_csv), - scenarioToJob: new Map(scenarioJobPairs), - }; + return inventory; } -const FREE_STANDING_JOBS_INVENTORY = readFreeStandingJobsInventory(); -const FREE_STANDING_VITEST_JOB_IDS = FREE_STANDING_JOBS_INVENTORY.allowedJobs; -const FREE_STANDING_VITEST_SCENARIO_TO_JOB = FREE_STANDING_JOBS_INVENTORY.scenarioToJob; +export function formatFreeStandingJobsInventoryForShell( + inventory: FreeStandingJobsInventory, +): string { + const scenarioJobMappings = [...inventory.scenarioToJob].map( + ([scenario, job]) => `${scenario}:${job}`, + ); + return [ + `allowed_jobs=${inventory.allowedJobs.join(",")}`, + `free_standing_scenarios_csv=${inventory.freeStandingScenarios.join(",")}`, + `free_standing_scenario_jobs_csv=${scenarioJobMappings.join(",")}`, + "", + ].join("\n"); +} export interface WorkflowDispatchSelectorEvaluation { valid: boolean; @@ -159,12 +160,6 @@ export interface WorkflowDispatchSelectorEvaluation { liveScenariosRuns: boolean; } -function asRecord(value: unknown): WorkflowRecord { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as WorkflowRecord) - : {}; -} - function asSteps(value: unknown): WorkflowStep[] { return Array.isArray(value) ? (value.filter((entry) => asRecord(entry) === entry) as WorkflowStep[]) @@ -186,6 +181,9 @@ export function evaluateE2eVitestWorkflowDispatchSelectors(input: { jobs?: string; scenarios?: string; }): WorkflowDispatchSelectorEvaluation { + const inventory = readFreeStandingJobsInventory(); + const freeStandingVitestJobIds = inventory.allowedJobs; + const freeStandingVitestScenarioToJob = inventory.scenarioToJob; const jobs = input.jobs ?? ""; const scenarios = input.scenarios ?? ""; const errors: string[] = []; @@ -201,7 +199,7 @@ export function evaluateE2eVitestWorkflowDispatchSelectors(input: { } if (jobs && SELECTOR_PATTERN.test(jobs)) { for (const job of splitSelector(jobs)) { - if (!FREE_STANDING_VITEST_JOB_IDS.includes(job)) { + if (!freeStandingVitestJobIds.includes(job)) { errors.push(`Unknown free-standing Vitest job: ${job}`); } } @@ -221,7 +219,7 @@ export function evaluateE2eVitestWorkflowDispatchSelectors(input: { return { valid: true, errors: [], - selectedFreeStandingJobs: [...FREE_STANDING_VITEST_JOB_IDS].sort(), + selectedFreeStandingJobs: [...freeStandingVitestJobIds].sort(), registryScenarios: [], liveScenariosRuns: true, }; @@ -240,7 +238,7 @@ export function evaluateE2eVitestWorkflowDispatchSelectors(input: { const selectedFreeStandingJobs = new Set(); const registryScenarios: string[] = []; for (const scenario of splitSelector(scenarios)) { - const job = FREE_STANDING_VITEST_SCENARIO_TO_JOB.get(scenario); + const job = freeStandingVitestScenarioToJob.get(scenario); if (job) selectedFreeStandingJobs.add(job); else registryScenarios.push(scenario); } @@ -365,12 +363,54 @@ function validateFreeStandingJobSelector( } } +function validateFreeStandingInventoryBoundary( + errors: string[], + jobs: WorkflowRecord, + inventory: FreeStandingJobsInventory, +): void { + const scenarioByJob = new Map( + [...inventory.scenarioToJob].map(([scenario, job]) => [job, scenario]), + ); + + for (const jobName of inventory.allowedJobs) { + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) continue; + + if (!FREE_STANDING_SELECTOR_SPECIAL_CASES.has(jobName)) { + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioByJob.get(jobName)); + } + + const jobEnv = asRecord(job.env); + for (const secret of COMMON_SECRET_ENV_NAMES) { + requireEnvDoesNotExposeSecret(errors, `${jobName} job`, jobEnv, secret); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + if (step.uses) { + requireFullShaAction( + errors, + step, + `${jobName} step '${step.name ?? step.uses}'`, + ); + } + if (/\$\{\{\s*secrets\./.test(stringValue(step.run))) { + errors.push( + `${jobName} step '${step.name ?? step.uses ?? ""}' run script must not interpolate secrets directly`, + ); + } + } + } +} + function validateFreeStandingInventoryCoverage( errors: string[], jobs: WorkflowRecord, reportNeeds: readonly unknown[], + inventory: FreeStandingJobsInventory, ): void { - for (const jobId of FREE_STANDING_VITEST_JOB_IDS) { + for (const jobId of inventory.allowedJobs) { if (!Object.hasOwn(jobs, jobId)) { errors.push(`free-standing inventory job missing workflow job: ${jobId}`); } @@ -378,8 +418,8 @@ function validateFreeStandingInventoryCoverage( errors.push(`report-to-pr job must wait for ${jobId}`); } } - for (const [scenario, jobId] of FREE_STANDING_VITEST_SCENARIO_TO_JOB) { - if (!FREE_STANDING_VITEST_JOB_IDS.includes(jobId)) { + for (const [scenario, jobId] of inventory.scenarioToJob) { + if (!inventory.allowedJobs.includes(jobId)) { errors.push(`free-standing inventory maps ${scenario} to unknown job ${jobId}`); continue; } @@ -1900,7 +1940,7 @@ function validateModelRouterProviderRoutedInferenceVitestJob( export function validateE2eVitestScenariosWorkflowBoundary( workflowPath = DEFAULT_VITEST_WORKFLOW_PATH, ): string[] { - const workflow = asRecord(YAML.parse(readFileSync(workflowPath, "utf-8"))); + const workflow = readWorkflowRecord(workflowPath); const errors: string[] = []; const triggers = asRecord(workflow.on ?? workflow[true as unknown as string]); @@ -1918,6 +1958,10 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (permissions.contents !== "read") errors.push("workflow permissions.contents must be read"); const jobs = asRecord(workflow.jobs); + const { errors: inventoryErrors, inventory: freeStandingInventory } = + deriveFreeStandingJobsInventoryFromJobs(jobs); + errors.push(...inventoryErrors); + validateFreeStandingInventoryBoundary(errors, jobs, freeStandingInventory); const generateMatrix = asRecord(jobs["generate-matrix"]); if (Object.keys(generateMatrix).length === 0) errors.push("workflow missing generate-matrix job"); if (generateMatrix["runs-on"] !== "ubuntu-latest") { @@ -1949,8 +1993,8 @@ export function validateE2eVitestScenariosWorkflowBoundary( if (generateEnv.SCENARIOS !== "${{ inputs.scenarios }}") { errors.push("matrix generation step must pass scenarios through SCENARIOS env"); } - requireRunContains(errors, generate, "tools/e2e-scenarios/free-standing-jobs.env"); - requireRunContains(errors, generate, "free-standing jobs inventory must be data-only key=value"); + requireRunContains(errors, generate, FREE_STANDING_WORKFLOW_INVENTORY_SCRIPT); + requireRunContains(errors, generate, "free-standing workflow inventory must be data-only key=value"); requireRunContains(errors, generate, "free_standing_scenarios_csv must match scenario mapping keys"); requireRunContains(errors, generate, "Free-standing scenario maps to unknown job"); requireRunContains(errors, generate, "Use either scenarios or jobs, not both"); @@ -2142,7 +2186,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( for (const required of ["generate-matrix", "live-scenarios"]) { if (!needs.includes(required)) errors.push(`report-to-pr job must wait for ${required}`); } - validateFreeStandingInventoryCoverage(errors, jobs, needs); + validateFreeStandingInventoryCoverage(errors, jobs, needs, freeStandingInventory); const reportSteps = asSteps(reportToPr.steps); const report = requireJobStep(errors, "report-to-pr", reportSteps, "Post Vitest scenario results to PR"); const reportEnv = asRecord(report?.env);