Skip to content

ci(e2e): derive free-standing selectors from workflow #5370

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cv merged 3 commits into from
Jun 13, 2026
+405 −220
Merged

ci(e2e): derive free-standing selectors from workflow #5370

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
37ef225
ci(e2e): derive free-standing selectors from workflow
cv Jun 12, 2026
0328dbe
ci(e2e): validate derived free-standing job boundaries
cv Jun 13, 2026
2f3ad0e
Merge branch 'main' into codex/derive-e2e-selectors-from-workflow
cv Jun 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 53 additions & 11 deletions .github/workflows/e2e-vitest-scenarios.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ jobs:
SCENARIOS: ${{ inputs.scenarios }}
run: |
set -euo pipefail
inventory_path="tools/e2e-scenarios/free-standing-jobs.env"
inventory_output="$(npx tsx tools/e2e-scenarios/free-standing-workflow-inventory.mts --shell)"
allowed_jobs=""
free_standing_scenarios_csv=""
free_standing_scenario_jobs_csv=""
Expand All @@ -68,13 +68,13 @@ jobs:
continue
fi
if [[ ! "${line}" =~ ^(allowed_jobs|free_standing_scenarios_csv|free_standing_scenario_jobs_csv)=([A-Za-z0-9_:-]+(,[A-Za-z0-9_:-]+)*)$ ]]; then
echo "::error::free-standing jobs inventory must be data-only key=value" >&2
echo "::error::free-standing workflow inventory must be data-only key=value" >&2
exit 1
fi
inventory_key="${BASH_REMATCH[1]}"
inventory_value="${BASH_REMATCH[2]}"
if [[ -n "${seen_inventory_keys[${inventory_key}]:-}" ]]; then
echo "::error::free-standing jobs inventory must not redefine ${inventory_key}" >&2
echo "::error::free-standing workflow inventory must not redefine ${inventory_key}" >&2
exit 1
fi
seen_inventory_keys["${inventory_key}"]=1
Expand All @@ -83,22 +83,22 @@ jobs:
free_standing_scenarios_csv) free_standing_scenarios_csv="${inventory_value}" ;;
free_standing_scenario_jobs_csv) free_standing_scenario_jobs_csv="${inventory_value}" ;;
esac
done < "${inventory_path}"
done <<< "${inventory_output}"
for required_inventory_key in allowed_jobs free_standing_scenarios_csv free_standing_scenario_jobs_csv; do
if [[ -z "${!required_inventory_key:-}" ]]; then
echo "::error::free-standing jobs inventory missing ${required_inventory_key}" >&2
echo "::error::free-standing workflow inventory missing ${required_inventory_key}" >&2
exit 1
fi
done
declare -A seen_allowed_jobs=()
IFS=',' read -r -a allowed_job_entries <<< "${allowed_jobs}"
for job in "${allowed_job_entries[@]}"; do
if [[ ! "${job}" =~ ^[A-Za-z0-9_-]+$ ]]; then
echo "::error::free-standing jobs inventory contains invalid job id" >&2
echo "::error::free-standing workflow inventory contains invalid job id" >&2
exit 1
fi
if [[ -n "${seen_allowed_jobs[${job}]:-}" ]]; then
echo "::error::free-standing jobs inventory repeats job id" >&2
echo "::error::free-standing workflow inventory repeats job id" >&2
exit 1
fi
seen_allowed_jobs["${job}"]=1
Expand All @@ -107,11 +107,11 @@ jobs:
IFS=',' read -r -a free_standing_scenario_entries <<< "${free_standing_scenarios_csv}"
for scenario in "${free_standing_scenario_entries[@]}"; do
if [[ ! "${scenario}" =~ ^[A-Za-z0-9_-]+$ ]]; then
echo "::error::free-standing jobs inventory contains invalid scenario id" >&2
echo "::error::free-standing workflow inventory contains invalid scenario id" >&2
exit 1
fi
if [[ -n "${seen_free_standing_scenarios[${scenario}]:-}" ]]; then
echo "::error::free-standing jobs inventory repeats scenario id" >&2
echo "::error::free-standing workflow inventory repeats scenario id" >&2
exit 1
fi
seen_free_standing_scenarios["${scenario}"]=1
Expand All @@ -127,7 +127,7 @@ jobs:
scenario="${entry%%:*}"
job="${entry#*:}"
if [[ -n "${seen_scenario_mappings[${scenario}]:-}" ]]; then
echo "::error::free-standing jobs inventory repeats scenario mapping" >&2
echo "::error::free-standing workflow inventory repeats scenario mapping" >&2
exit 1
fi
seen_scenario_mappings["${scenario}"]=1
Expand Down Expand Up @@ -323,6 +323,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 15
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "openshell-version-pin"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/openshell-version-pin
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
steps:
Expand Down Expand Up @@ -364,6 +366,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 15
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "onboard-negative-paths"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/onboard-negative-paths
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
steps:
Expand Down Expand Up @@ -412,6 +416,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 30
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "skill-agent"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/skill-agent
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -511,6 +517,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "hermes-root-entrypoint-smoke"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-root-entrypoint-smoke
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
steps:
Expand Down Expand Up @@ -553,6 +561,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "inference-routing"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/inference-routing
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
steps:
Expand Down Expand Up @@ -601,6 +611,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 120
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "issue-4434-tui-unreachable-inference"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/issue-4434-tui-unreachable-inference
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -699,6 +711,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 65
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "credential-sanitization"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/credential-sanitization
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -783,6 +797,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 50
env:
FREE_STANDING_VITEST_JOB: "1"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/credential-migration
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -861,6 +876,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "runtime-overrides"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/runtime-overrides
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
steps:
Expand Down Expand Up @@ -902,6 +919,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 75
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "hermes-e2e"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-e2e
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -954,6 +973,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 90
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "network-policy"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/network-policy
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -1016,6 +1037,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "shields-config"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/shields-config
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
Expand Down Expand Up @@ -1092,6 +1115,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 130
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "rebuild-openclaw"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/rebuild-openclaw
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -1184,6 +1209,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 90
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "sandbox-rebuild"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/sandbox-rebuild
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -1277,6 +1304,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 90
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "double-onboard"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/double-onboard
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -1364,6 +1393,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "token-rotation"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/token-rotation
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -1447,6 +1478,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 90
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "messaging-providers"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/messaging-providers
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -1534,6 +1567,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 30
env:
FREE_STANDING_VITEST_JOB: "1"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/launchable-smoke
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-launchable"
Expand Down Expand Up @@ -1608,6 +1642,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "model-router-provider-routed-inference"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/model-router-provider-routed-inference
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -1697,6 +1733,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 30
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "sandbox-survival"
DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-sandbox-survival
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/sandbox-survival
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -1788,6 +1826,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 75
env:
FREE_STANDING_VITEST_JOB: "1"
FREE_STANDING_SCENARIO_ID: "openclaw-tui-chat-correlation"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/openclaw-tui-chat-correlation
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
Expand Down Expand Up @@ -1868,10 +1908,10 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
FREE_STANDING_VITEST_JOB: "1"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/gateway-guard-recovery
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_E2E_SCENARIOS: "1"
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
# nemoclaw onboard registers the gateway under the canonical name
Expand Down Expand Up @@ -1910,6 +1950,8 @@ jobs:
run: bash scripts/install-openshell.sh

- name: Run Vitest gateway-guard-recovery scenario
env:
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
run: |
set -euo pipefail
# OpenShell installs to /usr/local/bin on GitHub-hosted runners
Expand Down
Loading
Loading