diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7cbde1baa0..8f5f9adbd9 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -4550,18 +4550,83 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - # Scheduled coverage for #6108 / #3513 / #3127. This exercises the complete - # managed v0.0.71 runtime with a custom plugin, then proves restart/rebuild - # persistence and target-side runtime-dependency replacement across devices. + # Scheduled release-baseline coverage for #6108. This runs independently of + # the current lifecycle lane so the exact v0.0.71 build does not extend that + # lane's critical path. + openclaw-plugin-runtime-exdev-release: + needs: generate-matrix + if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',openclaw-plugin-runtime-exdev-release,') || contains(format(',{0},', inputs.targets), ',openclaw-plugin-runtime-exdev-release,') }} + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 55 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "openclaw-plugin-runtime-exdev-release" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-openclaw-plugin-exdev-release" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ inputs.checkout_sha || github.sha }} + persist-credentials: false + + - *dockerhub-auth + + - name: Pre-pull release-matched Docker Hub builder image + shell: bash + run: | + set -euo pipefail + docker pull node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d + + - name: Remove Docker auth before release-pinned fixture + if: always() + shell: bash + run: | + set -euo pipefail + bash .github/scripts/docker-auth-cleanup.sh + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + + - name: Run OpenClaw custom-plugin release baseline live test + run: | + set -euo pipefail + test -n "${DOCKER_CONFIG:-}" + test ! -e "${DOCKER_CONFIG}" + test -z "${DOCKERHUB_USERNAME:-}" + test -z "${DOCKERHUB_TOKEN:-}" + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \ + npx tsx tools/e2e/live-vitest-invocation.mts run \ + --test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \ + --selector release-baseline + + - name: Upload OpenClaw plugin release baseline artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + + # Scheduled current-lifecycle coverage for #6108 / #3513 / #3127. This + # proves restart/rebuild persistence and target-side runtime-dependency + # replacement across devices without waiting for the release baseline. openclaw-plugin-runtime-exdev: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',openclaw-plugin-runtime-exdev,') || contains(format(',{0},', inputs.targets), ',openclaw-plugin-runtime-exdev,') }} runs-on: ubuntu-latest permissions: contents: read - # Three bounded 25-minute onboards plus the 20-minute rebuild and 15-minute - # Vitest buffer need 110 minutes; allow 20 more for setup and teardown. - timeout-minutes: 130 + # Two bounded 25-minute onboards plus the 20-minute rebuild and 15-minute + # Vitest buffer need 85 minutes; allow 20 more for setup and teardown. + timeout-minutes: 105 env: E2E_JOB: "1" E2E_TARGET_ID: "openclaw-plugin-runtime-exdev" @@ -4604,7 +4669,9 @@ jobs: test -z "${DOCKERHUB_USERNAME:-}" test -z "${DOCKERHUB_TOKEN:-}" env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \ - npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts + npx tsx tools/e2e/live-vitest-invocation.mts run \ + --test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \ + --selector current-lifecycle - name: Upload OpenClaw plugin runtime-deps EXDEV artifacts if: always() @@ -5622,6 +5689,7 @@ jobs: sandbox-survival, diagnostics, snapshot-commands, + openclaw-plugin-runtime-exdev-release, openclaw-plugin-runtime-exdev, openclaw-tui-chat-correlation, gateway-guard-recovery, diff --git a/.github/workflows/regression-e2e.yaml b/.github/workflows/regression-e2e.yaml index 85a4484657..2ce3184051 100644 --- a/.github/workflows/regression-e2e.yaml +++ b/.github/workflows/regression-e2e.yaml @@ -287,11 +287,63 @@ jobs: /tmp/nemoclaw-e2e-model-router-response.log if-no-files-found: ignore - # ── OpenClaw custom-plugin lifecycle and runtime-deps EXDEV E2E ─ - # Coverage guard for #6108 / #3513 / #3127. On Ubuntu/OpenShell sandbox layouts - # where /tmp and /sandbox can live on different filesystems, OpenClaw's - # first CLI bootstrap must not fail plugin runtime dependency installation - # with EXDEV cross-device rename errors. + # ── OpenClaw release-baseline custom-plugin E2E ──────────────── + # The exact v0.0.71 baseline runs in parallel with the current lifecycle + # contract so release provenance does not extend the EXDEV critical path. + openclaw-plugin-runtime-exdev-release-e2e: + needs: select_regression_jobs + if: >- + github.repository == 'NVIDIA/NemoClaw' && + needs.select_regression_jobs.outputs.openclaw_plugin_runtime_exdev == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 55 + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run OpenClaw custom-plugin release baseline Vitest test + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release + E2E_TARGET_ID: openclaw-plugin-runtime-exdev-release + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_SANDBOX_NAME: e2e-openclaw-plugin-exdev-release + run: | + set -euo pipefail + npx vitest run --project e2e-live \ + test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \ + -t release-baseline \ + --silent=false --reporter=default + + - name: Upload OpenClaw plugin release baseline artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openclaw-plugin-runtime-exdev-release-artifacts + path: e2e-artifacts/live/openclaw-plugin-runtime-exdev-release/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + # ── OpenClaw current lifecycle and runtime-deps EXDEV E2E ───── + # Coverage guard for #6108 / #3513 / #3127. On Ubuntu/OpenShell sandbox + # layouts where /tmp and /sandbox can live on different filesystems, the + # runtime dependency replacement must complete without EXDEV failures. openclaw-plugin-runtime-exdev-e2e: needs: select_regression_jobs if: >- @@ -300,9 +352,9 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - # Three bounded 25-minute onboards plus the 20-minute rebuild and 15-minute - # Vitest buffer need 110 minutes; allow 20 more for setup and teardown. - timeout-minutes: 130 + # Two bounded 25-minute onboards plus the 20-minute rebuild and 15-minute + # Vitest buffer need 85 minutes; allow 20 more for setup and teardown. + timeout-minutes: 105 steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -324,11 +376,13 @@ jobs: - name: Run OpenClaw custom-plugin lifecycle and runtime-deps EXDEV Vitest test env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev + E2E_TARGET_ID: openclaw-plugin-runtime-exdev NEMOCLAW_RUN_LIVE_E2E: "1" run: | set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \ + -t current-lifecycle \ --silent=false --reporter=default - name: Upload OpenClaw plugin runtime-deps EXDEV artifacts diff --git a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts index 84b08cfd1a..6a6e7c1a0d 100644 --- a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts +++ b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts @@ -13,7 +13,9 @@ import { hasRequiredOpenshellMessagingFeatures, REQUIRED_OPENSHELL_MCP_FEATURES, } from "../../../src/lib/onboard/openshell-feature-gate.ts"; +import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import { resultText, shellQuote } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { @@ -24,6 +26,7 @@ import { import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import type { TestProgress } from "../fixtures/progress.ts"; import { parseJsonFromText } from "./json-envelope.ts"; import { createOpenShellDriverConfigTestWrapper, @@ -70,6 +73,10 @@ const NEMOCLAW_RELEASE_OPENSHELL_VERSION = "0.0.71"; const CURRENT_OPENSHELL_VERSION = "0.0.85"; const NEMOCLAW_SOURCE_REPOSITORY = "https://github.com/NVIDIA/NemoClaw.git"; const SANDBOX_BASE_IMAGE_REF = "ghcr.io/nvidia/nemoclaw/sandbox-base:v0.0.71"; +const RELEASE_BUILDER_IMAGE_REF = + "node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d"; +const CURRENT_BUILDER_IMAGE_REF = + "node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba"; const TOOL_DISCLOSURE_ENV_REFERENCE = "${NEMOCLAW_TOOL_DISCLOSURE}"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-openclaw-plugin-exdev"; const ONBOARD_TIMEOUT_MS = 25 * 60_000; @@ -110,6 +117,9 @@ const EXDEV_PATTERNS = [ ]; type WeatherFixtureVersion = "v1" | "v2" | "v3"; +const RELEASE_BASELINE_TEST_SELECTOR = "release-baseline"; +const CURRENT_LIFECYCLE_TEST_SELECTOR = "current-lifecycle"; + const GATEWAY_CATALOG_CALL_SOURCE = String.raw` import { Buffer } from "node:buffer"; import { accessSync, constants, realpathSync } from "node:fs"; @@ -454,14 +464,19 @@ function writeCustomPluginVersion( ); } -function createCustomPluginDockerfile(context: CustomPluginBuildContext): void { +function createCustomPluginDockerfile( + context: CustomPluginBuildContext, + builderImageRef: string, +): void { const sourceDockerfile = path.join(context.sourceRoot, "Dockerfile"); const source = fs.readFileSync(sourceDockerfile, "utf8"); const baseImageAnchor = "ARG BASE_IMAGE=ghcr.io/nvidia/nemoclaw/sandbox-base:latest\n"; + const builderImageAnchor = `FROM ${RELEASE_BUILDER_IMAGE_REF} AS builder\n`; const runtimeAnchor = "FROM ${BASE_IMAGE}\n"; expect( source.match(/^ARG BASE_IMAGE=ghcr\.io\/nvidia\/nemoclaw\/sandbox-base:latest$/gm)?.length, ).toBe(1); + expect(source.split(builderImageAnchor)).toHaveLength(2); expect(source.match(/^FROM \$\{BASE_IMAGE\}$/gm)?.length, "expected one runtime stage").toBe(1); expect( source.match(/^ARG OPENCLAW_VERSION=([0-9.]+)$/m)?.[1], @@ -474,6 +489,7 @@ function createCustomPluginDockerfile(context: CustomPluginBuildContext): void { const runtime = source .replace(baseImageAnchor, `ARG BASE_IMAGE=${SANDBOX_BASE_IMAGE_REF}\n`) + .replace(builderImageAnchor, `FROM ${builderImageRef} AS builder\n`) .replace(runtimeAnchor, "FROM ${BASE_IMAGE} AS nemoclaw-runtime\n"); const pluginDirName = path.basename(context.pluginDirPath); const versionSourceName = path.basename(context.versionSourcePath); @@ -866,133 +882,161 @@ NODE`; const runtimeDepsReplacementProbe = trustedSandboxShellScript(runtimeDepsReplacementProbeSource); -test("a custom OpenClaw plugin survives restart, recreation, and rebuild without EXDEV failures (#6108)", { - timeout: ONBOARD_TIMEOUT_MS * 3 + REBUILD_TIMEOUT_MS + 15 * 60_000, +async function prepareCustomPluginSource( + host: HostCliClient, + cleanup: CleanupRegistry, + builderImageRef: string, +): Promise { + const context = createCustomPluginBuildContext(); + cleanup.add("remove v0.0.71 custom-plugin source worktree", () => + fs.rmSync(context.sourceParentDir, { recursive: true, force: true }), + ); + const cloneRelease = await host.command( + "git", + [ + "clone", + "--depth", + "1", + "--branch", + NEMOCLAW_RELEASE_TAG, + "--single-branch", + NEMOCLAW_SOURCE_REPOSITORY, + context.sourceRoot, + ], + { + artifactName: "clone-nemoclaw-v0-0-71-plugin-source", + env: liveEnv(), + timeoutMs: 180_000, + }, + ); + expect(cloneRelease.exitCode, resultText(cloneRelease)).toBe(0); + const releaseHead = await host.command("git", ["-C", context.sourceRoot, "rev-parse", "HEAD"], { + artifactName: "verify-nemoclaw-v0-0-71-plugin-source", + env: liveEnv(), + timeoutMs: 30_000, + }); + expect(releaseHead.exitCode, resultText(releaseHead)).toBe(0); + expect(releaseHead.stdout.trim()).toBe(NEMOCLAW_RELEASE_COMMIT); + createCustomPluginDockerfile(context, builderImageRef); + return context; +} + +async function startDeploymentFixture( + artifacts: ArtifactSink, + cleanup: CleanupRegistry, + progress: TestProgress, +): Promise { + const fake = await startFakeOpenAiCompatibleServer({ + apiKey: "nemoclaw-exdev-dummy-key", + host: "0.0.0.0", + model: "nemoclaw-exdev-probe", + progress, + publicHost: "host.openshell.internal", + responseText: "ok", + }); + await artifacts.writeJson("fake-openai-compatible.json", { baseUrl: fake.baseUrl }); + cleanup.add("close EXDEV compatible endpoint mock", async () => { + try { + await artifacts.writeJson("fake-openai-compatible-requests.json", fake.requests()); + } finally { + await fake.close(); + } + }); + + return liveEnv({ + COMPATIBLE_API_KEY: "nemoclaw-exdev-dummy-key", + NEMOCLAW_ENDPOINT_URL: fake.baseUrl, + NEMOCLAW_MODEL: "nemoclaw-exdev-probe", + NEMOCLAW_PROVIDER_KEY: "nemoclaw-exdev-dummy-key", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_SANDBOX_BASE_IMAGE_REF: SANDBOX_BASE_IMAGE_REF, + NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + }); +} + +async function requireDocker( + host: HostCliClient, + artifactName: string, + reason: string, + skip: (note?: string) => never, +): Promise { + const docker = await host.command("docker", ["info"], { + artifactName, + env: liveEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode === 0) return; + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`${reason}: ${resultText(docker)}`); + } + skip(reason); +} + +test("the release-baseline custom plugin loads with its exact NemoClaw and OpenShell versions (#6108)", { + timeout: ONBOARD_TIMEOUT_MS + 15 * 60_000, meta: { e2ePhases: [ - "confirm Docker CLI and clear the plugin sandbox", + "confirm Docker CLI and clear the release plugin sandbox", "clone and build the tagged plugin fixture", "install tagged OpenShell and onboard the release sandbox", - "install current OpenShell and onboard plugin v1", - "restart the gateway and confirm plugin v1", - "recreate the sandbox with plugin v2", - "rebuild the sandbox with plugin v3", - "prove cross-device runtime dependency replacement", + "verify the tagged plugin and remove the release sandbox", ], }, }, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { await artifacts.target.declare({ - id: "openclaw-plugin-runtime-exdev", + id: "openclaw-plugin-runtime-exdev-release", boundary: "fresh-openclaw-sandbox-exec", - regressionTargets: ["#6108", "#3513", "#3127"], + regressionTargets: ["#6108"], contract: [ "the exact NemoClaw v0.0.71 checkout installs, builds, and reports its tagged CLI version", "the tagged CLI uses OpenShell 0.0.71 with matching source, base image, and OpenClaw runtime", - "the current CLI reinstalls OpenShell 0.0.85 before current lifecycle coverage", "release-matched peer/dev dependencies prune private OpenClaw and link the host runtime", - "gateway log, runtime inspection, tools.catalog, and tools.invoke prove weather/get_weather", - "custom-plugin v1 survives restart, recreation installs v2, and rebuild installs v3", - "workspace state survives both onboarding recreation and rebuild", - `test-only driver config mounts tmpfs at ${EXDEV_TMPFS_MOUNT} without changing production policies`, - "stock OpenClaw policy source bytes remain unchanged through onboard and rebuild", - `sandbox proves ${EXDEV_TMPFS_SOURCE} and plugin-runtime-deps are distinct devices`, - `legacy source-side staging fails with EXDEV across the same ${EXDEV_TMPFS_SOURCE} to plugin-runtime-deps boundary`, - "OpenClaw-style target-side plugin runtime-deps replacement completes without EXDEV", + "the release weather plugin loads from the custom image without an EXDEV bootstrap failure", ], + selector: RELEASE_BASELINE_TEST_SELECTOR, nemoclawSourceRelease: NEMOCLAW_RELEASE_TAG, nemoclawSourceCommit: NEMOCLAW_RELEASE_COMMIT, taggedOpenshellVersion: NEMOCLAW_RELEASE_OPENSHELL_VERSION, - currentOpenshellVersion: CURRENT_OPENSHELL_VERSION, sandboxBaseImageRef: SANDBOX_BASE_IMAGE_REF, openclawVersion: WEATHER_OPENCLAW_VERSION, }); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-openclaw-plugin-exdev", - env: liveEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for the OpenClaw plugin EXDEV live guard: ${resultText(docker)}`, - ); - } - skip("Docker is required for the OpenClaw plugin EXDEV live guard"); - } - - expect( - fs.existsSync(CLI_ENTRYPOINT), - "bin/nemoclaw.js missing — run npm run build:cli before this live target", - ).toBe(true); + await requireDocker( + host, + "prereq-docker-info-openclaw-plugin-exdev-release", + "Docker is required for the OpenClaw plugin release baseline", + skip, + ); cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => sandbox.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-openshell-delete-openclaw-plugin-exdev", + artifactName: "cleanup-openshell-delete-openclaw-plugin-exdev-release", env: liveEnv(), timeoutMs: 60_000, }), ); cleanup.trackSandbox(host, SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy-openclaw-plugin-exdev", + artifactName: "cleanup-nemoclaw-destroy-openclaw-plugin-exdev-release", env: liveEnv(), timeoutMs: 120_000, }); - - await ignoreCleanupError(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "pre-cleanup-nemoclaw-destroy-openclaw-plugin-exdev", - env: liveEnv(), - timeoutMs: 120_000, - }), - ); await ignoreCleanupError(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "pre-cleanup-openshell-delete-openclaw-plugin-exdev", + artifactName: "pre-cleanup-openshell-delete-openclaw-plugin-exdev-release", env: liveEnv(), timeoutMs: 60_000, }), ); progress.phase("clone and build the tagged plugin fixture"); - const policySourceSnapshot = snapshotPolicySources(); - const customPluginContext = createCustomPluginBuildContext(); - cleanup.add("remove v0.0.71 custom-plugin source worktree", () => - fs.rmSync(customPluginContext.sourceParentDir, { recursive: true, force: true }), - ); - const cloneRelease = await host.command( - "git", - [ - "clone", - "--depth", - "1", - "--branch", - NEMOCLAW_RELEASE_TAG, - "--single-branch", - NEMOCLAW_SOURCE_REPOSITORY, - customPluginContext.sourceRoot, - ], - { - artifactName: "clone-nemoclaw-v0-0-71-plugin-source", - env: liveEnv(), - timeoutMs: 180_000, - }, - ); - expect(cloneRelease.exitCode, resultText(cloneRelease)).toBe(0); - const releaseHead = await host.command( - "git", - ["-C", customPluginContext.sourceRoot, "rev-parse", "HEAD"], - { - artifactName: "verify-nemoclaw-v0-0-71-plugin-source", - env: liveEnv(), - timeoutMs: 30_000, - }, + const customPluginContext = await prepareCustomPluginSource( + host, + cleanup, + RELEASE_BUILDER_IMAGE_REF, ); - expect(releaseHead.exitCode, resultText(releaseHead)).toBe(0); - expect(releaseHead.stdout.trim()).toBe(NEMOCLAW_RELEASE_COMMIT); - createCustomPluginDockerfile(customPluginContext); await buildAndVerifyTaggedCli(host, customPluginContext); - progress.phase("install tagged OpenShell and onboard the release sandbox"); await stopOpenShellGatewayBeforeVersionSwitch(host, "existing"); const taggedPinnedOpenshell = await installAndResolvePinnedOpenShell( @@ -1001,38 +1045,9 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without "v0-0-71", NEMOCLAW_RELEASE_OPENSHELL_VERSION, ); - // OpenShell 0.0.71 predates the current 0.0.85 MCP capability marker. - // The tagged CLI's own onboarding preflight below owns this compatibility check. const taggedOpenShellWrapper = createOpenShellTmpfsWrapper(taggedPinnedOpenshell.cli); cleanup.add("remove v0.0.71 EXDEV OpenShell PATH wrapper", taggedOpenShellWrapper.remove); - - // The OpenShell gateway reaches the fixture from its network namespace, so - // runner loopback is not routable from the readiness probe. - const fake = await startFakeOpenAiCompatibleServer({ - apiKey: "nemoclaw-exdev-dummy-key", - host: "0.0.0.0", - model: "nemoclaw-exdev-probe", - progress, - publicHost: "host.openshell.internal", - responseText: "ok", - }); - await artifacts.writeJson("fake-openai-compatible.json", { baseUrl: fake.baseUrl }); - cleanup.add("close EXDEV compatible endpoint mock", async () => { - await artifacts.writeJson("fake-openai-compatible-requests.json", fake.requests()); - await fake.close(); - }); - - const deploymentEnv = liveEnv({ - COMPATIBLE_API_KEY: "nemoclaw-exdev-dummy-key", - NEMOCLAW_ENDPOINT_URL: fake.baseUrl, - NEMOCLAW_MODEL: "nemoclaw-exdev-probe", - NEMOCLAW_PROVIDER_KEY: "nemoclaw-exdev-dummy-key", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_SANDBOX_BASE_IMAGE_REF: SANDBOX_BASE_IMAGE_REF, - NEMOCLAW_POLICY_MODE: "skip", - NEMOCLAW_PREFERRED_API: "openai-completions", - NEMOCLAW_PROVIDER: "custom", - }); + const deploymentEnv = await startDeploymentFixture(artifacts, cleanup, progress); const taggedSandboxEnv = withOpenShellWrapperEnv( deploymentEnv, taggedOpenShellWrapper, @@ -1063,6 +1078,8 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without const taggedOnboardText = resultText(taggedOnboard); expect(taggedOnboard.exitCode, taggedOnboardText).toBe(0); expect(taggedOnboardText).toContain("Deployment verified"); + + progress.phase("verify the tagged plugin and remove the release sandbox"); const taggedRuntimeVersion = await sandbox.exec(SANDBOX_NAME, ["openclaw", "--version"], { artifactName: "v0-0-71-openclaw-version", env: liveEnv(), @@ -1087,6 +1104,7 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without expect(taggedPluginInspect.plugin?.id).toBe("weather"); expect(taggedPluginInspect.plugin?.status).toBe("loaded"); expect(taggedPluginInspect.plugin?.toolNames).toContain("get_weather"); + const taggedDestroy = await host.command( "node", [customPluginContext.cliEntrypoint, SANDBOX_NAME, "destroy", "--yes"], @@ -1098,16 +1116,111 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without }, ); expect(taggedDestroy.exitCode, resultText(taggedDestroy)).toBe(0); + + await artifacts.target.complete({ + id: "openclaw-plugin-runtime-exdev-release", + taggedOnboardExitCode: taggedOnboard.exitCode, + taggedDestroyExitCode: taggedDestroy.exitCode, + assertions: { + taggedReleaseRuntimeMatched: taggedRuntimeVersionText.includes( + EXPECTED_RELEASE_OPENCLAW_VERSION, + ), + taggedReleasePluginLoaded: + taggedPluginInspect.plugin?.id === "weather" && + taggedPluginInspect.plugin?.status === "loaded" && + Array.isArray(taggedPluginInspect.plugin?.toolNames) && + taggedPluginInspect.plugin.toolNames.includes("get_weather"), + }, + }); +}); + +test("the current-lifecycle custom plugin survives restart, recreation, and rebuild without EXDEV failures (#6108)", { + timeout: ONBOARD_TIMEOUT_MS * 2 + REBUILD_TIMEOUT_MS + 15 * 60_000, + meta: { + e2ePhases: [ + "confirm Docker CLI and clear the current plugin sandbox", + "clone and prepare the current plugin fixture", + "install current OpenShell and onboard plugin v1", + "restart the gateway and confirm plugin v1", + "recreate the sandbox with plugin v2", + "rebuild the sandbox with plugin v3", + "prove cross-device runtime dependency replacement", + ], + }, +}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + await artifacts.target.declare({ + id: "openclaw-plugin-runtime-exdev", + boundary: "fresh-openclaw-sandbox-exec", + regressionTargets: ["#6108", "#3513", "#3127"], + contract: [ + "the current CLI uses OpenShell 0.0.85 for current lifecycle coverage", + "gateway log, runtime inspection, tools.catalog, and tools.invoke prove weather/get_weather", + "custom-plugin v1 survives restart, recreation installs v2, and rebuild installs v3", + "workspace state survives both onboarding recreation and rebuild", + `test-only driver config mounts tmpfs at ${EXDEV_TMPFS_MOUNT} without changing production policies`, + "stock OpenClaw policy source bytes remain unchanged through onboard and rebuild", + `sandbox proves ${EXDEV_TMPFS_SOURCE} and plugin-runtime-deps are distinct devices`, + `legacy source-side staging fails with EXDEV across the same ${EXDEV_TMPFS_SOURCE} to plugin-runtime-deps boundary`, + "OpenClaw-style target-side plugin runtime-deps replacement completes without EXDEV", + ], + selector: CURRENT_LIFECYCLE_TEST_SELECTOR, + nemoclawSourceRelease: NEMOCLAW_RELEASE_TAG, + nemoclawSourceCommit: NEMOCLAW_RELEASE_COMMIT, + currentOpenshellVersion: CURRENT_OPENSHELL_VERSION, + sandboxBaseImageRef: SANDBOX_BASE_IMAGE_REF, + openclawVersion: WEATHER_OPENCLAW_VERSION, + }); + + await requireDocker( + host, + "prereq-docker-info-openclaw-plugin-exdev", + "Docker is required for the OpenClaw plugin EXDEV live guard", + skip, + ); + + expect( + fs.existsSync(CLI_ENTRYPOINT), + "bin/nemoclaw.js missing — run npm run build:cli before this live target", + ).toBe(true); + + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete-openclaw-plugin-exdev", + env: liveEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-openclaw-plugin-exdev", + env: liveEnv(), + timeoutMs: 120_000, + }); + + await ignoreCleanupError(() => + host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy-openclaw-plugin-exdev", + env: liveEnv(), + timeoutMs: 120_000, + }), + ); await ignoreCleanupError(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "v0-0-71-openclaw-plugin-delete-fallback", - env: taggedSandboxEnv, + artifactName: "pre-cleanup-openshell-delete-openclaw-plugin-exdev", + env: liveEnv(), timeoutMs: 60_000, }), ); + progress.phase("clone and prepare the current plugin fixture"); + const policySourceSnapshot = snapshotPolicySources(); + const customPluginContext = await prepareCustomPluginSource( + host, + cleanup, + CURRENT_BUILDER_IMAGE_REF, + ); + const deploymentEnv = await startDeploymentFixture(artifacts, cleanup, progress); progress.phase("install current OpenShell and onboard plugin v1"); - await stopOpenShellGatewayBeforeVersionSwitch(host, "v0-0-71", taggedSandboxEnv); + await stopOpenShellGatewayBeforeVersionSwitch(host, "existing"); const pinnedOpenshell = await installAndResolvePinnedOpenShell( host, path.join(REPO_ROOT, "scripts", "install-openshell.sh"), @@ -1261,8 +1374,6 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without await artifacts.target.complete({ id: "openclaw-plugin-runtime-exdev", - taggedOnboardExitCode: taggedOnboard.exitCode, - taggedDestroyExitCode: taggedDestroy.exitCode, onboardExitCode: onboard.exitCode, restartExitCode: restart.exitCode, recreateExitCode: recreate.exitCode, @@ -1271,14 +1382,6 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without runtimeDepsProbeExitCode: probe.exitCode, testOnlyTmpfsSource: EXDEV_TMPFS_SOURCE, assertions: { - taggedReleaseRuntimeMatched: taggedRuntimeVersionText.includes( - EXPECTED_RELEASE_OPENCLAW_VERSION, - ), - taggedReleasePluginLoaded: - taggedPluginInspect.plugin?.id === "weather" && - taggedPluginInspect.plugin?.status === "loaded" && - Array.isArray(taggedPluginInspect.plugin?.toolNames) && - taggedPluginInspect.plugin.toolNames.includes("get_weather"), weatherAfterOnboard: weatherAfterOnboard.inspectLoaded && weatherAfterOnboard.catalogToolIds.includes("get_weather") && diff --git a/test/e2e/support/openclaw-plugin-runtime-exdev-workflow-boundary.test.ts b/test/e2e/support/openclaw-plugin-runtime-exdev-workflow-boundary.test.ts index 5d3600bcaa..337ccc3af8 100644 --- a/test/e2e/support/openclaw-plugin-runtime-exdev-workflow-boundary.test.ts +++ b/test/e2e/support/openclaw-plugin-runtime-exdev-workflow-boundary.test.ts @@ -77,7 +77,7 @@ describe("OpenClaw plugin runtime EXDEV workflow boundary", () => { expect(validateOpenClawPluginRuntimeExdevWorkflow(workflow)).toEqual( expect.arrayContaining([ "openclaw-plugin-runtime-exdev must run on ubuntu-latest", - "openclaw-plugin-runtime-exdev must retain its 130 minute runtime proof budget", + "openclaw-plugin-runtime-exdev must retain its 105 minute runtime proof budget", "openclaw-plugin-runtime-exdev must hold only contents: read", "openclaw-plugin-runtime-exdev must set E2E_ARTIFACT_DIR=${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev", "openclaw-plugin-runtime-exdev must remain enabled for scheduled and empty manual runs", diff --git a/test/regression-e2e-workflow.test.ts b/test/regression-e2e-workflow.test.ts index 1ce8247d9b..fa7fab5abb 100644 --- a/test/regression-e2e-workflow.test.ts +++ b/test/regression-e2e-workflow.test.ts @@ -137,6 +137,7 @@ describe("Regression E2E workflow contract", () => { // source-shape-contract: security -- Keeps the custom-plugin EXDEV regression immutable and free of repository secrets it("runs the OpenClaw custom-plugin lifecycle and EXDEV guard in a secret-free lane", () => { + const releaseJob = workflow.jobs?.["openclaw-plugin-runtime-exdev-release-e2e"]; const job = workflow.jobs?.["openclaw-plugin-runtime-exdev-e2e"]; const steps = job?.steps ?? []; const runText = steps.map((step) => step.run ?? "").join("\n"); @@ -151,7 +152,7 @@ describe("Regression E2E workflow contract", () => { const serializedJob = JSON.stringify(job); expect(job?.permissions).toEqual({ contents: "read" }); - expect(job?.["timeout-minutes"]).toBe(130); + expect(job?.["timeout-minutes"]).toBe(105); expect(checkoutStep?.uses).toMatch(FULL_SHA_ACTION); expect(checkoutStep?.with?.["persist-credentials"]).toBe(false); expect(setupNodeStep?.uses).toMatch(FULL_SHA_ACTION); @@ -166,8 +167,16 @@ describe("Regression E2E workflow contract", () => { } expect(runText).toContain("test/e2e/live/openclaw-plugin-runtime-exdev.test.ts"); + expect(runText).toContain("-t current-lifecycle"); expect(runText).toContain("npx vitest run --project e2e-live"); expect(runText).toContain("npm ci --ignore-scripts"); expect(runText).toContain("npm run build:cli"); + + const releaseRunText = (releaseJob?.steps ?? []).map((step) => step.run ?? "").join("\n"); + expect(releaseJob?.permissions).toEqual({ contents: "read" }); + expect(releaseJob?.["timeout-minutes"]).toBe(55); + expect(JSON.stringify(releaseJob)).not.toContain("${{ secrets."); + expect(releaseRunText).toContain("test/e2e/live/openclaw-plugin-runtime-exdev.test.ts"); + expect(releaseRunText).toContain("-t release-baseline"); }); }); diff --git a/tools/e2e/openclaw-plugin-runtime-exdev-workflow-boundary.mts b/tools/e2e/openclaw-plugin-runtime-exdev-workflow-boundary.mts index c07e220625..f0118b1416 100644 --- a/tools/e2e/openclaw-plugin-runtime-exdev-workflow-boundary.mts +++ b/tools/e2e/openclaw-plugin-runtime-exdev-workflow-boundary.mts @@ -11,10 +11,32 @@ import { UPLOAD_E2E_ARTIFACTS_ACTION } from "./upload-e2e-artifacts-workflow-bou const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); -const JOB_NAME = "openclaw-plugin-runtime-exdev"; const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; -const RELEASE_BUILDER_IMAGE = - "node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba"; + +const JOB_CONTRACTS = [ + { + jobName: "openclaw-plugin-runtime-exdev-release", + timeoutMinutes: 55, + artifactId: "openclaw-plugin-runtime-exdev-release", + sandboxName: "e2e-openclaw-plugin-exdev-release", + selector: "release-baseline", + runName: "Run OpenClaw custom-plugin release baseline live test", + uploadName: "Upload OpenClaw plugin release baseline artifacts", + builderImage: + "node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d", + }, + { + jobName: "openclaw-plugin-runtime-exdev", + timeoutMinutes: 105, + artifactId: "openclaw-plugin-runtime-exdev", + sandboxName: "e2e-openclaw-plugin-exdev", + selector: "current-lifecycle", + runName: "Run OpenClaw custom-plugin lifecycle and runtime-deps EXDEV live test", + uploadName: "Upload OpenClaw plugin runtime-deps EXDEV artifacts", + builderImage: + "node:22-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba", + }, +] as const; type WorkflowStep = { env?: Record; @@ -51,17 +73,19 @@ function findStep(job: WorkflowJob, name: string): WorkflowStep { function requireRunContains( errors: string[], + jobName: string, step: WorkflowStep, fragment: string, description = step.name ?? "", ): void { if (!step.run?.includes(fragment)) { - errors.push(`${JOB_NAME} step '${description}' must run: ${fragment}`); + errors.push(`${jobName} step '${description}' must run: ${fragment}`); } } function requireStepOrder( errors: string[], + jobName: string, steps: WorkflowStep[], beforeName: string, afterName: string, @@ -69,12 +93,13 @@ function requireStepOrder( const before = steps.findIndex((step) => step.name === beforeName); const after = steps.findIndex((step) => step.name === afterName); if (before < 0 || after < 0 || before >= after) { - errors.push(`${JOB_NAME} step '${beforeName}' must precede '${afterName}'`); + errors.push(`${jobName} step '${beforeName}' must precede '${afterName}'`); } } function requireAdjacentSteps( errors: string[], + jobName: string, steps: WorkflowStep[], beforeName: string, afterName: string, @@ -82,7 +107,7 @@ function requireAdjacentSteps( const before = steps.findIndex((step) => step.name === beforeName); const after = steps.findIndex((step) => step.name === afterName); if (before < 0 || after !== before + 1) { - errors.push(`${JOB_NAME} step '${beforeName}' must immediately precede '${afterName}'`); + errors.push(`${jobName} step '${beforeName}' must immediately precede '${afterName}'`); } } @@ -90,124 +115,136 @@ export function validateOpenClawPluginRuntimeExdevWorkflow( workflow: OpenClawPluginRuntimeExdevWorkflow, ): string[] { const errors: string[] = []; - const job = workflow.jobs[JOB_NAME]; - if (!job) return [`workflow is missing ${JOB_NAME}`]; - - if (job.needs !== "generate-matrix") { - errors.push(`${JOB_NAME} must depend on generate-matrix`); - } - if (job["runs-on"] !== "ubuntu-latest") { - errors.push(`${JOB_NAME} must run on ubuntu-latest`); - } - if (job["timeout-minutes"] !== 130) { - errors.push(`${JOB_NAME} must retain its 130 minute runtime proof budget`); - } - if (job.permissions?.contents !== "read" || Object.keys(job.permissions ?? {}).length !== 1) { - errors.push(`${JOB_NAME} must hold only contents: read`); - } - - const env = job.env ?? {}; - const expectedEnv = { - E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev", - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_RUN_LIVE_E2E: "1", - NEMOCLAW_SANDBOX_NAME: "e2e-openclaw-plugin-exdev", - OPENSHELL_GATEWAY: "nemoclaw", - }; - for (const [name, value] of Object.entries(expectedEnv)) { - if (env[name] !== value) errors.push(`${JOB_NAME} must set ${name}=${value}`); - } - if (Object.hasOwn(env, "E2E_DEFAULT_ENABLED")) { - errors.push(`${JOB_NAME} must remain enabled for scheduled and empty manual runs`); - } - for (const secret of [ - "DOCKERHUB_USERNAME", - "DOCKERHUB_TOKEN", - "GITHUB_TOKEN", - "NVIDIA_API_KEY", - "NVIDIA_INFERENCE_API_KEY", - ]) { - if (Object.hasOwn(env, secret)) - errors.push(`${JOB_NAME} must not expose ${secret} at job scope`); - } + for (const contract of JOB_CONTRACTS) { + const { + artifactId, + builderImage, + jobName, + runName, + sandboxName, + selector, + timeoutMinutes, + uploadName, + } = contract; + const job = workflow.jobs[jobName]; + if (!job) { + errors.push(`workflow is missing ${jobName}`); + continue; + } - const steps = job.steps ?? []; - for (const step of steps.filter((candidate) => candidate.uses)) { - if (!FULL_SHA_ACTION.test(step.uses ?? "")) { - errors.push(`${JOB_NAME} action '${step.name ?? step.uses}' must pin a full SHA`); + if (job.needs !== "generate-matrix") errors.push(`${jobName} must depend on generate-matrix`); + if (job["runs-on"] !== "ubuntu-latest") errors.push(`${jobName} must run on ubuntu-latest`); + if (job["timeout-minutes"] !== timeoutMinutes) { + errors.push(`${jobName} must retain its ${timeoutMinutes} minute runtime proof budget`); + } + if (job.permissions?.contents !== "read" || Object.keys(job.permissions ?? {}).length !== 1) { + errors.push(`${jobName} must hold only contents: read`); } - } - const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")) ?? {}; - if (checkout.with?.["persist-credentials"] !== false) { - errors.push(`${JOB_NAME} checkout must disable persisted credentials`); - } + const env = job.env ?? {}; + const expectedEnv = { + E2E_ARTIFACT_DIR: `\${{ github.workspace }}/e2e-artifacts/live/${artifactId}`, + E2E_TARGET_ID: artifactId, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_SANDBOX_NAME: sandboxName, + OPENSHELL_GATEWAY: "nemoclaw", + }; + for (const [name, value] of Object.entries(expectedEnv)) { + if (env[name] !== value) errors.push(`${jobName} must set ${name}=${value}`); + } + if (Object.hasOwn(env, "E2E_DEFAULT_ENABLED")) { + errors.push(`${jobName} must remain enabled for scheduled and empty manual runs`); + } + for (const secret of [ + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + "NVIDIA_API_KEY", + "NVIDIA_INFERENCE_API_KEY", + ]) { + if (Object.hasOwn(env, secret)) + errors.push(`${jobName} must not expose ${secret} at job scope`); + } - const prepare = findStep(job, "Prepare E2E workspace"); - if (prepare.uses !== PREPARE_E2E_ACTION) { - errors.push(`${JOB_NAME} must use the reviewed prepare-e2e action`); - } + const steps = job.steps ?? []; + for (const step of steps.filter((candidate) => candidate.uses)) { + if (!FULL_SHA_ACTION.test(step.uses ?? "")) { + errors.push(`${jobName} action '${step.name ?? step.uses}' must pin a full SHA`); + } + } + const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")) ?? {}; + if (checkout.with?.["persist-credentials"] !== false) { + errors.push(`${jobName} checkout must disable persisted credentials`); + } + const prepare = findStep(job, "Prepare E2E workspace"); + if (prepare.uses !== PREPARE_E2E_ACTION) { + errors.push(`${jobName} must use the reviewed prepare-e2e action`); + } - const prePull = findStep(job, "Pre-pull release-matched Docker Hub builder image"); - requireRunContains(errors, prePull, `docker pull ${RELEASE_BUILDER_IMAGE}`); + const prePull = findStep(job, "Pre-pull release-matched Docker Hub builder image"); + requireRunContains(errors, jobName, prePull, `docker pull ${builderImage}`); + const revoke = findStep(job, "Remove Docker auth before release-pinned fixture"); + if (revoke.if !== "always()") { + errors.push(`${jobName} must always revoke Docker auth before the release-pinned fixture`); + } + requireRunContains(errors, jobName, revoke, "bash .github/scripts/docker-auth-cleanup.sh"); - const revoke = findStep(job, "Remove Docker auth before release-pinned fixture"); - if (revoke.if !== "always()") { - errors.push(`${JOB_NAME} must always revoke Docker auth before the release-pinned fixture`); - } - requireRunContains(errors, revoke, "bash .github/scripts/docker-auth-cleanup.sh"); - - const runName = "Run OpenClaw custom-plugin lifecycle and runtime-deps EXDEV live test"; - const run = findStep(job, runName); - for (const fragment of [ - 'test -n "${DOCKER_CONFIG:-}"', - 'test ! -e "${DOCKER_CONFIG}"', - 'test -z "${DOCKERHUB_USERNAME:-}"', - 'test -z "${DOCKERHUB_TOKEN:-}"', - "env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN", - "tools/e2e/live-vitest-invocation.mts run --test-path", - "test/e2e/live/openclaw-plugin-runtime-exdev.test.ts", - ]) { - requireRunContains(errors, run, fragment, runName); - } - if (Object.keys(run.env ?? {}).length > 0 || JSON.stringify(run).includes("secrets.")) { - errors.push(`${JOB_NAME} runtime proof must not receive workflow credentials`); - } + const run = findStep(job, runName); + for (const fragment of [ + 'test -n "${DOCKER_CONFIG:-}"', + 'test ! -e "${DOCKER_CONFIG}"', + 'test -z "${DOCKERHUB_USERNAME:-}"', + 'test -z "${DOCKERHUB_TOKEN:-}"', + "env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN", + "tools/e2e/live-vitest-invocation.mts run", + "--test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts", + `--selector ${selector}`, + ]) { + requireRunContains(errors, jobName, run, fragment, runName); + } + if (Object.keys(run.env ?? {}).length > 0 || JSON.stringify(run).includes("secrets.")) { + errors.push(`${jobName} runtime proof must not receive workflow credentials`); + } + const upload = findStep(job, uploadName); + if (upload.uses !== UPLOAD_E2E_ARTIFACTS_ACTION || upload.if !== "always()") { + errors.push(`${jobName} must always use the reviewed artifact uploader`); + } - const upload = findStep(job, "Upload OpenClaw plugin runtime-deps EXDEV artifacts"); - if (upload.uses !== UPLOAD_E2E_ARTIFACTS_ACTION || upload.if !== "always()") { - errors.push(`${JOB_NAME} must always use the reviewed artifact uploader`); + requireStepOrder( + errors, + jobName, + steps, + "Pre-pull release-matched Docker Hub builder image", + "Remove Docker auth before release-pinned fixture", + ); + requireStepOrder( + errors, + jobName, + steps, + "Remove Docker auth before release-pinned fixture", + "Prepare E2E workspace", + ); + requireStepOrder(errors, jobName, steps, "Prepare E2E workspace", runName); + requireStepOrder(errors, jobName, steps, runName, uploadName); + requireAdjacentSteps( + errors, + jobName, + steps, + "Authenticate to Docker Hub", + "Pre-pull release-matched Docker Hub builder image", + ); + requireAdjacentSteps( + errors, + jobName, + steps, + "Pre-pull release-matched Docker Hub builder image", + "Remove Docker auth before release-pinned fixture", + ); } - requireStepOrder( - errors, - steps, - "Pre-pull release-matched Docker Hub builder image", - "Remove Docker auth before release-pinned fixture", - ); - requireStepOrder( - errors, - steps, - "Remove Docker auth before release-pinned fixture", - "Prepare E2E workspace", - ); - requireStepOrder(errors, steps, "Prepare E2E workspace", runName); - requireStepOrder(errors, steps, runName, "Upload OpenClaw plugin runtime-deps EXDEV artifacts"); - requireAdjacentSteps( - errors, - steps, - "Authenticate to Docker Hub", - "Pre-pull release-matched Docker Hub builder image", - ); - requireAdjacentSteps( - errors, - steps, - "Pre-pull release-matched Docker Hub builder image", - "Remove Docker auth before release-pinned fixture", - ); - return errors; }