diff --git a/.github/workflows/e2e-script.yaml b/.github/workflows/e2e-script.yaml index 8fdef8f6ff..0b729fead7 100644 --- a/.github/workflows/e2e-script.yaml +++ b/.github/workflows/e2e-script.yaml @@ -231,6 +231,7 @@ jobs: printf 'NEMOCLAW_ENDPOINT_URL=https://inference-api.nvidia.com/v1\n' printf 'NEMOCLAW_MODEL=nvidia/nvidia/nemotron-3-super-v3\n' printf 'NEMOCLAW_COMPAT_MODEL=nvidia/nvidia/nemotron-3-super-v3\n' + printf 'NEMOCLAW_PREFERRED_API=openai-completions\n' printf 'COMPATIBLE_API_KEY=%s\n' "${NVIDIA_INFERENCE_API_KEY}" } >> "$GITHUB_ENV" diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index b8a7c6816a..a77db9295d 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -992,6 +992,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions run: | set -euo pipefail npx vitest run --project e2e-scenarios-live \ diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index ce97dce38b..98619d67d4 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -470,6 +470,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -545,6 +546,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_ISSUE_4434_LIVE: "1" NEMOCLAW_NON_INTERACTIVE: "1" @@ -973,6 +975,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1270,6 +1273,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1576,6 +1580,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/credential-migration NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -1801,6 +1806,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1813,6 +1819,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1851,6 +1858,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1863,6 +1871,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1901,6 +1910,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1913,6 +1923,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1952,6 +1963,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1964,6 +1976,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -2003,6 +2016,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -2016,6 +2030,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -2057,6 +2072,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -2070,6 +2086,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -2149,6 +2166,7 @@ jobs: NEMOCLAW_ENDPOINT_URL: https://inference-api.nvidia.com/v1 NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" diff --git a/src/lib/onboard/providers.test.ts b/src/lib/onboard/providers.test.ts index 56090df090..c7411825f8 100644 --- a/src/lib/onboard/providers.test.ts +++ b/src/lib/onboard/providers.test.ts @@ -59,6 +59,7 @@ function withProviderEnv(next: Record, testBody: () "NEMOCLAW_ENDPOINT_URL", "NEMOCLAW_MODEL", "NEMOCLAW_COMPAT_MODEL", + "NEMOCLAW_PREFERRED_API", "NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL", "NEMOCLAW_E2E_USE_HOSTED_INFERENCE", "COMPATIBLE_API_KEY", @@ -303,11 +304,26 @@ describe("onboard provider helpers", () => { expect(process.env.NEMOCLAW_ENDPOINT_URL).toBe(HOSTED_INFERENCE_ENDPOINT_URL); expect(process.env.NEMOCLAW_MODEL).toBe(HOSTED_INFERENCE_MODEL); expect(process.env.NEMOCLAW_COMPAT_MODEL).toBe(HOSTED_INFERENCE_MODEL); + expect(process.env.NEMOCLAW_PREFERRED_API).toBe("openai-completions"); expect(process.env.COMPATIBLE_API_KEY).toBe("repo-hosted-key"); }, ); }); + it("does not override an explicit hosted inference API preference", () => { + withProviderEnv( + { + NVIDIA_INFERENCE_API_KEY: "repo-hosted-key", + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NEMOCLAW_PREFERRED_API: "openai-responses", + }, + () => { + expect(stageHostedInferenceSourceSecretEnv()).toBe(true); + expect(process.env.NEMOCLAW_PREFERRED_API).toBe("openai-responses"); + }, + ); + }); + it("keeps explicit cloud provider selection on the Build provider path", () => { withProviderEnv( { diff --git a/src/lib/onboard/providers.ts b/src/lib/onboard/providers.ts index 4ba749c2e1..18f835cb1b 100644 --- a/src/lib/onboard/providers.ts +++ b/src/lib/onboard/providers.ts @@ -254,6 +254,8 @@ function stageHostedInferenceSourceSecretEnv() { HOSTED_INFERENCE_MODEL; process.env.NEMOCLAW_MODEL = model; process.env.NEMOCLAW_COMPAT_MODEL = (process.env.NEMOCLAW_COMPAT_MODEL || "").trim() || model; + process.env.NEMOCLAW_PREFERRED_API = + (process.env.NEMOCLAW_PREFERRED_API || "").trim() || "openai-completions"; process.env[HOSTED_INFERENCE_CREDENTIAL_ENV] = sourceKey; return true; } diff --git a/test/e2e-advisor-dispatch.test.ts b/test/e2e-advisor-dispatch.test.ts index a8b7e31bab..8859fc49c0 100644 --- a/test/e2e-advisor-dispatch.test.ts +++ b/test/e2e-advisor-dispatch.test.ts @@ -19,6 +19,23 @@ jobs: cloud-e2e: if: github.event_name != 'workflow_dispatch' || inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',cloud-e2e,') steps: [] + cloud-onboard-e2e: + if: github.event_name != 'workflow_dispatch' || inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',cloud-onboard-e2e,') + uses: ./.github/workflows/e2e-script.yaml + with: + ref: \${{ inputs.target_ref || github.ref }} + nvidia_api_key: true + secrets: *nightly-e2e-default-secrets + launchable-smoke-e2e: + if: github.event_name != 'workflow_dispatch' || inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',launchable-smoke-e2e,') + runs-on: ubuntu-latest + steps: + - name: Run launchable install-flow smoke test + env: + NVIDIA_INFERENCE_API_KEY: \${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + COMPATIBLE_API_KEY: \${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} + run: bash test/e2e/test-launchable-smoke.sh report-to-pr: steps: [] notify-on-failure: @@ -68,6 +85,8 @@ describe("E2E advisor auto-dispatch planning", () => { expect(jobs).toContain("network-policy-e2e"); expect(jobs).toContain("cloud-e2e"); + expect(jobs).toContain("cloud-onboard-e2e"); + expect(jobs).toContain("launchable-smoke-e2e"); expect(jobs).not.toContain("report-to-pr"); expect(jobs).not.toContain("notify-on-failure"); expect(jobs).not.toContain("scorecard"); @@ -122,6 +141,56 @@ describe("E2E advisor auto-dispatch planning", () => { expect(plan.inputs?.jobs).toBe("network-policy-e2e,cloud-e2e"); }); + it("filters hosted-inference jobs that cannot receive secrets on target-ref dispatches", () => { + const workflowText = nightlyWorkflowText(); + const plan = planAutoDispatch({ + result: { + confidence: "high", + requiredTests: [ + { id: "network-policy-e2e", workflow: "nightly-e2e.yaml" }, + { id: "cloud-onboard-e2e", workflow: "nightly-e2e.yaml" }, + { id: "launchable-smoke-e2e", workflow: "nightly-e2e.yaml" }, + ], + }, + workflowText, + event: pullRequest("MEMBER"), + env: { + GITHUB_EVENT_NAME: "pull_request", + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + }, + }); + + expect(plan.status).toBe("ready"); + expect(plan.jobs).toEqual(["network-policy-e2e"]); + expect(plan.inputs?.jobs).toBe("network-policy-e2e"); + expect(plan.ignoredJobs).toEqual(["cloud-onboard-e2e", "launchable-smoke-e2e"]); + expect(plan.targetRefSecretBlockedJobs).toEqual(["cloud-onboard-e2e", "launchable-smoke-e2e"]); + }); + + it("skips cleanly when every recommended target-ref job requires withheld secrets", () => { + const workflowText = nightlyWorkflowText(); + const plan = planAutoDispatch({ + result: { + confidence: "high", + requiredTests: [ + { id: "cloud-onboard-e2e", workflow: "nightly-e2e.yaml" }, + { id: "launchable-smoke-e2e", workflow: "nightly-e2e.yaml" }, + ], + }, + workflowText, + event: pullRequest("MEMBER"), + env: { + GITHUB_EVENT_NAME: "pull_request", + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + }, + }); + + expect(plan.status).toBe("skipped"); + expect(plan.reason).toMatch(/hosted inference secrets are withheld/); + expect(plan.ignoredJobs).toEqual(["cloud-onboard-e2e", "launchable-smoke-e2e"]); + expect(plan.targetRefSecretBlockedJobs).toEqual(["cloud-onboard-e2e", "launchable-smoke-e2e"]); + }); + it("plans dispatch for allowlisted authors whose private org membership appears as contributor", () => { const workflowText = nightlyWorkflowText(); const plan = planAutoDispatch({ diff --git a/test/e2e-script-workflow.test.ts b/test/e2e-script-workflow.test.ts index 5fb007becb..5a0b151371 100644 --- a/test/e2e-script-workflow.test.ts +++ b/test/e2e-script-workflow.test.ts @@ -547,6 +547,7 @@ describe("E2E reusable workflow contract", () => { expect(runStep?.env?.NEMOCLAW_ENDPOINT_URL).toBe("https://inference-api.nvidia.com/v1"); expect(runStep?.env?.NEMOCLAW_MODEL).toBe("nvidia/nvidia/nemotron-3-super-v3"); expect(runStep?.env?.NEMOCLAW_COMPAT_MODEL).toBe("nvidia/nvidia/nemotron-3-super-v3"); + expect(runStep?.env?.NEMOCLAW_PREFERRED_API).toBe("openai-completions"); expect(runStep?.env?.COMPATIBLE_API_KEY).toBe(GUARDED_HOSTED_INFERENCE_SECRET); expect(runStep?.env?.GITHUB_TOKEN).toBeUndefined(); expect(runStep?.env?.NEMOCLAW_RUN_E2E_SCENARIOS).toBe("1"); @@ -905,6 +906,7 @@ describe("E2E reusable workflow contract", () => { expect(exportStep?.run).toContain("NEMOCLAW_ENDPOINT_URL=https://inference-api.nvidia.com/v1"); expect(exportStep?.run).toContain("NEMOCLAW_MODEL=nvidia/nvidia/nemotron-3-super-v3"); expect(exportStep?.run).toContain("NEMOCLAW_COMPAT_MODEL=nvidia/nvidia/nemotron-3-super-v3"); + expect(exportStep?.run).toContain("NEMOCLAW_PREFERRED_API=openai-completions"); expect(exportStep?.run).toContain("COMPATIBLE_API_KEY=%s"); expect(hostedJobs.length).toBeGreaterThan(20); @@ -962,6 +964,7 @@ describe("E2E reusable workflow contract", () => { expect(step.env?.NEMOCLAW_ENDPOINT_URL, jobName).toBe("https://inference-api.nvidia.com/v1"); expect(step.env?.NEMOCLAW_MODEL, jobName).toBe("nvidia/nvidia/nemotron-3-super-v3"); expect(step.env?.NEMOCLAW_COMPAT_MODEL, jobName).toBe("nvidia/nvidia/nemotron-3-super-v3"); + expect(step.env?.NEMOCLAW_PREFERRED_API, jobName).toBe("openai-completions"); expect(step.env?.COMPATIBLE_API_KEY, jobName).toBe(GUARDED_HOSTED_INFERENCE_SECRET); } diff --git a/test/e2e/lib/ci-compatible-inference.sh b/test/e2e/lib/ci-compatible-inference.sh index 94c3cb1867..01b677d26c 100755 --- a/test/e2e/lib/ci-compatible-inference.sh +++ b/test/e2e/lib/ci-compatible-inference.sh @@ -37,6 +37,7 @@ nemoclaw_e2e_configure_compatible_inference() { export NEMOCLAW_ENDPOINT_URL="${NEMOCLAW_ENDPOINT_URL:-https://inference-api.nvidia.com/v1}" export NEMOCLAW_MODEL="${NEMOCLAW_MODEL:-${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-$NEMOCLAW_E2E_COMPATIBLE_INFERENCE_MODEL_DEFAULT}}" export NEMOCLAW_COMPAT_MODEL="${NEMOCLAW_COMPAT_MODEL:-$NEMOCLAW_MODEL}" + export NEMOCLAW_PREFERRED_API="${NEMOCLAW_PREFERRED_API:-openai-completions}" export COMPATIBLE_API_KEY="$NVIDIA_INFERENCE_API_KEY" } diff --git a/tools/e2e-advisor/dispatch.mts b/tools/e2e-advisor/dispatch.mts index b5637d5de8..44718097fd 100644 --- a/tools/e2e-advisor/dispatch.mts +++ b/tools/e2e-advisor/dispatch.mts @@ -60,6 +60,7 @@ type DispatchPlan = { jobs?: string[]; ignoredJobs?: string[]; recommendedJobs?: string[]; + targetRefSecretBlockedJobs?: string[]; dispatchableJobCount?: number; prNumber?: number; targetRef?: string; @@ -260,16 +261,36 @@ export function planAutoDispatch({ }; } - const dispatchableJobs = extractDispatchableJobs(workflowText); + const targetRef = pr.head?.sha || pr.head?.ref || ""; + const dispatchableJobInfos = extractDispatchableJobInfos(workflowText); + const dispatchableJobs = dispatchableJobInfos.map((job) => job.id); const recommendedJobs = collectRecommendedJobs(result, targetWorkflow); - const jobs = unique(recommendedJobs.filter((job) => dispatchableJobs.includes(job))); - const ignoredJobs = unique(recommendedJobs.filter((job) => !dispatchableJobs.includes(job))); + const dispatchableRecommendedJobs = unique( + recommendedJobs.filter((job) => dispatchableJobs.includes(job)), + ); + const targetRefSecretBlockedJobs = + targetRef === "" + ? [] + : dispatchableRecommendedJobs.filter((job) => + dispatchableJobInfos.some((info) => info.id === job && info.targetRefSecretBlocked), + ); + const jobs = unique( + dispatchableRecommendedJobs.filter((job) => !targetRefSecretBlockedJobs.includes(job)), + ); + const ignoredJobs = unique([ + ...recommendedJobs.filter((job) => !dispatchableJobs.includes(job)), + ...targetRefSecretBlockedJobs, + ]); if (jobs.length === 0) { + const onlyBlockedByTargetRefSecrets = + dispatchableRecommendedJobs.length > 0 && + dispatchableRecommendedJobs.every((job) => targetRefSecretBlockedJobs.includes(job)); return { ...base, - reason: - "no required advisor recommendations matched dispatchable jobs in the target workflow", + reason: onlyBlockedByTargetRefSecrets + ? "no required advisor recommendations can run with target_ref because hosted inference secrets are withheld" + : "no required advisor recommendations matched dispatchable jobs in the target workflow", prNumber: pr.number, authorAssociation, authorLogin, @@ -277,10 +298,10 @@ export function planAutoDispatch({ dispatchableJobCount: dispatchableJobs.length, recommendedJobs, ignoredJobs, + targetRefSecretBlockedJobs, }; } - const targetRef = pr.head?.sha || pr.head?.ref || ""; const dispatchRef = env.E2E_ADVISOR_AUTO_DISPATCH_REF || pr.base?.ref || DEFAULT_DISPATCH_REF; const advisorDispatchId = buildAdvisorDispatchId(pr.number, env); const inputs = { @@ -299,6 +320,7 @@ export function planAutoDispatch({ inputs, jobs, ignoredJobs, + targetRefSecretBlockedJobs, dispatchableJobCount: dispatchableJobs.length, prNumber: pr.number, targetRef, @@ -311,11 +333,20 @@ export function planAutoDispatch({ } export function extractDispatchableJobs(workflowText: string): string[] { + return extractDispatchableJobInfos(workflowText).map((job) => job.id); +} + +type DispatchableJobInfo = { + id: string; + targetRefSecretBlocked: boolean; +}; + +function extractDispatchableJobInfos(workflowText: string): DispatchableJobInfo[] { const jobsBlockStart = workflowText.search(/^jobs:\s*$/m); if (jobsBlockStart === -1) return []; const lines = workflowText.slice(jobsBlockStart).split(/\r?\n/); - const jobs: string[] = []; + const jobs: DispatchableJobInfo[] = []; for (let index = 0; index < lines.length; index += 1) { const match = lines[index].match(/^ ([A-Za-z0-9_-]+):\s*$/); if (!match) continue; @@ -328,10 +359,26 @@ export function extractDispatchableJobs(workflowText: string): string[] { } const body = bodyLines.join("\n"); if (body.includes("inputs.jobs") && body.includes(`,${job},`)) { - jobs.push(job); + jobs.push({ + id: job, + targetRefSecretBlocked: isTargetRefSecretBlockedJob(body), + }); } } - return jobs.sort(); + return jobs.sort((left, right) => left.id.localeCompare(right.id)); +} + +function isTargetRefSecretBlockedJob(body: string): boolean { + const callsScriptRunnerWithHostedInference = + body.includes("uses: ./.github/workflows/e2e-script.yaml") && + /^\s+nvidia_api_key:\s*true\s*$/m.test(body); + if (callsScriptRunnerWithHostedInference) return true; + + const hasTargetRefGuardedInferenceSecret = + body.includes("inputs.target_ref == ''") && body.includes("secrets.NVIDIA_INFERENCE_API_KEY"); + const usesHostedInferenceSecret = + body.includes("NEMOCLAW_E2E_USE_HOSTED_INFERENCE") || body.includes("COMPATIBLE_API_KEY"); + return hasTargetRefGuardedInferenceSecret && usesHostedInferenceSecret; } export function collectRecommendedJobs( @@ -535,6 +582,14 @@ function renderDispatchSummary(result: DispatchPlan): string { `Ignored recommendations: ${result.ignoredJobs.map((job) => `\`${job}\``).join(", ")}`, ); } + if ( + Array.isArray(result.targetRefSecretBlockedJobs) && + result.targetRefSecretBlockedJobs.length > 0 + ) { + lines.push( + `Target-ref secret blocked jobs: ${result.targetRefSecretBlockedJobs.map((job) => `\`${job}\``).join(", ")}`, + ); + } lines.push(""); return `${lines.join("\n")}\n`; }