Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 84 additions & 10 deletions .github/workflows/pr-review-advisor.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -87,21 +87,30 @@ jobs:
TYPEBOX_VERSION: "1.1.38"
# Workflow-boundary modules parse YAML before the advisor session starts.
YAML_VERSION: "2.8.3"
# Embedded Pi SDK sessions use Pi's proxy-aware Undici transport.
UNDICI_VERSION: "8.5.0"
# Credential-free inventory discovery executes the trusted Vitest entrypoint.
VITEST_VERSION: "4.1.9"
FD_FIND_VERSION: "9.0.0-1"
RIPGREP_VERSION: "14.1.0-1"
OPENSHELL_GATEWAY_ENDPOINT: http://127.0.0.1:8080
PI_IMAGE: ghcr.io/nvidia/openshell-community/sandboxes/pi@sha256:00d0c5e9e733f94f6db3eaa2ab70d4fd75bcc4aace6b13a54535cbf2dd20dfcd
PR_REVIEW_ADVISOR_TIMEOUT_MS: "900000"
PR_REVIEW_ADVISOR_HEARTBEAT_MS: "60000"
PR_REVIEW_ADVISOR_SANDBOX_TIMEOUT_SECONDS: "2100"
PR_REVIEW_ADVISOR_MODEL: ${{ matrix.advisor.model }}
PR_REVIEW_ADVISOR_ARTIFACT_DIR: ${{ matrix.advisor.artifact_dir }}
PR_REVIEW_ADVISOR_RUN_ANALYSIS: ${{ github.event_name == 'workflow_dispatch' && inputs.run_analysis == false && '0' || '1' }}
PR_REVIEW_ADVISOR_COMMENT_MARKER: "<!-- nemoclaw-pr-review-advisor -->"
PR_REVIEW_ADVISOR_COMMENT_TITLE: PR Review Advisor
PR_REVIEW_ADVISOR_COMMENT_LABEL: PR review advisor
PR_REVIEW_ADVISOR_WORKFLOW_NAME: "PR Review / Advisor"
PR_REVIEW_ADVISOR_LOAD_PREVIOUS_REVIEW: "false"
SANDBOX_NAME: pr-advisor-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.advisor.id }}
# Only executable code from this checkout may run in the analysis job.
ADVISOR_DIR: ${{ github.workspace }}/advisor
TARGET_REPO: ${{ github.event_name == 'pull_request_target' && github.repository || inputs.target_repo || github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number || inputs.target_pr }}
steps:
- name: Checkout trusted advisor code (workflow revision)
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand Down Expand Up @@ -153,13 +162,14 @@ jobs:
PR_BASE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || '' }}
EXPECTED_HEAD_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}
GIT_LFS_SKIP_SMUDGE: "1"
TARGET_DIR: ${{ github.workspace }}/pr-workdir
run: |
node --experimental-strip-types \
"$ADVISOR_DIR/tools/pr-review-advisor/prepare-target-pr.mts"

# The advisor reads repository files while holding its model key. Remove
# worktree symlinks first so an untrusted link cannot redirect a read to
# runner state. Git still retains the link target in committed objects.
# The advisor reads repository files inside OpenShell. Remove worktree
# symlinks first so an untrusted link cannot redirect a read outside the
# uploaded repository. Git still retains the committed link target.
- name: Remove symlinks from analysis workspace
shell: bash
run: |
Expand Down Expand Up @@ -209,20 +219,65 @@ jobs:
npm ci --ignore-scripts --no-audit --no-fund
)

# Materialize GitHub metadata before sandbox creation. This is the only
# analysis-phase step with a GitHub token, and it has no model credential.
- name: Prepare advisor sandbox inputs
env:
GH_TOKEN: ${{ github.token }}
run: node --experimental-strip-types --no-warnings "$ADVISOR_DIR/tools/pr-review-advisor/openshell.mts" prepare

- name: Install OpenShell
if: ${{ env.PR_REVIEW_ADVISOR_RUN_ANALYSIS == '1' }}
run: |
env -u GITHUB_TOKEN -u GH_TOKEN -u PR_REVIEW_ADVISOR_API_KEY \
NEMOCLAW_NON_INTERACTIVE=1 \
bash "$ADVISOR_DIR/scripts/install-openshell.sh"

# The upstream model key exists only while the trusted host registers the
# provider. The sandbox uses inference.local and receives an inert key.
- name: Configure OpenShell inference
id: configure-openshell
if: ${{ env.PR_REVIEW_ADVISOR_RUN_ANALYSIS == '1' }}
continue-on-error: true
env:
OPENAI_API_KEY: ${{ secrets.PR_REVIEW_ADVISOR_API_KEY }}
run: node --experimental-strip-types --no-warnings "$ADVISOR_DIR/tools/pr-review-advisor/openshell.mts" configure

# Preserve a review artifact when the provider credential is unavailable
# or gateway configuration fails. This trusted host fallback has neither
# a GitHub token nor a model credential and never executes PR content.
- name: Write unavailable advisor artifacts
id: unavailable-analysis
if: ${{ always() && steps.configure-openshell.outcome != 'success' }}
env:
BASE_REF: ${{ github.event_name == 'pull_request_target' && 'target/base' || (github.event_name == 'workflow_dispatch' && inputs.target_repo != '' && inputs.target_pr != '' && 'target/base' || inputs.base_ref) }}
HEAD_REF: ${{ github.event_name == 'pull_request_target' && 'HEAD' || (github.event_name == 'workflow_dispatch' && inputs.target_repo != '' && inputs.target_pr != '' && 'HEAD' || inputs.head_ref) }}
PR_REVIEW_ADVISOR_UNAVAILABLE_REASON: ${{ env.PR_REVIEW_ADVISOR_RUN_ANALYSIS == '0' && 'PR_REVIEW_ADVISOR_RUN_ANALYSIS=0' || 'OpenShell inference configuration failed or the advisor credential is unavailable' }}
run: node --experimental-strip-types --no-warnings "$ADVISOR_DIR/tools/pr-review-advisor/openshell.mts" unavailable

- name: Create credential-free advisor sandbox
if: ${{ steps.configure-openshell.outcome == 'success' }}
run: node --experimental-strip-types --no-warnings "$ADVISOR_DIR/tools/pr-review-advisor/openshell.mts" create

- name: Run PR review advisor
id: analysis
if: ${{ steps.configure-openshell.outcome == 'success' }}
continue-on-error: true
env:
BASE_REF: ${{ github.event_name == 'pull_request_target' && 'target/base' || (github.event_name == 'workflow_dispatch' && inputs.target_repo != '' && inputs.target_pr != '' && 'target/base' || inputs.base_ref) }}
HEAD_REF: ${{ github.event_name == 'pull_request_target' && 'HEAD' || (github.event_name == 'workflow_dispatch' && inputs.target_repo != '' && inputs.target_pr != '' && 'HEAD' || inputs.head_ref) }}
PR_NUMBER: ${{ github.event.pull_request.number || inputs.target_pr }}
PR_REVIEW_ADVISOR_RUN_ANALYSIS: ${{ github.event_name == 'workflow_dispatch' && inputs.run_analysis == false && '0' || '1' }}
GH_TOKEN: ${{ github.token }}
PR_REVIEW_ADVISOR_API_KEY: ${{ secrets.PR_REVIEW_ADVISOR_API_KEY }}
run: |
cd "$ADVISOR_WORKDIR"
node --experimental-strip-types \
"$ADVISOR_DIR/tools/pr-review-advisor/run-analysis.mts"
run: node --experimental-strip-types --no-warnings "$ADVISOR_DIR/tools/pr-review-advisor/openshell.mts" run

- id: download-analysis
name: Download advisor artifacts from sandbox
if: ${{ always() && steps.configure-openshell.outcome == 'success' }}
continue-on-error: true
run: node --experimental-strip-types --no-warnings "$ADVISOR_DIR/tools/pr-review-advisor/openshell.mts" download

- name: Delete advisor sandbox
if: always()
run: node --experimental-strip-types --no-warnings "$ADVISOR_DIR/tools/pr-review-advisor/openshell.mts" delete

- name: Publish job summary
if: always()
Expand All @@ -246,11 +301,30 @@ jobs:
if: always()
env:
ANALYSIS_OUTCOME: ${{ steps.analysis.outcome }}
ANALYSIS_REQUESTED: ${{ env.PR_REVIEW_ADVISOR_RUN_ANALYSIS }}
CONFIGURE_OUTCOME: ${{ steps.configure-openshell.outcome }}
DOWNLOAD_OUTCOME: ${{ steps.download-analysis.outcome }}
UNAVAILABLE_OUTCOME: ${{ steps.unavailable-analysis.outcome }}
run: |
if [ "$CONFIGURE_OUTCOME" != "success" ]; then
if [ "$UNAVAILABLE_OUTCOME" != "success" ]; then
echo "::error::PR review advisor could not write unavailable artifacts: outcome=$UNAVAILABLE_OUTCOME"
exit 1
fi
if [ "$ANALYSIS_REQUESTED" = "0" ]; then
exit 0
fi
echo "::error::PR review advisor inference configuration did not complete: outcome=$CONFIGURE_OUTCOME"
exit 1
fi
if [ "$ANALYSIS_OUTCOME" != "success" ]; then
echo "::error::PR review advisor analysis did not complete: outcome=$ANALYSIS_OUTCOME"
exit 1
fi
if [ "$DOWNLOAD_OUTCOME" != "success" ]; then
echo "::error::PR review advisor artifacts were not downloaded: outcome=$DOWNLOAD_OUTCOME"
exit 1
fi

publish:
name: Publish PR review advisor
Expand Down
25 changes: 25 additions & 0 deletions ci/source-shape-test-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,31 @@
"test": "rejects malformed configured matrix shard selectors",
"category": "security"
},
{
"file": "test/pr-review-advisor-openshell-workflow-boundary.test.ts",
"test": "binds host-prepared GitHub context to the selected repository and pull request",
"category": "compatibility"
},
{
"file": "test/pr-review-advisor-openshell-workflow-boundary.test.ts",
"test": "fails closed around the credential-free OpenShell filesystem and network boundary",
"category": "security"
},
{
"file": "test/pr-review-advisor-openshell-workflow-boundary.test.ts",
"test": "keeps GitHub and upstream model credentials in separate host-only steps",
"category": "security"
},
{
"file": "test/pr-review-advisor-openshell-workflow-boundary.test.ts",
"test": "pins trusted helper and bind sources to their checkout directories",
"category": "security"
},
{
"file": "test/pr-review-advisor-openshell-workflow-boundary.test.ts",
"test": "preserves credential-free unavailable artifacts for manual dry runs",
"category": "compatibility"
},
{
"file": "test/pr-review-advisor-workflow-boundary.test.ts",
"test": "rejects deleting or weakening analysis-workspace symlink removal",
Expand Down
11 changes: 11 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@
"tsx": "^4.21.0",
"typebox": "1.1.38",
"typescript": "6.0.3",
"undici": "8.5.0",
"vitest": "^4.1.9"
}
}
95 changes: 95 additions & 0 deletions test/advisor-http-dispatcher.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { type ChildProcess, spawn } from "node:child_process";
import { once } from "node:events";
import http from "node:http";
import type { AddressInfo } from "node:net";

import { afterEach, describe, expect, it } from "vitest";

const servers: http.Server[] = [];
const children: ChildProcess[] = [];

afterEach(async () => {
for (const child of children.splice(0)) {
child.kill("SIGKILL");
}
await Promise.all(
servers.splice(0).map(
(server) =>
new Promise<void>((resolve, reject) => {
server.closeAllConnections();
server.close((error) => (error ? reject(error) : resolve()));
}),
),
);
});

describe("advisor HTTP dispatcher", () => {
it("routes embedded SDK fetch through the environment proxy", async () => {
const connectTargets: string[] = [];
const proxy = http.createServer();
proxy.on("connect", (request, socket) => {
connectTargets.push(request.url ?? "");
socket.end("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n");
});
proxy.listen(0, "127.0.0.1");
await once(proxy, "listening");
servers.push(proxy);
const address = proxy.address() as AddressInfo;

const moduleUrl = new URL("../tools/advisors/http-dispatcher.mts", import.meta.url).href;
const script = `
import { getGlobalDispatcher } from "undici";
import { configureAdvisorHttpDispatcher } from ${JSON.stringify(moduleUrl)};
const originalFetch = globalThis.fetch;
configureAdvisorHttpDispatcher();
if (globalThis.fetch === originalFetch) {
throw new Error("advisor transport did not install npm Undici fetch");
}
let fetchRejected = false;
try {
await fetch("https://advisor-transport.invalid/v1");
} catch {
fetchRejected = true;
}
if (!fetchRejected) {
throw new Error("advisor transport swallowed a failed fetch");
}
getGlobalDispatcher().emit("error", new Error("unpaired-dispatcher-failure"));
`;
const child = spawn(
process.execPath,
["--experimental-strip-types", "--no-warnings", "--input-type=module", "--eval", script],
{
env: {
...process.env,
HTTP_PROXY: `http://127.0.0.1:${address.port}`,
HTTPS_PROXY: `http://127.0.0.1:${address.port}`,
ALL_PROXY: "",
NO_PROXY: "",
NODE_USE_ENV_PROXY: "",
all_proxy: "",
http_proxy: `http://127.0.0.1:${address.port}`,
https_proxy: `http://127.0.0.1:${address.port}`,
no_proxy: "",
},
stdio: ["ignore", "pipe", "pipe"],
},
);
children.push(child);
const stderr: Buffer[] = [];
child.stderr?.on("data", (chunk: Buffer) => stderr.push(chunk));
const timeout = setTimeout(() => child.kill("SIGKILL"), 3_000);
timeout.unref();
const [exitCode] = (await once(child, "exit")) as [number | null];
Comment thread
coderabbitai[bot] marked this conversation as resolved.
clearTimeout(timeout);

expect(exitCode, Buffer.concat(stderr).toString("utf8")).toBe(0);
expect(connectTargets).toContain("advisor-transport.invalid:443");
expect(Buffer.concat(stderr).toString("utf8")).toContain(
"Advisor HTTP dispatcher error: Error: unpaired-dispatcher-failure",
);
});
});
33 changes: 33 additions & 0 deletions test/advisor-session-runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,12 +196,22 @@ const sdk = vi.hoisted(() => {
};
});

const transport = vi.hoisted(() => ({
configure: vi.fn(),
}));

vi.mock("@earendil-works/pi-coding-agent", async (importOriginal) => ({
...(await importOriginal()),
createAgentSession: sdk.createAgentSession,
}));

vi.mock("../tools/advisors/http-dispatcher.mts", () => ({
configureAdvisorHttpDispatcher: transport.configure,
}));

import {
ADVISOR_OPENAI_COMPATIBLE_BASE_URL,
ADVISOR_OPENSHELL_INFERENCE_BASE_URL,
type AdvisorPromptTurn,
advisorRetrySettings,
READ_ONLY_TOOLS,
Expand Down Expand Up @@ -276,6 +286,8 @@ async function run(promptTurns: AdvisorPromptTurn[]) {

afterEach(() => {
delete process.env.TEST_ADVISOR_KEY;
vi.unstubAllEnvs();
vi.clearAllMocks();
sdk.reset();
for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true });
});
Expand All @@ -294,6 +306,27 @@ describe("advisor session runner", () => {
expect(advisorRetrySettings("nvidia/nvidia/nemotron-3-ultra").baseDelayMs).toBe(9_000);
});

it("configures Pi's proxy transport before an OpenShell SDK session", async () => {
vi.stubEnv("PR_REVIEW_ADVISOR_BASE_URL", ADVISOR_OPENSHELL_INFERENCE_BASE_URL);

const result = await run([analysisTurn("only-analysis")]);

expect(result.fatalError).toBeUndefined();
expect(transport.configure).toHaveBeenCalledOnce();
expect(transport.configure.mock.invocationCallOrder[0]).toBeLessThan(
sdk.createAgentSession.mock.invocationCallOrder[0] as number,
);
});

it("leaves the global transport unchanged for hosted advisor inference", async () => {
vi.stubEnv("PR_REVIEW_ADVISOR_BASE_URL", ADVISOR_OPENAI_COMPATIBLE_BASE_URL);

const result = await run([analysisTurn("only-analysis")]);

expect(result.fatalError).toBeUndefined();
expect(transport.configure).not.toHaveBeenCalled();
});

it("clears a transient provider error after the same-session retry succeeds", async () => {
sdk.state.retryResponses = ["success"];
const result = await run([analysisTurn("only-analysis")]);
Expand Down
11 changes: 11 additions & 0 deletions test/helpers/vitest-watch-triggers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,17 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [
pattern: /(?:^|\/)\.github\/workflows\/pr-merge-conflict-fixer\.yaml$/,
testsToRun: runTests("test/pr-merge-conflict-fixer-workflow-boundary.test.ts"),
},
{
pattern: /(?:^|\/)\.github\/workflows\/pr-review-advisor\.yaml$/,
testsToRun: runTests(
"test/pr-review-advisor-workflow-boundary.test.ts",
"test/pr-review-advisor-openshell-workflow-boundary.test.ts",
),
},
{
pattern: /(?:^|\/)tools\/pr-review-advisor\/openshell-policy\.yaml$/,
testsToRun: runTests("test/pr-review-advisor-openshell-workflow-boundary.test.ts"),
},
{
pattern:
/(?:^|\/)\.github\/workflows\/(?:hosted-runner-recovery|wsl-e2e|macos-e2e|platform-vitest-main)\.yaml$/,
Expand Down
1 change: 1 addition & 0 deletions test/pr-merge-conflict-fixer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,7 @@ describe("PR merge conflict fixer", () => {
expect(configuration).toContain('bind_address = "127.0.0.1:8080"');
expect(configuration).toContain("allow_unauthenticated_users = true");
expect(configuration).toContain('supervisor_bin = "/trusted/bin/openshell-sandbox"');
expect(configuration).not.toContain("enable_bind_mounts");
expect(configuration).not.toContain("provider-secret");
expect(fs.statSync(configurationPath).mode & 0o777).toBe(0o600);

Expand Down
Loading
Loading