Skip to content

Commit ef9cade

Browse files
committed
fix(driver-vm): run sandbox supervisor as guest pid 1
Signed-off-by: Drew Newberry <anewberry@nvidia.com>
1 parent 709aa0f commit ef9cade

6 files changed

Lines changed: 93 additions & 22 deletions

File tree

architecture/compute-runtimes.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ Runtime-specific implementation notes belong in the driver crate README:
7373
- `crates/openshell-driver-kubernetes/README.md`
7474
- `crates/openshell-driver-vm/README.md`
7575

76+
The combined VM topology runs `openshell-sandbox` as guest PID 1. libkrun
77+
executes the driver-owned guest bootstrap as PID 1, and the bootstrap preserves
78+
that identity when it execs the supervisor after mounting and network setup.
79+
7680
## Supervisor Delivery
7781

7882
The supervisor must be available inside each sandbox workload:

crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@
99

1010
set -euo pipefail
1111

12+
# libkrun consumes this driver-owned switch before exec'ing this script as
13+
# PID 1. Do not leak the runtime control into the supervisor or workloads.
14+
unset KRUN_INIT_PID1
15+
1216
BOOT_START=$(date +%s%3N 2>/dev/null || date +%s)
1317
# gvisor-tap-vsock subnet layout:
1418
# 192.168.127.1 — gateway: gvproxy's DNS / DHCP / HTTP API. Does NOT

crates/openshell-driver-vm/src/runtime.rs

Lines changed: 60 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use std::time::{Duration, Instant};
1313
use crate::{embedded_runtime, ffi, nft_ruleset, procguard, rootfs};
1414

1515
pub const VM_RUNTIME_DIR_ENV: &str = "OPENSHELL_VM_RUNTIME_DIR";
16+
const KRUN_INIT_PID1_ENV: &str = "KRUN_INIT_PID1=1";
1617

1718
/// PID of the VM worker process (libkrun fork or QEMU). Zero when not running.
1819
/// Used by the SIGTERM/SIGINT handler to forward signals to the VM.
@@ -833,15 +834,7 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {
833834

834835
vm.set_console_output(&config.console_output)?;
835836

836-
let env = if config.env.is_empty() {
837-
vec![
838-
"HOME=/root".to_string(),
839-
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(),
840-
"TERM=xterm".to_string(),
841-
]
842-
} else {
843-
config.env.clone()
844-
};
837+
let env = libkrun_guest_env(config);
845838
vm.set_exec(&config.exec_path, &config.args, &env)?;
846839

847840
let pid = unsafe { libc::fork() };
@@ -889,6 +882,26 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {
889882
}
890883
}
891884

885+
fn libkrun_guest_env(config: &VmLaunchConfig) -> Vec<String> {
886+
let mut env = if config.env.is_empty() {
887+
vec![
888+
"HOME=/root".to_string(),
889+
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(),
890+
"TERM=xterm".to_string(),
891+
]
892+
} else {
893+
config.env.clone()
894+
};
895+
896+
// libkrun normally keeps /init.krun as PID 1 and forks the configured
897+
// executable. OpenShell's guest init is itself an init process and ends
898+
// by exec'ing the supervisor, so ask libkrun to exec it directly. Keep
899+
// this driver-owned setting authoritative over sandbox image/user env.
900+
env.retain(|value| !value.starts_with("KRUN_INIT_PID1="));
901+
env.push(KRUN_INIT_PID1_ENV.to_string());
902+
env
903+
}
904+
892905
pub fn validate_runtime_dir(dir: &Path) -> Result<(), String> {
893906
if !dir.is_dir() {
894907
return Err(format!(
@@ -1438,6 +1451,44 @@ mod tests {
14381451
assert!(env.contains(&"GPU_ENABLED=true".to_string()));
14391452
}
14401453

1454+
#[test]
1455+
fn libkrun_guest_env_runs_guest_init_as_pid_one() {
1456+
let env = libkrun_guest_env(&qemu_config());
1457+
1458+
assert!(env.contains(&"OPENSHELL_ENDPOINT=http://10.0.128.1:8080".to_string()));
1459+
assert!(env.contains(&KRUN_INIT_PID1_ENV.to_string()));
1460+
}
1461+
1462+
#[test]
1463+
fn libkrun_guest_env_overrides_caller_pid_one_setting() {
1464+
let mut config = qemu_config();
1465+
config.env.extend([
1466+
"KRUN_INIT_PID1=0".to_string(),
1467+
"KRUN_INIT_PID1=unexpected".to_string(),
1468+
]);
1469+
1470+
let env = libkrun_guest_env(&config);
1471+
let pid_one_settings = env
1472+
.iter()
1473+
.filter(|value| value.starts_with("KRUN_INIT_PID1="))
1474+
.collect::<Vec<_>>();
1475+
1476+
assert_eq!(pid_one_settings.len(), 1);
1477+
assert_eq!(pid_one_settings[0], KRUN_INIT_PID1_ENV);
1478+
}
1479+
1480+
#[test]
1481+
fn libkrun_guest_env_keeps_defaults_when_no_env_is_configured() {
1482+
let mut config = qemu_config();
1483+
config.env.clear();
1484+
1485+
let env = libkrun_guest_env(&config);
1486+
1487+
assert!(env.contains(&"HOME=/root".to_string()));
1488+
assert!(env.contains(&"TERM=xterm".to_string()));
1489+
assert!(env.contains(&KRUN_INIT_PID1_ENV.to_string()));
1490+
}
1491+
14411492
#[test]
14421493
fn kernel_cmdline_keeps_guest_init_metadata_out_of_proc_cmdline() {
14431494
let cmdline = build_kernel_cmdline(&qemu_config());

e2e/rust/Cargo.toml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ publish = false
1818
[features]
1919
e2e = []
2020
# Selects tests that rely on `host.openshell.internal` (the sandbox's stable
21-
# alias to the host running test fixtures). docker and podman wire the alias
22-
# unconditionally; the kube driver only does so when the chart's
21+
# alias to the host running test fixtures). docker, podman, and vm wire the
22+
# alias unconditionally; the kube driver only does so when the chart's
2323
# `server.hostGatewayIP` is set, so `e2e-kubernetes` does NOT imply this and
2424
# the helm wrapper opts in explicitly when it has resolved an IP.
2525
e2e-host-gateway = ["e2e"]
@@ -30,7 +30,7 @@ e2e-docker-gpu = ["e2e-docker", "e2e-gpu"]
3030
e2e-kubernetes = ["e2e"]
3131
e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"]
3232
e2e-podman-gpu = ["e2e-podman", "e2e-gpu"]
33-
e2e-vm = ["e2e"]
33+
e2e-vm = ["e2e", "e2e-host-gateway"]
3434

3535
[[test]]
3636
name = "custom_image"

e2e/rust/e2e-vm.sh

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
# 4. Writes a per-run gateway config with `[openshell.drivers.vm]`
3232
# settings, starts the gateway with `--config <run-state>/gateway.toml`
3333
# on a random free port, waits for `Server listening`, then runs the
34-
# selected Rust e2e test (`smoke` by default).
34+
# selected Rust e2e tests.
3535
# 5. Tears the gateway down and (on failure) preserves the gateway
3636
# log and every VM serial console log for post-mortem.
3737
#
@@ -47,7 +47,7 @@ source "${ROOT}/e2e/support/gateway-common.sh"
4747
COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed"
4848
GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway"
4949
DRIVER_BIN="${ROOT}/target/debug/openshell-driver-vm"
50-
E2E_TEST="${OPENSHELL_E2E_VM_TEST:-smoke}"
50+
E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}"
5151
E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}"
5252

5353
# The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`.
@@ -296,11 +296,23 @@ e2e_export_gateway_restart_metadata \
296296
# preparation; allow 180s for slower CI runners.
297297
export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}"
298298

299-
echo "==> Running e2e ${E2E_TEST} test (features: ${E2E_FEATURES}, endpoint: ${OPENSHELL_GATEWAY_ENDPOINT})"
300-
cargo test \
301-
--manifest-path "${ROOT}/e2e/rust/Cargo.toml" \
302-
--features "${E2E_FEATURES}" \
303-
--test "${E2E_TEST}" \
304-
-- --nocapture
299+
run_e2e_test() {
300+
local test_target="$1"
301+
shift
302+
303+
echo "==> Running e2e ${test_target} test (features: ${E2E_FEATURES}, endpoint: ${OPENSHELL_GATEWAY_ENDPOINT})"
304+
cargo test \
305+
--manifest-path "${ROOT}/e2e/rust/Cargo.toml" \
306+
--features "${E2E_FEATURES}" \
307+
--test "${test_target}" \
308+
"$@" \
309+
-- --nocapture
310+
echo "==> ${test_target} test passed."
311+
}
305312

306-
echo "==> ${E2E_TEST} test passed."
313+
if [ -n "${E2E_TEST_OVERRIDE}" ]; then
314+
run_e2e_test "${E2E_TEST_OVERRIDE}"
315+
else
316+
run_e2e_test smoke
317+
run_e2e_test host_gateway_alias
318+
fi

tasks/test.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" }
120120
run = "e2e/rust/e2e-kubernetes.sh"
121121

122122
["e2e:vm"]
123-
description = "Start openshell-gateway with the VM compute driver and run the cluster-agnostic smoke e2e"
123+
description = "Start openshell-gateway with the VM compute driver and run VM e2e tests"
124124
run = "e2e/rust/e2e-vm.sh"
125125

126126
["e2e:docker"]

0 commit comments

Comments
 (0)