Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions architecture/compute-runtimes.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ Runtime-specific implementation notes belong in the driver crate README:
- `crates/openshell-driver-kubernetes/README.md`
- `crates/openshell-driver-vm/README.md`

The combined VM topology runs `openshell-sandbox` as guest PID 1. libkrun
executes the driver-owned guest bootstrap as PID 1, and the bootstrap preserves
that identity when it execs the supervisor after mounting and network setup.

## Supervisor Delivery

The supervisor must be available inside each sandbox workload:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@

set -euo pipefail

# libkrun consumes this driver-owned switch before exec'ing this script as
# PID 1. Do not leak the runtime control into the supervisor or workloads.
unset KRUN_INIT_PID1

BOOT_START=$(date +%s%3N 2>/dev/null || date +%s)
# gvisor-tap-vsock subnet layout:
# 192.168.127.1 — gateway: gvproxy's DNS / DHCP / HTTP API. Does NOT
Expand Down
69 changes: 60 additions & 9 deletions crates/openshell-driver-vm/src/runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use std::time::{Duration, Instant};
use crate::{embedded_runtime, ffi, nft_ruleset, procguard, rootfs};

pub const VM_RUNTIME_DIR_ENV: &str = "OPENSHELL_VM_RUNTIME_DIR";
const KRUN_INIT_PID1_ENV: &str = "KRUN_INIT_PID1=1";

/// PID of the VM worker process (libkrun fork or QEMU). Zero when not running.
/// Used by the SIGTERM/SIGINT handler to forward signals to the VM.
Expand Down Expand Up @@ -833,15 +834,7 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {

vm.set_console_output(&config.console_output)?;

let env = if config.env.is_empty() {
vec![
"HOME=/root".to_string(),
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(),
"TERM=xterm".to_string(),
]
} else {
config.env.clone()
};
let env = libkrun_guest_env(config);
vm.set_exec(&config.exec_path, &config.args, &env)?;

let pid = unsafe { libc::fork() };
Expand Down Expand Up @@ -889,6 +882,26 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {
}
}

fn libkrun_guest_env(config: &VmLaunchConfig) -> Vec<String> {
let mut env = if config.env.is_empty() {
vec![
"HOME=/root".to_string(),
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(),
"TERM=xterm".to_string(),
]
} else {
config.env.clone()
};

// libkrun normally keeps /init.krun as PID 1 and forks the configured
// executable. OpenShell's guest init is itself an init process and ends
// by exec'ing the supervisor, so ask libkrun to exec it directly. Keep
// this driver-owned setting authoritative over sandbox image/user env.
env.retain(|value| !value.starts_with("KRUN_INIT_PID1="));
env.push(KRUN_INIT_PID1_ENV.to_string());
env
}

pub fn validate_runtime_dir(dir: &Path) -> Result<(), String> {
if !dir.is_dir() {
return Err(format!(
Expand Down Expand Up @@ -1438,6 +1451,44 @@ mod tests {
assert!(env.contains(&"GPU_ENABLED=true".to_string()));
}

#[test]
fn libkrun_guest_env_runs_guest_init_as_pid_one() {
let env = libkrun_guest_env(&qemu_config());

assert!(env.contains(&"OPENSHELL_ENDPOINT=http://10.0.128.1:8080".to_string()));
assert!(env.contains(&KRUN_INIT_PID1_ENV.to_string()));
}

#[test]
fn libkrun_guest_env_overrides_caller_pid_one_setting() {
let mut config = qemu_config();
config.env.extend([
"KRUN_INIT_PID1=0".to_string(),
"KRUN_INIT_PID1=unexpected".to_string(),
]);

let env = libkrun_guest_env(&config);
let pid_one_settings = env
.iter()
.filter(|value| value.starts_with("KRUN_INIT_PID1="))
.collect::<Vec<_>>();

assert_eq!(pid_one_settings.len(), 1);
assert_eq!(pid_one_settings[0], KRUN_INIT_PID1_ENV);
}

#[test]
fn libkrun_guest_env_keeps_defaults_when_no_env_is_configured() {
let mut config = qemu_config();
config.env.clear();

let env = libkrun_guest_env(&config);

assert!(env.contains(&"HOME=/root".to_string()));
assert!(env.contains(&"TERM=xterm".to_string()));
assert!(env.contains(&KRUN_INIT_PID1_ENV.to_string()));
}

#[test]
fn kernel_cmdline_keeps_guest_init_metadata_out_of_proc_cmdline() {
let cmdline = build_kernel_cmdline(&qemu_config());
Expand Down
6 changes: 3 additions & 3 deletions e2e/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ publish = false
[features]
e2e = []
# Selects tests that rely on `host.openshell.internal` (the sandbox's stable
# alias to the host running test fixtures). docker and podman wire the alias
# unconditionally; the kube driver only does so when the chart's
# alias to the host running test fixtures). docker, podman, and vm wire the
# alias unconditionally; the kube driver only does so when the chart's
# `server.hostGatewayIP` is set, so `e2e-kubernetes` does NOT imply this and
# the helm wrapper opts in explicitly when it has resolved an IP.
e2e-host-gateway = ["e2e"]
Expand All @@ -30,7 +30,7 @@ e2e-docker-gpu = ["e2e-docker", "e2e-gpu"]
e2e-kubernetes = ["e2e"]
e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"]
e2e-podman-gpu = ["e2e-podman", "e2e-gpu"]
e2e-vm = ["e2e"]
e2e-vm = ["e2e", "e2e-host-gateway"]

[[test]]
name = "custom_image"
Expand Down
30 changes: 21 additions & 9 deletions e2e/rust/e2e-vm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
# 4. Writes a per-run gateway config with `[openshell.drivers.vm]`
# settings, starts the gateway with `--config <run-state>/gateway.toml`
# on a random free port, waits for `Server listening`, then runs the
# selected Rust e2e test (`smoke` by default).
# selected Rust e2e tests.
# 5. Tears the gateway down and (on failure) preserves the gateway
# log and every VM serial console log for post-mortem.
#
Expand All @@ -47,7 +47,7 @@ source "${ROOT}/e2e/support/gateway-common.sh"
COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed"
GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway"
DRIVER_BIN="${ROOT}/target/debug/openshell-driver-vm"
E2E_TEST="${OPENSHELL_E2E_VM_TEST:-smoke}"
E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}"
E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}"

# The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`.
Expand Down Expand Up @@ -296,11 +296,23 @@ e2e_export_gateway_restart_metadata \
# preparation; allow 180s for slower CI runners.
export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}"

echo "==> Running e2e ${E2E_TEST} test (features: ${E2E_FEATURES}, endpoint: ${OPENSHELL_GATEWAY_ENDPOINT})"
cargo test \
--manifest-path "${ROOT}/e2e/rust/Cargo.toml" \
--features "${E2E_FEATURES}" \
--test "${E2E_TEST}" \
-- --nocapture
run_e2e_test() {
local test_target="$1"
shift

echo "==> Running e2e ${test_target} test (features: ${E2E_FEATURES}, endpoint: ${OPENSHELL_GATEWAY_ENDPOINT})"
cargo test \
--manifest-path "${ROOT}/e2e/rust/Cargo.toml" \
--features "${E2E_FEATURES}" \
--test "${test_target}" \
"$@" \
-- --nocapture
echo "==> ${test_target} test passed."
}

echo "==> ${E2E_TEST} test passed."
if [ -n "${E2E_TEST_OVERRIDE}" ]; then
run_e2e_test "${E2E_TEST_OVERRIDE}"
else
run_e2e_test smoke
run_e2e_test host_gateway_alias
fi
2 changes: 1 addition & 1 deletion tasks/test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" }
run = "e2e/rust/e2e-kubernetes.sh"

["e2e:vm"]
description = "Start openshell-gateway with the VM compute driver and run the cluster-agnostic smoke e2e"
description = "Start openshell-gateway with the VM compute driver and run VM e2e tests"
run = "e2e/rust/e2e-vm.sh"

["e2e:docker"]
Expand Down
Loading