From 411fc1f1657dceb9b76d54ece297347fc79f120c Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Mon, 6 Jul 2026 17:19:31 -0500 Subject: [PATCH] fix(podman): harden sandbox callbacks Pass gateway bind, TLS, and client certificate auth context into the Podman driver so rootless pasta callbacks use only bindability-checked listener addresses. Require TLS client certificate authentication before enabling the automatic non-loopback callback listener. Gateway JWT and OIDC alone do not protect the full listener surface exposed on host interfaces. Update callback listener tests and documentation to reflect the tightened protection boundary. Signed-off-by: Adam Miller --- Cargo.lock | 10 + Cargo.toml | 2 +- architecture/compute-runtimes.md | 2 +- crates/openshell-driver-podman/NETWORKING.md | 28 +- crates/openshell-driver-podman/README.md | 6 + crates/openshell-driver-podman/src/client.rs | 2 + crates/openshell-driver-podman/src/config.rs | 35 +- crates/openshell-driver-podman/src/driver.rs | 335 +++++++++++++++++- .../src/compute/driver_config.rs | 102 ++++++ crates/openshell-server/src/compute/mod.rs | 3 +- crates/openshell-server/src/config_file.rs | 6 +- crates/openshell-server/src/lib.rs | 78 +++- deploy/rpm/CONFIGURATION.md | 22 +- deploy/rpm/QUICKSTART.md | 10 +- deploy/rpm/TROUBLESHOOTING.md | 30 +- deploy/rpm/gateway.toml.default | 12 +- docs/reference/gateway-config.mdx | 10 + docs/reference/sandbox-compute-drivers.mdx | 22 +- e2e/with-podman-gateway.sh | 2 +- 19 files changed, 679 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13b670f558..62fc39b417 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3135,6 +3135,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.3" @@ -3282,6 +3291,7 @@ dependencies = [ "cfg-if", "cfg_aliases", "libc", + "memoffset", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f450cd5c8c..7b7e5a89b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ metrics = "0.24" metrics-exporter-prometheus = { version = "0.18", default-features = false, features = ["http-listener"] } # Unix/Process -nix = { version = "0.29", features = ["signal", "process", "user", "fs", "term"] } +nix = { version = "0.29", features = ["signal", "process", "user", "fs", "term", "net"] } rustix = { version = "1.1", features = ["process"] } # Serialization diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index f122fda5d7..c3aa29e712 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -31,7 +31,7 @@ when a sandbox create request asks for GPU resources. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| | Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. | -| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API, OCI image volumes, and CDI GPU devices when available. | +| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API, OCI image volumes, CDI GPU devices when available, and bindability-checked callback listeners for direct-host loopback gateway binds. Bridge-mode gateways can listen on the Podman bridge IP when bindable; rootless pasta can add a protected non-loopback local-interface listener when TLS client certificate authentication is enabled. Containerized gateways skip host listener discovery and require explicit port publishing. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a non-reserved custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. Reserved built-in names such as `vm`, `docker`, `podman`, and `kubernetes` cannot be used as unmanaged socket endpoints. The gateway connects to a UDS the operator already provisioned, runs `GetCapabilities`, logs the advertised `driver_name`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 8e202e18ab..4d22530490 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -255,6 +255,29 @@ if config.grpc_endpoint.is_empty() { The bridge gateway IP is not a stable substitute in rootless mode because it can live inside the user namespace rather than on the host. +### Rootless pasta callback listener + +When the gateway binds only to loopback, host-local clients can reach +`127.0.0.1:`, but sandbox containers do not share the host loopback +namespace. In rootless Podman with pasta, `host.containers.internal` reaches the +host through pasta's forwarding path and may not connect to a listener bound +only to `127.0.0.1`. + +For direct host gateways, the Podman driver detects rootless pasta from Podman +system info (`rootless=true` and `rootlessNetworkCmd=pasta`). If the gateway +bind address is loopback-only, TLS client certificate authentication is active, +and auto-discovery is enabled, the driver adds a second callback listener on a +bindable non-loopback local interface address. Candidate addresses come from +local interface enumeration via `getifaddrs`; the driver does not probe an +external IP to find this address. The supervisor endpoint remains +`host.containers.internal:`. + +Set `enable_auto_callback_listener = false` in `[openshell.drivers.podman]` or +`OPENSHELL_PODMAN_ENABLE_AUTO_CALLBACK_LISTENER=false` to disable the automatic +listener. Containerized gateways skip host listener discovery because their +network namespace is not the host namespace; publish the gateway container port +on an address sandbox containers can reach. + ### Layer 3 Inner Sandbox Network Namespace Inside the container, the supervisor creates another network namespace for the @@ -391,7 +414,10 @@ Gateway The gateway binds to `127.0.0.1:17670` by default in the RPM packaging. Client certificates are auto-generated by `openshell-gateway generate-certs` on first -start and bind-mounted into sandbox containers by the Podman driver. +start and bind-mounted into sandbox containers by the Podman driver. In +rootless pasta mode, the protected callback listener described above allows the +supervisor to keep using `host.containers.internal:` while the primary +listener remains loopback-only. ## Differences from the Kubernetes Driver diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 67b607db6a..70fb729289 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -216,6 +216,11 @@ Key points: host. Linux defaults to Podman's `host-gateway` resolver. macOS Podman machine defaults to gvproxy's host-loopback IP, `192.168.127.254`, because stale Podman machines may fail to resolve `host-gateway`. +- Callback listener: direct host gateways that bind to loopback can add a + protected non-loopback listener for rootless pasta callbacks when TLS client + certificate authentication is enabled. Disable this with + `enable_auto_callback_listener = false`. Gateway containers skip host + listener discovery and require explicit port publishing. - nsenter: the supervisor uses `nsenter --net=` instead of `ip netns exec` for namespace operations, avoiding the sysfs remount path that fails in rootless containers. @@ -338,6 +343,7 @@ Podman resources after out-of-band container removal or label drift. | `OPENSHELL_GATEWAY_PORT` | `--gateway-port` | `17670` | Gateway port used for endpoint auto-detection by the standalone binary. | | `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. | | `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. | +| `OPENSHELL_PODMAN_ENABLE_AUTO_CALLBACK_LISTENER` | none | `true` | Enable the protected non-loopback callback listener for rootless pasta when the gateway is loopback-bound and TLS client certificate authentication is enabled. | | `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. | | `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `10` | Container stop timeout in seconds. | | `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Set `0` to inherit Podman's runtime/default PID limit. | diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs index c93971b9f3..2b0706ad7e 100644 --- a/crates/openshell-driver-podman/src/client.rs +++ b/crates/openshell-driver-podman/src/client.rs @@ -245,6 +245,8 @@ pub struct HostInfo { #[serde(default)] pub network_backend: String, #[serde(default)] + pub rootless_network_cmd: String, + #[serde(default)] pub security: SecurityInfo, } diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 0e29f52dd2..973d85bf07 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use openshell_core::config::{DEFAULT_STOP_TIMEOUT_SECS, DEFAULT_SUPERVISOR_IMAGE}; -use std::net::IpAddr; +use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::str::FromStr; @@ -68,6 +68,7 @@ impl FromStr for ImagePullPolicy { #[derive(Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] +#[allow(clippy::struct_excessive_bools)] pub struct PodmanComputeConfig { /// Path to the Podman API Unix socket. /// Default: `$XDG_RUNTIME_DIR/podman/podman.sock` (Linux), @@ -89,6 +90,15 @@ pub struct PodmanComputeConfig { /// so the correct port is used even when `--port` differs from the /// default. Defaults to [`openshell_core::config::DEFAULT_SERVER_PORT`]. pub gateway_port: u16, + /// Full gateway bind address supplied by the gateway process at runtime. + #[serde(skip)] + pub gateway_bind_address: Option, + /// Whether the gateway has TLS enabled. Supplied by the gateway process. + #[serde(skip)] + pub gateway_tls_enabled: bool, + /// Whether gateway callbacks are protected by listener-level client certificate auth. + #[serde(skip)] + pub gateway_callback_client_cert_auth_enabled: bool, /// Unix socket path the in-container supervisor bridges relay traffic to. pub sandbox_ssh_socket_path: String, /// Name of the Podman bridge network. @@ -134,10 +144,19 @@ pub struct PodmanComputeConfig { /// Set to `0` to disable health checks entirely. /// Defaults to [`DEFAULT_HEALTH_CHECK_INTERVAL_SECS`] (10 seconds). pub health_check_interval_secs: u64, + /// Automatically add a non-loopback callback listener for rootless Podman + /// pasta when the primary gateway bind is loopback-only and TLS client + /// certificate authentication is active. + #[serde(default = "default_true")] + pub enable_auto_callback_listener: bool, } pub const DEFAULT_HEALTH_CHECK_INTERVAL_SECS: u64 = 10; +fn default_true() -> bool { + true +} + impl PodmanComputeConfig { /// Returns `true` when all three TLS paths are configured. #[must_use] @@ -251,6 +270,9 @@ impl Default for PodmanComputeConfig { image_pull_policy: ImagePullPolicy::default(), grpc_endpoint: String::new(), gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, + gateway_bind_address: None, + gateway_tls_enabled: false, + gateway_callback_client_cert_auth_enabled: false, sandbox_ssh_socket_path: "/run/openshell/ssh.sock".to_string(), network_name: DEFAULT_NETWORK_NAME.to_string(), host_gateway_ip: Self::default_host_gateway_ip(), @@ -262,6 +284,7 @@ impl Default for PodmanComputeConfig { sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, enable_bind_mounts: false, health_check_interval_secs: DEFAULT_HEALTH_CHECK_INTERVAL_SECS, + enable_auto_callback_listener: true, } } } @@ -274,6 +297,12 @@ impl std::fmt::Debug for PodmanComputeConfig { .field("image_pull_policy", &self.image_pull_policy.as_str()) .field("grpc_endpoint", &self.grpc_endpoint) .field("gateway_port", &self.gateway_port) + .field("gateway_bind_address", &self.gateway_bind_address) + .field("gateway_tls_enabled", &self.gateway_tls_enabled) + .field( + "gateway_callback_client_cert_auth_enabled", + &self.gateway_callback_client_cert_auth_enabled, + ) .field("sandbox_ssh_socket_path", &self.sandbox_ssh_socket_path) .field("network_name", &self.network_name) .field("host_gateway_ip", &self.host_gateway_ip) @@ -288,6 +317,10 @@ impl std::fmt::Debug for PodmanComputeConfig { "health_check_interval_secs", &self.health_check_interval_secs, ) + .field( + "enable_auto_callback_listener", + &self.enable_auto_callback_listener, + ) .finish() } } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 27d5102811..c5060fdc48 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -3,7 +3,7 @@ //! Podman compute driver. -use crate::client::{PodmanApiError, PodmanClient, VolumeInspect}; +use crate::client::{PodmanApiError, PodmanClient, SystemInfo, VolumeInspect}; use crate::config::PodmanComputeConfig; use crate::container::{self, LABEL_MANAGED_FILTER, LABEL_SANDBOX_ID, PodmanSandboxDriverConfig}; use crate::watcher::{ @@ -19,6 +19,7 @@ use openshell_core::gpu::{ use openshell_core::proto::compute::v1::{ DriverSandbox, GetCapabilitiesResponse, GpuResourceRequirements, }; +use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -42,6 +43,7 @@ pub struct PodmanComputeDriver { /// The host's IP on the bridge network. Sandbox containers use this to /// reach the gateway server when no explicit gRPC endpoint is configured. network_gateway_ip: Option, + uses_rootless_pasta: bool, gpu_selector: Arc, gpu_inventory_refresh: Arc (CdiGpuInventory, bool) + Send + Sync>, } @@ -226,6 +228,7 @@ impl PodmanComputeDriver { Ok(()) => break, Err(e) if attempts < MAX_PING_RETRIES => { attempts += 1; + let e = enrich_socket_connection_error(e); warn!( attempt = attempts, max_retries = MAX_PING_RETRIES, @@ -234,12 +237,12 @@ impl PodmanComputeDriver { ); tokio::time::sleep(PING_RETRY_DELAY).await; } - Err(e) => return Err(e), + Err(e) => return Err(enrich_socket_connection_error(e)), } } // Verify cgroups v2, detect rootless mode, and log system info. - match client.system_info().await { + let uses_rootless_pasta = match client.system_info().await { Ok(info) => { if info.host.cgroup_version != "v2" { return Err(PodmanApiError::Connection(format!( @@ -249,24 +252,30 @@ impl PodmanComputeDriver { info.host.cgroup_version ))); } + let uses_rootless_pasta = podman_uses_rootless_pasta(&info); info!( cgroup_version = %info.host.cgroup_version, network_backend = %info.host.network_backend, + rootless_network_cmd = %info.host.rootless_network_cmd, rootless = info.host.security.rootless, "Connected to Podman" ); + uses_rootless_pasta } Err(e) => { return Err(PodmanApiError::Connection(format!( "failed to query Podman system info: {e}" ))); } - } + }; // Rootless pre-flight: warn if subuid/subgid ranges look missing. // Not a hard error because some systems configure these via LDAP or // other mechanisms that /etc/subuid does not reflect. - if !cfg!(target_os = "macos") && rustix::process::getuid().as_raw() != 0 { + if !cfg!(target_os = "macos") + && rustix::process::getuid().as_raw() != 0 + && !running_inside_container() + { check_subuid_range(); } @@ -318,6 +327,7 @@ impl PodmanComputeDriver { client, config, network_gateway_ip, + uses_rootless_pasta, gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -335,6 +345,29 @@ impl PodmanComputeDriver { self.network_gateway_ip.as_deref() } + /// Extra host-side gateway listener address for direct host deployments. + /// + /// Direct host gateways that bind only to loopback can also listen on the + /// Podman bridge gateway IP so sandbox containers can reach + /// `host.containers.internal:`. Rootless pasta uses link-local + /// host-gateway forwarding instead, where the gateway should keep its + /// loopback listener and must not try to bind the link-local alias. + #[must_use] + pub fn gateway_bind_address(&self) -> Option { + gateway_bind_address( + self.network_gateway_ip.as_deref(), + self.config.gateway_bind_address, + self.config.gateway_port, + running_inside_container(), + self.uses_rootless_pasta, + self.config.enable_auto_callback_listener, + self.config.gateway_tls_enabled, + self.config.gateway_callback_client_cert_auth_enabled, + local_interface_bind_address, + socket_address_is_bindable, + ) + } + /// Report driver capabilities. pub fn capabilities(&self) -> Result { Ok(openshell_core::driver_utils::build_capabilities_response( @@ -798,6 +831,7 @@ impl PodmanComputeDriver { client, config, network_gateway_ip: None, + uses_rootless_pasta: false, gpu_selector: Arc::new(CdiGpuDefaultSelector::new( gpu_inventory, allow_all_default_gpu, @@ -809,6 +843,178 @@ impl PodmanComputeDriver { } } +/// Return true when the gateway process appears to be running inside a +/// container. Used only to tune host preflight checks whose local filesystem +/// or network namespace view is misleading from inside the gateway image. +fn running_inside_container() -> bool { + Path::new("/run/.containerenv").exists() || Path::new("/.dockerenv").exists() +} + +fn podman_uses_rootless_pasta(info: &SystemInfo) -> bool { + info.host.security.rootless && info.host.rootless_network_cmd == "pasta" +} + +#[allow(clippy::too_many_arguments, clippy::fn_params_excessive_bools)] +fn gateway_bind_address( + network_gateway_ip: Option<&str>, + gateway_bind_address: Option, + gateway_port: u16, + is_containerized_gateway: bool, + uses_rootless_pasta: bool, + enable_auto_callback_listener: bool, + gateway_tls_enabled: bool, + gateway_callback_client_cert_auth_enabled: bool, + callback_listener: impl Fn(u16) -> Option, + is_bindable: impl Fn(SocketAddr) -> bool, +) -> Option { + if is_containerized_gateway { + return None; + } + + if uses_rootless_pasta { + return pasta_gateway_bind_address( + gateway_bind_address, + gateway_port, + enable_auto_callback_listener, + gateway_tls_enabled, + gateway_callback_client_cert_auth_enabled, + callback_listener, + is_bindable, + ); + } + + let addr = bridge_gateway_bind_address_candidate(network_gateway_ip, gateway_port)?; + if is_bindable(addr) { + Some(addr) + } else { + warn!( + address = %addr, + "Podman bridge gateway address is not bindable from the gateway namespace; skipping extra gateway listener" + ); + None + } +} + +fn pasta_gateway_bind_address( + gateway_bind_address: Option, + gateway_port: u16, + enable_auto_callback_listener: bool, + gateway_tls_enabled: bool, + gateway_callback_client_cert_auth_enabled: bool, + callback_listener: impl Fn(u16) -> Option, + is_bindable: impl Fn(SocketAddr) -> bool, +) -> Option { + let gateway_bind_address = gateway_bind_address?; + if !gateway_bind_address.ip().is_loopback() { + return None; + } + if !enable_auto_callback_listener { + info!( + bind = %gateway_bind_address, + "Rootless Podman pasta detected with loopback gateway bind; automatic callback listener disabled by configuration" + ); + return None; + } + if !gateway_tls_enabled || !gateway_callback_client_cert_auth_enabled { + warn!( + bind = %gateway_bind_address, + tls_enabled = gateway_tls_enabled, + client_cert_auth_enabled = gateway_callback_client_cert_auth_enabled, + "Rootless Podman pasta detected with loopback gateway bind but TLS client certificate authentication is incomplete; skipping automatic non-loopback callback listener" + ); + return None; + } + + let addr = callback_listener(gateway_port)?; + if is_bindable(addr) { + info!( + bind = %gateway_bind_address, + callback_listener = %addr, + "Rootless Podman pasta detected with loopback gateway bind; adding callback listener for sandbox callbacks" + ); + Some(addr) + } else { + warn!( + bind = %gateway_bind_address, + callback_listener = %addr, + "Rootless Podman pasta callback listener address is not bindable; sandbox callbacks may fail" + ); + None + } +} + +fn bridge_gateway_bind_address_candidate( + network_gateway_ip: Option<&str>, + gateway_port: u16, +) -> Option { + let ip = network_gateway_ip?.parse().ok()?; + if openshell_core::net::is_link_local_ip(ip) { + return None; + } + Some(SocketAddr::new(ip, gateway_port)) +} + +fn local_interface_bind_address(gateway_port: u16) -> Option { + local_interface_callback_ips() + .into_iter() + .map(|ip| SocketAddr::new(ip, gateway_port)) + .find(|addr| socket_address_is_bindable(*addr)) +} + +fn local_interface_callback_ips() -> Vec { + let Ok(ifaddrs) = nix::ifaddrs::getifaddrs() else { + warn!("Failed to enumerate local interface addresses for Podman callback listener"); + return Vec::new(); + }; + + let mut ips = Vec::new(); + for iface in ifaddrs { + let Some(address) = iface.address else { + continue; + }; + let ip = address + .as_sockaddr_in() + .map(|addr| IpAddr::V4(addr.ip())) + .or_else(|| address.as_sockaddr_in6().map(|addr| IpAddr::V6(addr.ip()))); + let Some(ip) = ip else { + continue; + }; + if callback_listener_ip_is_usable(ip) && !ips.contains(&ip) { + ips.push(ip); + } + } + + ips.sort_by_key(|ip| match ip { + IpAddr::V4(_) => 0, + IpAddr::V6(_) => 1, + }); + ips +} + +fn callback_listener_ip_is_usable(ip: IpAddr) -> bool { + !ip.is_loopback() + && !ip.is_unspecified() + && !ip.is_multicast() + && !openshell_core::net::is_link_local_ip(ip) +} + +fn socket_address_is_bindable(addr: SocketAddr) -> bool { + std::net::TcpListener::bind(addr).is_ok() +} + +fn enrich_socket_connection_error(err: PodmanApiError) -> PodmanApiError { + match err { + PodmanApiError::Connection(message) + if running_inside_container() && message.contains("Permission denied") => + { + PodmanApiError::Connection(format!( + "{message}. Gateway is running inside a container and cannot access the mounted Podman socket; for rootless Podman socket deployments run the gateway container with --userns=keep-id so the container UID matches the host user that owns the socket" + )) + } + other => other, + } +} + fn supervisor_image_pull_policy(image: &str) -> &'static str { if supervisor_image_should_refresh(image) { "newer" @@ -897,6 +1103,125 @@ mod tests { } } + #[test] + fn gateway_bind_address_uses_non_link_local_bridge_ip() { + let addr = gateway_bind_address( + Some("10.89.1.1"), + Some("127.0.0.1:8080".parse().unwrap()), + 8080, + false, + false, + true, + true, + true, + |_| Some("192.0.2.10:8080".parse().unwrap()), + |_| true, + ); + assert_eq!(addr, Some("10.89.1.1:8080".parse().unwrap())); + } + + #[test] + fn gateway_bind_address_skips_link_local_bridge_gateway() { + let addr = gateway_bind_address( + Some("169.254.1.2"), + Some("127.0.0.1:8080".parse().unwrap()), + 8080, + false, + false, + true, + true, + true, + |_| Some("192.0.2.10:8080".parse().unwrap()), + |_| true, + ); + assert_eq!(addr, None); + } + + #[test] + fn gateway_bind_address_skips_containerized_gateway() { + let addr = gateway_bind_address( + Some("10.89.1.1"), + Some("127.0.0.1:8080".parse().unwrap()), + 8080, + true, + false, + true, + true, + true, + |_| Some("192.0.2.10:8080".parse().unwrap()), + |_| true, + ); + assert_eq!(addr, None); + } + + #[test] + fn gateway_bind_address_uses_local_interface_for_pasta_loopback_tls_auth() { + let addr = gateway_bind_address( + Some("10.89.1.1"), + Some("127.0.0.1:8080".parse().unwrap()), + 8080, + false, + true, + true, + true, + true, + |_| Some("192.0.2.10:8080".parse().unwrap()), + |_| true, + ); + assert_eq!(addr, Some("192.0.2.10:8080".parse().unwrap())); + } + + #[test] + fn gateway_bind_address_skips_pasta_when_auto_listener_disabled() { + let addr = gateway_bind_address( + Some("10.89.1.1"), + Some("127.0.0.1:8080".parse().unwrap()), + 8080, + false, + true, + false, + true, + true, + |_| Some("192.0.2.10:8080".parse().unwrap()), + |_| true, + ); + assert_eq!(addr, None); + } + + #[test] + fn gateway_bind_address_skips_pasta_without_tls_client_certificate_auth() { + let addr = gateway_bind_address( + Some("10.89.1.1"), + Some("127.0.0.1:8080".parse().unwrap()), + 8080, + false, + true, + true, + true, + false, + |_| Some("192.0.2.10:8080".parse().unwrap()), + |_| true, + ); + assert_eq!(addr, None); + } + + #[test] + fn gateway_bind_address_skips_pasta_non_loopback_primary_bind() { + let addr = gateway_bind_address( + Some("10.89.1.1"), + Some("192.0.2.10:8080".parse().unwrap()), + 8080, + false, + true, + true, + true, + true, + |_| Some("192.0.2.11:8080".parse().unwrap()), + |_| true, + ); + assert_eq!(addr, None); + } + #[test] fn podman_driver_error_from_conflict() { let err = ComputeDriverError::from(PodmanApiError::Conflict("exists".into())); diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index 294cec0ce7..03980583d3 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -15,6 +15,7 @@ use openshell_driver_kubernetes::KubernetesComputeConfig; use openshell_driver_podman::PodmanComputeConfig; use serde::Deserialize; use std::collections::BTreeMap; +use std::net::SocketAddr; use std::path::PathBuf; use super::VmComputeConfig; @@ -41,7 +42,9 @@ pub struct DriverStartupContext<'a> { pub file: Option<&'a config_file::ConfigFile>, pub guest_tls: Option<&'a GuestTlsPaths>, pub gateway_port: u16, + pub gateway_bind_address: SocketAddr, pub gateway_tls_enabled: bool, + pub gateway_callback_client_cert_auth_enabled: bool, pub endpoint_overrides: &'a BTreeMap, } @@ -152,6 +155,10 @@ fn apply_podman_runtime_defaults( context: DriverStartupContext<'_>, ) { podman.gateway_port = context.gateway_port; + podman.gateway_bind_address = Some(context.gateway_bind_address); + podman.gateway_tls_enabled = context.gateway_tls_enabled; + podman.gateway_callback_client_cert_auth_enabled = + context.gateway_callback_client_cert_auth_enabled; apply_podman_env_overrides(podman); apply_guest_tls_defaults_to_split_fields( &mut podman.guest_tls_ca, @@ -200,6 +207,27 @@ fn apply_podman_env_overrides(podman: &mut PodmanComputeConfig) { if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { podman.host_gateway_ip = ip; } + if let Ok(value) = std::env::var("OPENSHELL_PODMAN_ENABLE_AUTO_CALLBACK_LISTENER") + && let Some(enabled) = + parse_bool_env(&value, "OPENSHELL_PODMAN_ENABLE_AUTO_CALLBACK_LISTENER") + { + podman.enable_auto_callback_listener = enabled; + } +} + +fn parse_bool_env(value: &str, name: &str) -> Option { + match value.to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => Some(true), + "0" | "false" | "no" | "off" => Some(false), + _ => { + tracing::warn!( + env = name, + value, + "Ignoring invalid boolean environment override" + ); + None + } + } } fn apply_remote_driver_overrides( @@ -241,8 +269,36 @@ fn apply_guest_tls_defaults_to_split_fields( #[cfg(test)] mod tests { use super::*; + use crate::TEST_ENV_LOCK as ENV_LOCK; use std::collections::BTreeMap; + struct EnvVarGuard { + key: &'static str, + original: Option, + } + + impl EnvVarGuard { + #[allow(unsafe_code)] + fn set(key: &'static str, value: &str) -> Self { + let original = std::env::var(key).ok(); + // SAFETY: tests serialize environment mutation with ENV_LOCK. + unsafe { std::env::set_var(key, value) }; + Self { key, original } + } + } + + impl Drop for EnvVarGuard { + #[allow(unsafe_code)] + fn drop(&mut self) { + match self.original.as_deref() { + // SAFETY: tests serialize environment mutation with ENV_LOCK. + Some(value) => unsafe { std::env::set_var(self.key, value) }, + // SAFETY: tests serialize environment mutation with ENV_LOCK. + None => unsafe { std::env::remove_var(self.key) }, + } + } + } + fn test_context(file: Option<&config_file::ConfigFile>) -> DriverStartupContext<'_> { static EMPTY_ENDPOINT_OVERRIDES: std::sync::LazyLock> = std::sync::LazyLock::new(BTreeMap::new); @@ -257,7 +313,10 @@ mod tests { file, guest_tls: None, gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, + gateway_bind_address: ([127, 0, 0, 1], openshell_core::config::DEFAULT_SERVER_PORT) + .into(), gateway_tls_enabled: false, + gateway_callback_client_cert_auth_enabled: false, endpoint_overrides, } } @@ -306,6 +365,49 @@ enable_bind_mounts = true assert!(cfg.enable_bind_mounts); } + #[test] + fn podman_auto_callback_listener_env_override_disables_default() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _guard = EnvVarGuard::set("OPENSHELL_PODMAN_ENABLE_AUTO_CALLBACK_LISTENER", "false"); + + let cfg = podman_config_from_context(test_context(None)).expect("podman config"); + + assert!(!cfg.enable_auto_callback_listener); + } + + #[test] + fn podman_auto_callback_listener_invalid_env_preserves_configured_false() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _guard = EnvVarGuard::set("OPENSHELL_PODMAN_ENABLE_AUTO_CALLBACK_LISTENER", "fales"); + let file: config_file::ConfigFile = toml::from_str( + r" +[openshell.drivers.podman] +enable_auto_callback_listener = false +", + ) + .expect("valid config"); + + let cfg = podman_config_from_context(test_context(Some(&file))).expect("podman config"); + + assert!(!cfg.enable_auto_callback_listener); + } + + #[test] + fn podman_auto_callback_listener_invalid_env_preserves_default_true() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _guard = EnvVarGuard::set("OPENSHELL_PODMAN_ENABLE_AUTO_CALLBACK_LISTENER", "fales"); + + let cfg = podman_config_from_context(test_context(None)).expect("podman config"); + + assert!(cfg.enable_auto_callback_listener); + } + #[test] fn docker_config_reads_bind_mount_opt_in_from_driver_table() { let file: config_file::ConfigFile = toml::from_str( diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 3a92cd209a..d0792bc933 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -434,6 +434,7 @@ impl ComputeRuntime { let driver = PodmanComputeDriver::new(config) .await .map_err(|err| ComputeError::Message(err.to_string()))?; + let gateway_bind_addresses = driver.gateway_bind_address().into_iter().collect(); let driver: SharedComputeDriver = Arc::new(PodmanDriverService::new(driver)); Self::from_driver( ComputeDriverKind::Podman.as_str().to_string(), @@ -446,7 +447,7 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - Vec::new(), + gateway_bind_addresses, ) .await } diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index b65b5f3b09..a580bb4fa7 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -637,9 +637,9 @@ version = 2 .bind_address .expect("bind_address must be explicitly set in the RPM default config"); assert!( - addr.ip().is_unspecified(), - "RPM default bind_address must be 0.0.0.0 so Podman sandbox containers \ - can reach the gateway over the host network bridge, got {addr}" + addr.ip().is_loopback(), + "RPM default bind_address must be loopback; the Podman driver adds a \ + protected callback listener for rootless pasta when needed, got {addr}" ); assert_eq!( addr.port(), diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 6462ccbbf3..8340f70093 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -201,6 +201,13 @@ impl ServerState { } } +fn gateway_callback_client_cert_auth_enabled(config: &Config) -> bool { + config + .tls + .as_ref() + .is_some_and(|tls| tls.require_client_auth) +} + /// Run the `OpenShell` server. /// /// This starts a multiplexed gRPC/HTTP server on the configured bind address. @@ -250,7 +257,11 @@ pub(crate) async fn run_server( file: config_file.as_ref(), guest_tls: guest_tls.as_ref(), gateway_port: config.bind_address.port(), + gateway_bind_address: config.bind_address, gateway_tls_enabled: config.tls.is_some(), + gateway_callback_client_cert_auth_enabled: gateway_callback_client_cert_auth_enabled( + &config, + ), endpoint_overrides: &config.compute_driver_endpoints, }; let compute = build_compute_runtime( @@ -900,8 +911,9 @@ mod tests { use super::{ ConfiguredComputeDriver, ConnectionProtocol, MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, classify_initial_bytes, configured_compute_driver, - gateway_listener_addresses, is_benign_tls_handshake_failure, - kubernetes_sandbox_jwt_expiry_disabled, serve_gateway_listener, + gateway_callback_client_cert_auth_enabled, gateway_listener_addresses, + is_benign_tls_handshake_failure, kubernetes_sandbox_jwt_expiry_disabled, + serve_gateway_listener, }; use openshell_core::{ ComputeDriverKind, Config, @@ -926,7 +938,10 @@ mod tests { file, guest_tls: None, gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, + gateway_bind_address: ([127, 0, 0, 1], openshell_core::config::DEFAULT_SERVER_PORT) + .into(), gateway_tls_enabled: false, + gateway_callback_client_cert_auth_enabled: false, endpoint_overrides: &config.compute_driver_endpoints, } } @@ -1371,6 +1386,65 @@ mod tests { assert!(!kubernetes_sandbox_jwt_expiry_disabled(&Config::new(None))); } + fn tls_config(require_client_auth: bool) -> openshell_core::TlsConfig { + openshell_core::TlsConfig { + cert_path: "/tmp/server.crt".into(), + key_path: "/tmp/server.key".into(), + client_ca_path: Some("/tmp/ca.crt".into()), + require_client_auth, + } + } + + #[test] + fn gateway_callback_client_cert_auth_ignores_gateway_jwt_only() { + let mut config = Config::new(Some(tls_config(false))); + config.gateway_jwt = Some(openshell_core::GatewayJwtConfig { + signing_key_path: "/tmp/signing.pem".into(), + public_key_path: "/tmp/public.pem".into(), + kid_path: "/tmp/kid".into(), + gateway_id: "openshell".to_string(), + ttl_secs: 3600, + }); + + assert!(!gateway_callback_client_cert_auth_enabled(&config)); + } + + #[test] + fn gateway_callback_client_cert_auth_ignores_oidc_only() { + let config = Config::new(Some(tls_config(false))).with_oidc(openshell_core::OidcConfig { + issuer: "https://issuer.example".to_string(), + audience: "openshell".to_string(), + jwks_ttl_secs: 3600, + roles_claim: "roles".to_string(), + admin_role: "openshell-admin".to_string(), + user_role: "openshell-user".to_string(), + scopes_claim: String::new(), + }); + + assert!(!gateway_callback_client_cert_auth_enabled(&config)); + } + + #[test] + fn gateway_callback_client_cert_auth_rejects_missing_tls() { + assert!(!gateway_callback_client_cert_auth_enabled(&Config::new( + None + ))); + } + + #[test] + fn gateway_callback_client_cert_auth_rejects_tls_without_required_client_auth() { + assert!(!gateway_callback_client_cert_auth_enabled(&Config::new( + Some(tls_config(false),) + ))); + } + + #[test] + fn gateway_callback_client_cert_auth_accepts_required_tls_client_auth() { + assert!(gateway_callback_client_cert_auth_enabled(&Config::new( + Some(tls_config(true),) + ))); + } + #[test] fn gateway_listener_addresses_skip_driver_address_covered_by_wildcard() { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); diff --git a/deploy/rpm/CONFIGURATION.md b/deploy/rpm/CONFIGURATION.md index a144cac8ed..1bb89228b3 100644 --- a/deploy/rpm/CONFIGURATION.md +++ b/deploy/rpm/CONFIGURATION.md @@ -20,14 +20,17 @@ The defaults are tuned for rootless Podman use: version = 1 [openshell.gateway] -bind_address = "0.0.0.0:17670" +bind_address = "127.0.0.1:17670" compute_drivers = ["podman"] ``` -`bind_address = "0.0.0.0:17670"` is required because Podman sandbox -containers reach the gateway over the host network bridge and cannot -connect to `127.0.0.1` inside the gateway's network namespace. mTLS is -enabled by default and protects all connections. +`bind_address = "127.0.0.1:17670"` keeps the user-facing gateway listener on +loopback by default. For rootless Podman/pasta, the Podman driver adds a +bindability-checked callback listener on a non-loopback host address when +TLS client certificate authentication is enabled so sandbox supervisors can +reach the gateway. Set +`enable_auto_callback_listener = false` under `[openshell.drivers.podman]` to +opt out, or bind explicitly to another address when operating remotely. `compute_drivers = ["podman"]` pins the compute driver to Podman. Without this, the gateway auto-detects in order: Kubernetes, Podman, Docker. Pinning @@ -64,7 +67,7 @@ systemctl --user edit openshell-gateway The RPM enables mutual TLS by default. The gateway requires a valid client certificate for all API connections and listens on -`0.0.0.0:17670` by default (see "Default configuration" above). +`127.0.0.1:17670` by default (see "Default configuration" above). ### Auto-generated certificates @@ -214,13 +217,14 @@ overrides that persist across package upgrades. | TOML option | Default | Description | |-------------|---------|-------------| -| `bind_address` | `0.0.0.0:17670` (RPM default) | Address for the gRPC/HTTP API. | +| `bind_address` | `127.0.0.1:17670` (RPM default) | Address for the gRPC/HTTP API. | | `compute_drivers` | `["podman"]` (RPM default) | When unset, the gateway auto-detects Kubernetes, then Podman, then Docker. The RPM default pins to Podman. | | `default_image` | `ghcr.io/nvidia/openshell-community/sandboxes/base:latest` | Default sandbox image. | | `supervisor_image` | `ghcr.io/nvidia/openshell/supervisor:latest` | Supervisor image mounted into Podman sandboxes. | | `guest_tls_ca`, `guest_tls_cert`, `guest_tls_key` | auto-generated paths | Client TLS material bind-mounted into sandbox containers. | | `[openshell.gateway.tls]` paths | auto-generated paths | Server TLS certificate, key, and client CA. | | `disable_tls` | unset | Set to `true` to disable TLS. | +| `[openshell.drivers.podman].enable_auto_callback_listener` | `true` | Adds a protected non-loopback callback listener for rootless Podman/pasta loopback-bound gateways. | The database URL is not accepted in TOML. When `OPENSHELL_DB_URL` is unset, the gateway uses `sqlite:$XDG_STATE_HOME/openshell/gateway/openshell.db`. @@ -235,7 +239,7 @@ settings: version = 1 [openshell.gateway] -bind_address = "0.0.0.0:17670" +bind_address = "127.0.0.1:17670" compute_drivers = ["podman"] default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" @@ -243,6 +247,8 @@ default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" image_pull_policy = "missing" network_name = "openshell" stop_timeout_secs = 10 +# Set false to keep the gateway loopback-only even for rootless Podman/pasta. +enable_auto_callback_listener = true ``` ### Image management diff --git a/deploy/rpm/QUICKSTART.md b/deploy/rpm/QUICKSTART.md index c6634ced95..fead35fe21 100644 --- a/deploy/rpm/QUICKSTART.md +++ b/deploy/rpm/QUICKSTART.md @@ -65,11 +65,11 @@ On first start, the gateway automatically generates: - A self-signed PKI bundle (CA, server cert, client cert) for mTLS -> **Note:** The RPM default configuration binds to `0.0.0.0:17670` so -> Podman sandbox containers can reach the gateway over the host network -> bridge. Mutual TLS (mTLS) is enabled automatically on first start, -> requiring a valid client certificate for every connection. See -> CONFIGURATION.md for details. +> **Note:** The RPM default configuration binds to `127.0.0.1:17670`. +> For rootless Podman/pasta, the gateway adds a protected callback listener +> on a bindable host interface so sandbox supervisors can reach it without +> exposing every interface. Mutual TLS (mTLS) is enabled automatically on +> first start. See CONFIGURATION.md for details. Verify the service is running: diff --git a/deploy/rpm/TROUBLESHOOTING.md b/deploy/rpm/TROUBLESHOOTING.md index 68a1f49464..dfce5967b1 100644 --- a/deploy/rpm/TROUBLESHOOTING.md +++ b/deploy/rpm/TROUBLESHOOTING.md @@ -79,6 +79,30 @@ openshell gateway add --local https://127.0.0.1:17670 openshell status ``` +### Sandbox callbacks fail with loopback gateway bind + +The RPM default binds the primary gateway listener to `127.0.0.1:17670`. When +Podman uses rootless pasta, sandbox containers call back through +`host.containers.internal` and may not reach host loopback directly. The Podman +driver normally adds a protected non-loopback callback listener when TLS client +certificate authentication is enabled. + +Check the gateway logs: + +```shell +journalctl --user -u openshell-gateway -b | grep 'callback listener' +``` + +A working rootless pasta setup logs that it is adding a callback listener. If +the logs say TLS client certificate authentication is incomplete, restore the +generated local TLS/mTLS configuration or bind explicitly to a reachable address. If you intentionally +disabled the listener, re-enable it: + +```toml +[openshell.drivers.podman] +enable_auto_callback_listener = true +``` + ### Option 2: Externally-managed certificates Generate certificates that include the server's hostname or IP in the @@ -272,9 +296,9 @@ Other breaking changes in this release: openshell gateway add --local https://127.0.0.1:17670 ``` -- **Default bind address changed from `0.0.0.0` to `127.0.0.1`.** If - you relied on network-accessible access without an explicit bind - address, add the following to `~/.config/openshell/gateway.toml`: +- **The default bind address is `127.0.0.1`.** If you rely on + network-accessible access without the Podman auto callback listener, + add the following to `~/.config/openshell/gateway.toml`: ```toml [openshell.gateway] diff --git a/deploy/rpm/gateway.toml.default b/deploy/rpm/gateway.toml.default index d853799640..9419d57c56 100644 --- a/deploy/rpm/gateway.toml.default +++ b/deploy/rpm/gateway.toml.default @@ -18,11 +18,13 @@ version = 1 [openshell.gateway] -# Podman sandbox containers reach the gateway over the host network bridge, -# which requires binding to all interfaces. Override to 127.0.0.1:17670 if -# you don't use Podman or want loopback-only access (e.g. behind a reverse -# proxy). mTLS is enabled by default and protects all connections. -bind_address = "0.0.0.0:17670" +# Bind to loopback by default. For rootless Podman/pasta, the Podman driver +# adds a bindability-checked callback listener when TLS client certificate +# authentication is enabled so sandbox supervisors can reach the gateway +# without exposing every interface. +# Set enable_auto_callback_listener = false under [openshell.drivers.podman] +# to opt out, or bind explicitly to another address when operating remotely. +bind_address = "127.0.0.1:17670" # Pin to the Podman compute driver. Without this, the gateway auto-detects # in order: Kubernetes, Podman, Docker. Pinning prevents unexpected driver diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 88b82870de..06a5b1e59f 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -268,6 +268,9 @@ network_name = "openshell" # Omit for the platform default: empty on Linux, 192.168.127.254 on macOS Podman machine. # Set "" to force Podman's host-gateway resolver. # host_gateway_ip = "192.168.127.254" +# Disable to prevent the gateway from adding a non-loopback callback listener +# for rootless Podman pasta when the primary bind address is loopback-only. +enable_auto_callback_listener = true sandbox_ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 10 supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" @@ -286,6 +289,13 @@ sandbox_pids_limit = 2048 health_check_interval_secs = 10 ``` +`enable_auto_callback_listener` defaults to `true`. When rootless Podman uses +pasta and the gateway bind address is loopback-only, the driver may add a +bindability-checked non-loopback callback listener if TLS client certificate +authentication is enabled. Set it to `false`, or set +`OPENSHELL_PODMAN_ENABLE_AUTO_CALLBACK_LISTENER=false`, +to opt out. + ### MicroVM Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 714ddd6c33..72a135176f 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -180,10 +180,30 @@ The gateway talks to the Podman API socket. The Podman driver requires Podman 5. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). -Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.podman]`. +Select Podman with `compute_drivers = ["podman"]` in `[openshell.gateway]`. Configure Podman driver values such as `socket_path`, `network_name`, `supervisor_image`, `stop_timeout_secs`, `image_pull_policy`, `grpc_endpoint`, `host_gateway_ip`, `sandbox_ssh_socket_path`, `sandbox_pids_limit`, `enable_auto_callback_listener`, and `guest_tls_*` in `[openshell.drivers.podman]`. On macOS with `podman machine`, the driver uses gvproxy's host-loopback IP, `192.168.127.254`, for sandbox host aliases by default. Set `host_gateway_ip` only when your Podman machine uses a non-standard host-loopback address. On Linux, an empty `host_gateway_ip` keeps Podman's `host-gateway` resolver behavior. +### Podman callback listener + +The RPM gateway binds the primary listener to loopback by default. Rootless +Podman sandboxes call back through `host.containers.internal`, which may not +reach host loopback when Podman uses pasta. When TLS client certificate +authentication is enabled, the Podman driver can add a bindability-checked +listener on a non-loopback local interface for sandbox callbacks. The driver +selects this address from local interfaces and does not probe external IPs. + +Disable this behavior with: + +```toml +[openshell.drivers.podman] +enable_auto_callback_listener = false +``` + +Containerized gateways do not use this automatic listener because their network +namespace is not the host namespace. Publish the gateway container port on an +address sandbox containers can reach. + ### Podman Driver Config Mounts Podman driver config accepts user-supplied `volume`, `tmpfs`, and `image` diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 77b4c1324c..e3b6da778c 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -400,7 +400,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" # Start from the RPM default template so this e2e test exercises the same # TOML config path that RPM users get on first start. The template sets -# bind_address = "0.0.0.0:17670" and compute_drivers = ["podman"]; those +# bind_address = "127.0.0.1:17670" and compute_drivers = ["podman"]; those # values must be correct for Podman e2e to pass, which means a regression # to the template (wrong bind address, wrong driver) will surface here. #