Skip to content

Commit 28a8c31

Browse files
committed
feat(driver-docker): isolate sandboxes on dedicated bridge
Signed-off-by: Drew Newberry <anewberry@nvidia.com>
1 parent ebbd9de commit 28a8c31

7 files changed

Lines changed: 717 additions & 27 deletions

File tree

architecture/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,8 @@ This opens an interactive SSH session into the sandbox, with all provider creden
299299
| [Docs Site Architecture](docs-site.md) | Documentation source layout, navigation structure, local validation and preview workflow, and publish pipeline. |
300300
| [Policy Language](security-policy.md) | The YAML/Rego policy system that governs sandbox behavior. |
301301
| [Inference Routing](inference-routing.md) | Transparent interception and sandbox-local routing of AI inference API calls to configured backends. |
302+
| [Docker Driver](docker-driver.md) | Docker compute driver implementation, dedicated network, multi-bind connectivity. |
303+
| [Podman Driver](podman-driver.md) | Podman compute driver implementation, rootless networking, secret injection. |
302304
| [System Architecture](system-architecture.md) | Top-level system architecture diagram with all deployable components and communication flows. |
303305
| [Gateway Settings Channel](gateway-settings.md) | Runtime settings channel: two-tier key-value configuration, global policy override, settings registry, CLI/TUI commands. |
304306
| [TUI](tui.md) | Terminal user interface for sandbox interaction. |

architecture/docker-driver.md

Lines changed: 342 additions & 0 deletions
Large diffs are not rendered by default.

crates/openshell-core/src/config.rs

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,20 @@ pub struct Config {
105105
#[serde(default)]
106106
pub metrics_bind_address: Option<SocketAddr>,
107107

108+
/// Additional bind addresses that serve the same multiplexed gRPC/HTTP
109+
/// surface as `bind_address`.
110+
///
111+
/// Compute drivers may register extra listeners during startup so that
112+
/// sandbox workloads can call back into the gateway over an interface
113+
/// that the operator-supplied `bind_address` does not expose. The
114+
/// canonical case is the Docker driver: when the gateway only binds to
115+
/// loopback for the CLI, the driver appends the Docker bridge gateway
116+
/// IP here so containers reaching the host via `host.openshell.internal`
117+
/// (which Docker maps to `host-gateway`) can complete the gRPC
118+
/// handshake.
119+
#[serde(default)]
120+
pub extra_bind_addresses: Vec<SocketAddr>,
121+
108122
/// Log level (trace, debug, info, warn, error).
109123
#[serde(default = "default_log_level")]
110124
pub log_level: String,
@@ -230,6 +244,7 @@ impl Config {
230244
bind_address: default_bind_address(),
231245
health_bind_address: None,
232246
metrics_bind_address: None,
247+
extra_bind_addresses: Vec::new(),
233248
log_level: default_log_level(),
234249
tls,
235250
database_url: String::new(),
@@ -270,6 +285,19 @@ impl Config {
270285
self
271286
}
272287

288+
/// Append an extra listener address to the multiplex service.
289+
///
290+
/// Duplicate entries (matching `bind_address` or any existing entry) are
291+
/// silently dropped so callers can naively push driver-derived addresses
292+
/// without checking for collisions.
293+
#[must_use]
294+
pub fn with_extra_bind_address(mut self, addr: SocketAddr) -> Self {
295+
if addr != self.bind_address && !self.extra_bind_addresses.contains(&addr) {
296+
self.extra_bind_addresses.push(addr);
297+
}
298+
self
299+
}
300+
273301
/// Create a new configuration with the given log level.
274302
#[must_use]
275303
pub fn with_log_level(mut self, level: impl Into<String>) -> Self {

crates/openshell-driver-docker/src/lib.rs

Lines changed: 201 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use bollard::Docker;
99
use bollard::errors::Error as BollardError;
1010
use bollard::models::{
1111
ContainerCreateBody, ContainerSummary, ContainerSummaryStateEnum, HostConfig, Mount,
12-
MountTypeEnum, RestartPolicy, RestartPolicyNameEnum,
12+
MountTypeEnum, NetworkCreateRequest, RestartPolicy, RestartPolicyNameEnum,
1313
};
1414
use bollard::query_parameters::{
1515
CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder,
@@ -56,7 +56,6 @@ const TLS_CERT_MOUNT_PATH: &str = "/etc/openshell/tls/client/tls.crt";
5656
const TLS_KEY_MOUNT_PATH: &str = "/etc/openshell/tls/client/tls.key";
5757
const SANDBOX_COMMAND: &str = "sleep infinity";
5858
const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal";
59-
const HOST_DOCKER_INTERNAL: &str = "host.docker.internal";
6059

6160
/// Default image holding the Linux `openshell-sandbox` binary. The gateway
6261
/// pulls this image and extracts the binary to a host-side cache when no
@@ -118,8 +117,124 @@ pub trait SupervisorReadiness: Send + Sync + 'static {
118117
fn is_supervisor_connected(&self, sandbox_id: &str) -> bool;
119118
}
120119

120+
/// Ensure a user-defined Docker bridge network with `name` exists, creating
121+
/// it if necessary.
122+
///
123+
/// If the network already exists with a different driver (e.g. `overlay`)
124+
/// the existing definition is reused with a warning — the daemon-managed
125+
/// network may have been pre-provisioned by an operator who knows what
126+
/// they're doing, and refusing to start would break that workflow. Other
127+
/// inspect failures are surfaced as configuration errors so misconfigured
128+
/// daemons fail fast.
129+
pub async fn ensure_network(docker: &Docker, name: &str) -> CoreResult<()> {
130+
match docker
131+
.inspect_network(
132+
name,
133+
None::<bollard::query_parameters::InspectNetworkOptions>,
134+
)
135+
.await
136+
{
137+
Ok(existing) => {
138+
let driver = existing.driver.as_deref().unwrap_or("<unknown>");
139+
if driver == "bridge" {
140+
info!(network = %name, "Reusing existing Docker network");
141+
} else {
142+
warn!(
143+
network = %name,
144+
driver = %driver,
145+
"Reusing existing Docker network with non-bridge driver; sandboxes may behave unexpectedly"
146+
);
147+
}
148+
return Ok(());
149+
}
150+
Err(BollardError::DockerResponseServerError {
151+
status_code: 404, ..
152+
}) => {}
153+
Err(err) => {
154+
return Err(Error::execution(format!(
155+
"failed to inspect Docker network '{name}': {err}"
156+
)));
157+
}
158+
}
159+
160+
docker
161+
.create_network(NetworkCreateRequest {
162+
name: name.to_string(),
163+
driver: Some("bridge".to_string()),
164+
attachable: Some(true),
165+
..Default::default()
166+
})
167+
.await
168+
.map_err(|err| {
169+
Error::execution(format!("failed to create Docker network '{name}': {err}"))
170+
})?;
171+
info!(network = %name, "Created Docker network");
172+
Ok(())
173+
}
174+
175+
/// Query the Docker daemon for the IPv4 gateway address of `network_name`.
176+
///
177+
/// This is the address that `host-gateway` resolves to inside sandbox
178+
/// containers attached to that network — i.e. the only host-side address
179+
/// that `host.openshell.internal` is guaranteed to reach for them.
180+
///
181+
/// The gateway uses this to bind only the network-facing interface so that
182+
/// sandboxes can call back without exposing the gateway on every host
183+
/// interface.
184+
///
185+
/// Returns `Err` if the daemon is unreachable, the network has no IPAM
186+
/// configuration, or the gateway field is missing/unparseable.
187+
pub async fn host_bridge_gateway_address(
188+
docker: &Docker,
189+
network_name: &str,
190+
) -> CoreResult<std::net::IpAddr> {
191+
let network = docker
192+
.inspect_network(
193+
network_name,
194+
None::<bollard::query_parameters::InspectNetworkOptions>,
195+
)
196+
.await
197+
.map_err(|err| {
198+
Error::execution(format!(
199+
"failed to inspect Docker network '{network_name}': {err}"
200+
))
201+
})?;
202+
let ipam = network.ipam.ok_or_else(|| {
203+
Error::execution(format!(
204+
"Docker network '{network_name}' has no IPAM configuration"
205+
))
206+
})?;
207+
let configs = ipam.config.unwrap_or_default();
208+
let gateway = configs
209+
.into_iter()
210+
.find_map(|c| c.gateway.filter(|g| !g.is_empty()))
211+
.ok_or_else(|| {
212+
Error::execution(format!(
213+
"Docker network '{network_name}' has no IPv4 gateway configured"
214+
))
215+
})?;
216+
gateway.parse().map_err(|err| {
217+
Error::execution(format!(
218+
"Docker network '{network_name}' gateway '{gateway}' is not a valid IP address: {err}"
219+
))
220+
})
221+
}
222+
223+
/// Convenience wrapper that connects to the local Docker daemon, ensures
224+
/// the sandbox network exists, and returns its IPv4 gateway address.
225+
///
226+
/// Used by `openshell-server` to set up the multi-bind listener without
227+
/// taking a direct dependency on `bollard`. Equivalent to calling
228+
/// [`ensure_network`] followed by [`host_bridge_gateway_address`].
229+
pub async fn ensure_network_and_get_gateway(network_name: &str) -> CoreResult<std::net::IpAddr> {
230+
let docker = Docker::connect_with_local_defaults()
231+
.map_err(|err| Error::execution(format!("failed to create Docker client: {err}")))?;
232+
ensure_network(&docker, network_name).await?;
233+
host_bridge_gateway_address(&docker, network_name).await
234+
}
235+
121236
/// Gateway-local configuration for the Docker compute driver.
122-
#[derive(Debug, Clone, Default)]
237+
#[derive(Debug, Clone)]
123238
pub struct DockerComputeConfig {
124239
/// Optional override for the Linux `openshell-sandbox` binary mounted into containers.
125240
pub supervisor_bin: Option<PathBuf>,
@@ -138,6 +253,28 @@ pub struct DockerComputeConfig {
138253

139254
/// Host-side private key for Docker sandbox mTLS.
140255
pub guest_tls_key: Option<PathBuf>,
256+
257+
/// Name of the user-defined Docker bridge network the driver creates and
258+
/// attaches sandbox containers to. Isolates sandboxes at L2 from
259+
/// unrelated workloads on the host's default `docker0` bridge.
260+
///
261+
/// Defaults to [`openshell_core::config::DEFAULT_NETWORK_NAME`]. Override
262+
/// via `OPENSHELL_NETWORK_NAME` (the same env var the Podman driver
263+
/// honours).
264+
pub network_name: String,
265+
}
266+
267+
impl Default for DockerComputeConfig {
268+
fn default() -> Self {
269+
Self {
270+
supervisor_bin: None,
271+
supervisor_image: None,
272+
guest_tls_ca: None,
273+
guest_tls_cert: None,
274+
guest_tls_key: None,
275+
network_name: openshell_core::config::DEFAULT_NETWORK_NAME.to_string(),
276+
}
277+
}
141278
}
142279

143280
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -159,6 +296,17 @@ struct DockerDriverRuntimeConfig {
159296
supervisor_bin: PathBuf,
160297
guest_tls: Option<DockerGuestTlsPaths>,
161298
daemon_version: String,
299+
network_name: String,
300+
/// IPv4 gateway address of the sandbox network. Injected into each
301+
/// container's `/etc/hosts` as `host.openshell.internal -> <ip>`.
302+
///
303+
/// Docker's `host-gateway` alias resolves to whatever the daemon was
304+
/// configured with (typically the default `docker0` IP), which is
305+
/// **not** the gateway of our user-defined `openshell` network. We
306+
/// therefore inject the address explicitly so sandboxes attached to
307+
/// the openshell network can reach the gateway listener bound on its
308+
/// bridge.
309+
network_gateway_ip: std::net::IpAddr,
162310
}
163311

164312
#[derive(Clone)]
@@ -189,6 +337,11 @@ impl DockerComputeDriver {
189337
"grpc_endpoint is required when using the docker compute driver",
190338
));
191339
}
340+
if docker_config.network_name.trim().is_empty() {
341+
return Err(Error::config(
342+
"network_name is required when using the docker compute driver",
343+
));
344+
}
192345

193346
let docker = Docker::connect_with_local_defaults()
194347
.map_err(|err| Error::execution(format!("failed to create Docker client: {err}")))?;
@@ -199,6 +352,17 @@ impl DockerComputeDriver {
199352
let supervisor_bin = resolve_supervisor_bin(&docker, docker_config, &daemon_arch).await?;
200353
let guest_tls = docker_guest_tls_paths(config, docker_config)?;
201354

355+
// Ensure the dedicated sandbox network exists before any sandbox is
356+
// created so containers can be attached to it deterministically.
357+
ensure_network(&docker, &docker_config.network_name).await?;
358+
let network_gateway_ip =
359+
host_bridge_gateway_address(&docker, &docker_config.network_name).await?;
360+
info!(
361+
network = %docker_config.network_name,
362+
gateway = %network_gateway_ip,
363+
"Resolved sandbox network gateway"
364+
);
365+
202366
let driver = Self {
203367
docker: Arc::new(docker),
204368
config: DockerDriverRuntimeConfig {
@@ -212,6 +376,8 @@ impl DockerComputeDriver {
212376
supervisor_bin,
213377
guest_tls,
214378
daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()),
379+
network_name: docker_config.network_name.clone(),
380+
network_gateway_ip,
215381
},
216382
events: broadcast::channel(WATCH_BUFFER).0,
217383
supervisor_readiness,
@@ -928,10 +1094,38 @@ fn build_container_create_body(
9281094
"SYS_PTRACE".to_string(),
9291095
"SYSLOG".to_string(),
9301096
]),
931-
extra_hosts: Some(vec![
932-
format!("{HOST_DOCKER_INTERNAL}:host-gateway"),
933-
format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"),
934-
]),
1097+
// The sandbox supervisor needs to bind-mount `/run/netns`,
1098+
// mark it shared, and create per-process network namespaces.
1099+
// Docker's default AppArmor profile (`docker-default`) denies
1100+
// these mount operations even with CAP_SYS_ADMIN, so we opt
1101+
// out of AppArmor confinement for sandbox containers. The
1102+
// sandbox enforces its own security boundary via Landlock,
1103+
// seccomp, OPA policy evaluation, and the dedicated network
1104+
// namespace it sets up for the agent — AppArmor at the
1105+
// container layer is redundant relative to those controls
1106+
// and conflicts with them in this case.
1107+
security_opt: Some(vec!["apparmor=unconfined".to_string()]),
1108+
// Attach to the dedicated openshell sandbox network so
1109+
// containers are isolated at L2 from unrelated workloads on
1110+
// the default `docker0` bridge.
1111+
network_mode: Some(config.network_name.clone()),
1112+
// Map the canonical OpenShell hostname to the IPv4 gateway of
1113+
// the sandbox network. We deliberately do *not* use Docker's
1114+
// `host-gateway` alias here: that magic value resolves to the
1115+
// daemon-configured host-gateway IP (typically the default
1116+
// `docker0` address), which is the wrong host-side address
1117+
// when the sandbox sits on a user-defined network. The
1118+
// gateway listener for sandboxes is bound to this network's
1119+
// bridge IP, so we inject it explicitly.
1120+
//
1121+
// We deliberately do not inject `host.docker.internal` —
1122+
// that alias is a Docker Desktop convention, not part of the
1123+
// OpenShell sandbox contract, and operators who need it can
1124+
// add it themselves at the daemon level.
1125+
extra_hosts: Some(vec![format!(
1126+
"{HOST_OPENSHELL_INTERNAL}:{}",
1127+
config.network_gateway_ip
1128+
)]),
9351129
..Default::default()
9361130
}),
9371131
..Default::default()

crates/openshell-driver-docker/src/tests.rs

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ fn runtime_config() -> DockerDriverRuntimeConfig {
5252
key: PathBuf::from("/tmp/tls.key"),
5353
}),
5454
daemon_version: "28.0.0".to_string(),
55+
network_name: openshell_core::config::DEFAULT_NETWORK_NAME.to_string(),
56+
network_gateway_ip: "172.18.0.1".parse().unwrap(),
5557
}
5658
}
5759

@@ -188,6 +190,37 @@ fn require_sandbox_identifier_rejects_when_id_and_name_are_empty() {
188190
require_sandbox_identifier("sbx-1", "demo").expect("id and name is accepted");
189191
}
190192

193+
#[test]
194+
fn build_container_create_body_attaches_to_dedicated_network() {
195+
// Sandboxes must opt out of Docker's default `bridge` and join the
196+
// user-defined network that the driver provisions at startup. This
197+
// keeps them L2-isolated from unrelated containers running on the
198+
// host's docker0 bridge and matches the Podman driver's posture.
199+
let mut config = runtime_config();
200+
config.network_name = "openshell-test".to_string();
201+
config.network_gateway_ip = "10.99.0.1".parse().unwrap();
202+
let create_body = build_container_create_body(&test_sandbox(), &config).unwrap();
203+
let host_config = create_body.host_config.expect("host_config is populated");
204+
205+
assert_eq!(
206+
host_config.network_mode,
207+
Some("openshell-test".to_string()),
208+
"sandbox must attach to the configured openshell network"
209+
);
210+
211+
let extra_hosts = host_config.extra_hosts.expect("extra_hosts is populated");
212+
assert!(
213+
extra_hosts.contains(&"host.openshell.internal:10.99.0.1".to_string()),
214+
"host.openshell.internal must resolve to the sandbox network's gateway IP, not Docker's host-gateway alias (got {extra_hosts:?})"
215+
);
216+
assert!(
217+
!extra_hosts
218+
.iter()
219+
.any(|h| h.starts_with("host.docker.internal:")),
220+
"the docker-specific alias should not be injected (got {extra_hosts:?})"
221+
);
222+
}
223+
191224
#[test]
192225
fn build_container_create_body_uses_runtime_namespace_label() {
193226
// Regression test: the namespace label must come from the driver's

0 commit comments

Comments
 (0)