Skip to content

Commit 189470c

Browse files
authored
fix(cli): pass cluster name to ssh-proxy child process for correct TLS path resolution (#52)
1 parent 734ec5e commit 189470c

10 files changed

Lines changed: 221 additions & 78 deletions

File tree

.env.example

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
# Navigator local development environment
22
# Copy to .env and customise. Mise loads .env automatically.
33
#
4-
# Local clusters bind Kubernetes API port 6443 on the host, so only one local
5-
# Navigator cluster can run at a time on a given Docker host.
6-
# Use unique CLUSTER_NAME/GATEWAY_PORT values per worktree to make switching
7-
# predictable; `mise run cluster` will recreate as needed.
4+
# Use unique CLUSTER_NAME/GATEWAY_PORT values per worktree to run
5+
# multiple clusters simultaneously; `mise run cluster` will recreate as needed.
86

97
# ---------- Cluster identity ----------
108

@@ -22,3 +20,9 @@
2220
# Host port mapped to the k3s NodePort (30051) where the Navigator gateway
2321
# listens. The CLI connects here. Must be unique per cluster.
2422
#GATEWAY_PORT=8080
23+
24+
# Expose the Kubernetes control plane on this host port (maps to 6443
25+
# inside the container). Leave unset to skip the binding — this is the
26+
# default and allows multiple clusters to coexist without port conflicts.
27+
# Set this only when you need direct kubectl access to the cluster.
28+
#KUBE_PORT=6443

build/scripts/cluster-bootstrap.sh

Lines changed: 12 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -85,13 +85,6 @@ pick_random_port() {
8585
return 1
8686
}
8787

88-
list_6443_conflicts() {
89-
docker ps --format '{{.Names}} {{.Ports}}' | awk '
90-
$0 ~ /0\.0\.0\.0:6443->6443\/tcp/ || $0 ~ /:::6443->6443\/tcp/ {
91-
print $1
92-
}'
93-
}
94-
9588
CLUSTER_NAME=${CLUSTER_NAME:-$(basename "$PWD")}
9689

9790
if [ -n "${GATEWAY_PORT:-}" ]; then
@@ -210,25 +203,6 @@ CONTAINER_NAME="navigator-cluster-${CLUSTER_NAME}"
210203
VOLUME_NAME="navigator-cluster-${CLUSTER_NAME}"
211204

212205
if [ "${MODE}" = "fast" ]; then
213-
mapfile -t port_conflicts < <(list_6443_conflicts || true)
214-
for cname in "${port_conflicts[@]:-}"; do
215-
[ -n "${cname}" ] || continue
216-
if [ "${cname}" = "${CONTAINER_NAME}" ]; then
217-
continue
218-
fi
219-
220-
if [[ "${cname}" == navigator-cluster-* ]]; then
221-
other_cluster=${cname#navigator-cluster-}
222-
echo "Removing conflicting local cluster '${other_cluster}' (holds host port 6443)..."
223-
nav cluster admin destroy --name "${other_cluster}"
224-
continue
225-
fi
226-
227-
echo "Error: container '${cname}' is using host port 6443, which Navigator clusters require." >&2
228-
echo "Stop/remove that container, then retry 'mise run cluster'." >&2
229-
exit 1
230-
done
231-
232206
if docker inspect "${CONTAINER_NAME}" >/dev/null 2>&1 || docker volume inspect "${VOLUME_NAME}" >/dev/null 2>&1; then
233207
echo "Recreating cluster '${CLUSTER_NAME}' from scratch..."
234208
nav cluster admin destroy --name "${CLUSTER_NAME}"
@@ -243,9 +217,10 @@ elif [ "${MODE}" = "build" ] || [ "${MODE}" = "fast" ]; then
243217
done
244218
fi
245219

246-
GATEWAY_HOST_ARGS=()
220+
DEPLOY_CMD=(nav cluster admin deploy --name "${CLUSTER_NAME}" --port "${GATEWAY_PORT}" --update-kube-config)
221+
247222
if [ -n "${GATEWAY_HOST:-}" ]; then
248-
GATEWAY_HOST_ARGS+=(--gateway-host "${GATEWAY_HOST}")
223+
DEPLOY_CMD+=(--gateway-host "${GATEWAY_HOST}")
249224

250225
# Ensure the gateway host resolves from the current environment.
251226
# On Linux CI runners host.docker.internal is not set automatically
@@ -260,7 +235,15 @@ if [ -n "${GATEWAY_HOST:-}" ]; then
260235
fi
261236
fi
262237

263-
nav cluster admin deploy --name "${CLUSTER_NAME}" --port "${GATEWAY_PORT}" "${GATEWAY_HOST_ARGS[@]}" --update-kube-config
238+
if [ -n "${KUBE_PORT:-}" ]; then
239+
# Explicit port requested — pass it through.
240+
DEPLOY_CMD+=(--kube-port "${KUBE_PORT}")
241+
else
242+
# Auto-select a free port so kubectl works during local development.
243+
DEPLOY_CMD+=(--kube-port)
244+
fi
245+
246+
"${DEPLOY_CMD[@]}"
264247

265248
echo ""
266249
echo "Cluster '${CLUSTER_NAME}' is ready."

crates/navigator-bootstrap/src/docker.rs

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ pub async fn ensure_container(
198198
extra_sans: &[String],
199199
ssh_gateway_host: Option<&str>,
200200
gateway_port: u16,
201+
kube_port: Option<u16>,
201202
) -> Result<()> {
202203
let container_name = container_name(name);
203204

@@ -256,21 +257,26 @@ pub async fn ensure_container(
256257
}
257258

258259
let mut port_bindings = HashMap::new();
259-
port_bindings.insert(
260-
"6443/tcp".to_string(),
261-
Some(vec![PortBinding {
262-
host_ip: Some("0.0.0.0".to_string()),
263-
host_port: Some("6443".to_string()),
264-
}]),
265-
);
260+
if let Some(kp) = kube_port {
261+
port_bindings.insert(
262+
"6443/tcp".to_string(),
263+
Some(vec![PortBinding {
264+
host_ip: Some("0.0.0.0".to_string()),
265+
host_port: Some(kp.to_string()),
266+
}]),
267+
);
268+
}
266269
port_bindings.insert(
267270
"30051/tcp".to_string(),
268271
Some(vec![PortBinding {
269272
host_ip: Some("0.0.0.0".to_string()),
270273
host_port: Some(gateway_port.to_string()),
271274
}]),
272275
);
273-
let exposed_ports = vec!["6443/tcp".to_string(), "30051/tcp".to_string()];
276+
let mut exposed_ports = vec!["30051/tcp".to_string()];
277+
if kube_port.is_some() {
278+
exposed_ports.push("6443/tcp".to_string());
279+
}
274280

275281
let host_config = HostConfig {
276282
privileged: Some(true),

crates/navigator-bootstrap/src/kubeconfig.rs

Lines changed: 44 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -94,26 +94,36 @@ pub fn store_kubeconfig(path: &Path, contents: &str) -> Result<()> {
9494

9595
/// Rewrite kubeconfig for remote deployment.
9696
///
97-
/// The kubeconfig points to 127.0.0.1:6443, which works with an SSH tunnel:
98-
/// `ssh -L 6443:127.0.0.1:6443 user@host`
97+
/// The kubeconfig points to `127.0.0.1:<kube_port>`, which works with an SSH tunnel:
98+
/// `ssh -L <kube_port>:127.0.0.1:6443 user@host`
9999
///
100100
/// The cluster name includes "-remote" suffix to distinguish from local clusters.
101-
pub fn rewrite_kubeconfig_remote(contents: &str, cluster_name: &str, _destination: &str) -> String {
102-
// Use the same rewrite logic but with a remote-specific cluster name
103-
// The server address stays as 127.0.0.1:6443 because users will use SSH tunneling
101+
pub fn rewrite_kubeconfig_remote(
102+
contents: &str,
103+
cluster_name: &str,
104+
_destination: &str,
105+
kube_port: Option<u16>,
106+
) -> String {
104107
let remote_name = format!("{cluster_name}-remote");
105-
rewrite_kubeconfig(contents, &remote_name)
108+
rewrite_kubeconfig(contents, &remote_name, kube_port)
106109
}
107110

108-
pub fn rewrite_kubeconfig(contents: &str, cluster_name: &str) -> String {
111+
/// Rewrite the raw k3s kubeconfig for use on the host.
112+
///
113+
/// When `kube_port` is `Some`, the server URL is rewritten to
114+
/// `https://127.0.0.1:<kube_port>`. When `None`, the original server URL
115+
/// is left intact (the control plane is not exposed on the host).
116+
pub fn rewrite_kubeconfig(contents: &str, cluster_name: &str, kube_port: Option<u16>) -> String {
109117
let mut replaced = Vec::new();
110118
for line in contents.lines() {
111119
let trimmed = line.trim_start();
112-
if trimmed.starts_with("server:") {
113-
let indent_len = line.len() - trimmed.len();
114-
let indent = &line[..indent_len];
115-
replaced.push(format!("{indent}server: https://127.0.0.1:6443"));
116-
continue;
120+
if let Some(kp) = kube_port {
121+
if trimmed.starts_with("server:") {
122+
let indent_len = line.len() - trimmed.len();
123+
let indent = &line[..indent_len];
124+
replaced.push(format!("{indent}server: https://127.0.0.1:{kp}"));
125+
continue;
126+
}
117127
}
118128
// Rename default cluster/context/user to the cluster name
119129
// Handle both "name: default" and "- name: default" (YAML list item)
@@ -237,16 +247,34 @@ mod tests {
237247
use super::rewrite_kubeconfig;
238248

239249
#[test]
240-
fn rewrite_updates_server_address() {
250+
fn rewrite_updates_server_address_with_kube_port() {
241251
let input = "apiVersion: v1\nclusters:\n- cluster:\n server: https://10.0.0.1:6443\n";
242-
let output = rewrite_kubeconfig(input, "test-cluster");
252+
let output = rewrite_kubeconfig(input, "test-cluster", Some(6443));
243253
assert!(output.contains("server: https://127.0.0.1:6443"));
244254
}
245255

256+
#[test]
257+
fn rewrite_updates_server_address_custom_port() {
258+
let input = "apiVersion: v1\nclusters:\n- cluster:\n server: https://10.0.0.1:6443\n";
259+
let output = rewrite_kubeconfig(input, "test-cluster", Some(7443));
260+
assert!(output.contains("server: https://127.0.0.1:7443"));
261+
assert!(!output.contains("server: https://127.0.0.1:6443"));
262+
}
263+
264+
#[test]
265+
fn rewrite_preserves_server_when_no_kube_port() {
266+
let input = "apiVersion: v1\nclusters:\n- cluster:\n server: https://10.0.0.1:6443\n";
267+
let output = rewrite_kubeconfig(input, "test-cluster", None);
268+
assert!(
269+
output.contains("server: https://10.0.0.1:6443"),
270+
"server address should be preserved when kube_port is None"
271+
);
272+
}
273+
246274
#[test]
247275
fn rewrite_preserves_trailing_newline() {
248276
let input = "apiVersion: v1\nserver: https://10.0.0.1\n";
249-
let output = rewrite_kubeconfig(input, "test-cluster");
277+
let output = rewrite_kubeconfig(input, "test-cluster", Some(6443));
250278
assert!(output.ends_with('\n'));
251279
}
252280

@@ -266,7 +294,7 @@ mod tests {
266294
- name: default
267295
current-context: default
268296
";
269-
let output = rewrite_kubeconfig(input, "my-cluster");
297+
let output = rewrite_kubeconfig(input, "my-cluster", Some(6443));
270298
assert!(
271299
output.contains("name: my-cluster"),
272300
"should contain 'name: my-cluster'"

crates/navigator-bootstrap/src/lib.rs

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,15 @@ impl RemoteOptions {
7676
/// Default host port that maps to the k3s `NodePort` (30051) for the gateway.
7777
pub const DEFAULT_GATEWAY_PORT: u16 = 8080;
7878

79+
/// Find a random available TCP port by binding to port 0.
80+
///
81+
/// Binds to `127.0.0.1:0` and returns the OS-assigned port.
82+
pub fn pick_available_port() -> Result<u16> {
83+
let listener = std::net::TcpListener::bind("127.0.0.1:0").into_diagnostic()?;
84+
let port = listener.local_addr().into_diagnostic()?.port();
85+
Ok(port)
86+
}
87+
7988
#[derive(Debug, Clone)]
8089
pub struct DeployOptions {
8190
pub name: String,
@@ -90,6 +99,10 @@ pub struct DeployOptions {
9099
/// Useful in CI where `127.0.0.1` is not reachable from the test runner
91100
/// (e.g., `host.docker.internal`).
92101
pub gateway_host: Option<String>,
102+
/// Host port to expose the k3s Kubernetes control plane on.
103+
/// When `None`, the control plane port (6443) is not exposed on the host,
104+
/// allowing multiple clusters to run simultaneously without port conflicts.
105+
pub kube_port: Option<u16>,
93106
}
94107

95108
impl DeployOptions {
@@ -100,6 +113,7 @@ impl DeployOptions {
100113
remote: None,
101114
port: DEFAULT_GATEWAY_PORT,
102115
gateway_host: None,
116+
kube_port: None,
103117
}
104118
}
105119

@@ -123,6 +137,14 @@ impl DeployOptions {
123137
self.gateway_host = Some(host.into());
124138
self
125139
}
140+
141+
/// Set the host port for the k3s Kubernetes control plane.
142+
/// When set, the control plane is accessible via `kubectl` at this port.
143+
#[must_use]
144+
pub fn with_kube_port(mut self, kube_port: u16) -> Self {
145+
self.kube_port = Some(kube_port);
146+
self
147+
}
126148
}
127149

128150
#[derive(Debug, Clone)]
@@ -184,6 +206,7 @@ where
184206
let image_ref = options.image_ref.unwrap_or_else(default_cluster_image_ref);
185207
let port = options.port;
186208
let gateway_host = options.gateway_host;
209+
let kube_port = options.kube_port;
187210
let kubeconfig_path = stored_kubeconfig_path(&name)?;
188211

189212
// Wrap on_log in Arc<Mutex<>> so we can share it with pull_remote_image
@@ -273,6 +296,7 @@ where
273296
&extra_sans,
274297
ssh_gateway_host.as_deref(),
275298
port,
299+
kube_port,
276300
)
277301
.await?;
278302
log("[status] Starting cluster container".to_string());
@@ -283,8 +307,8 @@ where
283307

284308
// Rewrite kubeconfig based on deployment mode
285309
let rewritten = remote_opts.as_ref().map_or_else(
286-
|| rewrite_kubeconfig(&raw_kubeconfig, &name),
287-
|opts| rewrite_kubeconfig_remote(&raw_kubeconfig, &name, &opts.destination),
310+
|| rewrite_kubeconfig(&raw_kubeconfig, &name, kube_port),
311+
|opts| rewrite_kubeconfig_remote(&raw_kubeconfig, &name, &opts.destination, kube_port),
288312
);
289313
log("[status] Writing kubeconfig".to_string());
290314
store_kubeconfig(&kubeconfig_path, &rewritten)?;
@@ -384,6 +408,7 @@ where
384408
&name,
385409
remote_opts.as_ref(),
386410
port,
411+
kube_port,
387412
ssh_gateway_host.as_deref(),
388413
);
389414
store_cluster_metadata(&name, &metadata)?;
@@ -407,9 +432,9 @@ pub fn cluster_handle(name: &str, remote: Option<&RemoteOptions>) -> Result<Clus
407432
};
408433
let kubeconfig_path = stored_kubeconfig_path(name)?;
409434
// Try to load existing metadata, fall back to creating new metadata
410-
// with the default port (the actual port is only known at deploy time).
435+
// with the default ports (the actual ports are only known at deploy time).
411436
let metadata = load_cluster_metadata(name)
412-
.unwrap_or_else(|_| create_cluster_metadata(name, remote, DEFAULT_GATEWAY_PORT));
437+
.unwrap_or_else(|_| create_cluster_metadata(name, remote, DEFAULT_GATEWAY_PORT, None));
413438
Ok(ClusterHandle {
414439
name: name.to_string(),
415440
kubeconfig_path,

0 commit comments

Comments
 (0)