fix(kubernetes): stage sidecar mtls files before proxy chown

Signed-off-by: Taylor Mutch <taylormutch@gmail.com>

Author: TaylorMutch

.agents/skills/helm-dev-environment/SKILL.md

@@ -65,6 +65,11 @@ mise run helm:skaffold:run
 mise run helm:skaffold:run:sidecar
 ```
 
+**Supervisor sidecar topology with TLS/mTLS enabled** (build once and leave running):
+```bash
+mise run helm:skaffold:run:sidecar-mtls
+```
+
 Both commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm
 chart. The sidecar profile renders an `openshell-network-init` init container for
 nftables setup and a non-root `openshell-supervisor-network` runtime sidecar for
@@ -265,6 +270,7 @@ for dependencies still declared in `Chart.yaml`.
 | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) |
 | `deploy/helm/openshell/ci/values-keycloak.yaml` | Keycloak OIDC overlay |
 | `deploy/helm/openshell/ci/values-sidecar.yaml` | Supervisor sidecar topology overlay for Kubernetes e2e/dev |
+| `deploy/helm/openshell/ci/values-sidecar-mtls.yaml` | Supervisor sidecar topology overlay with built-in TLS/mTLS re-enabled after Skaffold dev values |
 | `deploy/helm/openshell/ci/values-spire.yaml` | SPIFFE/SPIRE provider token grant overlay |
 | `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev |
 | `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) |

crates/openshell-sandbox/src/main.rs

@@ -43,6 +43,16 @@ const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client";
 const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client";
 #[cfg(target_os = "linux")]
 const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"];
+#[cfg(target_os = "linux")]
+const SIDECAR_STATE_DIR_MODE: u32 = 0o2775;
+#[cfg(target_os = "linux")]
+const SIDECAR_TLS_DIR_MODE: u32 = 0o755;
+#[cfg(target_os = "linux")]
+const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700;
+#[cfg(target_os = "linux")]
+const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750;
+#[cfg(target_os = "linux")]
+const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400;
 
 /// Which supervisor leaves are enabled in this process.
 ///
@@ -256,6 +266,35 @@ fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Resu
     Ok(())
 }
 
+#[cfg(target_os = "linux")]
+fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> {
+    use miette::Context as _;
+    use nix::unistd::{Gid, Uid, chown};
+    use std::os::unix::fs::PermissionsExt;
+
+    let uid = Uid::current();
+    let gid = Gid::current();
+    std::fs::create_dir_all(path)
+        .into_diagnostic()
+        .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?;
+    chown(path, Some(uid), Some(gid))
+        .into_diagnostic()
+        .wrap_err_with(|| {
+            format!(
+                "failed to chown sidecar directory {} to {}:{}",
+                path.display(),
+                uid.as_raw(),
+                gid.as_raw()
+            )
+        })?;
+    let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions();
+    perms.set_mode(mode);
+    std::fs::set_permissions(path, perms)
+        .into_diagnostic()
+        .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?;
+    Ok(())
+}
+
 #[cfg(target_os = "linux")]
 fn copy_sidecar_client_tls_if_present(
     source_dir: &Path,
@@ -272,7 +311,7 @@ fn copy_sidecar_client_tls_if_present(
     }
 
     let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR);
-    prepare_sidecar_directory(&dest_dir, uid, gid, 0o750)?;
+    prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?;
     for file_name in CLIENT_TLS_FILES {
         let source = source_dir.join(file_name);
         if !source.exists() {
@@ -282,6 +321,13 @@ fn copy_sidecar_client_tls_if_present(
             ));
         }
         let dest = dest_dir.join(file_name);
+        if dest.exists() {
+            std::fs::remove_file(&dest)
+                .into_diagnostic()
+                .wrap_err_with(|| {
+                    format!("failed to remove stale client TLS file {}", dest.display())
+                })?;
+        }
         std::fs::copy(&source, &dest)
             .into_diagnostic()
             .wrap_err_with(|| {
@@ -292,7 +338,7 @@ fn copy_sidecar_client_tls_if_present(
                 )
             })?;
         let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions();
-        perms.set_mode(0o400);
+        perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE);
         std::fs::set_permissions(&dest, perms)
             .into_diagnostic()
             .wrap_err_with(|| {
@@ -308,6 +354,8 @@ fn copy_sidecar_client_tls_if_present(
             })?;
     }
 
+    prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?;
+
     Ok(())
 }
 
@@ -337,19 +385,23 @@ fn run_network_init(
         sidecar_state_dir,
         proxy_user_id,
         proxy_primary_group_id,
-        0o2775,
+        SIDECAR_STATE_DIR_MODE,
     )?;
-    prepare_sidecar_directory(
+    // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the
+    // TLS work directory owned by the init user until the client cert copy is
+    // complete, then hand it to the long-running proxy UID.
+    prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?;
+    copy_sidecar_client_tls_if_present(
+        Path::new(CLIENT_TLS_DIR),
         sidecar_tls_dir,
         proxy_user_id,
         proxy_primary_group_id,
-        0o755,
     )?;
-    copy_sidecar_client_tls_if_present(
-        Path::new(CLIENT_TLS_DIR),
+    prepare_sidecar_directory(
         sidecar_tls_dir,
         proxy_user_id,
         proxy_primary_group_id,
+        SIDECAR_TLS_DIR_MODE,
     )?;
     openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id)
 }
@@ -623,4 +675,13 @@ mod tests {
         let err = "".parse::<Mode>().unwrap_err();
         assert!(err.contains("at least one"));
     }
+
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() {
+        assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755);
+        assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700);
+        assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750);
+        assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400);
+    }
 }

deploy/helm/openshell/ci/values-sidecar-mtls.yaml

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology with
+# the built-in PKI init job and gateway TLS/mTLS enabled.
+#
+# Merge after ci/values-skaffold.yaml so local image pull policy and developer
+# auth settings remain active while this file restores TLS:
+#   helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-sidecar-mtls.yaml
+#
+# Or use the Skaffold profile:
+#   mise run helm:skaffold:run:sidecar-mtls
+server:
+  disableTls: false
+
+supervisor:
+  topology: sidecar

deploy/helm/openshell/skaffold.yaml

@@ -134,3 +134,8 @@ profiles:
       - op: add
         path: /deploy/helm/releases/0/valuesFiles/-
         value: ci/values-sidecar.yaml
+  - name: sidecar-mtls
+    patches:
+      - op: add
+        path: /deploy/helm/releases/0/valuesFiles/-
+        value: ci/values-sidecar-mtls.yaml

tasks/helm.toml

@@ -60,6 +60,11 @@ description = "Run skaffold dev with the Kubernetes supervisor sidecar topology"
 dir = "deploy/helm/openshell"
 run = "skaffold dev -p sidecar"
 
+["helm:skaffold:dev:sidecar-mtls"]
+description = "Run skaffold dev with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled"
+dir = "deploy/helm/openshell"
+run = "skaffold dev -p sidecar-mtls"
+
 ["helm:skaffold:run"]
 description = "Run skaffold run for deploy/helm/openshell (one-shot deploy)"
 dir = "deploy/helm/openshell"
@@ -70,6 +75,11 @@ description = "Run skaffold run with the Kubernetes supervisor sidecar topology"
 dir = "deploy/helm/openshell"
 run = "skaffold run -p sidecar"
 
+["helm:skaffold:run:sidecar-mtls"]
+description = "Run skaffold run with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled"
+dir = "deploy/helm/openshell"
+run = "skaffold run -p sidecar-mtls"
+
 ["helm:skaffold:delete"]
 description = "Run skaffold delete for deploy/helm/openshell"
 dir = "deploy/helm/openshell"
@@ -80,6 +90,11 @@ description = "Run skaffold delete for the Kubernetes supervisor sidecar topolog
 dir = "deploy/helm/openshell"
 run = "skaffold delete -p sidecar"
 
+["helm:skaffold:delete:sidecar-mtls"]
+description = "Run skaffold delete for the Kubernetes supervisor sidecar topology with TLS/mTLS enabled"
+dir = "deploy/helm/openshell"
+run = "skaffold delete -p sidecar-mtls"
+
 ["helm:skaffold:diagnose"]
 description = "Run skaffold diagnose for deploy/helm/openshell"
 dir = "deploy/helm/openshell"