fix(nvsnap): recreate the source's runtime directories on warm restore

src/compute-plane-services/nvsnap/cmd/nvsnap-rootfs-restore/main.go

@@ -74,6 +74,10 @@ import (
 
 const mergedRoot = "/nvsnap-merged"
 
+// envRuntimeDirs carries the capture's EntryRuntimeDirs as a JSON array.
+// Must match the name the webhook injects.
+const envRuntimeDirs = "NVSNAP_RUNTIME_DIRS"
+
 type volMount struct {
 	Name      string `json:"name"`
 	MountPath string `json:"mountPath"`
@@ -129,6 +133,8 @@ func runNoOverlay() error {
 		fmt.Fprintln(os.Stderr, "nvsnap-rootfs-restore: page-cache prewarm disabled (NVSNAP_PREWARM=0)")
 	}
 
+	recreateRuntimeDirs(os.Getenv, runtimeDirRoots)
+
 	if err := unix.Chdir(cwd); err != nil {
 		if err2 := unix.Chdir("/"); err2 != nil {
 			return fmt.Errorf("chdir %q (and / fallback): %w", cwd, err2)
@@ -145,6 +151,90 @@ func runNoOverlay() error {
 	return nil
 }
 
+// runtimeDirRoots are the only trees the shim will create directories in.
+// Must match the roots the capture side collects from
+// (internal/rootfsonly.runtimeDirRoots).
+var runtimeDirRoots = []string{"/run", "/var/run"}
+
+// underAllowedRoot reports whether p is an absolute, already-clean path at or
+// below one of roots. Requiring the path to be clean is what rejects traversal:
+// "/run/../etc" is not equal to its own Clean(), so it never reaches the
+// prefix check.
+func underAllowedRoot(p string, roots []string) bool {
+	if p == "" || !filepath.IsAbs(p) || filepath.Clean(p) != p {
+		return false
+	}
+	for _, r := range roots {
+		if p == r || strings.HasPrefix(p, r+string(filepath.Separator)) {
+			return true
+		}
+	}
+	return false
+}
+
+// runtimeDir mirrors checkpointstore.EntryRuntimeDir. Declared here rather
+// than imported so the shim stays a standalone static binary the webhook can
+// drop into any workload image.
+type runtimeDir struct {
+	Path string `json:"path"`
+	Mode uint32 `json:"mode"`
+	UID  uint32 `json:"uid"`
+	GID  uint32 `json:"gid"`
+}
+
+// recreateRuntimeDirs recreates the ephemeral directories the source
+// container had before its entrypoint ran.
+//
+// The commands that created them are not recoverable: a container started as
+// `bash -c 'mkdir -p /var/run/vllm; vllm serve ...'` runs the mkdir as a child
+// and then replaces itself with vllm, so /proc/1/cmdline at capture time holds
+// only the engine. Restoring into a pristine container therefore leaves the
+// directory missing, and anything binding a unix socket under it fails with
+// ENOENT (vLLM's ZMQ IPC socket).
+//
+// Best-effort by design: most workloads need none of these, so a directory we
+// cannot create is reported and skipped rather than failing the restore.
+func recreateRuntimeDirs(getenv func(string) string, allowedRoots []string) {
+	raw := getenv(envRuntimeDirs)
+	if raw == "" {
+		return
+	}
+	var dirs []runtimeDir
+	if err := json.Unmarshal([]byte(raw), &dirs); err != nil {
+		fmt.Fprintf(os.Stderr, "nvsnap-rootfs-restore: ignoring malformed %s: %v\n", envRuntimeDirs, err)
+		return
+	}
+	for _, d := range dirs {
+		// Confine to the roots capture collects from, and require an already
+		// clean path. This runs as root in the workload's mount namespace, so
+		// it must not be a general "create any directory" primitive just
+		// because the manifest asked for one.
+		if !underAllowedRoot(d.Path, allowedRoots) {
+			fmt.Fprintf(os.Stderr, "nvsnap-rootfs-restore: refusing runtime dir outside the allowed roots: %q\n", d.Path)
+			continue
+		}
+		// Capture always serializes Mode, so 0 means the source directory
+		// really was 0000 -- reproduce it rather than widening to 0755 and
+		// quietly loosening the source's access policy. MkdirAll still needs a
+		// traversable mode to create intermediate parents, so create with 0755
+		// and narrow to the recorded mode immediately afterwards.
+		mode := os.FileMode(d.Mode).Perm()
+		if err := os.MkdirAll(d.Path, 0o755); err != nil {
+			fmt.Fprintf(os.Stderr, "nvsnap-rootfs-restore: runtime dir %s: %v\n", d.Path, err)
+			continue
+		}
+		// MkdirAll applies the umask, so set the recorded mode explicitly --
+		// a group-writable runtime dir must stay writable for a workload that
+		// drops privileges after start.
+		if err := os.Chmod(d.Path, mode); err != nil {
+			fmt.Fprintf(os.Stderr, "nvsnap-rootfs-restore: chmod %s: %v\n", d.Path, err)
+		}
+		if err := os.Chown(d.Path, int(d.UID), int(d.GID)); err != nil {
+			fmt.Fprintf(os.Stderr, "nvsnap-rootfs-restore: chown %s: %v\n", d.Path, err)
+		}
+	}
+}
+
 func parseConfig(getenv func(string) string) (config, error) {
 	var c config
 	c.capturedDir = getenv("NVSNAP_CAPTURED_DIR")
@@ -374,6 +464,9 @@ func run() error {
 	}
 	_ = os.Remove("/.nvsnap-oldroot")
 
+	// After pivot_root, so the paths resolve inside the restored tree.
+	recreateRuntimeDirs(os.Getenv, runtimeDirRoots)
+
 	// chdir into the captured working directory so the entrypoint's
 	// relative paths resolve as they did pre-capture. Fall back to "/"
 	// if the recorded cwd no longer exists in the merged tree.

src/compute-plane-services/nvsnap/cmd/nvsnap-rootfs-restore/main_test.go

@@ -18,6 +18,8 @@ limitations under the License.
 package main
 
 import (
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 )
@@ -130,3 +132,83 @@ func TestParseMountpoints(t *testing.T) {
 		}
 	}
 }
+
+// TestRecreateRuntimeDirs covers the vLLM ENOENT case: the source container's
+// entrypoint created /var/run/vllm before starting, bash exec'd itself away so
+// the mkdir is absent from the recorded argv, and restore must recreate the
+// directory or the engine's unix socket bind fails.
+func TestRecreateRuntimeDirs(t *testing.T) {
+	sandbox := t.TempDir()
+	target := filepath.Join(sandbox, "vllm")
+
+	recreateRuntimeDirs(envFunc(`[{"path":"`+target+`","mode":493,"uid":0,"gid":0}]`), []string{sandbox})
+
+	fi, err := os.Stat(target)
+	if err != nil {
+		t.Fatalf("runtime dir not recreated: %v", err)
+	}
+	if !fi.IsDir() {
+		t.Fatalf("%s is not a directory", target)
+	}
+	if got := fi.Mode().Perm(); got != 0o755 {
+		t.Errorf("mode = %o, want 755 (umask must not narrow it)", got)
+	}
+}
+
+// A recorded 0000 directory must come back as 0000, not widened to 0755.
+func TestRecreateRuntimeDirsPreservesZeroMode(t *testing.T) {
+	sandbox := t.TempDir()
+	target := filepath.Join(sandbox, "locked")
+
+	recreateRuntimeDirs(envFunc(`[{"path":"`+target+`","mode":0,"uid":0,"gid":0}]`), []string{sandbox})
+
+	fi, err := os.Stat(target)
+	if err != nil {
+		t.Fatalf("dir not created: %v", err)
+	}
+	if got := fi.Mode().Perm(); got != 0 {
+		t.Errorf("mode = %o, want 0 (an explicit 0000 must not be widened)", got)
+	}
+}
+
+// Malformed or hostile input must not abort a restore, and must not create
+// anything. The traversal string is built by concatenation, NOT filepath.Join,
+// because Join normalizes ".." away and would silently make this a clean path
+// that legitimately gets created -- which is exactly how an earlier version of
+// this test passed against a guard that did not actually block traversal.
+func TestRecreateRuntimeDirsIgnoresBadInput(t *testing.T) {
+	sandbox := t.TempDir()
+	escapeTarget := filepath.Join(sandbox, "etc", "pwn")
+	traversal := sandbox + "/run/../etc/pwn"
+	outside := filepath.Join(t.TempDir(), "elsewhere")
+
+	for name, val := range map[string]string{
+		"empty":          "",
+		"not json":       "{{{",
+		"wrong type":     `{"path":"/x"}`,
+		"relative":       `[{"path":"var/run/x","mode":493}]`,
+		"parent escape":  `[{"path":"` + traversal + `","mode":493}]`,
+		"outside root":   `[{"path":"` + outside + `","mode":493}]`,
+		"root itself":    `[{"path":"/","mode":493}]`,
+		"prefix look-al": `[{"path":"` + sandbox + `-evil/x","mode":493}]`,
+	} {
+		t.Run(name, func(t *testing.T) {
+			recreateRuntimeDirs(envFunc(val), []string{sandbox})
+		})
+	}
+
+	for _, p := range []string{escapeTarget, outside, sandbox + "-evil/x"} {
+		if _, err := os.Stat(p); !os.IsNotExist(err) {
+			t.Errorf("created %s (err=%v); the path guard regressed", p, err)
+		}
+	}
+}
+
+func envFunc(val string) func(string) string {
+	return func(k string) string {
+		if k == envRuntimeDirs {
+			return val
+		}
+		return ""
+	}
+}

src/compute-plane-services/nvsnap/internal/checkpointstore/store.go

@@ -35,7 +35,12 @@ import (
 // CaptureFormatVersion is bumped whenever the on-disk schema for a capture
 // changes (manifest format, layout, included metadata). Hashes are recomputed
 // across versions, so old captures stop matching.
-const CaptureFormatVersion = 1
+// 2: added EntryRuntimeDirs. A capture taken before this has no recorded
+// runtime directories, so restoring it cannot recreate them and workloads that
+// need one still fail. Without the bump those captures hash identically to new
+// ones and would be reused forever after an upgrade -- silently, since the
+// agent reports the reuse as a successful capture.
+const CaptureFormatVersion = 2
 
 // ErrNotFound is returned by Stat / Get when no capture is stored under the
 // given hash.
@@ -258,6 +263,34 @@ type Manifest struct {
 	// time. The shim chdir()s here before exec so relative paths in the
 	// entrypoint resolve as they did pre-capture. Empty → "/".
 	EntryCwd string `json:"entry_cwd,omitempty"`
+
+	// EntryRuntimeDirs are directories that existed under the source
+	// container's ephemeral runtime roots (/run, /var/run) at capture time.
+	//
+	// EntryArgv is /proc/<pid>/cmdline, which is the process image AFTER any
+	// exec. A container started as `bash -c 'mkdir -p /var/run/vllm; vllm
+	// serve ...'` runs the mkdir as a child and then, via bash's last-command
+	// exec optimization, REPLACES itself with vllm -- so by capture time the
+	// mkdir is no longer anywhere in the process image and cannot be
+	// recovered from argv. Restore then execs into a pristine container where
+	// the directory does not exist, and anything binding a unix socket there
+	// fails with ENOENT (vLLM's ZMQ IPC socket, observed).
+	//
+	// We cannot replay the setup commands (they are gone), and we cannot
+	// prefer the Pod's command/args instead -- ENTRYPOINT-only images carry
+	// only args, or nothing, in the Pod spec. So we record the directories
+	// themselves and recreate them before exec.
+	EntryRuntimeDirs []EntryRuntimeDir `json:"entry_runtime_dirs,omitempty"`
+}
+
+// EntryRuntimeDir is one directory to recreate in the restored container
+// before the entrypoint is exec'd. Mode and ownership are carried so a
+// workload running as a non-root UID can still write inside it.
+type EntryRuntimeDir struct {
+	Path string `json:"path"`
+	Mode uint32 `json:"mode"`
+	UID  uint32 `json:"uid"`
+	GID  uint32 `json:"gid"`
 }
 
 // VolumeMeta is a single captured volume's metadata. Volume.Name == "rootfs"

src/compute-plane-services/nvsnap/internal/rootfsonly/BUILD.bazel

@@ -35,6 +35,7 @@ go_test(
         "classify_test.go",
         "composer_test.go",
         "entry_argv_test.go",
+        "entry_runtime_dirs_test.go",
         "enumerate_test.go",
         "orchestrator_test.go",
         "pidresolver_test.go",