fix: tasks are started with a cwd in a veiled directory

CHANGELOG.md

@@ -1,5 +1,13 @@
 ## UNRELEASED
 
+BREAKING CHANGES:
+
+* Task working directory is now explicitly set to `$NOMAD_TASK_DIR`. Jobs using relative args (e.g. `args = ["local/run.sh"]`) must switch to absolute paths (e.g. `command = "${NOMAD_TASK_DIR}/run.sh"`). [[GH-97](https://github.com/hashicorp/nomad-driver-exec2/pull/97)]
+
+IMPROVEMENTS:
+
+* Added optional `work_dir` task config field to override the default CWD. Accepts an absolute path or a path relative to the task directory parent. [[GH-97](https://github.com/hashicorp/nomad-driver-exec2/pull/97)]
+
 BUG FIXES:
 
 * Error messages from the `unshare`/`nsenter` shim processes now appear in the allocation logs. [[GH-95](https://github.com/hashicorp/nomad-driver-exec2/pull/95)]

e2e/basic_test.go

@@ -291,6 +291,7 @@ func TestBasic_ProcessNamespace(t *testing.T) {
 	defer purge(t, ctx, "ps")()
 
 	_ = run(t, ctx, "nomad", "job", "run", "./jobs/ps.hcl")
+	wait(t, ctx, "ps")
 
 	logs := logs2(t, ctx, "ps", "ps")
 	lines := strings.Split(logs, "\n") // header + shim + ps

e2e/jobs/java.hcl

@@ -41,7 +41,7 @@ public class Test {
 
       config {
         command = "${var.javabin}/javac"
-        args    = ["-d", "${NOMAD_ALLOC_DIR}", "local/Test.java"]
+        args    = ["-d", "${NOMAD_ALLOC_DIR}", "${NOMAD_TASK_DIR}/Test.java"]
         unveil  = ["r:${var.etcjava}"]
       }
 

pkg/shim/shim.go

@@ -33,6 +33,7 @@ type Options struct {
 	UnveilPaths    []string
 	UnveilDefaults bool
 	OOMScoreAdj    int
+	WorkDir        string // working directory for the task; defaults to TaskDir
 }
 
 // Environment represents runtime configuration.
@@ -42,6 +43,7 @@ type Environment struct {
 	ErrPipe      string            // io pipe path for stderr
 	Env          map[string]string // environment variables
 	TaskDir      string            // task directory
+	WorkDir      string            // working directory for the task; defaults to TaskDir
 	Cgroup       string            // task cgroup path
 	Net          string            // allocation network namespace path
 	Memory       uint64            // memory in megabytes
@@ -293,7 +295,7 @@ func (e *exe) writeCG(file, content string) error {
 	return f.Close()
 }
 
-func flatten(user, home string, env map[string]string) []string {
+func flatten(user, home, workDir string, env map[string]string) []string {
 	result := make([]string, 0, len(env))
 
 	// override and remove some variables
@@ -314,6 +316,13 @@ func flatten(user, home string, env map[string]string) []string {
 	tmp := filepath.Join(parent, "tmp")
 	env["TMPDIR"] = tmp
 
+	// set the working directory; defaults to NOMAD_TASK_DIR when not overridden
+	if workDir != "" {
+		env["NOMAD_WORK_DIR"] = workDir
+	} else {
+		env["NOMAD_WORK_DIR"] = env["NOMAD_TASK_DIR"]
+	}
+
 	// copy environment variables into list form
 	for k, v := range env {
 		switch {
@@ -425,8 +434,13 @@ func (e *exe) prepare(ctx context.Context, home string, fd, uid, gid int) (*exec
 	cmd.Stderr = errfd
 	e.errfd = errfd
 
-	cmd.Env = flatten(e.env.User, home, e.env.Env)
-	cmd.Dir = e.env.TaskDir
+	cmd.Env = flatten(e.env.User, home, e.env.WorkDir, e.env.Env)
+	// use work_dir when set, otherwise fall back to the task directory
+	if e.env.WorkDir != "" {
+		cmd.Dir = e.env.WorkDir
+	} else {
+		cmd.Dir = e.env.TaskDir
+	}
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		UseCgroupFD: true, // clone directly into cgroup
 		CgroupFD:    fd,   // cgroup file descriptor

pkg/shim/z_shim_cmd.go

@@ -95,8 +95,10 @@ func init() {
 		}
 
 		// invoke the task command with its args
-		// the environment has already been set for us by the exec2 driver
+		// the environment has already been set for us by the exec2 driver;
+		// NOMAD_WORK_DIR is set to work_dir if configured, otherwise NOMAD_TASK_DIR
 		cmd := exec.Command(cmdpath, commands[1:]...)
+		cmd.Dir = os.Getenv("NOMAD_WORK_DIR")
 		cmd.Stdout = stdout
 		cmd.Stderr = stderr
 

plugin/about.go

@@ -60,6 +60,7 @@ var taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{
 	"args":          hclspec.NewAttr("args", "list(string)", false),
 	"unveil":        hclspec.NewAttr("unveil", "list(string)", false),
 	"oom_score_adj": hclspec.NewAttr("oom_score_adj", "number", false),
+	"work_dir":      hclspec.NewAttr("work_dir", "string", false),
 })
 
 var capabilities = &drivers.Capabilities{
@@ -91,4 +92,5 @@ type TaskConfig struct {
 	Args        []string `codec:"args"`
 	Unveil      []string `codec:"unveil"`
 	OOMScoreAdj int      `codec:"oom_score_adj"`
+	WorkDir     string   `codec:"work_dir"`
 }

plugin/driver.go

@@ -253,6 +253,7 @@ func (p *Plugin) StartTask(config *drivers.TaskConfig) (*drivers.TaskHandle, *dr
 		ErrPipe:      errPipe,
 		Env:          config.Env,
 		TaskDir:      config.TaskDir().Dir,
+		WorkDir:      opts.WorkDir,
 		User:         config.User,
 		Cgroup:       cgroup,
 		Net:          netns(config),
@@ -326,6 +327,7 @@ func (p *Plugin) RecoverTask(handle *drivers.TaskHandle) error {
 		User:    handle.Config.User,
 		Cgroup:  cgroup,
 	}
+	// WorkDir is not needed for recovery (task is already running)
 
 	taskLogger := p.logger.With(
 		"alloc_id", taskState.TaskConfig.AllocID,
@@ -526,6 +528,15 @@ func (p *Plugin) setOptions(driverTaskConfig *drivers.TaskConfig) (*shim.Options
 		return nil, fmt.Errorf("failed to decode driver task config: %w", err)
 	}
 
+	// if work_dir is set, resolve a relative path against the parent of
+	// NOMAD_TASK_DIR (the task working directory: <alloc>/<task-name>/) so
+	// that job authors can write portable relative paths like "local/subdir"
+	// without needing to know the runtime absolute allocation path.
+	if taskConfig.WorkDir != "" && !filepath.IsAbs(taskConfig.WorkDir) {
+		taskParent := filepath.Dir(driverTaskConfig.Env["NOMAD_TASK_DIR"])
+		taskConfig.WorkDir = filepath.Join(taskParent, taskConfig.WorkDir)
+	}
+
 	// combine paths to unveil from plugin config, task config (if enabled),
 	// and some task/alloc directory default paths
 	unveil := slices.Clone(p.config.UnveilPaths)
@@ -541,6 +552,29 @@ func (p *Plugin) setOptions(driverTaskConfig *drivers.TaskConfig) (*shim.Options
 		unveil = append(unveil, "rwxc:"+filepath.Join(parent, "tmp"))
 	}
 
+	// if work_dir is set, it must be accessible under Landlock — the task
+	// cannot chdir into a veiled directory.
+	//
+	// work_dir that resolves inside the alloc directory is already unveiled by
+	// the defaults block above — no gate is needed and no extra unveil entry
+	// is required. work_dir outside the alloc directory expands the filesystem
+	// surface, so it uses the same unveil_by_task gate as task-specified
+	// unveil paths.
+	//
+	// childEscapesParentDir uses os.OpenRoot so the kernel enforces the
+	// boundary — symlinks pointing outside the alloc root cannot bypass this.
+	if taskConfig.WorkDir != "" {
+		// alloc root is the grandparent of NOMAD_TASK_DIR:
+		// NOMAD_TASK_DIR = <alloc>/<task>/local  →  alloc root = <alloc>
+		allocRoot := filepath.Dir(filepath.Dir(driverTaskConfig.Env["NOMAD_TASK_DIR"]))
+		if err := childEscapesParentDir(allocRoot, taskConfig.WorkDir); err != nil {
+			if !p.config.UnveilByTask {
+				return nil, fmt.Errorf("task set work_dir outside sandbox but driver config does not allow this")
+			}
+			unveil = append(unveil, "rwxc:"+taskConfig.WorkDir)
+		}
+	}
+
 	if len(taskConfig.Unveil) > 0 {
 		if !p.config.UnveilByTask {
 			// if task.config.unveil is set, the plugin config must allow it
@@ -556,5 +590,31 @@ func (p *Plugin) setOptions(driverTaskConfig *drivers.TaskConfig) (*shim.Options
 		UnveilPaths:    unveil,
 		UnveilDefaults: p.config.UnveilDefaults,
 		OOMScoreAdj:    taskConfig.OOMScoreAdj,
+		WorkDir:        taskConfig.WorkDir,
 	}, nil
 }
+
+// childEscapesParentDir reports whether child escapes parent by returning a
+// non-nil error. Uses os.OpenRoot so that the kernel enforces the sandbox
+// boundary, a symlink inside parent that points outside it cannot bypass
+// this check. This mirrors the escapingfs.ChildEscapesParentDir helper
+// introduced in Nomad v2.0.5
+func childEscapesParentDir(parent, child string) error {
+	if filepath.IsAbs(child) {
+		var err error
+		child, err = filepath.Rel(parent, child)
+		if err != nil {
+			return err
+		}
+	}
+	root, err := os.OpenRoot(parent)
+	if err != nil {
+		return err
+	}
+	defer root.Close()
+	_, err = root.Stat(child)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}

plugin/driver_test.go

@@ -153,6 +153,7 @@ func TestFunctional_cases(t *testing.T) {
 		command string
 		args    []string
 		unveil  []string
+		workDir string // relative or absolute path; empty defaults to NOMAD_TASK_DIR
 
 		// plugin config
 		unveilDefaults bool
@@ -439,6 +440,48 @@ func TestFunctional_cases(t *testing.T) {
 			exp:            &drivers.ExitResult{ExitCode: 0},
 			stdoutRe:       regexp.MustCompile(`\w+/tmp/tmp\.\w+`),
 		},
+		// cwd is the task directory (not in a veiled parent path)
+		{
+			name:           "cwd is task dir",
+			user:           "nomad-83000",
+			command:        "sh",
+			args:           []string{"-c", `test "$(pwd)" = "$NOMAD_TASK_DIR"`},
+			unveilDefaults: true,
+			exp:            &drivers.ExitResult{ExitCode: 0},
+		},
+		// work_dir inside sandbox — works without unveil_by_task because the
+		// alloc dir is already unveiled by defaults
+		{
+			name:           "work_dir overrides cwd to alloc dir",
+			user:           "nomad-84000",
+			command:        "sh",
+			args:           []string{"-c", `test "$(pwd)" = "$NOMAD_ALLOC_DIR"`},
+			workDir:        "alloc", // resolves to <alloc>/alloc == NOMAD_ALLOC_DIR
+			unveilDefaults: true,
+			unveilByTask:   false, // no gate needed — inside sandbox
+			exp:            &drivers.ExitResult{ExitCode: 0},
+		},
+		// work_dir inside sandbox — relative path to task dir, no gate needed
+		{
+			name:           "work_dir relative path resolved",
+			user:           "nomad-86000",
+			command:        "sh",
+			args:           []string{"-c", `test "$(pwd)" = "$NOMAD_TASK_DIR"`},
+			workDir:        "local", // resolves to <alloc>/<task>/local == NOMAD_TASK_DIR
+			unveilDefaults: true,
+			unveilByTask:   false, // no gate needed — inside sandbox
+			exp:            &drivers.ExitResult{ExitCode: 0},
+		},
+		// work_dir outside sandbox without unveil_by_task — must be rejected
+		{
+			name:           "work_dir outside sandbox rejected without unveil_by_task",
+			user:           "nomad-85000",
+			command:        "pwd",
+			workDir:        "/tmp", // outside alloc dir — needs gate
+			unveilByTask:   false,
+			unveilDefaults: true,
+			exp:            nil, // StartTask itself returns an error; no exit result
+		},
 	}
 
 	for _, tc := range cases {
@@ -449,12 +492,6 @@ func TestFunctional_cases(t *testing.T) {
 				UnveilPaths:    tc.unveilPaths,
 			}
 
-			taskConfig := &TaskConfig{
-				Command: tc.command,
-				Args:    tc.args,
-				Unveil:  tc.unveil,
-			}
-
 			allocID := uuid.Generate()
 			taskName := "test_cases_" + uuid.Short()
 
@@ -466,15 +503,28 @@ func TestFunctional_cases(t *testing.T) {
 				Resources: basicResources(allocID, taskName),
 			}
 
-			must.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig))
-
 			harness := newTestHarness(t, pluginConfig)
 			harness.MakeTaskCgroup(task.AllocID, task.Name)
 			cleanup := harness.MkAllocDir(task, true)
 			defer cleanup()
 
+			taskConfig := &TaskConfig{
+				Command: tc.command,
+				Args:    tc.args,
+				Unveil:  tc.unveil,
+				WorkDir: tc.workDir,
+			}
+
+			must.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig))
+
 			// Start the task
 			_, _, err := harness.StartTask(task)
+
+			// cases with exp==nil expect StartTask itself to return an error
+			if tc.exp == nil {
+				must.Error(t, err)
+				return
+			}
 			must.NoError(t, err)
 
 			defer func() { _ = harness.DestroyTask(task.ID, true) }()