diff --git a/CHANGELOG.md b/CHANGELOG.md index d5b570e..f0a03ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## UNRELEASED +BREAKING CHANGES: + +* Task working directory is now explicitly set to `$NOMAD_TASK_DIR`. Jobs using relative args (e.g. `args = ["local/run.sh"]`) must switch to absolute paths (e.g. `command = "${NOMAD_TASK_DIR}/run.sh"`). [[GH-97](https://github.com/hashicorp/nomad-driver-exec2/pull/97)] + +IMPROVEMENTS: + +* Added optional `work_dir` task config field to override the default CWD. Accepts an absolute path or a path relative to the task directory parent. [[GH-97](https://github.com/hashicorp/nomad-driver-exec2/pull/97)] + BUG FIXES: * Error messages from the `unshare`/`nsenter` shim processes now appear in the allocation logs. [[GH-95](https://github.com/hashicorp/nomad-driver-exec2/pull/95)] diff --git a/e2e/basic_test.go b/e2e/basic_test.go index fdb7a85..8d06cae 100644 --- a/e2e/basic_test.go +++ b/e2e/basic_test.go @@ -291,6 +291,7 @@ func TestBasic_ProcessNamespace(t *testing.T) { defer purge(t, ctx, "ps")() _ = run(t, ctx, "nomad", "job", "run", "./jobs/ps.hcl") + wait(t, ctx, "ps") logs := logs2(t, ctx, "ps", "ps") lines := strings.Split(logs, "\n") // header + shim + ps diff --git a/e2e/jobs/java.hcl b/e2e/jobs/java.hcl index 7a88777..476ee2b 100644 --- a/e2e/jobs/java.hcl +++ b/e2e/jobs/java.hcl @@ -41,7 +41,7 @@ public class Test { config { command = "${var.javabin}/javac" - args = ["-d", "${NOMAD_ALLOC_DIR}", "local/Test.java"] + args = ["-d", "${NOMAD_ALLOC_DIR}", "${NOMAD_TASK_DIR}/Test.java"] unveil = ["r:${var.etcjava}"] } diff --git a/pkg/shim/shim.go b/pkg/shim/shim.go index 7d94d4e..281d05c 100644 --- a/pkg/shim/shim.go +++ b/pkg/shim/shim.go @@ -33,6 +33,7 @@ type Options struct { UnveilPaths []string UnveilDefaults bool OOMScoreAdj int + WorkDir string // working directory for the task; defaults to TaskDir } // Environment represents runtime configuration. @@ -42,6 +43,7 @@ type Environment struct { ErrPipe string // io pipe path for stderr Env map[string]string // environment variables TaskDir string // task directory + WorkDir string // working directory for the task; defaults to TaskDir Cgroup string // task cgroup path Net string // allocation network namespace path Memory uint64 // memory in megabytes @@ -293,7 +295,7 @@ func (e *exe) writeCG(file, content string) error { return f.Close() } -func flatten(user, home string, env map[string]string) []string { +func flatten(user, home, workDir string, env map[string]string) []string { result := make([]string, 0, len(env)) // override and remove some variables @@ -314,6 +316,13 @@ func flatten(user, home string, env map[string]string) []string { tmp := filepath.Join(parent, "tmp") env["TMPDIR"] = tmp + // set the working directory; defaults to NOMAD_TASK_DIR when not overridden + if workDir != "" { + env["NOMAD_WORK_DIR"] = workDir + } else { + env["NOMAD_WORK_DIR"] = env["NOMAD_TASK_DIR"] + } + // copy environment variables into list form for k, v := range env { switch { @@ -425,8 +434,13 @@ func (e *exe) prepare(ctx context.Context, home string, fd, uid, gid int) (*exec cmd.Stderr = errfd e.errfd = errfd - cmd.Env = flatten(e.env.User, home, e.env.Env) - cmd.Dir = e.env.TaskDir + cmd.Env = flatten(e.env.User, home, e.env.WorkDir, e.env.Env) + // use work_dir when set, otherwise fall back to the task directory + if e.env.WorkDir != "" { + cmd.Dir = e.env.WorkDir + } else { + cmd.Dir = e.env.TaskDir + } cmd.SysProcAttr = &syscall.SysProcAttr{ UseCgroupFD: true, // clone directly into cgroup CgroupFD: fd, // cgroup file descriptor diff --git a/pkg/shim/z_shim_cmd.go b/pkg/shim/z_shim_cmd.go index b2faf57..c09baff 100644 --- a/pkg/shim/z_shim_cmd.go +++ b/pkg/shim/z_shim_cmd.go @@ -95,8 +95,10 @@ func init() { } // invoke the task command with its args - // the environment has already been set for us by the exec2 driver + // the environment has already been set for us by the exec2 driver; + // NOMAD_WORK_DIR is set to work_dir if configured, otherwise NOMAD_TASK_DIR cmd := exec.Command(cmdpath, commands[1:]...) + cmd.Dir = os.Getenv("NOMAD_WORK_DIR") cmd.Stdout = stdout cmd.Stderr = stderr diff --git a/plugin/about.go b/plugin/about.go index ef64792..52ccffa 100644 --- a/plugin/about.go +++ b/plugin/about.go @@ -60,6 +60,7 @@ var taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{ "args": hclspec.NewAttr("args", "list(string)", false), "unveil": hclspec.NewAttr("unveil", "list(string)", false), "oom_score_adj": hclspec.NewAttr("oom_score_adj", "number", false), + "work_dir": hclspec.NewAttr("work_dir", "string", false), }) var capabilities = &drivers.Capabilities{ @@ -91,4 +92,5 @@ type TaskConfig struct { Args []string `codec:"args"` Unveil []string `codec:"unveil"` OOMScoreAdj int `codec:"oom_score_adj"` + WorkDir string `codec:"work_dir"` } diff --git a/plugin/driver.go b/plugin/driver.go index e045031..5cd9de5 100644 --- a/plugin/driver.go +++ b/plugin/driver.go @@ -253,6 +253,7 @@ func (p *Plugin) StartTask(config *drivers.TaskConfig) (*drivers.TaskHandle, *dr ErrPipe: errPipe, Env: config.Env, TaskDir: config.TaskDir().Dir, + WorkDir: opts.WorkDir, User: config.User, Cgroup: cgroup, Net: netns(config), @@ -326,6 +327,7 @@ func (p *Plugin) RecoverTask(handle *drivers.TaskHandle) error { User: handle.Config.User, Cgroup: cgroup, } + // WorkDir is not needed for recovery (task is already running) taskLogger := p.logger.With( "alloc_id", taskState.TaskConfig.AllocID, @@ -526,6 +528,15 @@ func (p *Plugin) setOptions(driverTaskConfig *drivers.TaskConfig) (*shim.Options return nil, fmt.Errorf("failed to decode driver task config: %w", err) } + // if work_dir is set, resolve a relative path against the parent of + // NOMAD_TASK_DIR (the task working directory: //) so + // that job authors can write portable relative paths like "local/subdir" + // without needing to know the runtime absolute allocation path. + if taskConfig.WorkDir != "" && !filepath.IsAbs(taskConfig.WorkDir) { + taskParent := filepath.Dir(driverTaskConfig.Env["NOMAD_TASK_DIR"]) + taskConfig.WorkDir = filepath.Join(taskParent, taskConfig.WorkDir) + } + // combine paths to unveil from plugin config, task config (if enabled), // and some task/alloc directory default paths unveil := slices.Clone(p.config.UnveilPaths) @@ -541,6 +552,29 @@ func (p *Plugin) setOptions(driverTaskConfig *drivers.TaskConfig) (*shim.Options unveil = append(unveil, "rwxc:"+filepath.Join(parent, "tmp")) } + // if work_dir is set, it must be accessible under Landlock — the task + // cannot chdir into a veiled directory. + // + // work_dir that resolves inside the alloc directory is already unveiled by + // the defaults block above — no gate is needed and no extra unveil entry + // is required. work_dir outside the alloc directory expands the filesystem + // surface, so it uses the same unveil_by_task gate as task-specified + // unveil paths. + // + // childEscapesParentDir uses os.OpenRoot so the kernel enforces the + // boundary — symlinks pointing outside the alloc root cannot bypass this. + if taskConfig.WorkDir != "" { + // alloc root is the grandparent of NOMAD_TASK_DIR: + // NOMAD_TASK_DIR = //local → alloc root = + allocRoot := filepath.Dir(filepath.Dir(driverTaskConfig.Env["NOMAD_TASK_DIR"])) + if err := childEscapesParentDir(allocRoot, taskConfig.WorkDir); err != nil { + if !p.config.UnveilByTask { + return nil, fmt.Errorf("task set work_dir outside sandbox but driver config does not allow this") + } + unveil = append(unveil, "rwxc:"+taskConfig.WorkDir) + } + } + if len(taskConfig.Unveil) > 0 { if !p.config.UnveilByTask { // if task.config.unveil is set, the plugin config must allow it @@ -556,5 +590,31 @@ func (p *Plugin) setOptions(driverTaskConfig *drivers.TaskConfig) (*shim.Options UnveilPaths: unveil, UnveilDefaults: p.config.UnveilDefaults, OOMScoreAdj: taskConfig.OOMScoreAdj, + WorkDir: taskConfig.WorkDir, }, nil } + +// childEscapesParentDir reports whether child escapes parent by returning a +// non-nil error. Uses os.OpenRoot so that the kernel enforces the sandbox +// boundary, a symlink inside parent that points outside it cannot bypass +// this check. This mirrors the escapingfs.ChildEscapesParentDir helper +// introduced in Nomad v2.0.5 +func childEscapesParentDir(parent, child string) error { + if filepath.IsAbs(child) { + var err error + child, err = filepath.Rel(parent, child) + if err != nil { + return err + } + } + root, err := os.OpenRoot(parent) + if err != nil { + return err + } + defer root.Close() + _, err = root.Stat(child) + if err != nil && !os.IsNotExist(err) { + return err + } + return nil +} diff --git a/plugin/driver_test.go b/plugin/driver_test.go index 482528a..0dcdc08 100644 --- a/plugin/driver_test.go +++ b/plugin/driver_test.go @@ -153,6 +153,7 @@ func TestFunctional_cases(t *testing.T) { command string args []string unveil []string + workDir string // relative or absolute path; empty defaults to NOMAD_TASK_DIR // plugin config unveilDefaults bool @@ -439,6 +440,48 @@ func TestFunctional_cases(t *testing.T) { exp: &drivers.ExitResult{ExitCode: 0}, stdoutRe: regexp.MustCompile(`\w+/tmp/tmp\.\w+`), }, + // cwd is the task directory (not in a veiled parent path) + { + name: "cwd is task dir", + user: "nomad-83000", + command: "sh", + args: []string{"-c", `test "$(pwd)" = "$NOMAD_TASK_DIR"`}, + unveilDefaults: true, + exp: &drivers.ExitResult{ExitCode: 0}, + }, + // work_dir inside sandbox — works without unveil_by_task because the + // alloc dir is already unveiled by defaults + { + name: "work_dir overrides cwd to alloc dir", + user: "nomad-84000", + command: "sh", + args: []string{"-c", `test "$(pwd)" = "$NOMAD_ALLOC_DIR"`}, + workDir: "alloc", // resolves to /alloc == NOMAD_ALLOC_DIR + unveilDefaults: true, + unveilByTask: false, // no gate needed — inside sandbox + exp: &drivers.ExitResult{ExitCode: 0}, + }, + // work_dir inside sandbox — relative path to task dir, no gate needed + { + name: "work_dir relative path resolved", + user: "nomad-86000", + command: "sh", + args: []string{"-c", `test "$(pwd)" = "$NOMAD_TASK_DIR"`}, + workDir: "local", // resolves to //local == NOMAD_TASK_DIR + unveilDefaults: true, + unveilByTask: false, // no gate needed — inside sandbox + exp: &drivers.ExitResult{ExitCode: 0}, + }, + // work_dir outside sandbox without unveil_by_task — must be rejected + { + name: "work_dir outside sandbox rejected without unveil_by_task", + user: "nomad-85000", + command: "pwd", + workDir: "/tmp", // outside alloc dir — needs gate + unveilByTask: false, + unveilDefaults: true, + exp: nil, // StartTask itself returns an error; no exit result + }, } for _, tc := range cases { @@ -449,12 +492,6 @@ func TestFunctional_cases(t *testing.T) { UnveilPaths: tc.unveilPaths, } - taskConfig := &TaskConfig{ - Command: tc.command, - Args: tc.args, - Unveil: tc.unveil, - } - allocID := uuid.Generate() taskName := "test_cases_" + uuid.Short() @@ -466,15 +503,28 @@ func TestFunctional_cases(t *testing.T) { Resources: basicResources(allocID, taskName), } - must.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig)) - harness := newTestHarness(t, pluginConfig) harness.MakeTaskCgroup(task.AllocID, task.Name) cleanup := harness.MkAllocDir(task, true) defer cleanup() + taskConfig := &TaskConfig{ + Command: tc.command, + Args: tc.args, + Unveil: tc.unveil, + WorkDir: tc.workDir, + } + + must.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig)) + // Start the task _, _, err := harness.StartTask(task) + + // cases with exp==nil expect StartTask itself to return an error + if tc.exp == nil { + must.Error(t, err) + return + } must.NoError(t, err) defer func() { _ = harness.DestroyTask(task.ID, true) }()