From 95766a382dfe20607c226ee4faa9bc0574b15ae7 Mon Sep 17 00:00:00 2001 From: Ritesh Harihar Date: Fri, 7 Aug 2026 11:08:05 +0530 Subject: [PATCH 1/5] inject GOMAXPROCS from CPU bandwidth to prevent Go runtime over-threading --- plugin/driver.go | 17 +++++++++++++++++ plugin/driver_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/plugin/driver.go b/plugin/driver.go index e045031..fea99c9 100644 --- a/plugin/driver.go +++ b/plugin/driver.go @@ -11,6 +11,7 @@ import ( "os/exec" "path/filepath" "slices" + "strconv" "time" "github.com/hashicorp/go-hclog" @@ -229,6 +230,22 @@ func (p *Plugin) StartTask(config *drivers.TaskConfig) (*drivers.TaskHandle, *dr cpuset := config.Resources.LinuxResources.CpusetCpus p.logger.Trace("resources", "memory", memory, "memory_max", memoryMax, "compute", bandwidth, "cpuset", cpuset) + // compute and inject GOMAXPROCS so Go workloads see the correct parallelism + // limit rather than defaulting to the total host CPU count. The Go runtime + // cannot reliably auto-detect this inside an exec2 task because Landlock + // restricts access to /sys/fs/cgroup and /sys/devices/system/cpu by default. + // + // bandwidth is the cgroup cpu.max quota in microseconds per 100_000 µs period. + // Ceiling integer division by the period gives the whole-core equivalent: + // cores=N tasks: bandwidth = N × 100_000 → result = N (exact) + // cpu=M MHz tasks: bandwidth = M×100_000/per_core_MHz → result = ceil(fraction) + // + // Only inject if the operator has not already set it explicitly. + if _, alreadySet := config.Env["GOMAXPROCS"]; !alreadySet { + gomaxprocs := max(1, int((bandwidth+99_999)/100_000)) + config.Env["GOMAXPROCS"] = strconv.Itoa(gomaxprocs) + } + // with cgroups v2 this is just the task cgroup cgroup := config.Resources.LinuxResources.CpusetCgroupPath diff --git a/plugin/driver_test.go b/plugin/driver_test.go index 482528a..2feb4c9 100644 --- a/plugin/driver_test.go +++ b/plugin/driver_test.go @@ -152,6 +152,7 @@ func TestFunctional_cases(t *testing.T) { user string command string args []string + env map[string]string unveil []string // plugin config @@ -439,6 +440,28 @@ func TestFunctional_cases(t *testing.T) { exp: &drivers.ExitResult{ExitCode: 0}, stdoutRe: regexp.MustCompile(`\w+/tmp/tmp\.\w+`), }, + // GOMAXPROCS is injected from CPU bandwidth allocation so Go workloads + // see the correct parallelism limit rather than the host CPU count. + { + name: "GOMAXPROCS injected from bandwidth", + user: "root", + command: "printenv", + args: []string{"GOMAXPROCS"}, + unveilDefaults: true, + exp: &drivers.ExitResult{ExitCode: 0}, + stdoutRe: regexp.MustCompile(`^1$`), + }, + // Operator-supplied GOMAXPROCS must not be overwritten by the driver. + { + name: "GOMAXPROCS operator override preserved", + user: "root", + command: "printenv", + args: []string{"GOMAXPROCS"}, + env: map[string]string{"GOMAXPROCS": "8"}, + unveilDefaults: true, + exp: &drivers.ExitResult{ExitCode: 0}, + stdoutRe: regexp.MustCompile(`^8$`), + }, } for _, tc := range cases { @@ -458,11 +481,16 @@ func TestFunctional_cases(t *testing.T) { allocID := uuid.Generate() taskName := "test_cases_" + uuid.Short() + taskEnv := tc.env + if taskEnv == nil { + taskEnv = map[string]string{} + } task := &drivers.TaskConfig{ User: tc.user, ID: uuid.Generate(), Name: taskName, AllocID: allocID, + Env: taskEnv, Resources: basicResources(allocID, taskName), } From ba313875b5a6827d522a9d5bd112519e527c9c8c Mon Sep 17 00:00:00 2001 From: Ritesh Harihar Date: Fri, 7 Aug 2026 13:16:18 +0530 Subject: [PATCH 2/5] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5b570e..a67842f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ BUG FIXES: +* Fixed `GOMAXPROCS` to prevent Go workloads from being over-threaded against the host CPU capacity. [[GH-98](https://github.com/hashicorp/nomad-driver-exec2/pull/98)] * Error messages from the `unshare`/`nsenter` shim processes now appear in the allocation logs. [[GH-95](https://github.com/hashicorp/nomad-driver-exec2/pull/95)] ## 0.1.2 (May 12, 2026) From 05fd92b3b6657a41a7cf8f6f2003253bd5e04f71 Mon Sep 17 00:00:00 2001 From: Ritesh Harihar Date: Wed, 12 Aug 2026 02:11:25 +0530 Subject: [PATCH 3/5] added task's cgroup in unvielDefaults --- pkg/shim/sandbox.go | 5 +++++ plugin/driver.go | 12 +++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/shim/sandbox.go b/pkg/shim/sandbox.go index 73f05a9..30f0de7 100644 --- a/pkg/shim/sandbox.go +++ b/pkg/shim/sandbox.go @@ -52,6 +52,11 @@ func lockdown(defaults bool, elements []string) error { landlock.Dir("/usr/bin", "rx"), landlock.Dir("/usr/local/bin", "rx"), ) + // expose /proc read-only so runtimes (Go 1.25+, JVM, dotnet) can read + // /proc/self/cgroup and /proc/self/mountinfo to discover their cgroup + // CPU and memory limits. unshare --mount-proc creates + // an isolated /proc scoped to the task's PID namespace. + paths = append(paths, landlock.Dir("/proc", "r")) } return landlock.New(paths...).Lock(landlock.Mandatory) diff --git a/plugin/driver.go b/plugin/driver.go index fea99c9..b44c9aa 100644 --- a/plugin/driver.go +++ b/plugin/driver.go @@ -231,9 +231,13 @@ func (p *Plugin) StartTask(config *drivers.TaskConfig) (*drivers.TaskHandle, *dr p.logger.Trace("resources", "memory", memory, "memory_max", memoryMax, "compute", bandwidth, "cpuset", cpuset) // compute and inject GOMAXPROCS so Go workloads see the correct parallelism - // limit rather than defaulting to the total host CPU count. The Go runtime - // cannot reliably auto-detect this inside an exec2 task because Landlock - // restricts access to /sys/fs/cgroup and /sys/devices/system/cpu by default. + // limit rather than defaulting to the total host CPU count. + // + // Go 1.25+ can read cpu.max from the task's own cgroup scope (now always + // unveiled read-only via setOptions) and auto-detect correctly. However Go + // 1.24 and below never read cgroup at all — they fall back to + // sched_getaffinity() which returns all host cores for cpu=N MHz tasks. + // The env var injection covers all Go versions uniformly. // // bandwidth is the cgroup cpu.max quota in microseconds per 100_000 µs period. // Ceiling integer division by the period gives the whole-core equivalent: @@ -550,6 +554,8 @@ func (p *Plugin) setOptions(driverTaskConfig *drivers.TaskConfig) (*shim.Options // if the plugin config.unveil_defaults value is set to true (very common) // then automatically unveil the sandbox directories if p.config.UnveilDefaults { + // Expose the task's cgroup read-only so runtimes can read resource limits. + unveil = append(unveil, "r:"+driverTaskConfig.Resources.LinuxResources.CpusetCgroupPath) unveil = append(unveil, "rwxc:"+driverTaskConfig.Env["NOMAD_TASK_DIR"]) unveil = append(unveil, "rwxc:"+driverTaskConfig.Env["NOMAD_ALLOC_DIR"]) unveil = append(unveil, "rx:"+driverTaskConfig.Env["NOMAD_ALLOC_DIR"]+"/logs") From f15f0c8e47c78231ce2d149362102146b646b5a5 Mon Sep 17 00:00:00 2001 From: Ritesh Harihar Date: Wed, 12 Aug 2026 18:56:18 +0530 Subject: [PATCH 4/5] removed env-var fix --- plugin/driver.go | 21 ------------------- plugin/driver_test.go | 47 +++++++++---------------------------------- 2 files changed, 10 insertions(+), 58 deletions(-) diff --git a/plugin/driver.go b/plugin/driver.go index b44c9aa..d30cbe1 100644 --- a/plugin/driver.go +++ b/plugin/driver.go @@ -11,7 +11,6 @@ import ( "os/exec" "path/filepath" "slices" - "strconv" "time" "github.com/hashicorp/go-hclog" @@ -230,26 +229,6 @@ func (p *Plugin) StartTask(config *drivers.TaskConfig) (*drivers.TaskHandle, *dr cpuset := config.Resources.LinuxResources.CpusetCpus p.logger.Trace("resources", "memory", memory, "memory_max", memoryMax, "compute", bandwidth, "cpuset", cpuset) - // compute and inject GOMAXPROCS so Go workloads see the correct parallelism - // limit rather than defaulting to the total host CPU count. - // - // Go 1.25+ can read cpu.max from the task's own cgroup scope (now always - // unveiled read-only via setOptions) and auto-detect correctly. However Go - // 1.24 and below never read cgroup at all — they fall back to - // sched_getaffinity() which returns all host cores for cpu=N MHz tasks. - // The env var injection covers all Go versions uniformly. - // - // bandwidth is the cgroup cpu.max quota in microseconds per 100_000 µs period. - // Ceiling integer division by the period gives the whole-core equivalent: - // cores=N tasks: bandwidth = N × 100_000 → result = N (exact) - // cpu=M MHz tasks: bandwidth = M×100_000/per_core_MHz → result = ceil(fraction) - // - // Only inject if the operator has not already set it explicitly. - if _, alreadySet := config.Env["GOMAXPROCS"]; !alreadySet { - gomaxprocs := max(1, int((bandwidth+99_999)/100_000)) - config.Env["GOMAXPROCS"] = strconv.Itoa(gomaxprocs) - } - // with cgroups v2 this is just the task cgroup cgroup := config.Resources.LinuxResources.CpusetCgroupPath diff --git a/plugin/driver_test.go b/plugin/driver_test.go index 2feb4c9..5f41fe7 100644 --- a/plugin/driver_test.go +++ b/plugin/driver_test.go @@ -152,7 +152,6 @@ func TestFunctional_cases(t *testing.T) { user string command string args []string - env map[string]string unveil []string // plugin config @@ -440,28 +439,6 @@ func TestFunctional_cases(t *testing.T) { exp: &drivers.ExitResult{ExitCode: 0}, stdoutRe: regexp.MustCompile(`\w+/tmp/tmp\.\w+`), }, - // GOMAXPROCS is injected from CPU bandwidth allocation so Go workloads - // see the correct parallelism limit rather than the host CPU count. - { - name: "GOMAXPROCS injected from bandwidth", - user: "root", - command: "printenv", - args: []string{"GOMAXPROCS"}, - unveilDefaults: true, - exp: &drivers.ExitResult{ExitCode: 0}, - stdoutRe: regexp.MustCompile(`^1$`), - }, - // Operator-supplied GOMAXPROCS must not be overwritten by the driver. - { - name: "GOMAXPROCS operator override preserved", - user: "root", - command: "printenv", - args: []string{"GOMAXPROCS"}, - env: map[string]string{"GOMAXPROCS": "8"}, - unveilDefaults: true, - exp: &drivers.ExitResult{ExitCode: 0}, - stdoutRe: regexp.MustCompile(`^8$`), - }, } for _, tc := range cases { @@ -479,20 +456,16 @@ func TestFunctional_cases(t *testing.T) { } allocID := uuid.Generate() - taskName := "test_cases_" + uuid.Short() - - taskEnv := tc.env - if taskEnv == nil { - taskEnv = map[string]string{} - } - task := &drivers.TaskConfig{ - User: tc.user, - ID: uuid.Generate(), - Name: taskName, - AllocID: allocID, - Env: taskEnv, - Resources: basicResources(allocID, taskName), - } + taskName := "test_cases_" + uuid.Short() + + task := &drivers.TaskConfig{ + User: tc.user, + ID: uuid.Generate(), + Name: taskName, + AllocID: allocID, + Env: map[string]string{}, + Resources: basicResources(allocID, taskName), + } must.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig)) From 036be3a7032d6f4a65f02fda33c045c9fd02e41a Mon Sep 17 00:00:00 2001 From: Ritesh Harihar Date: Wed, 12 Aug 2026 19:07:02 +0530 Subject: [PATCH 5/5] updated tests --- plugin/driver_test.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/plugin/driver_test.go b/plugin/driver_test.go index 5f41fe7..482528a 100644 --- a/plugin/driver_test.go +++ b/plugin/driver_test.go @@ -456,16 +456,15 @@ func TestFunctional_cases(t *testing.T) { } allocID := uuid.Generate() - taskName := "test_cases_" + uuid.Short() - - task := &drivers.TaskConfig{ - User: tc.user, - ID: uuid.Generate(), - Name: taskName, - AllocID: allocID, - Env: map[string]string{}, - Resources: basicResources(allocID, taskName), - } + taskName := "test_cases_" + uuid.Short() + + task := &drivers.TaskConfig{ + User: tc.user, + ID: uuid.Generate(), + Name: taskName, + AllocID: allocID, + Resources: basicResources(allocID, taskName), + } must.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig))