operator: add gpu plugin's by-path option

Signed-off-by: Tuomas Katila <[email protected]>
Co-authored-by: Eero Tamminen <[email protected]>

Author: tkatila

cmd/gpu_plugin/README.md

@@ -275,8 +275,8 @@ To support possible all use cases, GPU plugin allows changing the by-path mounti
 * `single` - Symlinks are individually mounted per device. Default.
   * Mostly Works, but is known to have issues with some pytorch workloads. See [issue](https://github.com/intel/intel-device-plugins-for-kubernetes/issues/2158).
 * `none` - No symlinks are mounted.
-  * Aligned with docker use where devices are included with privileged mode.
-* `all` - All symlinks are mounted even if only one is allocated by the container.
+  * Aligned with Docker `privileged` mode devices usage.
+* `all` - Mounts whole DRM `by-path` directory.  Pro: symlink file types are preserved.  Con: symlinks are present for all devices.
   * Optimal for scale-up workloads where all the GPUs are used by the workload.
 
 ### Issues with media workloads on multi-GPU setups

cmd/gpu_plugin/gpu_plugin.go

@@ -811,7 +811,7 @@ func main() {
 	flag.StringVar(&prefix, "prefix", "", "Prefix for devfs & sysfs paths")
 	flag.BoolVar(&opts.enableMonitoring, "enable-monitoring", false, "whether to enable '*_monitoring' (= all GPUs) resource")
 	flag.BoolVar(&opts.healthManagement, "health-management", false, "enable GPU health management")
-	flag.StringVar(&opts.bypathMount, "bypath", bypathOptionSingle, "bypath mounting options: single, none, all. Default: single")
+	flag.StringVar(&opts.bypathMount, "bypath", bypathOptionSingle, "DRI device 'by-path/' directory mounting options: single, none, all. Default: single")
 	flag.BoolVar(&opts.wslScan, "wsl", false, "scan for / use WSL devices")
 	flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device")
 	flag.IntVar(&opts.globalTempLimit, "temp-limit", 100, "Global temperature limit at which device is marked unhealthy")

deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml

@@ -62,6 +62,16 @@ spec:
                   The list can contain IDs in the form of '0x1234,0x49a4,0x50b4'.
                   Cannot be used together with DenyIDs.
                 type: string
+              bypathMode:
+                description: |-
+                  ByPathMode changes how plugin handles the DRM by-path/ directory mounting for GPU devices.
+                  See GPU plugin documentation for detailed description of the modes.
+                  If left empty, it defaults to 'single'.
+                enum:
+                - none
+                - single
+                - all
+                type: string
               denyIDs:
                 description: |-
                   DenyIDs is a comma-separated list of PCI IDs of GPU devices that should only be denied by the plugin.

pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go

@@ -51,6 +51,12 @@ type GpuDevicePluginSpec struct {
 	// +kubebuilder:validation:Enum=balanced;packed;none
 	PreferredAllocationPolicy string `json:"preferredAllocationPolicy,omitempty"`
 
+	// ByPathMode changes how plugin handles the DRM by-path/-dir mounting for GPU devices.
+	// See GPU plugin documentation for detailed description of the modes.
+	// If left empty, it defaults to 'single'.
+	// +kubebuilder:validation:Enum=none;single;all
+	ByPathMode string `json:"bypathMode,omitempty"`
+
 	// Specialized nodes (e.g., with accelerators) can be Tainted to make sure unwanted pods are not scheduled on them. Tolerations can be set for the plugin pod to neutralize the Taint.
 	Tolerations []v1.Toleration `json:"tolerations,omitempty"`
 

pkg/controllers/gpu/controller.go

@@ -285,5 +285,9 @@ func getPodArgs(gdp *devicepluginv1.GpuDevicePlugin) []string {
 		args = append(args, "-deny-ids", gdp.Spec.DenyIDs)
 	}
 
+	if gdp.Spec.ByPathMode != "" {
+		args = append(args, "-bypath", gdp.Spec.ByPathMode)
+	}
+
 	return args
 }