diff --git a/CHANGELOG.md b/CHANGELOG.md index f02e2dbd9..33980e17f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 than 560, keep reporting `N/A`. The `GPU Fabric GUID` row of the same block is not modelled and now renders `0x0000000000000000` where it used to read `N/A`. (#642) +- The rendered PCI sysfs tree now reaches consumers written in Go. `lspci` and + other libc consumers found it through the `libpcimocksys.so` `LD_PRELOAD` + shim, but Go reads sysfs with direct `openat` syscalls no shim can intercept, + so GPU Feature Discovery and the NVIDIA DRA driver read the node's real + `/sys` and saw no mock GPUs — GFD logged `unable to read PCI device vendor id` + and labelled the node `nvidia.com/gpu.mode=unknown`. The staged + `sys/devices` and `sys/bus/pci/devices` directories are now bind-mounted + read-only onto the kernel paths, through both the CDI spec the DaemonSet + generates and the NRI plugin's container adjustment. Both mounts go together: + the PCI entries are relative symlinks into `../../../devices/pciDDDD:BB`, so + mounting one alone leaves every attribute read failing with `ENOENT`. + `/sys/devices` is necessarily mounted whole — it cannot be narrowed to the + profile's root complexes, because a bind mount at a path sysfs lacks needs a + mountpoint the runtime cannot create on a read-only `/sys` — which hides the + host's other device classes from served containers. Under NRI, which injects + node-wide, exempt a workload that needs the host's real device tree with the + `nvml-mock.nvidia.com/inject: "false"` pod annotation, or a whole namespace + with `nri.excludedNamespaces`. The rendered tree also carries the node's DMI + attributes in `sys/devices/virtual/dmi/id`, because shadowing `/sys/devices` + shadows the directory `/sys/class/dmi/id` resolves into: kind's + `mount-product-files.sh` createContainer hook bind-mounts the node's product + files there for every container, and `mount(8)` cannot create a target on a + read-only sysfs. `product_name` is mirrored by value; `product_uuid` is an + empty stand-in, since kind mounts the node's own copy over it and the value is + a node identifier the kernel exposes to root alone. The attributes are + mirrored, not mocked: `nvidia.com/gpu.machine` still reports what the node + itself reports, tracked in #681. Each render now replaces the previous tree + instead of adding to it, so re-profiling a node no longer serves both + profiles' devices — including a re-profile onto a config that declares no PCI + devices at all, which clears the tree rather than leaving the previous one to + describe the node — and both mount channels gate on a completion marker the + renderer writes last — the mounted directories exist from the start of a + render, so their presence alone would serve a tree still missing the bind + targets kind's hook needs. (#673) - mocknvml: configured `processes:` now surface in nvidia-smi — the default table's Processes box, `-q`, and `--query-compute-apps` all report the configured PIDs, names and GPU memory instead of always reporting none. diff --git a/cmd/render-pci-sysfs/main.go b/cmd/render-pci-sysfs/main.go index 0ee9d7e64..ae84451bb 100644 --- a/cmd/render-pci-sysfs/main.go +++ b/cmd/render-pci-sysfs/main.go @@ -33,55 +33,92 @@ import ( "github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render" ) -//nolint:cyclop // existing complexity; refactor deferred +// defaultDMISource is where the kernel exposes the node's SMBIOS identity. +// The rendered tree mirrors it because serving the tree means bind-mounting +// it over /sys/devices, which would otherwise hide the DMI directory that +// /sys/class/dmi/id resolves into. +const defaultDMISource = "/sys/class/dmi/id" + func main() { var ( - cfgPath = flag.String("config", "", "path to mock-nvml profile YAML") - outDir = flag.String("output", "", "fake-root directory; tree is written under /sys/...") - strict = flag.Bool("strict", false, "fail if the profile does not declare `pcie_topology:`") - dryRun = flag.Bool("dry-run", false, "validate the config and exit without writing files") + opts options + dmiSource = flag.String("dmi-source", defaultDMISource, + "kernel DMI directory to mirror into the tree; empty mirrors nothing") ) + flag.StringVar(&opts.configPath, "config", "", "path to mock-nvml profile YAML") + flag.StringVar(&opts.outputDir, "output", "", "fake-root directory; tree is written under /sys/...") + flag.BoolVar(&opts.strict, "strict", false, "fail if the profile does not declare `pcie_topology:`") + flag.BoolVar(&opts.dryRun, "dry-run", false, "validate the config and exit without writing files") flag.Parse() + opts.dmiSource = *dmiSource - if *cfgPath == "" || *outDir == "" { + if opts.configPath == "" || opts.outputDir == "" { fmt.Fprintln(os.Stderr, "usage: render-pci-sysfs --config --output [--strict] [--dry-run]") os.Exit(2) } - data, err := os.ReadFile(*cfgPath) + if err := run(opts); err != nil { + fatalf("%v", err) + } +} + +// options is the resolved command line. +type options struct { + configPath string + outputDir string + dmiSource string + strict bool + dryRun bool +} + +func run(o options) error { + data, err := os.ReadFile(o.configPath) if err != nil { - fatalf("read config: %v", err) + return fmt.Errorf("read config: %w", err) } var prof config.Profile if err := yaml.Unmarshal(data, &prof); err != nil { - fatalf("parse config: %v", err) + return fmt.Errorf("parse config: %w", err) } if err := prof.Validate(); err != nil { - fatalf("%v", err) + return err } topo := prof.EffectiveTopology() - if topo == nil { - fmt.Fprintf(os.Stderr, "render-pci-sysfs: no devices in %s, nothing to render\n", *cfgPath) - return + if topo != nil && o.strict && prof.PCIeTopology == nil { + return fmt.Errorf("--strict: profile %s does not declare `pcie_topology:`", o.configPath) } - if *strict && prof.PCIeTopology == nil { - fatalf("--strict: profile %s does not declare `pcie_topology:`", *cfgPath) + if o.dryRun { + reportDryRun(o.configPath, topo) + return nil } - if *dryRun { - fmt.Fprintf(os.Stderr, "render-pci-sysfs: %d root complex(es), %d device(s) — config OK\n", - len(topo.RootComplexes), countDevices(topo)) - return + // A profile with no devices still goes through Render, rather than + // returning here: a tree rendered from a previous profile is on disk and + // still served, and Render is what clears it along with its completion + // marker, so setup.sh's gate cannot flip on for devices this profile does + // not declare. + if topo == nil { + fmt.Fprintf(os.Stderr, "render-pci-sysfs: no devices in %s, clearing any previously rendered tree\n", o.configPath) } - if err := render.Render(render.Options{ Topology: topo, Identities: prof.DeviceIdentities(), - Output: *outDir, + Output: o.outputDir, + DMISource: o.dmiSource, }); err != nil { - fatalf("render: %v", err) + return fmt.Errorf("render: %w", err) + } + return nil +} + +func reportDryRun(configPath string, topo *config.PCIeTopology) { + if topo == nil { + fmt.Fprintf(os.Stderr, "render-pci-sysfs: no devices in %s, nothing to render — config OK\n", configPath) + return } + fmt.Fprintf(os.Stderr, "render-pci-sysfs: %d root complex(es), %d device(s) — config OK\n", + len(topo.RootComplexes), countDevices(topo)) } func countDevices(t *config.PCIeTopology) int { diff --git a/cmd/render-pci-sysfs/main_test.go b/cmd/render-pci-sysfs/main_test.go new file mode 100644 index 000000000..52636e204 --- /dev/null +++ b/cmd/render-pci-sysfs/main_test.go @@ -0,0 +1,81 @@ +// Copyright 2026 NVIDIA CORPORATION +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render" +) + +const profileWithDevices = ` +devices: + - index: 0 + pci: + bus_id: "0000:07:00.0" +` + +// A profile whose devices declare no bus_id renders no topology. It is +// reachable through gpu.customConfig, and it used to return before Render. +const profileWithoutBusIDs = ` +devices: + - index: 0 +` + +// TestRun_ClearsTreeWhenProfileDeclaresNoDevices covers re-profiling a node +// onto a profile with nothing to render. The tree and the completion marker +// left by the previous profile would otherwise stay on disk, and both serving +// channels would keep mounting devices this profile does not declare — +// setup.sh gates on the marker, which said "rendered" about the old tree. +func TestRun_ClearsTreeWhenProfileDeclaresNoDevices(t *testing.T) { + out := t.TempDir() + require.NoError(t, run(options{ + configPath: writeProfile(t, profileWithDevices), + outputDir: out, + }), "render a profile with devices") + require.FileExists(t, filepath.Join(out, render.MarkerRelPath), "marker after the first render") + + require.NoError(t, run(options{ + configPath: writeProfile(t, profileWithoutBusIDs), + outputDir: out, + }), "render a profile without devices") + + entries, err := os.ReadDir(filepath.Join(out, render.PCIDevicesRelPath)) + require.NoError(t, err, "read devices dir") + require.Empty(t, entries, "the previous profile's devices are still served") + require.NoFileExists(t, filepath.Join(out, render.MarkerRelPath), + "the marker still claims a rendered tree") +} + +// TestRun_DryRunWritesNothing pins that --dry-run stays a validation pass on +// both paths, including the one that now prunes. +func TestRun_DryRunWritesNothing(t *testing.T) { + for name, profile := range map[string]string{ + "with devices": profileWithDevices, + "without devices": profileWithoutBusIDs, + } { + t.Run(name, func(t *testing.T) { + out := t.TempDir() + require.NoError(t, run(options{ + configPath: writeProfile(t, profile), + outputDir: out, + dryRun: true, + }), "dry run") + entries, err := os.ReadDir(out) + require.NoError(t, err, "read output dir") + require.Empty(t, entries, "--dry-run wrote to the output directory") + }) + } +} + +func writeProfile(t *testing.T, contents string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(path, []byte(contents), 0o644), "write profile") + return path +} diff --git a/deployments/nvml-mock/helm/nvml-mock/README.md b/deployments/nvml-mock/helm/nvml-mock/README.md index f9814729a..7736e32e3 100644 --- a/deployments/nvml-mock/helm/nvml-mock/README.md +++ b/deployments/nvml-mock/helm/nvml-mock/README.md @@ -21,10 +21,11 @@ Deploys a DaemonSet that creates on every node: - A fake PCI sysfs tree at `/var/lib/nvml-mock/sys/bus/pci/devices/...` (symlinks into `/var/lib/nvml-mock/sys/devices/pciDDDD:BB/...`) so C consumers of the PCI sysfs — `lspci` and anything else reaching it through libc — resolve the - PCIe root complex via a standard `readlink()`. The NVIDIA DRA driver is a Go - binary and does not see this tree, so `dra.k8s.io/pcieRoot` is still absent - from its ResourceSlices; see [Known Limitations](#known-limitations) and - issue [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265) + PCIe root complex via a standard `readlink()`. Consumers written in Go read + sysfs with direct syscalls no `LD_PRELOAD` shim can intercept, so the two + directories are additionally bind-mounted onto `/sys/bus/pci/devices` and + `/sys/devices` in served containers (see + [PCI sysfs in containers](#pci-sysfs-in-containers)) Consumers (DRA driver, device plugin) point at `/var/lib/nvml-mock/driver` as the NVIDIA driver root and discover GPUs through standard NVML APIs. @@ -708,6 +709,53 @@ DaemonSet under `set -e` if it finds a typo: If a profile omits `pcie_topology:` entirely the renderer falls back to a flat single-root layout (every device under `pci0000:00`, NUMA 0). +### PCI sysfs in containers + +Reaching the tree through `MOCK_PCI_ROOT` requires the `libpcimocksys.so` +`LD_PRELOAD` shim, which only works for consumers that go through libc. A Go +program does not: `os.Open` issues `openat` directly, the shim never sees it, +and the process reads the node's real `/sys` — where the mock GPUs do not +exist. GPU Feature Discovery and the NVIDIA DRA driver are both Go. + +So the rendered directories are bind-mounted read-only onto the kernel paths +in containers the mock serves, through the CDI spec the DaemonSet generates at +`/var/run/cdi/nvidia.yaml` and, when `nri.enabled=true`, through the NRI +plugin's container adjustment: + +| Host | Container | +|---|---| +| `/var/lib/nvml-mock/sys/devices` | `/sys/devices` | +| `/var/lib/nvml-mock/sys/bus/pci/devices` | `/sys/bus/pci/devices` | + +Both are needed together: the entries under `/sys/bus/pci/devices` are +relative symlinks into `../../../devices/pciDDDD:BB`, so mounting only that +directory yields entries that list but whose every attribute read fails with +`ENOENT`. + +**Trade-off:** `/sys/devices` is mounted whole, which hides the host's other +device classes (CPU topology among them) from those containers. It cannot be +narrowed to the profile's root complexes — a bind mount at a path sysfs does +not already have needs a mountpoint, and the runtime cannot create one on a +read-only `/sys`. A node running nvml-mock is simulating GPU hardware, so +serving the tree is not itself configurable. Through CDI a container is served +only if it requests a mock GPU. Through NRI, which injects ambiently, the two +existing escape hatches cover it: the pod annotation +`nvml-mock.nvidia.com/inject: "false"` (`nri.optOutAnnotation`) exempts a +single workload, and `nri.excludedNamespaces` exempts a whole namespace. + +`/sys/devices/virtual/dmi/id` — the directory `/sys/class/dmi/id` resolves +into — is shadowed along with the rest, so the renderer mirrors the node's +`product_name` there and leaves an empty `product_uuid` beside it. This is not +cosmetic: kind's `mount-product-files.sh` createContainer hook bind-mounts the +node's copies of both onto every container it starts, and `mount(8)` cannot +create a target on a read-only sysfs, so a missing attribute fails container +creation for every pod the mock serves. `product_uuid` is a node identifier the +kernel exposes to root alone and kind mounts its own copy over it, so only the +target is reproduced, never the value. `product_name` is mirrored, not mocked — +a node keeps reporting its own machine type, which under kind is the literal +`kind`, so `nvidia.com/gpu.machine` does not follow the profile. Tracked in +[#681](https://github.com/NVIDIA/k8s-test-infra/issues/681). + ### Cross-node `ibping` Sysfs mocking alone lets `ibstat` / `iblinkinfo` work, but real `ibping` @@ -1362,8 +1410,8 @@ discovery and monitoring. Some host-level subsystems are not mocked: | What's Missing | Affected Consumer | Impact | |----------------|-------------------|--------| -| `/sys/bus/pci/devices/{busID}` sysfs entries **as a Go program reads them** | DRA driver | The tree is rendered and `lspci` reads it, but the driver is a Go binary: Go's `os` package issues raw syscalls that the `LD_PRELOAD` shim cannot intercept, so it reads the host's real sysfs instead. `dra.k8s.io/pcieRoot` stays absent from ResourceSlices — **blocks topology-aware scheduling demos** (e.g., GPU + SR-IOV VF alignment). Tracked in [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265) | -| `/sys/bus/pci/devices/{busID}/numa_node` | Device plugin | NUMA-aware topology hints unavailable; scheduling works but NUMA affinity not enforced | +| `/sys/bus/pci/devices/{busID}` sysfs entries in containers the mock does **not** serve | DRA driver | The tree is now bind-mounted onto the kernel paths for CDI- and NRI-served containers, which is what Go consumers need (see [PCI sysfs in containers](#pci-sysfs-in-containers)). A consumer deployed outside those channels still reads the host's real sysfs; whether `dra.k8s.io/pcieRoot` reaches ResourceSlices is tracked in [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265) | +| `/sys/bus/pci/devices/{busID}/numa_node` in containers the mock does **not** serve | Device plugin | The renderer writes `numa_node` for every device and it arrives through the same mount, so a served device plugin does get NUMA hints. Outside those channels the hints are unavailable: scheduling works but NUMA affinity is not enforced | | `/sys/bus/pci/devices/*/vendor,device,class` **as NFD reads them** (`/host-sys/…`, fixed at link time) | NFD (Node Feature Discovery) | PCI feature labels not auto-detected. `nvidia.com/gpu.present` is written directly by nvml-mock; `pci-10de.present` is created by NFD from a feature file nvml-mock drops in `nodeLabels.featuresDir` — see [Node Labels](#node-labels) | ### PCIe Root Complex (DRA driver) @@ -1377,11 +1425,12 @@ W0319 11:41:21.314205 1 nvlib.go:491] error getting PCIe root for device 0 readlink /sys/bus/pci/devices/0000:07:00.0: no such file or directory ``` -**This warning is expected** but has real impact. The DRA driver resolves PCIe -root complex topology by reading sysfs symlinks. Since nvml-mock provides a mock -NVML library (not a full kernel driver), these sysfs entries don't exist. GPUs -appear in ResourceSlices and are fully allocatable, but the -`dra.k8s.io/pcieRoot` topology attribute is absent. +The DRA driver resolves PCIe root complex topology by reading sysfs symlinks. +The rendered tree now reaches served containers at `/sys/bus/pci/devices` (see +[PCI sysfs in containers](#pci-sysfs-in-containers)), so a driver the mock +serves resolves the root complex; one deployed outside the CDI and NRI paths +still reads the host's sysfs and logs the warning above, with GPUs allocatable +but `dra.k8s.io/pcieRoot` absent. **What this blocks:** DRA topology-aware scheduling that uses `pcieRoot` to align devices on the same PCIe root complex — for example, co-scheduling a GPU diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml index 210b69cef..99ef06eac 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml @@ -434,10 +434,10 @@ infiniband: # PCIe topology - 2 NUMA nodes (dual EPYC), 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml index a92c03c0a..ccf6756b6 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml @@ -431,10 +431,10 @@ infiniband: # PCIe topology - B200, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml index f1975350d..76be781f4 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml @@ -513,10 +513,10 @@ infiniband: # PCIe topology - GB200, 4 Grace CPU pairs -> 4 NUMA nodes, 2 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml index a74180be1..6a8db6067 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml @@ -444,10 +444,10 @@ infiniband: # PCIe topology - HGX H100, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml index b38ad30c3..d9e79619b 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/l40s.yaml @@ -391,10 +391,10 @@ infiniband: # PCIe topology - L40S, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml index 44a826903..aa65b16e6 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/t4.yaml @@ -369,10 +369,10 @@ infiniband: # PCIe topology - T4 inference card, single NUMA node, 4 GPUs. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap index 6d58c6d6f..35aaab9fc 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap +++ b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap @@ -436,10 +436,10 @@ should match snapshot with b200 profile: # PCIe topology - B200, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -907,10 +907,10 @@ should match snapshot with default a100 profile: # PCIe topology - 2 NUMA nodes (dual EPYC), 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -1457,10 +1457,10 @@ should match snapshot with gb200 profile: # PCIe topology - GB200, 4 Grace CPU pairs -> 4 NUMA nodes, 2 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -2500,10 +2500,10 @@ should match snapshot with h100 profile: # PCIe topology - HGX H100, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -2928,10 +2928,10 @@ should match snapshot with l40s profile: # PCIe topology - L40S, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: @@ -3334,10 +3334,10 @@ should match snapshot with t4 profile: # PCIe topology - T4 inference card, single NUMA node, 4 GPUs. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root - # complex through these symlinks. The NVIDIA DRA driver does NOT: it is a - # Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD - # shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the - # ResourceSlices it publishes. See issue #265. + # complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU + # Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot + # intercept, so this tree is bind-mounted at /sys/bus/pci/devices and + # /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap index 98f9fba4f..22d37bc09 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap +++ b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap @@ -19,7 +19,7 @@ should match snapshot with all overrides: template: metadata: annotations: - checksum/config: 1893be873c970041a080b07af3cc840fb218a12192a6f0e5c305ba70a12ce620 + checksum/config: 0a18dc0ae54a3a137218785bddf06ebc287c39fb6547e51ea7a2b650e956511b labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: custom @@ -166,7 +166,7 @@ should match snapshot with b200 profile: template: metadata: annotations: - checksum/config: 25749368d74f6ae7ce538ddfa371e30c93ec11a4968b155c683b914e05f76b81 + checksum/config: c9174f0176a7b6f1deeb10de59ce2f8811e9d25a391a628f247102ee978832f6 labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: RELEASE-NAME @@ -292,7 +292,7 @@ should match snapshot with default values: template: metadata: annotations: - checksum/config: 4a1427c36fb9a24923ec692eb82cd2ba449715be5e194942ae26549a1d1f81b3 + checksum/config: 1bdbe9385e4c347ca3f4d2678fbbfc885d7539e4340a6a8c2b93fe78c1ed44c5 labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: RELEASE-NAME diff --git a/deployments/nvml-mock/scripts/setup.sh b/deployments/nvml-mock/scripts/setup.sh index 4e5394167..8cf73112e 100644 --- a/deployments/nvml-mock/scripts/setup.sh +++ b/deployments/nvml-mock/scripts/setup.sh @@ -114,14 +114,49 @@ CAPS_EOF echo "Mock IMEX surface ready: $IMEX_CHANNELS channels, major $IMEX_MAJOR, proc-devices at $IMEX_DIR/proc-devices" fi -# 3b. Generate CDI spec for nvidia-container-runtime CDI mode. +# 3b. Render fake PCI sysfs tree (consumed by topology-aware DRA / device +# plugins that resolve PCIe root complex via a readlink on +# /sys/bus/pci/devices/, and by GPU Feature Discovery, which reads +# each device's `vendor` / `class` to derive nvidia.com/gpu.mode). The +# renderer parses the profile's `pcie_topology:` block; profiles without +# one get a flat default covering every device under a single root +# complex (`pci0000:00`, NUMA 0). It also mirrors the node's DMI identity +# into the tree, which the mount below depends on. Each run replaces the +# previous tree rather than adding to it, so re-profiling a node does not +# leave it serving both profiles' devices. Failures are fatal +# under `set -e` for the same reason as the IB render below — a topology +# typo otherwise yields silently malformed sysfs that downstream +# `dra.k8s.io/pcieRoot` attributes would inherit. +# +# This runs before the CDI spec below because that spec bind-mounts the +# rendered directories into consumers: a bind mount whose source is +# missing fails container creation for the whole pod. +PCI_ROOT="$HOST" +mkdir -p "$PCI_ROOT" +# Keep in sync with render.MarkerRelPath (pkg/system/mockpcisysfs/render): the +# renderer writes it last, once the whole tree is on disk, and removes it when +# the profile declares no PCI devices at all. Gating on the directories instead +# would say "rendered" partway through a render, and would keep serving the +# previous profile's devices to a profile that renders none of its own. +PCI_SYSFS_MARKER=sys/.rendered +PCI_SYSFS_RENDERED=off +if [ -x /usr/local/bin/render-pci-sysfs ]; then + /usr/local/bin/render-pci-sysfs \ + --config /etc/nvml-mock/config.yaml \ + --output "$PCI_ROOT" + if [ -f "$PCI_ROOT/$PCI_SYSFS_MARKER" ]; then + PCI_SYSFS_RENDERED=on + fi +fi + +# 3c. Generate CDI spec for nvidia-container-runtime CDI mode. # This allows the toolkit to inject our mock libs into containers without # needing libnvidia-container or kernel modules. CDI_DIR=/host/var/run/cdi mkdir -p "$CDI_DIR" # Resolve fabricmanager enablement once, here, because it influences both the -# CDI spec (below) and the daemon launch (step 11). Validate early so a typo +# CDI spec (below) and the daemon launch (step 10). Validate early so a typo # fails the pod with a clear message rather than silently disabling the gate. MOCK_FM_MODE=$(printf '%s' "${MOCK_FABRICMANAGER:-off}" | tr '[:upper:]' '[:lower:]') case "$MOCK_FM_MODE" in @@ -164,6 +199,40 @@ containerEdits: options: [ro, nosuid, nodev, bind] CDI_HEADER +# Fake PCI sysfs, mounted at the kernel paths. Consumers written in Go +# (GPU Feature Discovery, the DRA driver) read sysfs with direct syscalls, +# so the LD_PRELOAD redirector never sees their opens and MOCK_PCI_ROOT +# does nothing for them — only a real mount at /sys/bus/pci/devices works. +# Without this, GFD resolves a mock GPU's BDF from NVML, fails to read +# /sys/bus/pci/devices//vendor, and labels the node +# nvidia.com/gpu.mode=unknown. +# +# +# Both mounts are needed: the entries under sys/bus/pci/devices are +# relative symlinks into ../../../devices/pciDDDD:BB, which only resolve +# when the rendered sys/devices is mounted too. That second mount hides the +# host's other device classes (CPU topology among them) from the container. +# The alternative — mounting only the root complexes the profile declares — +# is not available: sysfs is read-only inside the container, so the runtime +# cannot create a mountpoint like /sys/devices/pci0000:80 that the host +# does not already have, and container creation fails outright. +# +# Shadowing sys/devices also replaces virtual/dmi/id, which +# /sys/class/dmi/id resolves into. kind's mount-product-files.sh +# createContainer hook bind-mounts the node's product_name / product_uuid +# there for every container, and mount(8) cannot create a target on a +# read-only sysfs — hence the renderer mirroring those attributes above. +if [ "$PCI_SYSFS_RENDERED" = "on" ]; then + cat >> "$CDI_DIR/nvidia.yaml" << PCI_SYSFS_MOUNT_EOF + - hostPath: /var/lib/nvml-mock/sys/devices + containerPath: /sys/devices + options: [ro, nosuid, nodev, bind] + - hostPath: /var/lib/nvml-mock/sys/bus/pci/devices + containerPath: /sys/bus/pci/devices + options: [ro, nosuid, nodev, bind] +PCI_SYSFS_MOUNT_EOF +fi + # When fabricmanager is enabled, bind-mount the node-local readiness marker # directory into CDI-injected workloads and point the mock NVML library at it. # Without this, the mock .so loaded inside user pods sees an empty @@ -252,7 +321,7 @@ done echo "CDI spec generated at $CDI_DIR/nvidia.yaml ($GPU_COUNT devices, index + UUID keyed)" -# 3c. Generate the CDI spec the NRI plugin injects (issue #436). +# 3d. Generate the CDI spec the NRI plugin injects (issue #436). # # This is deliberately a SECOND spec, not a reuse of nvidia.yaml above: # @@ -547,7 +616,7 @@ if [ "$PCI_LABEL_MODE" = "on" ]; then # step 8's /host/run/nvidia/driver symlink, crash-looping the whole mock for # an optional, gated feature. When the write fails no label appears, which is # the honest state (#505), and nothing downstream is corrupted — unlike the - # IB and PCI renders in steps 9 and 10, which are deliberately fatal because + # IB render (step 9) and PCI render (step 3b), which are deliberately fatal because # a partial tree silently misleads its consumers. A failing `if` CONDITION # does not trip `set -e`, so this form warns and continues rather than # swallowing the error the way `|| true` would. @@ -662,24 +731,7 @@ if [ "$MOCK_IB_MODE" != "off" ] && [ -x /usr/local/bin/mock-ib ]; then fi fi -# 10. Render fake PCI sysfs tree (consumed by topology-aware DRA / device -# plugins that resolve PCIe root complex via a readlink on -# /sys/bus/pci/devices/). The renderer parses the profile's -# `pcie_topology:` block; profiles without one get a flat default -# covering every device under a single root complex (`pci0000:00`, -# NUMA 0). Failures are fatal under `set -e` for the same reason as -# the IB block above — a topology typo otherwise yields silently -# malformed sysfs that downstream `dra.k8s.io/pcieRoot` attributes -# would inherit. -PCI_ROOT="$HOST" -mkdir -p "$PCI_ROOT" -if [ -x /usr/local/bin/render-pci-sysfs ]; then - /usr/local/bin/render-pci-sysfs \ - --config /etc/nvml-mock/config.yaml \ - --output "$PCI_ROOT" -fi - -# 11. Fabric Manager: on NVSwitch platforms (HGX H100 / GB200 / GB300) the +# 10. Fabric Manager: on NVSwitch platforms (HGX H100 / GB200 / GB300) the # real nvidia-fabricmanager registers the GPUs with the NVSwitch fabric # before they are usable. When MOCK_FABRICMANAGER is enabled we start the # fake daemon, which writes a node-local readiness marker under diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-a100.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-a100.yaml index c0cace998..259a5e9b9 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-a100.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-a100.yaml @@ -398,10 +398,10 @@ nvlink: # PCIe topology - 2 NUMA nodes (dual EPYC), 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-b200.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-b200.yaml index d5f9d2881..92d453f16 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-b200.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-b200.yaml @@ -390,10 +390,10 @@ nvlink: # PCIe topology - B200, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml index ca9bb033b..4b71213ac 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml @@ -432,10 +432,10 @@ nvlink: # PCIe topology - GB200, 4 Grace CPU pairs -> 4 NUMA nodes, 2 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml index c6d7778b0..4880bf65a 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml @@ -410,10 +410,10 @@ nvlink: # PCIe topology - HGX H100, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-l40s.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-l40s.yaml index d2cacb8c0..07e5c9a31 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-l40s.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-l40s.yaml @@ -365,10 +365,10 @@ devices: # PCIe topology - L40S, 2 NUMA nodes, 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-t4.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-t4.yaml index 10c6df2f2..c45f25628 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-t4.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-t4.yaml @@ -337,10 +337,10 @@ devices: # PCIe topology - T4 inference card, single NUMA node, 4 GPUs. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: diff --git a/pkg/nri/nvmlmock/adjust.go b/pkg/nri/nvmlmock/adjust.go index 84ea462f8..d65668847 100644 --- a/pkg/nri/nvmlmock/adjust.go +++ b/pkg/nri/nvmlmock/adjust.go @@ -11,6 +11,8 @@ import ( "path/filepath" "sort" "strings" + + "github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render" ) // warnf logs a non-fatal condition. It is a package var so tests can capture @@ -42,6 +44,15 @@ const ( // MOCK_TOPOLOGY_CONFIG env). defaultTopologyRelPath = "topology/topology.yaml" + // pciDevicesContainerPath and sysDevicesContainerPath are the kernel + // paths the tree must appear at inside the container. Unlike the + // LD_PRELOAD-based redirection (MOCK_PCI_ROOT), these cannot be + // relocated: Go consumers such as GPU Feature Discovery and the DRA + // driver hard-code them and read them with direct syscalls, which no + // libc shim can intercept. + pciDevicesContainerPath = "/sys/bus/pci/devices" + sysDevicesContainerPath = "/sys/devices" + // DeviceInjectionModeRaw stages the mock /dev/nvidiaN nodes directly in the // adjustment. It is the default: MEP-0002 requires the raw path to stay // reachable, and it is the only mode that works on a runtime whose CDI @@ -207,6 +218,7 @@ func Adjust(cfg Config, container Container) (Adjustment, bool, error) { }, Env: buildEnv(cfg, container.Env, topologyInjectable(cfg)), } + adjustment.Mounts = append(adjustment.Mounts, pciSysfsMounts(cfg)...) if strings.EqualFold(container.PodAnnotations[cfg.DeviceAnnotation], "true") { switch { @@ -338,6 +350,69 @@ func topologyInjectable(cfg Config) bool { return err == nil } +// pciSysfsMounts maps the rendered PCI tree onto the kernel paths inside the +// container. It returns both mounts or neither: /sys/bus/pci/devices holds +// relative symlinks into ../../../devices/pciDDDD:BB, so without +// /sys/devices every entry dangles and reads fail with ENOENT — the same +// symptom as no mount at all, only harder to diagnose. +// +// Mounting /sys/devices necessarily hides the host's other device classes +// (CPU topology among them) from the container. That is the price of serving +// consumers that resolve GPUs through sysfs: the tree cannot be assembled +// per root complex instead, because a bind mount at a path sysfs does not +// already have (say /sys/devices/pci0000:80) needs a mountpoint the runtime +// cannot create on a read-only sysfs. It also shadows virtual/dmi/id, which +// is why the renderer mirrors the node's DMI attributes into the tree: kind's +// createContainer hook bind-mounts the node's product files there, and a +// missing target fails container creation. +// +// An unfinished tree is skipped rather than reported: it is staged by the +// main nvml-mock DaemonSet and nothing orders this plugin after it, and a +// mount the tree cannot honour fails container creation for the whole pod. +// Silence rather than a warning because this runs for every container on the +// node, staged or not. +// +// "Finished" is the renderer's marker, not the presence of the directories +// mounted here: those exist from the start of a render while the DMI +// attributes kind's hook needs are written at its end, so a tree caught +// mid-render would otherwise pass and then fail every container on the node. +func pciSysfsMounts(cfg Config) []Mount { + if cfg.HostOverlayPath == "" { + return nil + } + if _, err := os.Stat(filepath.Join(cfg.HostOverlayPath, render.MarkerRelPath)); err != nil { + return nil + } + // Paths come from the renderer that writes them: a guard statting a path + // nothing renders would fail open, dropping the mounts silently. + sysDevices := filepath.Join(cfg.HostOverlayPath, render.SysDevicesRelPath) + pciDevices := filepath.Join(cfg.HostOverlayPath, render.PCIDevicesRelPath) + for _, dir := range []string{sysDevices, pciDevices} { + info, err := os.Stat(dir) + if err != nil || !info.IsDir() { + return nil + } + } + + // /sys/devices first: containerd orders mounts parent-before-child, but + // emitting them in dependency order keeps the adjustment readable and + // correct under any runtime that applies them verbatim. + return []Mount{ + { + Source: sysDevices, + Destination: sysDevicesContainerPath, + Type: "bind", + Options: []string{"rbind", "ro", "nosuid", "nodev"}, + }, + { + Source: pciDevices, + Destination: pciDevicesContainerPath, + Type: "bind", + Options: []string{"rbind", "ro", "nosuid", "nodev"}, + }, + } +} + func shouldSkip(cfg Config, container Container) bool { if strings.EqualFold(container.PodAnnotations[cfg.OptOutAnnotation], "false") { return true diff --git a/pkg/nri/nvmlmock/adjust_test.go b/pkg/nri/nvmlmock/adjust_test.go index fa195b545..a1b5ae502 100644 --- a/pkg/nri/nvmlmock/adjust_test.go +++ b/pkg/nri/nvmlmock/adjust_test.go @@ -11,6 +11,8 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render" ) func TestAdjustPlainContainerAddsOverlayAndEnvironment(t *testing.T) { @@ -48,6 +50,114 @@ func TestAdjustPlainContainerAddsOverlayAndEnvironment(t *testing.T) { requireNoEnvKey(t, adjustment.Env, "MOCK_IB") } +// TestAdjustMountsPCISysfsWhenStaged pins the pair of mounts that let Go +// consumers (GPU Feature Discovery, the DRA driver) see the mock PCI tree. +// They read sysfs with direct syscalls, so the LD_PRELOAD redirector never +// sees their opens and only a real mount at the canonical path works. +// +// Both mounts are required together: /sys/bus/pci/devices holds symlinks +// pointing at ../../../devices/pciDDDD:BB/, which only resolve when +// the rendered sys/devices is mounted too. +func TestAdjustMountsPCISysfsWhenStaged(t *testing.T) { + overlay := t.TempDir() + stagePCISysfs(t, overlay) + + cfg := DefaultConfig() + cfg.HostOverlayPath = overlay + + adjustment, ok, err := Adjust(cfg, Container{Namespace: "gpu-operator"}) + require.NoError(t, err) + require.True(t, ok) + + require.Contains(t, adjustment.Mounts, Mount{ + Source: filepath.Join(overlay, "sys/devices"), + Destination: "/sys/devices", + Type: "bind", + Options: []string{"rbind", "ro", "nosuid", "nodev"}, + }) + require.Contains(t, adjustment.Mounts, Mount{ + Source: filepath.Join(overlay, "sys/bus/pci/devices"), + Destination: "/sys/bus/pci/devices", + Type: "bind", + Options: []string{"rbind", "ro", "nosuid", "nodev"}, + }) +} + +// TestAdjustSkipsPCISysfsMountsWhenNotStaged is the fail-open case: the +// tree is staged by the main DaemonSet and nothing orders this plugin after +// it. A bind mount with a missing source fails container creation outright, +// so an unstaged node must simply get no sysfs mounts. +func TestAdjustSkipsPCISysfsMountsWhenNotStaged(t *testing.T) { + cfg := DefaultConfig() + cfg.HostOverlayPath = t.TempDir() + + adjustment, ok, err := Adjust(cfg, Container{Namespace: "default"}) + require.NoError(t, err) + require.True(t, ok) + + for _, mount := range adjustment.Mounts { + require.NotContains(t, mount.Destination, "/sys/", + "unstaged node must not get sysfs mounts, got %+v", mount) + } +} + +// TestAdjustSkipsPCIDevicesMountWithoutSysDevices guards the half-rendered +// case. Mounting the symlink directory alone yields dangling symlinks, +// which reads report as ENOENT — the exact failure the mounts exist to +// fix, but harder to diagnose because the entries appear to be there. +func TestAdjustSkipsPCIDevicesMountWithoutSysDevices(t *testing.T) { + overlay := t.TempDir() + stagePCISysfs(t, overlay) + require.NoError(t, os.RemoveAll(filepath.Join(overlay, render.SysDevicesRelPath))) + + cfg := DefaultConfig() + cfg.HostOverlayPath = overlay + + adjustment, ok, err := Adjust(cfg, Container{Namespace: "default"}) + require.NoError(t, err) + require.True(t, ok) + + for _, mount := range adjustment.Mounts { + require.NotContains(t, mount.Destination, "/sys/", + "a tree without sys/devices must yield no sysfs mounts, got %+v", mount) + } +} + +// TestAdjustSkipsPCISysfsMountsWhileRenderIncomplete covers the window inside +// a render: the directories these mounts name are created at its start, while +// the DMI attributes kind's createContainer hook bind-mounts the node's +// product files onto are written at its end. Mounting in between hands the +// container a tree missing those targets, and mount(8) cannot create one on a +// read-only sysfs — container creation fails, which is the failure the guard +// exists to prevent. The renderer's completion marker is what distinguishes +// the two states. +func TestAdjustSkipsPCISysfsMountsWhileRenderIncomplete(t *testing.T) { + overlay := t.TempDir() + stagePCISysfs(t, overlay) + require.NoError(t, os.Remove(filepath.Join(overlay, render.MarkerRelPath))) + + cfg := DefaultConfig() + cfg.HostOverlayPath = overlay + + adjustment, ok, err := Adjust(cfg, Container{Namespace: "default"}) + require.NoError(t, err) + require.True(t, ok) + + for _, mount := range adjustment.Mounts { + require.NotContains(t, mount.Destination, "/sys/", + "an incomplete tree must yield no sysfs mounts, got %+v", mount) + } +} + +// stagePCISysfs stages a completely rendered PCI sysfs tree in the overlay, +// as the main DaemonSet's render-pci-sysfs run leaves it. +func stagePCISysfs(t *testing.T, overlay string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Join(overlay, render.PCIDevicesRelPath), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(overlay, render.SysDevicesRelPath), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(overlay, render.MarkerRelPath), nil, 0o644)) +} + func TestAdjustEmitsOnlyAddedOrChangedEnv(t *testing.T) { container := Container{ Namespace: "default", diff --git a/pkg/system/mockpcisysfs/config/types.go b/pkg/system/mockpcisysfs/config/types.go index d978fa192..d16a78d13 100644 --- a/pkg/system/mockpcisysfs/config/types.go +++ b/pkg/system/mockpcisysfs/config/types.go @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 // Package config defines the YAML schema for the `pcie_topology:` block -// embedded in mock-nvml profile configs. The renderer consumes this to +// embedded in mock-nvml profile configs. The renderer consumes it to // populate a fake `/sys/bus/pci/devices` + `/sys/devices/pciDDDD:BB` tree // under MOCK_PCI_ROOT. // diff --git a/pkg/system/mockpcisysfs/render/render.go b/pkg/system/mockpcisysfs/render/render.go index 0c9b8a135..e39875d7f 100644 --- a/pkg/system/mockpcisysfs/render/render.go +++ b/pkg/system/mockpcisysfs/render/render.go @@ -52,33 +52,126 @@ type Options struct { // Output is the fake-root directory. The renderer writes under // /sys/... — Output itself is created if missing. // - // When Topology is nil or has no root complexes, Render is a no-op - // even if Output is empty (so setup.sh can invoke the renderer - // unconditionally). A non-nil Topology with a non-empty Output is - // required; otherwise Render returns an error. + // A non-nil Topology requires a non-empty Output; otherwise Render + // returns an error. When Topology is nil or has no root complexes there + // is nothing to write, and Render instead empties whatever a previous + // profile left under Output and drops the completion marker — so a caller + // can invoke the renderer unconditionally without leaving a tree that + // describes the wrong profile. With Output empty too, Render does nothing. Output string + + // DMISource is the directory holding the node's kernel DMI identity, + // normally /sys/class/dmi/id. The attributes found there are mirrored + // into the tree so that bind-mounting sys/devices over the kernel's + // does not take the DMI directory with it — see renderDMI. Empty + // mirrors nothing. + DMISource string } -// Render writes the entire tree. It is idempotent: existing directories -// are reused, existing files are truncated and rewritten, and existing -// symlinks are removed and recreated so a stale relative target does not -// linger across re-renders. +// MarkerRelPath is written last, once the whole tree — topology and mirrored +// DMI attributes alike — is on disk. Consumers that bind-mount the tree onto +// the kernel paths gate on it rather than on the directories they mount: +// those are created at the start of a render, so their presence says nothing +// about whether the render finished, and serving a half-rendered tree fails +// container creation on a bind target that is not there yet. +// +// It sits outside both mounted subtrees, so it is not visible to a container +// the tree is served to. +const MarkerRelPath = "sys/.rendered" + +// PCIDevicesRelPath and SysDevicesRelPath are the two halves of the tree, +// relative to Options.Output. They are exported for the consumers that +// bind-mount them onto the kernel paths, so the layout has one definition +// rather than a copy per consumer. +const ( + PCIDevicesRelPath = "sys/bus/pci/devices" + SysDevicesRelPath = "sys/devices" +) + +// Render writes the entire tree, replacing whatever a previous render left +// behind, and marks it complete with MarkerRelPath. Within a render existing +// files are truncated and rewritten, and existing symlinks are removed and +// recreated so a stale relative target cannot linger. func Render(o Options) error { - if o.Topology == nil || len(o.Topology.RootComplexes) == 0 { - // Nothing to do — caller decided to render a profile with no - // declared topology and no devices. Treat as a no-op so the - // renderer can be invoked unconditionally from setup.sh. - return nil + if !o.hasTopology() { + // Nothing to render — the caller passed a profile with no declared + // topology and no devices, which setup.sh does unconditionally. A tree + // left here by a previous profile would still describe the node, so it + // is emptied rather than kept. + if o.Output == "" { + return nil + } + return pruneTree(o.Output) } if o.Output == "" { return errors.New("pcisysfs render: Output is required") } + if err := pruneTree(o.Output); err != nil { + return err + } + if err := renderTopology(o); err != nil { + return err + } + if err := renderDMI(o.Output, o.DMISource); err != nil { + return err + } + return writeFile(o.Output, MarkerRelPath, "") +} + +func (o Options) hasTopology() bool { + return o.Topology != nil && len(o.Topology.RootComplexes) > 0 +} + +// pruneTree drops the devices a previous render left behind. Rendering only +// ever added entries, so without this a re-profiled node keeps both profiles' +// devices (an a100 and an h100 share no BDFs) and consumers see their union +// mounted at /sys/bus/pci/devices — more GPUs than the node simulates, some +// under a root complex no profile declares. +// +// What survives is deliberate: the two directories consumers bind-mount, and +// the DMI directory holding the targets kind's createContainer hook needs. A +// container created while a render is in flight then still finds every mount's +// source and every target in place — it may see fewer devices than the profile +// declares, but it starts. Consumers served through CDI have no way to wait +// for MarkerRelPath, since the runtime applies the spec's mounts unconditionally. +func pruneTree(root string) error { + if err := os.RemoveAll(filepath.Join(root, MarkerRelPath)); err != nil { + return fmt.Errorf("clear %s: %w", MarkerRelPath, err) + } + if err := removeEntries(filepath.Join(root, PCIDevicesRelPath), ""); err != nil { + return err + } + return removeEntries(filepath.Join(root, SysDevicesRelPath), dmiVirtualDirName) +} + +// removeEntries empties dir, keeping the entry named keep (if any). A missing +// dir is not an error: there is nothing to prune on a first render. +func removeEntries(dir, keep string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("read %s: %w", dir, err) + } + for _, entry := range entries { + if entry.Name() == keep { + continue + } + if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil { + return fmt.Errorf("clear %s: %w", filepath.Join(dir, entry.Name()), err) + } + } + return nil +} + +func renderTopology(o Options) error { root := o.Output - if err := mkdirAll(root, "sys/bus/pci/devices"); err != nil { + if err := mkdirAll(root, PCIDevicesRelPath); err != nil { return err } - if err := mkdirAll(root, "sys/devices"); err != nil { + if err := mkdirAll(root, SysDevicesRelPath); err != nil { return err } @@ -90,8 +183,71 @@ func Render(o Options) error { return nil } +// dmiIDDir is where the kernel materializes the SMBIOS identity; the +// familiar /sys/class/dmi/id path is only a symlink into it. +// dmiVirtualDirName names its top-level directory under sys/devices, which +// pruneTree keeps so the mount targets inside it never go missing. +const ( + dmiVirtualDirName = "virtual" + dmiIDDir = SysDevicesRelPath + "/" + dmiVirtualDirName + "/dmi/id" +) + +// dmiMirroredAttrs are the DMI attributes kind's mount-product-files.sh +// createContainer hook bind-mounts the node's copies onto, for every +// container on the node. Each has to exist in the tree as a mount target; +// byValue says whether the node's value travels with it. +var dmiMirroredAttrs = []struct { + name string + byValue bool +}{ + // The node's machine type, which consumers do read: GFD's default + // machine-type file resolves here. + {name: "product_name", byValue: true}, + // A node identifier the kernel deliberately exposes 0400 to root alone. + // Only its existence matters, since kind mounts the node's own copy over + // it, so the tree carries an empty stand-in rather than republishing the + // value world-readable into every served container. + {name: "product_uuid"}, +} + +// renderDMI mirrors the node's DMI identity into the tree. Serving the tree +// means bind-mounting it over /sys/devices, which also replaces +// virtual/dmi/id — the directory /sys/class/dmi/id resolves into. Any +// attribute missing from the replacement is a bind-mount target that no +// longer exists, and mount(8) cannot create one on a read-only sysfs, so +// kind's hook fails and every injected container fails to start. +// +// Mirroring rather than mocking keeps the node's identity intact: kind +// already reports its own ("kind" as the product name, a random UUID), and +// overriding that is a separate concern with its own consumers. +func renderDMI(root, source string) error { + if source == "" { + return nil + } + for _, attr := range dmiMirroredAttrs { + src := filepath.Join(source, attr.name) + if _, err := os.Stat(src); err != nil { + // The kernel exposes no such attribute, so nothing bind-mounts + // it either and a stand-in would only invent an identity. + continue + } + var contents []byte + if attr.byValue { + // A read failure still has to leave the file behind: the target + // matters more than the value, and mount(8) cannot create one. + if value, err := os.ReadFile(src); err == nil { + contents = value + } + } + if err := writeFile(root, filepath.Join(dmiIDDir, attr.name), string(contents)); err != nil { + return err + } + } + return nil +} + func renderRootComplex(root string, rc config.RootComplex, ids map[string]config.PCI) error { - rcDir := filepath.Join("sys/devices", rc.ID) + rcDir := filepath.Join(SysDevicesRelPath, rc.ID) if err := mkdirAll(root, rcDir); err != nil { return err } @@ -122,11 +278,11 @@ func renderRootComplex(root string, rc config.RootComplex, ids map[string]config // Relative target matches what the kernel emits, so any // readlink() consumer (`realpath`, deviceattribute, etc.) // resolves to the same canonical path it would on real Linux. - linkPath := filepath.Join(root, "sys/bus/pci/devices", bdfLC) + linkPath := filepath.Join(root, PCIDevicesRelPath, bdfLC) linkTarget := filepath.Join("..", "..", "..", "devices", rc.ID, bdfLC) if err := replaceSymlink(linkPath, linkTarget); err != nil { return fmt.Errorf("symlink %s -> %s: %w", - filepath.Join("sys/bus/pci/devices", bdfLC), linkTarget, err) + filepath.Join(PCIDevicesRelPath, bdfLC), linkTarget, err) } } return nil diff --git a/pkg/system/mockpcisysfs/render/render_test.go b/pkg/system/mockpcisysfs/render/render_test.go index a14016ea1..a11fa1643 100644 --- a/pkg/system/mockpcisysfs/render/render_test.go +++ b/pkg/system/mockpcisysfs/render/render_test.go @@ -200,6 +200,111 @@ func TestRender_IdempotentRerender(t *testing.T) { require.Equal(t, "3\n", string(got), "numa_node not updated") } +// TestRender_PrunesStaleDevices covers re-profiling a node: an a100 and an +// h100 profile share no BDFs, and the renderer used to only add entries, so +// both sets stayed under the tree and consumers saw their union mounted at +// /sys/bus/pci/devices — more GPUs than the node simulates, some of them +// pointing at a root complex no profile declares. +func TestRender_PrunesStaleDevices(t *testing.T) { + dir := t.TempDir() + previous := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: previous, Output: dir}), "Render previous profile") + + current := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:c0", NUMANode: 3, + Devices: []string{"0000:1a:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: current, Output: dir}), "Render current profile") + + entries, err := os.ReadDir(filepath.Join(dir, "sys/bus/pci/devices")) + require.NoError(t, err, "read devices dir") + require.Len(t, entries, 1, "stale device symlinks survived the re-render") + require.Equal(t, "0000:1a:00.0", entries[0].Name(), "device") + + _, err = os.Stat(filepath.Join(dir, "sys/devices/pci0000:00")) + require.True(t, os.IsNotExist(err), "stale root complex survived, got err=%v", err) +} + +// TestRender_MarkerFollowsTheTree pins the signal consumers gate their bind +// mounts on. The mounted directories are created at the start of a render and +// the DMI attributes are written at its end, so their presence cannot mean +// "complete" — and mounting an incomplete tree fails container creation on a +// bind target that is not there yet. +func TestRender_MarkerFollowsTheTree(t *testing.T) { + dir := t.TempDir() + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: topo, Output: dir}), "Render") + _, err := os.Stat(filepath.Join(dir, MarkerRelPath)) + require.NoError(t, err, "marker missing after a complete render") + + // A profile with nothing to render must leave no marker: a stale one would + // keep consumers mounting the previous profile's devices. + require.NoError(t, Render(Options{Output: dir}), "Render without topology") + _, err = os.Stat(filepath.Join(dir, MarkerRelPath)) + require.True(t, os.IsNotExist(err), "marker survived an empty render, got err=%v", err) + entries, err := os.ReadDir(filepath.Join(dir, "sys/bus/pci/devices")) + require.NoError(t, err, "read devices dir") + require.Empty(t, entries, "devices survived an empty render") +} + +// TestRender_KeepsMountedPathsWhilePruning pins what a re-render must not take +// away. Both mounted directories, and the DMI attributes kind's +// createContainer hook bind-mounts the node's product files onto, are targets +// of mounts already in effect for containers the CDI spec serves — and the CDI +// path cannot wait for the marker, since the runtime applies the spec's mounts +// unconditionally. Removing them mid-render would fail container creation for +// pods that have nothing to do with the re-profiling. +func TestRender_KeepsMountedPathsWhilePruning(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "product_name"), []byte("kind\n"), 0o644), "stage product_name") + require.NoError(t, os.WriteFile(filepath.Join(src, "product_uuid"), []byte("dead-beef\n"), 0o644), "stage product_uuid") + + dir := t.TempDir() + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: src}), "Render") + + // Watch the directories across a prune: they must be the same inodes + // afterwards, since a bind mount whose source was replaced still resolves + // to the vanished original. + sysDevices := statOrFail(t, filepath.Join(dir, "sys/devices")) + pciDevices := statOrFail(t, filepath.Join(dir, "sys/bus/pci/devices")) + + require.NoError(t, pruneTree(dir), "pruneTree") + + require.True(t, os.SameFile(sysDevices, statOrFail(t, filepath.Join(dir, "sys/devices"))), + "sys/devices replaced by the prune") + require.True(t, os.SameFile(pciDevices, statOrFail(t, filepath.Join(dir, "sys/bus/pci/devices"))), + "sys/bus/pci/devices replaced by the prune") + for _, attr := range []string{"product_name", "product_uuid"} { + _, err := os.Stat(filepath.Join(dir, "sys/devices/virtual/dmi/id", attr)) + require.NoError(t, err, "%s removed by the prune", attr) + } +} + +func statOrFail(t *testing.T, path string) os.FileInfo { + t.Helper() + info, err := os.Stat(path) + require.NoError(t, err, "stat %s", path) + return info +} + func TestRender_NormalizesUppercaseBDF(t *testing.T) { dir := t.TempDir() topo := &config.PCIeTopology{ @@ -216,6 +321,80 @@ func TestRender_NormalizesUppercaseBDF(t *testing.T) { require.NoError(t, err, "expected lowercase symlink") } +// TestRender_MirrorsKernelDMI covers the reason the tree carries a DMI +// directory at all: bind-mounting it over /sys/devices hides the real +// virtual/dmi/id, which kind's mount-product-files.sh hook bind-mounts the +// node's product files onto for every container. Both attributes must exist +// as mount targets; only product_name travels by value. product_uuid is a +// node identifier the kernel exposes 0400 to root alone, and kind mounts the +// node's own copy over it anyway, so the tree must not republish it into +// every served container. +func TestRender_MirrorsKernelDMI(t *testing.T) { + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "product_name"), []byte("kind\n"), 0o644), "stage product_name") + require.NoError(t, os.WriteFile(filepath.Join(src, "product_uuid"), []byte("dead-beef\n"), 0o644), "stage product_uuid") + + dir := t.TempDir() + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: src}), "Render") + + dmi := filepath.Join(dir, "sys/devices/virtual/dmi/id") + name, err := os.ReadFile(filepath.Join(dmi, "product_name")) + require.NoError(t, err, "read product_name") + require.Equal(t, "kind\n", string(name), "product_name") + + uuid, err := os.ReadFile(filepath.Join(dmi, "product_uuid")) + require.NoError(t, err, "read product_uuid") + require.Empty(t, uuid, "product_uuid exists as a mount target, without the node's value") +} + +// TestRender_StandsInForUnreadableDMI covers an attribute the renderer cannot +// read — product_name is mode 0444 on every kernel we know of, but a mirror +// that failed on a permission error would leave the mount target missing, and +// mount(8) cannot create one on a read-only sysfs. +func TestRender_StandsInForUnreadableDMI(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root reads any mode; the permission branch is unreachable") + } + src := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(src, "product_name"), []byte("secret\n"), 0o000), "stage product_name") + + dir := t.TempDir() + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: src}), "Render") + + got, err := os.ReadFile(filepath.Join(dir, "sys/devices/virtual/dmi/id/product_name")) + require.NoError(t, err, "read product_name") + require.Empty(t, got, "an unreadable attribute renders as an empty stand-in") +} + +// TestRender_NoDMIWithoutKernelDMI pins the behavior on kernels that expose +// no DMI at all (Docker Desktop's linuxkit VM, for one): nothing bind-mounts +// product files there, so inventing them would only mislead consumers. +func TestRender_NoDMIWithoutKernelDMI(t *testing.T) { + dir := t.TempDir() + topo := &config.PCIeTopology{ + RootComplexes: []config.RootComplex{{ + ID: "pci0000:00", NUMANode: 0, + Devices: []string{"0000:07:00.0"}, + }}, + } + require.NoError(t, Render(Options{Topology: topo, Output: dir, DMISource: filepath.Join(t.TempDir(), "absent")}), "Render") + + _, err := os.Stat(filepath.Join(dir, "sys/devices/virtual")) + require.True(t, os.IsNotExist(err), "expected no DMI directory, got err=%v", err) +} + // --- Config / Validate tests -------------------------------------------------- func TestValidate_AcceptsCanonicalProfile(t *testing.T) { diff --git a/tests/e2e/go/assertions/pcisysfs.go b/tests/e2e/go/assertions/pcisysfs.go index 18b0644f6..08d720f51 100644 --- a/tests/e2e/go/assertions/pcisysfs.go +++ b/tests/e2e/go/assertions/pcisysfs.go @@ -65,6 +65,41 @@ func PCISysfs(ctx context.Context, k *kube.Client, pod kube.PodRef, gpuCount, ex "distinct PCI root complexes\n%s", roots.Combined()) } +// KernelPCIDevicesDir is where the kernel exposes PCI devices and where the +// mock tree must appear for consumers that cannot be redirected: Go binaries +// read sysfs with direct syscalls, so the libpcimocksys.so shim never sees +// their opens and MOCK_PCI_ROOT has no effect on them. +const KernelPCIDevicesDir = "/sys/bus/pci/devices" + +// PCISysfsAtKernelPath asserts, from inside a container the mock serves (a GPU +// Operator operand, for instance), that the rendered tree arrived at the real +// kernel paths rather than only in the overlay: +// - /sys/bus/pci/devices holds exactly the mock GPUs, so the host's own PCI +// devices are masked and consumers enumerate the profile, +// - reading a device's `vendor` yields NVIDIA, which only works when +// /sys/devices is mounted too (the entries are relative symlinks into it). +func PCISysfsAtKernelPath(ctx context.Context, k *kube.Client, pod kube.PodRef, gpuCount int) { + ginkgo.GinkgoHelper() + + ginkgo.By(fmt.Sprintf("%d mock PCI devices visible at %s", gpuCount, KernelPCIDevicesDir)) + res, err := k.ExecSh(ctx, pod, "ls "+KernelPCIDevicesDir+" 2>/dev/null | wc -l") + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "listing %s: %s", KernelPCIDevicesDir, res.Combined()) + gomega.Expect(atoiTrim(res.Stdout)).To(gomega.Equal(gpuCount), + "device count at %s — the mock tree is not mounted there\n%s", KernelPCIDevicesDir, res.Combined()) + + ginkgo.By("a device's vendor reads through the symlink into /sys/devices") + first, err := k.ExecSh(ctx, pod, "ls "+KernelPCIDevicesDir+" | sort | head -1") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + dev := strings.TrimSpace(first.Stdout) + gomega.Expect(dev).NotTo(gomega.BeEmpty(), "no PCI devices at %s", KernelPCIDevicesDir) + + vendor, err := k.ExecSh(ctx, pod, "cat "+KernelPCIDevicesDir+"/"+dev+"/vendor") + gomega.Expect(err).NotTo(gomega.HaveOccurred(), + "reading vendor for %s — a dangling symlink means /sys/devices is missing", dev) + gomega.Expect(strings.TrimSpace(vendor.Stdout)).To(gomega.Equal("0x10de"), + "vendor for %s\n%s", dev, vendor.Combined()) +} + func atoiTrim(s string) int { n, _ := strconv.Atoi(strings.TrimSpace(s)) return n diff --git a/tests/e2e/go/framework/kube/kube.go b/tests/e2e/go/framework/kube/kube.go index 82b493c36..f263f7fd2 100644 --- a/tests/e2e/go/framework/kube/kube.go +++ b/tests/e2e/go/framework/kube/kube.go @@ -79,6 +79,10 @@ type objectMeta struct { Name string `json:"name"` Labels map[string]string `json:"labels"` Annotations map[string]string `json:"annotations"` + // DeletionTimestamp is set once the object is being deleted. For a pod + // that is the only marker of "Terminating": the phase stays Running + // until its containers exit. + DeletionTimestamp string `json:"deletionTimestamp"` } type nodeCondition struct { @@ -285,6 +289,28 @@ func (c *Client) RunningPodNames(ctx context.Context, ns, selector string) ([]st return out, nil } +// RunningPodOnNode returns a Running pod matching the selector on a given +// node, skipping pods that are on their way out. Callers that exec into a +// DaemonSet's pod need all three filters: a Pending pod matches the selector +// as readily as a Running one, a terminating pod keeps reporting phase +// Running until its containers exit (deletionTimestamp is what marks it), and +// a pod on another node answers about hardware the assertion is not about. +// +// A rollout can still leave no candidate at all, so callers poll. +func (c *Client) RunningPodOnNode(ctx context.Context, ns, selector, node string) (string, error) { + var pl podList + if err := c.getJSON(ctx, &pl, "pods", "-n", ns, "-l", selector, + "--field-selector", "spec.nodeName="+node); err != nil { + return "", err + } + for _, p := range pl.Items { + if p.Status.Phase == "Running" && p.Metadata.DeletionTimestamp == "" { + return p.Metadata.Name, nil + } + } + return "", fmt.Errorf("no Running pod in ns %q matching %q on node %q", ns, selector, node) +} + // PodNode returns the Kubernetes node a pod is scheduled on. func (c *Client) PodNode(ctx context.Context, ns, name string) (string, error) { var p podObj diff --git a/tests/e2e/go/scenario_gpu_operator_test.go b/tests/e2e/go/scenario_gpu_operator_test.go index ccfa7c456..7dfdd5c63 100644 --- a/tests/e2e/go/scenario_gpu_operator_test.go +++ b/tests/e2e/go/scenario_gpu_operator_test.go @@ -18,6 +18,7 @@ import ( "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/config" "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/harness" "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/helm" + "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/kube" "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/runner" "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/profile" ) @@ -74,6 +75,29 @@ var _ = Describe("nvml-mock GPU Operator", Label("gpu-operator"), Ordered, func( assertions.WaitAllocatableGPU(ctx, h.Kube, node, p.ExpectedGPUs(), config.ReadyTimeout(), config.PollInterval()) }) + It("serves the rendered PCI tree to the GFD container at the kernel paths", Label("device-plugin"), func(ctx SpecContext) { + // The NVML-derived labels above cannot distinguish "GFD read + // the mock tree" from "GFD read the host's sysfs and happened + // to agree". Reading the tree from inside the container pins + // the delivery itself, independent of what GFD makes of it. + // The GFD pod on `node`, and only while it is Running and not + // terminating: the specs above assert about that node's + // labels, and a pod the exec cannot reach fails the spec for a + // reason it is not about. Polled because the operator replaces + // its operands a reconcile after nvml-mock rolls (#602), the + // same reason waitOperatorValidatorRunning polls. + var pod string + Eventually(func() (string, error) { + p, err := h.Kube.RunningPodOnNode(ctx, gpuOperatorNamespace, "app=gpu-feature-discovery", node) + pod = p + return p, err + }).WithContext(ctx).WithTimeout(config.ReadyTimeout()).WithPolling(config.PollInterval()). + ShouldNot(BeEmpty(), "no Running gpu-feature-discovery pod on %s", node) + assertions.PCISysfsAtKernelPath(ctx, h.Kube, + kube.PodRef{Namespace: gpuOperatorNamespace, Pod: pod, Container: "gpu-feature-discovery"}, + p.ExpectedGPUs()) + }) + It("exports DCGM device metrics that vary over time", Label("dcgm"), func(ctx SpecContext) { assertions.DCGMDeviceMetrics(ctx, h.Kube, gpuOperatorNamespace, p.DisplayName, p.ExpectedGPUs(), gpmProfiles[name], diff --git a/tests/mocknvml/util-test-config.yaml b/tests/mocknvml/util-test-config.yaml index feb692bd9..9cc6d428b 100644 --- a/tests/mocknvml/util-test-config.yaml +++ b/tests/mocknvml/util-test-config.yaml @@ -395,10 +395,10 @@ nvlink: # PCIe topology - 2 NUMA nodes (dual EPYC), 4 GPUs each. # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root -# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a -# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD -# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the -# ResourceSlices it publishes. See issue #265. +# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU +# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot +# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and +# /sys/devices for them instead. See issues #265 and #673. # ============================================================================= pcie_topology: root_complexes: