feat(nvml-mock): serve the rendered PCI sysfs tree and a DMI identity to Go consumers

CHANGELOG.md

@@ -22,6 +22,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   than 560, keep reporting `N/A`. The `GPU Fabric GUID` row of the same block is
   not modelled and now renders `0x0000000000000000` where it used to read `N/A`.
   (#642)
+- The rendered PCI sysfs tree now reaches consumers written in Go. `lspci` and
+  other libc consumers found it through the `libpcimocksys.so` `LD_PRELOAD`
+  shim, but Go reads sysfs with direct `openat` syscalls no shim can intercept,
+  so GPU Feature Discovery and the NVIDIA DRA driver read the node's real
+  `/sys` and saw no mock GPUs — GFD logged `unable to read PCI device vendor id`
+  and labelled the node `nvidia.com/gpu.mode=unknown`. The staged
+  `sys/devices` and `sys/bus/pci/devices` directories are now bind-mounted
+  read-only onto the kernel paths, through both the CDI spec the DaemonSet
+  generates and the NRI plugin's container adjustment. Both mounts go together:
+  the PCI entries are relative symlinks into `../../../devices/pciDDDD:BB`, so
+  mounting one alone leaves every attribute read failing with `ENOENT`.
+  `/sys/devices` is necessarily mounted whole — it cannot be narrowed to the
+  profile's root complexes, because a bind mount at a path sysfs lacks needs a
+  mountpoint the runtime cannot create on a read-only `/sys` — which hides the
+  host's other device classes from served containers. Under NRI, which injects
+  node-wide, exempt a workload that needs the host's real device tree with the
+  `nvml-mock.nvidia.com/inject: "false"` pod annotation, or a whole namespace
+  with `nri.excludedNamespaces`. The rendered tree also carries the node's DMI
+  attributes in `sys/devices/virtual/dmi/id`, because shadowing `/sys/devices`
+  shadows the directory `/sys/class/dmi/id` resolves into: kind's
+  `mount-product-files.sh` createContainer hook bind-mounts the node's product
+  files there for every container, and `mount(8)` cannot create a target on a
+  read-only sysfs. `product_name` is mirrored by value; `product_uuid` is an
+  empty stand-in, since kind mounts the node's own copy over it and the value is
+  a node identifier the kernel exposes to root alone. The attributes are
+  mirrored, not mocked: `nvidia.com/gpu.machine` still reports what the node
+  itself reports, tracked in #681. Each render now replaces the previous tree
+  instead of adding to it, so re-profiling a node no longer serves both
+  profiles' devices — including a re-profile onto a config that declares no PCI
+  devices at all, which clears the tree rather than leaving the previous one to
+  describe the node — and both mount channels gate on a completion marker the
+  renderer writes last — the mounted directories exist from the start of a
+  render, so their presence alone would serve a tree still missing the bind
+  targets kind's hook needs. (#673)
 - mocknvml: configured `processes:` now surface in nvidia-smi — the default
   table's Processes box, `-q`, and `--query-compute-apps` all report the
   configured PIDs, names and GPU memory instead of always reporting none.

cmd/render-pci-sysfs/main.go

@@ -33,55 +33,92 @@ import (
 	"github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render"
 )
 
-//nolint:cyclop // existing complexity; refactor deferred
+// defaultDMISource is where the kernel exposes the node's SMBIOS identity.
+// The rendered tree mirrors it because serving the tree means bind-mounting
+// it over /sys/devices, which would otherwise hide the DMI directory that
+// /sys/class/dmi/id resolves into.
+const defaultDMISource = "/sys/class/dmi/id"
+
 func main() {
 	var (
-		cfgPath = flag.String("config", "", "path to mock-nvml profile YAML")
-		outDir  = flag.String("output", "", "fake-root directory; tree is written under <output>/sys/...")
-		strict  = flag.Bool("strict", false, "fail if the profile does not declare `pcie_topology:`")
-		dryRun  = flag.Bool("dry-run", false, "validate the config and exit without writing files")
+		opts      options
+		dmiSource = flag.String("dmi-source", defaultDMISource,
+			"kernel DMI directory to mirror into the tree; empty mirrors nothing")
 	)
+	flag.StringVar(&opts.configPath, "config", "", "path to mock-nvml profile YAML")
+	flag.StringVar(&opts.outputDir, "output", "", "fake-root directory; tree is written under <output>/sys/...")
+	flag.BoolVar(&opts.strict, "strict", false, "fail if the profile does not declare `pcie_topology:`")
+	flag.BoolVar(&opts.dryRun, "dry-run", false, "validate the config and exit without writing files")
 	flag.Parse()
+	opts.dmiSource = *dmiSource
 
-	if *cfgPath == "" || *outDir == "" {
+	if opts.configPath == "" || opts.outputDir == "" {
 		fmt.Fprintln(os.Stderr, "usage: render-pci-sysfs --config <yaml> --output <dir> [--strict] [--dry-run]")
 		os.Exit(2)
 	}
 
-	data, err := os.ReadFile(*cfgPath)
+	if err := run(opts); err != nil {
+		fatalf("%v", err)
+	}
+}
+
+// options is the resolved command line.
+type options struct {
+	configPath string
+	outputDir  string
+	dmiSource  string
+	strict     bool
+	dryRun     bool
+}
+
+func run(o options) error {
+	data, err := os.ReadFile(o.configPath)
 	if err != nil {
-		fatalf("read config: %v", err)
+		return fmt.Errorf("read config: %w", err)
 	}
 	var prof config.Profile
 	if err := yaml.Unmarshal(data, &prof); err != nil {
-		fatalf("parse config: %v", err)
+		return fmt.Errorf("parse config: %w", err)
 	}
 	if err := prof.Validate(); err != nil {
-		fatalf("%v", err)
+		return err
 	}
 
 	topo := prof.EffectiveTopology()
-	if topo == nil {
-		fmt.Fprintf(os.Stderr, "render-pci-sysfs: no devices in %s, nothing to render\n", *cfgPath)
-		return
+	if topo != nil && o.strict && prof.PCIeTopology == nil {
+		return fmt.Errorf("--strict: profile %s does not declare `pcie_topology:`", o.configPath)
 	}
-	if *strict && prof.PCIeTopology == nil {
-		fatalf("--strict: profile %s does not declare `pcie_topology:`", *cfgPath)
+	if o.dryRun {
+		reportDryRun(o.configPath, topo)
+		return nil
 	}
 
-	if *dryRun {
-		fmt.Fprintf(os.Stderr, "render-pci-sysfs: %d root complex(es), %d device(s) — config OK\n",
-			len(topo.RootComplexes), countDevices(topo))
-		return
+	// A profile with no devices still goes through Render, rather than
+	// returning here: a tree rendered from a previous profile is on disk and
+	// still served, and Render is what clears it along with its completion
+	// marker, so setup.sh's gate cannot flip on for devices this profile does
+	// not declare.
+	if topo == nil {
+		fmt.Fprintf(os.Stderr, "render-pci-sysfs: no devices in %s, clearing any previously rendered tree\n", o.configPath)
 	}
-
 	if err := render.Render(render.Options{
 		Topology:   topo,
 		Identities: prof.DeviceIdentities(),
-		Output:     *outDir,
+		Output:     o.outputDir,
+		DMISource:  o.dmiSource,
 	}); err != nil {
-		fatalf("render: %v", err)
+		return fmt.Errorf("render: %w", err)
+	}
+	return nil
+}
+
+func reportDryRun(configPath string, topo *config.PCIeTopology) {
+	if topo == nil {
+		fmt.Fprintf(os.Stderr, "render-pci-sysfs: no devices in %s, nothing to render — config OK\n", configPath)
+		return
 	}
+	fmt.Fprintf(os.Stderr, "render-pci-sysfs: %d root complex(es), %d device(s) — config OK\n",
+		len(topo.RootComplexes), countDevices(topo))
 }
 
 func countDevices(t *config.PCIeTopology) int {

cmd/render-pci-sysfs/main_test.go

@@ -0,0 +1,81 @@
+// Copyright 2026 NVIDIA CORPORATION
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/NVIDIA/k8s-test-infra/pkg/system/mockpcisysfs/render"
+)
+
+const profileWithDevices = `
+devices:
+  - index: 0
+    pci:
+      bus_id: "0000:07:00.0"
+`
+
+// A profile whose devices declare no bus_id renders no topology. It is
+// reachable through gpu.customConfig, and it used to return before Render.
+const profileWithoutBusIDs = `
+devices:
+  - index: 0
+`
+
+// TestRun_ClearsTreeWhenProfileDeclaresNoDevices covers re-profiling a node
+// onto a profile with nothing to render. The tree and the completion marker
+// left by the previous profile would otherwise stay on disk, and both serving
+// channels would keep mounting devices this profile does not declare —
+// setup.sh gates on the marker, which said "rendered" about the old tree.
+func TestRun_ClearsTreeWhenProfileDeclaresNoDevices(t *testing.T) {
+	out := t.TempDir()
+	require.NoError(t, run(options{
+		configPath: writeProfile(t, profileWithDevices),
+		outputDir:  out,
+	}), "render a profile with devices")
+	require.FileExists(t, filepath.Join(out, render.MarkerRelPath), "marker after the first render")
+
+	require.NoError(t, run(options{
+		configPath: writeProfile(t, profileWithoutBusIDs),
+		outputDir:  out,
+	}), "render a profile without devices")
+
+	entries, err := os.ReadDir(filepath.Join(out, render.PCIDevicesRelPath))
+	require.NoError(t, err, "read devices dir")
+	require.Empty(t, entries, "the previous profile's devices are still served")
+	require.NoFileExists(t, filepath.Join(out, render.MarkerRelPath),
+		"the marker still claims a rendered tree")
+}
+
+// TestRun_DryRunWritesNothing pins that --dry-run stays a validation pass on
+// both paths, including the one that now prunes.
+func TestRun_DryRunWritesNothing(t *testing.T) {
+	for name, profile := range map[string]string{
+		"with devices":    profileWithDevices,
+		"without devices": profileWithoutBusIDs,
+	} {
+		t.Run(name, func(t *testing.T) {
+			out := t.TempDir()
+			require.NoError(t, run(options{
+				configPath: writeProfile(t, profile),
+				outputDir:  out,
+				dryRun:     true,
+			}), "dry run")
+			entries, err := os.ReadDir(out)
+			require.NoError(t, err, "read output dir")
+			require.Empty(t, entries, "--dry-run wrote to the output directory")
+		})
+	}
+}
+
+func writeProfile(t *testing.T, contents string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte(contents), 0o644), "write profile")
+	return path
+}

deployments/nvml-mock/helm/nvml-mock/README.md

@@ -21,10 +21,11 @@ Deploys a DaemonSet that creates on every node:
 - A fake PCI sysfs tree at `/var/lib/nvml-mock/sys/bus/pci/devices/...` (symlinks
   into `/var/lib/nvml-mock/sys/devices/pciDDDD:BB/...`) so C consumers of the
   PCI sysfs — `lspci` and anything else reaching it through libc — resolve the
-  PCIe root complex via a standard `readlink()`. The NVIDIA DRA driver is a Go
-  binary and does not see this tree, so `dra.k8s.io/pcieRoot` is still absent
-  from its ResourceSlices; see [Known Limitations](#known-limitations) and
-  issue [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265)
+  PCIe root complex via a standard `readlink()`. Consumers written in Go read
+  sysfs with direct syscalls no `LD_PRELOAD` shim can intercept, so the two
+  directories are additionally bind-mounted onto `/sys/bus/pci/devices` and
+  `/sys/devices` in served containers (see
+  [PCI sysfs in containers](#pci-sysfs-in-containers))
 
 Consumers (DRA driver, device plugin) point at `/var/lib/nvml-mock/driver`
 as the NVIDIA driver root and discover GPUs through standard NVML APIs.
@@ -708,6 +709,53 @@ DaemonSet under `set -e` if it finds a typo:
 If a profile omits `pcie_topology:` entirely the renderer falls back to
 a flat single-root layout (every device under `pci0000:00`, NUMA 0).
 
+### PCI sysfs in containers
+
+Reaching the tree through `MOCK_PCI_ROOT` requires the `libpcimocksys.so`
+`LD_PRELOAD` shim, which only works for consumers that go through libc. A Go
+program does not: `os.Open` issues `openat` directly, the shim never sees it,
+and the process reads the node's real `/sys` — where the mock GPUs do not
+exist. GPU Feature Discovery and the NVIDIA DRA driver are both Go.
+
+So the rendered directories are bind-mounted read-only onto the kernel paths
+in containers the mock serves, through the CDI spec the DaemonSet generates at
+`/var/run/cdi/nvidia.yaml` and, when `nri.enabled=true`, through the NRI
+plugin's container adjustment:
+
+| Host | Container |
+|---|---|
+| `/var/lib/nvml-mock/sys/devices` | `/sys/devices` |
+| `/var/lib/nvml-mock/sys/bus/pci/devices` | `/sys/bus/pci/devices` |
+
+Both are needed together: the entries under `/sys/bus/pci/devices` are
+relative symlinks into `../../../devices/pciDDDD:BB`, so mounting only that
+directory yields entries that list but whose every attribute read fails with
+`ENOENT`.
+
+**Trade-off:** `/sys/devices` is mounted whole, which hides the host's other
+device classes (CPU topology among them) from those containers. It cannot be
+narrowed to the profile's root complexes — a bind mount at a path sysfs does
+not already have needs a mountpoint, and the runtime cannot create one on a
+read-only `/sys`. A node running nvml-mock is simulating GPU hardware, so
+serving the tree is not itself configurable. Through CDI a container is served
+only if it requests a mock GPU. Through NRI, which injects ambiently, the two
+existing escape hatches cover it: the pod annotation
+`nvml-mock.nvidia.com/inject: "false"` (`nri.optOutAnnotation`) exempts a
+single workload, and `nri.excludedNamespaces` exempts a whole namespace.
+
+`/sys/devices/virtual/dmi/id` — the directory `/sys/class/dmi/id` resolves
+into — is shadowed along with the rest, so the renderer mirrors the node's
+`product_name` there and leaves an empty `product_uuid` beside it. This is not
+cosmetic: kind's `mount-product-files.sh` createContainer hook bind-mounts the
+node's copies of both onto every container it starts, and `mount(8)` cannot
+create a target on a read-only sysfs, so a missing attribute fails container
+creation for every pod the mock serves. `product_uuid` is a node identifier the
+kernel exposes to root alone and kind mounts its own copy over it, so only the
+target is reproduced, never the value. `product_name` is mirrored, not mocked —
+a node keeps reporting its own machine type, which under kind is the literal
+`kind`, so `nvidia.com/gpu.machine` does not follow the profile. Tracked in
+[#681](https://github.com/NVIDIA/k8s-test-infra/issues/681).
+
 ### Cross-node `ibping`
 
 Sysfs mocking alone lets `ibstat` / `iblinkinfo` work, but real `ibping`
@@ -1362,8 +1410,8 @@ discovery and monitoring. Some host-level subsystems are not mocked:
 
 | What's Missing | Affected Consumer | Impact |
 |----------------|-------------------|--------|
-| `/sys/bus/pci/devices/{busID}` sysfs entries **as a Go program reads them** | DRA driver | The tree is rendered and `lspci` reads it, but the driver is a Go binary: Go's `os` package issues raw syscalls that the `LD_PRELOAD` shim cannot intercept, so it reads the host's real sysfs instead. `dra.k8s.io/pcieRoot` stays absent from ResourceSlices — **blocks topology-aware scheduling demos** (e.g., GPU + SR-IOV VF alignment). Tracked in [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265) |
-| `/sys/bus/pci/devices/{busID}/numa_node` | Device plugin | NUMA-aware topology hints unavailable; scheduling works but NUMA affinity not enforced |
+| `/sys/bus/pci/devices/{busID}` sysfs entries in containers the mock does **not** serve | DRA driver | The tree is now bind-mounted onto the kernel paths for CDI- and NRI-served containers, which is what Go consumers need (see [PCI sysfs in containers](#pci-sysfs-in-containers)). A consumer deployed outside those channels still reads the host's real sysfs; whether `dra.k8s.io/pcieRoot` reaches ResourceSlices is tracked in [#265](https://github.com/NVIDIA/k8s-test-infra/issues/265) |
+| `/sys/bus/pci/devices/{busID}/numa_node` in containers the mock does **not** serve | Device plugin | The renderer writes `numa_node` for every device and it arrives through the same mount, so a served device plugin does get NUMA hints. Outside those channels the hints are unavailable: scheduling works but NUMA affinity is not enforced |
 | `/sys/bus/pci/devices/*/vendor,device,class` **as NFD reads them** (`/host-sys/…`, fixed at link time) | NFD (Node Feature Discovery) | PCI feature labels not auto-detected. `nvidia.com/gpu.present` is written directly by nvml-mock; `pci-10de.present` is created by NFD from a feature file nvml-mock drops in `nodeLabels.featuresDir` — see [Node Labels](#node-labels) |
 
 ### PCIe Root Complex (DRA driver)
@@ -1377,11 +1425,12 @@ W0319 11:41:21.314205       1 nvlib.go:491] error getting PCIe root for device 0
   readlink /sys/bus/pci/devices/0000:07:00.0: no such file or directory
 ```
 
-**This warning is expected** but has real impact. The DRA driver resolves PCIe
-root complex topology by reading sysfs symlinks. Since nvml-mock provides a mock
-NVML library (not a full kernel driver), these sysfs entries don't exist. GPUs
-appear in ResourceSlices and are fully allocatable, but the
-`dra.k8s.io/pcieRoot` topology attribute is absent.
+The DRA driver resolves PCIe root complex topology by reading sysfs symlinks.
+The rendered tree now reaches served containers at `/sys/bus/pci/devices` (see
+[PCI sysfs in containers](#pci-sysfs-in-containers)), so a driver the mock
+serves resolves the root complex; one deployed outside the CDI and NRI paths
+still reads the host's sysfs and logs the warning above, with GPUs allocatable
+but `dra.k8s.io/pcieRoot` absent.
 
 **What this blocks:** DRA topology-aware scheduling that uses `pcieRoot` to
 align devices on the same PCIe root complex — for example, co-scheduling a GPU

deployments/nvml-mock/helm/nvml-mock/profiles/a100.yaml

@@ -434,10 +434,10 @@ infiniband:
 # PCIe topology - 2 NUMA nodes (dual EPYC), 4 GPUs each.
 # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree
 # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root
-# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a
-# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD
-# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the
-# ResourceSlices it publishes. See issue #265.
+# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU
+# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot
+# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and
+# /sys/devices for them instead. See issues #265 and #673.
 # =============================================================================
 pcie_topology:
   root_complexes:

deployments/nvml-mock/helm/nvml-mock/profiles/b200.yaml

@@ -431,10 +431,10 @@ infiniband:
 # PCIe topology - B200, 2 NUMA nodes, 4 GPUs each.
 # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree
 # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root
-# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a
-# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD
-# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the
-# ResourceSlices it publishes. See issue #265.
+# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU
+# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot
+# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and
+# /sys/devices for them instead. See issues #265 and #673.
 # =============================================================================
 pcie_topology:
   root_complexes:

deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml

@@ -513,10 +513,10 @@ infiniband:
 # PCIe topology - GB200, 4 Grace CPU pairs -> 4 NUMA nodes, 2 GPUs each.
 # Consumed by `render-pci-sysfs` to materialize a fake /sys/bus/pci tree
 # under MOCK_PCI_ROOT. C consumers such as `lspci` resolve the PCIe root
-# complex through these symlinks. The NVIDIA DRA driver does NOT: it is a
-# Go binary, and Go's os package issues raw syscalls that the LD_PRELOAD
-# shim cannot intercept, so `dra.k8s.io/pcieRoot` stays absent from the
-# ResourceSlices it publishes. See issue #265.
+# complex through these symlinks. Go consumers (the NVIDIA DRA driver, GPU
+# Feature Discovery) issue raw syscalls the LD_PRELOAD shim cannot
+# intercept, so this tree is bind-mounted at /sys/bus/pci/devices and
+# /sys/devices for them instead. See issues #265 and #673.
 # =============================================================================
 pcie_topology:
   root_complexes: