feat(mocknvml): decode GPU fabric health for nvidia-smi -q

CHANGELOG.md

@@ -40,6 +40,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `nvmlDeviceGetMemoryErrorCounter` now honours its `locationType` and
   `counterType` arguments instead of returning the DRAM counter for every
   location. (#641)
+- mocknvml: GPU fabric health is now decoded and configurable, so
+  `nvidia-smi -q` reports `Summary : Healthy` / `Bandwidth : Full` on the
+  shipped Grace-Blackwell profiles instead of `N/A` for every row of the
+  `Fabric` → `Health` block. A `fabric:` block with no health keys means a
+  healthy fabric; individual conditions (`degraded_bandwidth`,
+  `route_recovery`, `route_unhealthy`, `access_timeout_recovery`,
+  `incorrect_configuration`) can be faulted under `fabric.health`, and the
+  `Summary` row is derived from them (degraded bandwidth alone reports
+  `Limited Capacity`) unless pinned with `fabric.health_summary`. The raw
+  `fabric.health_mask` stays available as an escape hatch and is no longer
+  silently dropped — it previously had no effect on the rendered output
+  because the health summary was always zero. Fabric health can also be
+  degraded and restored while a workload runs, with
+  `nvml-mock-ctl fabric-health --gpu <idx> route_unhealthy` and
+  `... fabric-health --gpu <idx> healthy`. `Partition Assigned` is reported as
+  NOT_SUPPORTED, matching hardware, which newer `nvidia-smi` builds render as
+  `N/A`. v1/v2 `nvmlDeviceGetGpuFabricInfo` callers are unaffected by the
+  now non-zero summary: the field lives past the end of the v2 struct, pinned
+  by a unit test on the struct-tail boundary. (#677)
 - mocknvml: configured `processes:` now surface in nvidia-smi — the default
   table's Processes box, `-q`, and `--query-compute-apps` all report the
   configured PIDs, names and GPU memory instead of always reporting none.

cmd/nvml-mock-ctl/main.go

@@ -78,6 +78,11 @@ commands:
   sram-ecc --gpu <idx|all|uuid> <count> [--type correctable|parity|secded]
            [--source l2|sm|microcontroller|pcie|other] [--threshold-exceeded]
                                            inject SRAM ECC errors (0 heals)
+  fabric-health --gpu <idx|all|uuid> <condition>[ condition ...]  degrade NVLink fabric health ('healthy' clears)
+         conditions: degraded_bandwidth, route_recovery, route_unhealthy,
+         access_timeout_recovery, or a misconfiguration (no_partition,
+         insufficient_nvlinks, incompatible_gpu_fw, invalid_location,
+         incorrect_sysguid, incorrect_chassis_sn, gpu_state_invalid)
   set    --gpu <idx|all|uuid> key.path=value [key.path=value ...]
   status [--gpu <idx>]
   reset  [--gpu <idx|all|uuid>]
@@ -163,7 +168,7 @@ func run(args []string, stdout, stderr io.Writer) int {
 	case "status":
 		return doStatus(configOverridePath, gpu, stdout, stderr)
 	case "fail", "temp", "temperature", "power", "fan", "util", "utilization",
-		"clocks", "throttle", "pstate", "nvlink-error", "sram-ecc", "set", "reset":
+		"clocks", "throttle", "pstate", "nvlink-error", "sram-ecc", "fabric-health", "set", "reset":
 		sram := sramECCOptions{errorType: sramErrorType, source: sramSource, thresholdExceeded: sramThresholdExceeded}
 		return mutate(cmd, configOverridePath, gpu, mode, links, afterCalls, xid, positional, sram, cfg, base, stdout, stderr)
 	case "watch-allocations":
@@ -327,6 +332,19 @@ func mutate(cmd, configOverridePath, gpu, mode, links string, afterCalls int, xi
 		if code := applyPatch(doc, target, base, patch, stderr); code != 0 {
 			return code
 		}
+	case "fabric-health":
+		if len(positional) == 0 {
+			fprintln(stderr, "fabric-health requires at least one condition (or 'healthy')")
+			return 2
+		}
+		patch, perr := mockctl.FabricHealthPatch(positional)
+		if perr != nil {
+			fprintf(stderr, "%v\n", perr)
+			return 2
+		}
+		if code := applyPatch(doc, target, base, patch, stderr); code != 0 {
+			return code
+		}
 	case "set":
 		if len(positional) == 0 {
 			fprintln(stderr, "set requires at least one key.path=value")

cmd/nvml-mock-ctl/main_test.go

@@ -170,30 +170,54 @@ func TestCLI_SramECCZeroHeals(t *testing.T) {
 	require.Contains(t, readConfigOverride(t, configOverride), "uncorrectable_secded: 0")
 }
 
+func TestCLI_FabricHealthDegradesOneCondition(t *testing.T) {
+	dir := t.TempDir()
+	configOverride := filepath.Join(dir, "overrides.yaml")
+	_, e, c := runCLI(t, configOverride, "fabric-health", "--gpu", "0", "route_unhealthy")
+	require.Equalf(t, 0, c, "fabric-health exited %d: %s", c, e)
+	s := readConfigOverride(t, configOverride)
+	require.Contains(t, s, "route_unhealthy: true")
+	require.Contains(t, s, "route_recovery: false", "fabric-health should write authoritative false conditions")
+	require.Contains(t, s, "health_summary: auto", "the summary must follow the injected conditions")
+}
+
+func TestCLI_FabricHealthHealthyClears(t *testing.T) {
+	dir := t.TempDir()
+	configOverride := filepath.Join(dir, "overrides.yaml")
+	_, e, c := runCLI(t, configOverride, "fabric-health", "--gpu", "0", "route_unhealthy")
+	require.Equalf(t, 0, c, "fabric-health exited %d: %s", c, e)
+	_, e, c = runCLI(t, configOverride, "fabric-health", "--gpu", "0", "healthy")
+	require.Equalf(t, 0, c, "fabric-health healthy exited %d: %s", c, e)
+	require.Contains(t, readConfigOverride(t, configOverride), "route_unhealthy: false")
+}
+
 func TestCLI_ConvenienceArgValidation(t *testing.T) {
 	dir := t.TempDir()
 	configOverride := filepath.Join(dir, "overrides.yaml")
 	cases := [][]string{
-		{"temp", "--gpu", "0"},                              // missing value
-		{"temp", "--gpu", "0", "hot"},                       // non-integer
-		{"fan", "--gpu", "0", "150"},                        // out of range
-		{"power", "--gpu", "0", "--", "-5"},                 // negative watts (-- so it reaches the guard, not flag.Parse)
-		{"power", "--gpu", "0", "NaN"},                      // non-finite watts
-		{"power", "--gpu", "0", "Inf"},                      // non-finite watts
-		{"power", "--gpu", "0", "10000000"},                 // watts overflow guard
-		{"power", "--gpu", "0", "1", "2"},                   // too many values
-		{"util", "--gpu", "0", "150"},                       // out of range
-		{"pstate", "--gpu", "0", "16"},                      // out of range
-		{"throttle", "--gpu", "0"},                          // missing reason
-		{"throttle", "--gpu", "0", "nope"},                  // unknown reason
-		{"throttle", "--gpu", "0", "none", "thermal"},       // none + reason
-		{"nvlink-error", "--gpu", "0"},                      // missing rate
-		{"nvlink-error", "--gpu", "0", "-5"},                // negative rate (flag.Parse stops at -5 -> missing value)
-		{"nvlink-error", "--gpu", "0", "2000000000"},        // rate over cap
-		{"nvlink-error", "--gpu", "0", "--links", "x", "1"}, // non-integer link id
-		{"sram-ecc", "--gpu", "0"},                          // missing count
-		{"sram-ecc", "--gpu", "0", "--type", "nope", "1"},   // unknown error type
-		{"sram-ecc", "--gpu", "0", "--source", "nope", "1"}, // unknown source
+		{"temp", "--gpu", "0"},                                       // missing value
+		{"temp", "--gpu", "0", "hot"},                                // non-integer
+		{"fan", "--gpu", "0", "150"},                                 // out of range
+		{"power", "--gpu", "0", "--", "-5"},                          // negative watts (-- so it reaches the guard, not flag.Parse)
+		{"power", "--gpu", "0", "NaN"},                               // non-finite watts
+		{"power", "--gpu", "0", "Inf"},                               // non-finite watts
+		{"power", "--gpu", "0", "10000000"},                          // watts overflow guard
+		{"power", "--gpu", "0", "1", "2"},                            // too many values
+		{"util", "--gpu", "0", "150"},                                // out of range
+		{"pstate", "--gpu", "0", "16"},                               // out of range
+		{"throttle", "--gpu", "0"},                                   // missing reason
+		{"throttle", "--gpu", "0", "nope"},                           // unknown reason
+		{"throttle", "--gpu", "0", "none", "thermal"},                // none + reason
+		{"nvlink-error", "--gpu", "0"},                               // missing rate
+		{"nvlink-error", "--gpu", "0", "-5"},                         // negative rate (flag.Parse stops at -5 -> missing value)
+		{"nvlink-error", "--gpu", "0", "2000000000"},                 // rate over cap
+		{"nvlink-error", "--gpu", "0", "--links", "x", "1"},          // non-integer link id
+		{"sram-ecc", "--gpu", "0"},                                   // missing count
+		{"sram-ecc", "--gpu", "0", "--type", "nope", "1"},            // unknown error type
+		{"sram-ecc", "--gpu", "0", "--source", "nope", "1"},          // unknown source
+		{"fabric-health", "--gpu", "0"},                              // missing condition
+		{"fabric-health", "--gpu", "0", "nope"},                      // unknown condition
+		{"fabric-health", "--gpu", "0", "healthy", "route_recovery"}, // healthy + fault
 		// The per-source breakdown only covers uncorrectable errors.
 		{"sram-ecc", "--gpu", "0", "--type", "correctable", "--source", "sm", "1"},
 	}

deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml

@@ -55,7 +55,10 @@ device_defaults:
     cluster_uuid: "00000000-0000-0000-0000-000000000001"
     clique_id: 0
     state: "auto"                   # couple GPU fabric registration to fake fabricmanager readiness
-    health_mask: 0
+    # Health defaults to a healthy fabric (nvidia-smi -q reports Summary:
+    # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific
+    # condition, or use `nvml-mock-ctl fabric-health` at runtime.
+    # See docs/configuration.md#fabric-health.
 
   # ---------------------------------------------------------------------------
   # Platform identity — where this node sits in the NVL72 rack, which

deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml

@@ -56,7 +56,10 @@ device_defaults:
     cluster_uuid: "00000000-0000-0000-0000-000000000001"
     clique_id: 0
     state: "auto"                   # couple GPU fabric registration to fake fabricmanager readiness
-    health_mask: 0
+    # Health defaults to a healthy fabric (nvidia-smi -q reports Summary:
+    # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific
+    # condition, or use `nvml-mock-ctl fabric-health` at runtime.
+    # See docs/configuration.md#fabric-health.
 
   # ---------------------------------------------------------------------------
   # Platform identity — where this node sits in the NVL72 rack, which

deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml

@@ -46,7 +46,10 @@ device_defaults:
     cluster_uuid: "00000000-0000-0000-0000-000000000001"
     clique_id: 0
     state: "auto"
-    health_mask: 0
+    # Health defaults to a healthy fabric (nvidia-smi -q reports Summary:
+    # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific
+    # condition, or use `nvml-mock-ctl fabric-health` at runtime.
+    # See docs/configuration.md#fabric-health.
 
   # ---------------------------------------------------------------------------
   # InfoROM versions

deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap

@@ -1023,7 +1023,10 @@ should match snapshot with gb200 profile:
             cluster_uuid: "00000000-0000-0000-0000-000000000001"
             clique_id: 0
             state: "auto"                   # couple GPU fabric registration to fake fabricmanager readiness
-            health_mask: 0
+            # Health defaults to a healthy fabric (nvidia-smi -q reports Summary:
+            # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific
+            # condition, or use `nvml-mock-ctl fabric-health` at runtime.
+            # See docs/configuration.md#fabric-health.
 
           # ---------------------------------------------------------------------------
           # Platform identity — where this node sits in the NVL72 rack, which
@@ -1592,7 +1595,10 @@ should match snapshot with gb300 profile:
             cluster_uuid: "00000000-0000-0000-0000-000000000001"
             clique_id: 0
             state: "auto"                   # couple GPU fabric registration to fake fabricmanager readiness
-            health_mask: 0
+            # Health defaults to a healthy fabric (nvidia-smi -q reports Summary:
+            # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific
+            # condition, or use `nvml-mock-ctl fabric-health` at runtime.
+            # See docs/configuration.md#fabric-health.
 
           # ---------------------------------------------------------------------------
           # Platform identity — where this node sits in the NVL72 rack, which
@@ -2150,7 +2156,10 @@ should match snapshot with h100 profile:
             cluster_uuid: "00000000-0000-0000-0000-000000000001"
             clique_id: 0
             state: "auto"
-            health_mask: 0
+            # Health defaults to a healthy fabric (nvidia-smi -q reports Summary:
+            # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific
+            # condition, or use `nvml-mock-ctl fabric-health` at runtime.
+            # See docs/configuration.md#fabric-health.
 
           # ---------------------------------------------------------------------------
           # InfoROM versions

deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap

@@ -19,7 +19,7 @@ should match snapshot with all overrides:
       template:
         metadata:
           annotations:
-            checksum/config: 3c7c93d7e37bb5597589d285c34278a619be2dfedc35d204ba0bb2dd04c09a35
+            checksum/config: fb921dcb2b78ce2a7f41a6b6bf55586804e92655fcf1a29449d8515db3c44e97
           labels:
             app.kubernetes.io/component: daemon
             app.kubernetes.io/instance: custom

docs/configuration.md

@@ -511,6 +511,86 @@ is deliberately *not* an "NVSwitch entity health" knob — DCGM's NVSwitch/SXID
 health is NSCQ/kernel-log sourced and cannot be driven from a `libnvidia-ml.so`
 mock.
 
+## Fabric Health
+
+`fabric:` describes the GPU's NVLink fabric attachment — the identity fields
+(`cluster_uuid`, `clique_id`, `state`) plus the health `nvidia-smi -q` renders
+under `Fabric` → `Health`. A `fabric:` block with no health keys means a healthy
+fabric, so the shipped Grace-Blackwell profiles report `Summary: Healthy` and
+`Bandwidth: Full` without configuring anything:
+
+```yaml
+device_defaults:
+  fabric:
+    cluster_uuid: "00000000-0000-0000-0000-000000000001"
+    clique_id: 0
+    state: "auto"      # couple registration to fake fabricmanager readiness
+```
+
+Fault a single condition by naming it. Every other condition stays healthy, so a
+consumer sees exactly the fault you configured:
+
+```yaml
+devices:
+  - index: 3
+    fabric:
+      health:
+        route_unhealthy: true
+```
+
+| `health:` key | `nvidia-smi -q` row | values |
+| ------------- | ------------------- | ------ |
+| `degraded_bandwidth` | `Bandwidth` | `false` → `Full`, `true` → `Degraded` |
+| `route_recovery` | `Route Recovery in progress` | `false` / `true` |
+| `route_unhealthy` | `Route Unhealthy` | `false` / `true` |
+| `access_timeout_recovery` | `Access Timeout Recovery` | `false` / `true` |
+| `incorrect_configuration` | `Incorrect Configuration` | `none` (default), `no_partition`, `insufficient_nvlinks`, `incompatible_gpu_fw`, `invalid_location`, `incorrect_sysguid`, `incorrect_chassis_sn`, `gpu_state_invalid` |
+
+### Health summary
+
+The `Summary` row is derived from the conditions above, so an injected fault
+moves it: all clear → `Healthy`, degraded bandwidth alone → `Limited Capacity`,
+any other fault → `Unhealthy`. Pin it with `health_summary` when you need a
+summary that disagrees with the conditions (a driver that reports a fault
+without classifying it, say):
+
+```yaml
+fabric:
+  health_summary: "limited_capacity"   # healthy | unhealthy | limited_capacity
+                                       # | not_supported | auto (default)
+```
+
+`not_supported` reproduces the pre-#677 rendering: `nvidia-smi` treats an
+unreported summary as "no health data" and prints `N/A` for the whole `Health`
+block.
+
+### Raw `health_mask`
+
+`health_mask` sets the NVML v2/v3 health bitmask directly, for encodings the
+`health:` keys cannot express. It replaces the derived mask wholesale, and the
+summary is derived from it:
+
+```yaml
+fabric:
+  health_mask: 0x1aa    # what `health:` with everything clear produces
+```
+
+An explicit `health_mask: 0` means "the driver reported no health at all" and
+renders the whole block as `N/A` — that is why the shipped profiles no longer
+set it.
+
+### `Partition Assigned`
+
+The mock reports this field as `NOT_SUPPORTED`, which is what a real GB300 tray
+in a healthy rack reports (the driver does not answer it). Whether the row
+appears at all is up to the `nvidia-smi` build: 580.173.02 prints
+`Partition Assigned : N/A`, while the 580.65.06 binary the mock image bundles has
+no such label and omits the row for every mask value.
+
+Fabric health can also be degraded and restored at runtime with
+[`nvml-mock-ctl fabric-health`](nvml-mock-ctl.md#fabric-health--degrade-nvlink-fabric-health),
+without restarting the consumer.
+
 ## Available GPU Profiles
 
 Standalone configuration files are provided for each supported GPU model: