From 1e36db1547121c6b068fd8288c6b7d34a6a9bbbe Mon Sep 17 00:00:00 2001 From: Giulio Calzolari Date: Wed, 19 Aug 2026 14:39:13 +0200 Subject: [PATCH] feat(mocknvml): decode GPU fabric health for nvidia-smi -q (#677) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every row of the nvidia-smi -q Fabric.Health block read N/A, so a consumer could not tell a healthy fabric from an unknown one and degraded-fabric handling could not be exercised at all: FabricInfo carried a HealthSummary the bridge already copied into the v3 struct, but nothing ever set it, and a zero summary makes nvidia-smi treat the whole block as unreported — which also made the existing fabric.health_mask key silently ineffective. Make fabric health a configured, decoded state. A fabric: block with no health keys means a healthy fabric, so the shipped Grace-Blackwell profiles now report Summary: Healthy and Bandwidth: Full; fabric.health faults one named condition at a time, the summary is derived from the conditions unless pinned with fabric.health_summary, and a raw fabric.health_mask stays available as an escape hatch (an explicit 0 still means "no health reported"). nvml-mock-ctl fabric-health degrades and restores it while a workload runs, authoritatively like throttle, so one condition flips its own row and only its own row. v1/v2 nvmlDeviceGetGpuFabricInfo callers are unaffected now that the summary is non-zero: the field starts exactly where a v2 buffer ends, pinned by a unit test on the struct-tail boundary alongside the shared-field offsets the v2 branch's reinterpret cast depends on. The C ABI harness exercises all three struct versions against a healthy and a faulted device. The e2e assertions read the -q -x document (#658) rather than the human table: the fabric health elements are decoded in schema.go, and the checks compare a whole block per GPU, so one observation covers both the degraded device and its untouched neighbours. A captured document with GPU 1 route-unhealthy is the fixture, and the pre-#677 capture already in testdata is the defect the check must reject. Partition Assigned has no element or row to assert: the label does not exist in the nvidia-smi the mock image bundles, which omits it for every mask value, while the 580.173.02 reference tray prints it as N/A. The mock therefore reports the field as NOT_SUPPORTED, which is what that tray reports. Signed-off-by: Giulio Calzolari --- CHANGELOG.md | 19 + cmd/nvml-mock-ctl/main.go | 20 +- cmd/nvml-mock-ctl/main_test.go | 64 +- .../helm/nvml-mock/profiles/gb200.yaml | 5 +- .../helm/nvml-mock/profiles/gb300.yaml | 5 +- .../helm/nvml-mock/profiles/h100.yaml | 5 +- .../__snapshot__/configmap_test.yaml.snap | 15 +- .../__snapshot__/daemonset_test.yaml.snap | 2 +- docs/configuration.md | 80 +++ docs/nvml-mock-ctl.md | 55 ++ pkg/gpu/mockctl/config_override.go | 77 +++ pkg/gpu/mockctl/config_override_test.go | 74 +++ pkg/gpu/mocknvml/bridge/fabric_tail_test.go | 94 +++ .../configs/mock-nvml-config-gb200.yaml | 2 +- .../configs/mock-nvml-config-gb300.yaml | 2 +- .../configs/mock-nvml-config-h100.yaml | 2 +- pkg/gpu/mocknvml/engine/config_types.go | 42 +- pkg/gpu/mocknvml/engine/fabric.go | 12 +- pkg/gpu/mocknvml/engine/fabric_health.go | 186 ++++++ pkg/gpu/mocknvml/engine/fabric_health_test.go | 294 +++++++++ pkg/gpu/mocknvml/engine/fabric_test.go | 3 +- tests/e2e/go/assertions/nvidiasmi/exec.go | 13 + .../go/assertions/nvidiasmi/fabric_health.go | 107 +++ .../nvidiasmi/fabric_health_test.go | 90 +++ tests/e2e/go/assertions/nvidiasmi/schema.go | 30 + .../assertions/nvidiasmi/testdata/README.md | 2 + .../testdata/qx-gb200-fabric-degraded.xml | 618 ++++++++++++++++++ tests/e2e/go/scenario_fabric_health.go | 121 ++++ tests/e2e/go/scenario_standalone_test.go | 22 + tests/mocknvml/bridge_tests.go | 120 ++++ tests/mocknvml/util-test-config.yaml | 19 + 31 files changed, 2161 insertions(+), 39 deletions(-) create mode 100644 pkg/gpu/mocknvml/bridge/fabric_tail_test.go create mode 100644 pkg/gpu/mocknvml/engine/fabric_health.go create mode 100644 pkg/gpu/mocknvml/engine/fabric_health_test.go create mode 100644 tests/e2e/go/assertions/nvidiasmi/fabric_health.go create mode 100644 tests/e2e/go/assertions/nvidiasmi/fabric_health_test.go create mode 100644 tests/e2e/go/assertions/nvidiasmi/testdata/qx-gb200-fabric-degraded.xml create mode 100644 tests/e2e/go/scenario_fabric_health.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d2d593ad..ab63a016f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `nvmlDeviceGetMemoryErrorCounter` now honours its `locationType` and `counterType` arguments instead of returning the DRAM counter for every location. (#641) +- mocknvml: GPU fabric health is now decoded and configurable, so + `nvidia-smi -q` reports `Summary : Healthy` / `Bandwidth : Full` on the + shipped Grace-Blackwell profiles instead of `N/A` for every row of the + `Fabric` → `Health` block. A `fabric:` block with no health keys means a + healthy fabric; individual conditions (`degraded_bandwidth`, + `route_recovery`, `route_unhealthy`, `access_timeout_recovery`, + `incorrect_configuration`) can be faulted under `fabric.health`, and the + `Summary` row is derived from them (degraded bandwidth alone reports + `Limited Capacity`) unless pinned with `fabric.health_summary`. The raw + `fabric.health_mask` stays available as an escape hatch and is no longer + silently dropped — it previously had no effect on the rendered output + because the health summary was always zero. Fabric health can also be + degraded and restored while a workload runs, with + `nvml-mock-ctl fabric-health --gpu route_unhealthy` and + `... fabric-health --gpu healthy`. `Partition Assigned` is reported as + NOT_SUPPORTED, matching hardware, which newer `nvidia-smi` builds render as + `N/A`. v1/v2 `nvmlDeviceGetGpuFabricInfo` callers are unaffected by the + now non-zero summary: the field lives past the end of the v2 struct, pinned + by a unit test on the struct-tail boundary. (#677) - mocknvml: configured `processes:` now surface in nvidia-smi — the default table's Processes box, `-q`, and `--query-compute-apps` all report the configured PIDs, names and GPU memory instead of always reporting none. diff --git a/cmd/nvml-mock-ctl/main.go b/cmd/nvml-mock-ctl/main.go index 0667c764f..0d9683774 100644 --- a/cmd/nvml-mock-ctl/main.go +++ b/cmd/nvml-mock-ctl/main.go @@ -78,6 +78,11 @@ commands: sram-ecc --gpu [--type correctable|parity|secded] [--source l2|sm|microcontroller|pcie|other] [--threshold-exceeded] inject SRAM ECC errors (0 heals) + fabric-health --gpu [ condition ...] degrade NVLink fabric health ('healthy' clears) + conditions: degraded_bandwidth, route_recovery, route_unhealthy, + access_timeout_recovery, or a misconfiguration (no_partition, + insufficient_nvlinks, incompatible_gpu_fw, invalid_location, + incorrect_sysguid, incorrect_chassis_sn, gpu_state_invalid) set --gpu key.path=value [key.path=value ...] status [--gpu ] reset [--gpu ] @@ -163,7 +168,7 @@ func run(args []string, stdout, stderr io.Writer) int { case "status": return doStatus(configOverridePath, gpu, stdout, stderr) case "fail", "temp", "temperature", "power", "fan", "util", "utilization", - "clocks", "throttle", "pstate", "nvlink-error", "sram-ecc", "set", "reset": + "clocks", "throttle", "pstate", "nvlink-error", "sram-ecc", "fabric-health", "set", "reset": sram := sramECCOptions{errorType: sramErrorType, source: sramSource, thresholdExceeded: sramThresholdExceeded} return mutate(cmd, configOverridePath, gpu, mode, links, afterCalls, xid, positional, sram, cfg, base, stdout, stderr) case "watch-allocations": @@ -327,6 +332,19 @@ func mutate(cmd, configOverridePath, gpu, mode, links string, afterCalls int, xi if code := applyPatch(doc, target, base, patch, stderr); code != 0 { return code } + case "fabric-health": + if len(positional) == 0 { + fprintln(stderr, "fabric-health requires at least one condition (or 'healthy')") + return 2 + } + patch, perr := mockctl.FabricHealthPatch(positional) + if perr != nil { + fprintf(stderr, "%v\n", perr) + return 2 + } + if code := applyPatch(doc, target, base, patch, stderr); code != 0 { + return code + } case "set": if len(positional) == 0 { fprintln(stderr, "set requires at least one key.path=value") diff --git a/cmd/nvml-mock-ctl/main_test.go b/cmd/nvml-mock-ctl/main_test.go index 78f8d18a7..683c3ebab 100644 --- a/cmd/nvml-mock-ctl/main_test.go +++ b/cmd/nvml-mock-ctl/main_test.go @@ -170,30 +170,54 @@ func TestCLI_SramECCZeroHeals(t *testing.T) { require.Contains(t, readConfigOverride(t, configOverride), "uncorrectable_secded: 0") } +func TestCLI_FabricHealthDegradesOneCondition(t *testing.T) { + dir := t.TempDir() + configOverride := filepath.Join(dir, "overrides.yaml") + _, e, c := runCLI(t, configOverride, "fabric-health", "--gpu", "0", "route_unhealthy") + require.Equalf(t, 0, c, "fabric-health exited %d: %s", c, e) + s := readConfigOverride(t, configOverride) + require.Contains(t, s, "route_unhealthy: true") + require.Contains(t, s, "route_recovery: false", "fabric-health should write authoritative false conditions") + require.Contains(t, s, "health_summary: auto", "the summary must follow the injected conditions") +} + +func TestCLI_FabricHealthHealthyClears(t *testing.T) { + dir := t.TempDir() + configOverride := filepath.Join(dir, "overrides.yaml") + _, e, c := runCLI(t, configOverride, "fabric-health", "--gpu", "0", "route_unhealthy") + require.Equalf(t, 0, c, "fabric-health exited %d: %s", c, e) + _, e, c = runCLI(t, configOverride, "fabric-health", "--gpu", "0", "healthy") + require.Equalf(t, 0, c, "fabric-health healthy exited %d: %s", c, e) + require.Contains(t, readConfigOverride(t, configOverride), "route_unhealthy: false") +} + func TestCLI_ConvenienceArgValidation(t *testing.T) { dir := t.TempDir() configOverride := filepath.Join(dir, "overrides.yaml") cases := [][]string{ - {"temp", "--gpu", "0"}, // missing value - {"temp", "--gpu", "0", "hot"}, // non-integer - {"fan", "--gpu", "0", "150"}, // out of range - {"power", "--gpu", "0", "--", "-5"}, // negative watts (-- so it reaches the guard, not flag.Parse) - {"power", "--gpu", "0", "NaN"}, // non-finite watts - {"power", "--gpu", "0", "Inf"}, // non-finite watts - {"power", "--gpu", "0", "10000000"}, // watts overflow guard - {"power", "--gpu", "0", "1", "2"}, // too many values - {"util", "--gpu", "0", "150"}, // out of range - {"pstate", "--gpu", "0", "16"}, // out of range - {"throttle", "--gpu", "0"}, // missing reason - {"throttle", "--gpu", "0", "nope"}, // unknown reason - {"throttle", "--gpu", "0", "none", "thermal"}, // none + reason - {"nvlink-error", "--gpu", "0"}, // missing rate - {"nvlink-error", "--gpu", "0", "-5"}, // negative rate (flag.Parse stops at -5 -> missing value) - {"nvlink-error", "--gpu", "0", "2000000000"}, // rate over cap - {"nvlink-error", "--gpu", "0", "--links", "x", "1"}, // non-integer link id - {"sram-ecc", "--gpu", "0"}, // missing count - {"sram-ecc", "--gpu", "0", "--type", "nope", "1"}, // unknown error type - {"sram-ecc", "--gpu", "0", "--source", "nope", "1"}, // unknown source + {"temp", "--gpu", "0"}, // missing value + {"temp", "--gpu", "0", "hot"}, // non-integer + {"fan", "--gpu", "0", "150"}, // out of range + {"power", "--gpu", "0", "--", "-5"}, // negative watts (-- so it reaches the guard, not flag.Parse) + {"power", "--gpu", "0", "NaN"}, // non-finite watts + {"power", "--gpu", "0", "Inf"}, // non-finite watts + {"power", "--gpu", "0", "10000000"}, // watts overflow guard + {"power", "--gpu", "0", "1", "2"}, // too many values + {"util", "--gpu", "0", "150"}, // out of range + {"pstate", "--gpu", "0", "16"}, // out of range + {"throttle", "--gpu", "0"}, // missing reason + {"throttle", "--gpu", "0", "nope"}, // unknown reason + {"throttle", "--gpu", "0", "none", "thermal"}, // none + reason + {"nvlink-error", "--gpu", "0"}, // missing rate + {"nvlink-error", "--gpu", "0", "-5"}, // negative rate (flag.Parse stops at -5 -> missing value) + {"nvlink-error", "--gpu", "0", "2000000000"}, // rate over cap + {"nvlink-error", "--gpu", "0", "--links", "x", "1"}, // non-integer link id + {"sram-ecc", "--gpu", "0"}, // missing count + {"sram-ecc", "--gpu", "0", "--type", "nope", "1"}, // unknown error type + {"sram-ecc", "--gpu", "0", "--source", "nope", "1"}, // unknown source + {"fabric-health", "--gpu", "0"}, // missing condition + {"fabric-health", "--gpu", "0", "nope"}, // unknown condition + {"fabric-health", "--gpu", "0", "healthy", "route_recovery"}, // healthy + fault // The per-source breakdown only covers uncorrectable errors. {"sram-ecc", "--gpu", "0", "--type", "correctable", "--source", "sm", "1"}, } diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml index c208a1fc1..e1d018ebc 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/gb200.yaml @@ -55,7 +55,10 @@ device_defaults: cluster_uuid: "00000000-0000-0000-0000-000000000001" clique_id: 0 state: "auto" # couple GPU fabric registration to fake fabricmanager readiness - health_mask: 0 + # Health defaults to a healthy fabric (nvidia-smi -q reports Summary: + # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific + # condition, or use `nvml-mock-ctl fabric-health` at runtime. + # See docs/configuration.md#fabric-health. # --------------------------------------------------------------------------- # Platform identity — where this node sits in the NVL72 rack, which diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml index e04e2951f..b4b94af51 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/gb300.yaml @@ -56,7 +56,10 @@ device_defaults: cluster_uuid: "00000000-0000-0000-0000-000000000001" clique_id: 0 state: "auto" # couple GPU fabric registration to fake fabricmanager readiness - health_mask: 0 + # Health defaults to a healthy fabric (nvidia-smi -q reports Summary: + # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific + # condition, or use `nvml-mock-ctl fabric-health` at runtime. + # See docs/configuration.md#fabric-health. # --------------------------------------------------------------------------- # Platform identity — where this node sits in the NVL72 rack, which diff --git a/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml b/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml index 143afc596..0feaef577 100644 --- a/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml +++ b/deployments/nvml-mock/helm/nvml-mock/profiles/h100.yaml @@ -46,7 +46,10 @@ device_defaults: cluster_uuid: "00000000-0000-0000-0000-000000000001" clique_id: 0 state: "auto" - health_mask: 0 + # Health defaults to a healthy fabric (nvidia-smi -q reports Summary: + # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific + # condition, or use `nvml-mock-ctl fabric-health` at runtime. + # See docs/configuration.md#fabric-health. # --------------------------------------------------------------------------- # InfoROM versions diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap index fab078b2f..a00833728 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap +++ b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/configmap_test.yaml.snap @@ -1023,7 +1023,10 @@ should match snapshot with gb200 profile: cluster_uuid: "00000000-0000-0000-0000-000000000001" clique_id: 0 state: "auto" # couple GPU fabric registration to fake fabricmanager readiness - health_mask: 0 + # Health defaults to a healthy fabric (nvidia-smi -q reports Summary: + # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific + # condition, or use `nvml-mock-ctl fabric-health` at runtime. + # See docs/configuration.md#fabric-health. # --------------------------------------------------------------------------- # Platform identity — where this node sits in the NVL72 rack, which @@ -1592,7 +1595,10 @@ should match snapshot with gb300 profile: cluster_uuid: "00000000-0000-0000-0000-000000000001" clique_id: 0 state: "auto" # couple GPU fabric registration to fake fabricmanager readiness - health_mask: 0 + # Health defaults to a healthy fabric (nvidia-smi -q reports Summary: + # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific + # condition, or use `nvml-mock-ctl fabric-health` at runtime. + # See docs/configuration.md#fabric-health. # --------------------------------------------------------------------------- # Platform identity — where this node sits in the NVL72 rack, which @@ -2150,7 +2156,10 @@ should match snapshot with h100 profile: cluster_uuid: "00000000-0000-0000-0000-000000000001" clique_id: 0 state: "auto" - health_mask: 0 + # Health defaults to a healthy fabric (nvidia-smi -q reports Summary: + # Healthy, Bandwidth: Full). Add a `health:` block to fault a specific + # condition, or use `nvml-mock-ctl fabric-health` at runtime. + # See docs/configuration.md#fabric-health. # --------------------------------------------------------------------------- # InfoROM versions diff --git a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap index dd3c474ba..35b51c0ac 100644 --- a/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap +++ b/deployments/nvml-mock/helm/nvml-mock/tests/__snapshot__/daemonset_test.yaml.snap @@ -19,7 +19,7 @@ should match snapshot with all overrides: template: metadata: annotations: - checksum/config: 3c7c93d7e37bb5597589d285c34278a619be2dfedc35d204ba0bb2dd04c09a35 + checksum/config: fb921dcb2b78ce2a7f41a6b6bf55586804e92655fcf1a29449d8515db3c44e97 labels: app.kubernetes.io/component: daemon app.kubernetes.io/instance: custom diff --git a/docs/configuration.md b/docs/configuration.md index 426f51663..13b4adf64 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -511,6 +511,86 @@ is deliberately *not* an "NVSwitch entity health" knob — DCGM's NVSwitch/SXID health is NSCQ/kernel-log sourced and cannot be driven from a `libnvidia-ml.so` mock. +## Fabric Health + +`fabric:` describes the GPU's NVLink fabric attachment — the identity fields +(`cluster_uuid`, `clique_id`, `state`) plus the health `nvidia-smi -q` renders +under `Fabric` → `Health`. A `fabric:` block with no health keys means a healthy +fabric, so the shipped Grace-Blackwell profiles report `Summary: Healthy` and +`Bandwidth: Full` without configuring anything: + +```yaml +device_defaults: + fabric: + cluster_uuid: "00000000-0000-0000-0000-000000000001" + clique_id: 0 + state: "auto" # couple registration to fake fabricmanager readiness +``` + +Fault a single condition by naming it. Every other condition stays healthy, so a +consumer sees exactly the fault you configured: + +```yaml +devices: + - index: 3 + fabric: + health: + route_unhealthy: true +``` + +| `health:` key | `nvidia-smi -q` row | values | +| ------------- | ------------------- | ------ | +| `degraded_bandwidth` | `Bandwidth` | `false` → `Full`, `true` → `Degraded` | +| `route_recovery` | `Route Recovery in progress` | `false` / `true` | +| `route_unhealthy` | `Route Unhealthy` | `false` / `true` | +| `access_timeout_recovery` | `Access Timeout Recovery` | `false` / `true` | +| `incorrect_configuration` | `Incorrect Configuration` | `none` (default), `no_partition`, `insufficient_nvlinks`, `incompatible_gpu_fw`, `invalid_location`, `incorrect_sysguid`, `incorrect_chassis_sn`, `gpu_state_invalid` | + +### Health summary + +The `Summary` row is derived from the conditions above, so an injected fault +moves it: all clear → `Healthy`, degraded bandwidth alone → `Limited Capacity`, +any other fault → `Unhealthy`. Pin it with `health_summary` when you need a +summary that disagrees with the conditions (a driver that reports a fault +without classifying it, say): + +```yaml +fabric: + health_summary: "limited_capacity" # healthy | unhealthy | limited_capacity + # | not_supported | auto (default) +``` + +`not_supported` reproduces the pre-#677 rendering: `nvidia-smi` treats an +unreported summary as "no health data" and prints `N/A` for the whole `Health` +block. + +### Raw `health_mask` + +`health_mask` sets the NVML v2/v3 health bitmask directly, for encodings the +`health:` keys cannot express. It replaces the derived mask wholesale, and the +summary is derived from it: + +```yaml +fabric: + health_mask: 0x1aa # what `health:` with everything clear produces +``` + +An explicit `health_mask: 0` means "the driver reported no health at all" and +renders the whole block as `N/A` — that is why the shipped profiles no longer +set it. + +### `Partition Assigned` + +The mock reports this field as `NOT_SUPPORTED`, which is what a real GB300 tray +in a healthy rack reports (the driver does not answer it). Whether the row +appears at all is up to the `nvidia-smi` build: 580.173.02 prints +`Partition Assigned : N/A`, while the 580.65.06 binary the mock image bundles has +no such label and omits the row for every mask value. + +Fabric health can also be degraded and restored at runtime with +[`nvml-mock-ctl fabric-health`](nvml-mock-ctl.md#fabric-health--degrade-nvlink-fabric-health), +without restarting the consumer. + ## Available GPU Profiles Standalone configuration files are provided for each supported GPU model: diff --git a/docs/nvml-mock-ctl.md b/docs/nvml-mock-ctl.md index 1b29241c5..91e3e9c60 100644 --- a/docs/nvml-mock-ctl.md +++ b/docs/nvml-mock-ctl.md @@ -108,6 +108,11 @@ commands: sram-ecc --gpu [--type correctable|parity|secded] [--source l2|sm|microcontroller|pcie|other] [--threshold-exceeded] inject SRAM ECC errors (0 heals) + fabric-health --gpu [ condition ...] degrade NVLink fabric health ('healthy' clears) + conditions: degraded_bandwidth, route_recovery, route_unhealthy, + access_timeout_recovery, or a misconfiguration (no_partition, + insufficient_nvlinks, incompatible_gpu_fw, invalid_location, + incorrect_sysguid, incorrect_chassis_sn, gpu_state_invalid) set --gpu key.path=value [key.path=value ...] status [--gpu ] reset [--gpu ] @@ -274,6 +279,46 @@ Values surface in `nvidia-smi -q` under `ECC Errors` (both scopes, `SRAM Threshold Exceeded` and `Aggregate Uncorrectable SRAM Sources`) and through the per-location field values DCGM reads. +### `fabric-health` — degrade NVLink fabric health + +Degrades the target's NVLink *fabric* health — the block `nvidia-smi -q` renders +under `Fabric` → `Health`, and the first thing a fault-handling controller reads +to decide whether a GPU's fabric attachment is usable. Positional arguments name +the conditions to report: + +```bash +# a route on GPU 0 is unhealthy +kubectl -n nvml-mock exec "$POD" -- nvml-mock-ctl fabric-health --gpu 0 route_unhealthy +# degraded fabric bandwidth while a route recovers +kubectl -n nvml-mock exec "$POD" -- nvml-mock-ctl fabric-health --gpu 0 degraded_bandwidth route_recovery +# the fabric manager gave this GPU no partition +kubectl -n nvml-mock exec "$POD" -- nvml-mock-ctl fabric-health --gpu 0 no_partition +# recover +kubectl -n nvml-mock exec "$POD" -- nvml-mock-ctl fabric-health --gpu 0 healthy +``` + +| condition | reported as | +| --------- | ----------- | +| `degraded_bandwidth` | `Bandwidth: Degraded`, `Summary: Limited Capacity` | +| `route_recovery` | `Route Recovery in progress: True` | +| `route_unhealthy` | `Route Unhealthy: True` | +| `access_timeout_recovery` | `Access Timeout Recovery: True` | +| `no_partition`, `insufficient_nvlinks`, `incompatible_gpu_fw`, `invalid_location`, `incorrect_sysguid`, `incorrect_chassis_sn`, `gpu_state_invalid` | `Incorrect Configuration: ` | +| `healthy` | clears every condition | + +Like `throttle`, it is *authoritative*: the named conditions are turned on and +every other condition is turned off, so repeated calls replace rather than +accumulate. `healthy` (on its own) clears them all — recovery does not need +`reset`, so the device's other overrides survive it. + +The `Summary` row follows the conditions (`Unhealthy` for a fault, `Limited +Capacity` for degraded bandwidth alone, `Healthy` when clear), including for a +profile that pinned `fabric.health_summary`: injecting a fault releases the pin, +because a summary that stays `Healthy` while a condition reports a fault is not a +state hardware can be in. See +[fabric health configuration](configuration.md#fabric-health) for the static +equivalent. + ### `set` — set arbitrary fields `set` takes one or more `key.path=value` pairs. The path is the YAML/JSON path @@ -335,6 +380,7 @@ the pristine profile within one TTL. | ------ | --------------------------- | ------ | | `nvml-mock-ctl reset [--gpu ]` | clears the targeted bucket(s) from `overrides.yaml` | device(s) revert to pristine profile within one TTL | | `nvml-mock-ctl fail --gpu --mode healthy` | removes just the `failure` block for the target | that device recovers within one TTL; other overrides stay | +| `nvml-mock-ctl fabric-health --gpu healthy` | clears just the fabric health conditions for the target | that device's fabric reports healthy within one TTL; other overrides stay | | DaemonSet pod restart | `setup.sh` deletes `overrides.yaml` on startup | **all** overrides wiped; back to pristine profile | | Consumer pod restart | none — the config override lives on the node, not in the consumer | consumer re-reads and picks up the *current* config override (does **not** reset it) | | `helm upgrade` (profile/values change) | rolls the DaemonSet pod (config checksum + `RollingUpdate`), so `setup.sh` wipes `overrides.yaml` on the new pod | **all** overrides reset to the new pristine config; only an upgrade that does not recreate the nvml-mock pod leaves an config override in place | @@ -385,6 +431,15 @@ kubectl exec -- nvidia-smi --id=0 \ kubectl -n nvml-mock exec "$POD" -- nvml-mock-ctl throttle --gpu 0 none ``` +```bash +# 3d) Degrade GPU 0's NVLink fabric route while the workload keeps running. +kubectl -n nvml-mock exec "$POD" -- nvml-mock-ctl fabric-health --gpu 0 route_unhealthy +# verify from any consumer pod (Fabric -> Health block): +kubectl exec -- nvidia-smi --id=0 -q | grep -A 7 Health +# recover, keeping the device's other overrides: +kubectl -n nvml-mock exec "$POD" -- nvml-mock-ctl fabric-health --gpu 0 healthy +``` + ```bash # 4) Set several fields on GPU 0 in one call kubectl -n nvml-mock exec "$POD" -- nvml-mock-ctl set --gpu 0 \ diff --git a/pkg/gpu/mockctl/config_override.go b/pkg/gpu/mockctl/config_override.go index 2210844f5..165680e5f 100644 --- a/pkg/gpu/mockctl/config_override.go +++ b/pkg/gpu/mockctl/config_override.go @@ -327,6 +327,83 @@ func SramECCPatch(count uint64, errorType, source string, thresholdExceeded bool }, nil } +// fabricHealthConditionKeys maps CLI-friendly fabric health condition names to +// the fabric.health boolean they set. The canonical JSON keys are accepted +// directly; the aliases cover how operators say them out loud. +var fabricHealthConditionKeys = map[string]string{ + "degraded_bandwidth": "degraded_bandwidth", + "bandwidth": "degraded_bandwidth", + "degraded_bw": "degraded_bandwidth", + "route_recovery": "route_recovery", + "route_unhealthy": "route_unhealthy", + "access_timeout_recovery": "access_timeout_recovery", + "access_timeout": "access_timeout_recovery", +} + +// allFabricHealthConditionKeys is the full set of fabric.health booleans, used +// to write an authoritative all-clear baseline so a FabricHealthPatch +// represents exactly the requested conditions. +var allFabricHealthConditionKeys = []string{ + "degraded_bandwidth", "route_recovery", "route_unhealthy", "access_timeout_recovery", +} + +// FabricHealthPatch builds a config override patch that sets a device's NVLink +// fabric health conditions, so a fabric can degrade under a running workload +// (#677). conditions are CLI-friendly names (see fabricHealthConditionKeys) +// plus the incorrect-configuration values the engine accepts +// (engine.FabricIncorrectConfigNames, e.g. "no_partition"); the special +// condition "healthy" clears everything and cannot be combined with others. +// +// Like ThrottlePatch the block is authoritative: every condition is written +// (requested ones faulted, the rest clear) so repeated invocations replace +// rather than accumulate. It also releases fabric.health_summary to "auto", +// because a profile that pins a summary would otherwise keep reporting it no +// matter which condition is injected. +func FabricHealthPatch(conditions []string) (map[string]any, error) { + if len(conditions) == 0 { + return nil, errors.New("fabric-health requires at least one condition (or 'healthy')") + } + health := map[string]any{"incorrect_configuration": "none"} + for _, k := range allFabricHealthConditionKeys { + health[k] = false + } + for _, c := range conditions { + key := strings.ToLower(strings.TrimSpace(c)) + if key == "healthy" || key == "none" { + if len(conditions) != 1 { + return nil, errors.New("fabric-health 'healthy' cannot be combined with other conditions") + } + break + } + if canonical, ok := fabricHealthConditionKeys[key]; ok { + health[canonical] = true + continue + } + if misconfig, ok := fabricIncorrectConfigName(key); ok { + health["incorrect_configuration"] = misconfig + continue + } + return nil, fmt.Errorf("unknown fabric health condition %q", c) + } + return map[string]any{"fabric": map[string]any{ + "health": health, + "health_summary": "auto", + }}, nil +} + +// fabricIncorrectConfigName reports whether name is one of the +// incorrect-configuration values the engine decodes. "not_supported" is +// excluded: it means "this driver does not answer the question", which is a +// profile-level statement rather than something to inject at runtime. +func fabricIncorrectConfigName(name string) (string, bool) { + for _, known := range engine.FabricIncorrectConfigNames() { + if known == name && known != "none" && known != "not_supported" { + return known, true + } + } + return "", false +} + // throttleReasonKeys maps CLI-friendly throttle reason names to the // clocks_throttle_reasons config field they enable. The canonical JSON keys are // accepted directly; short aliases cover the common thermal/power cases. diff --git a/pkg/gpu/mockctl/config_override_test.go b/pkg/gpu/mockctl/config_override_test.go index 742064f98..707c689fc 100644 --- a/pkg/gpu/mockctl/config_override_test.go +++ b/pkg/gpu/mockctl/config_override_test.go @@ -298,6 +298,80 @@ func TestThrottlePatch_Errors(t *testing.T) { require.Error(t, err, "expected error combining none with other reasons") } +func TestFabricHealthPatch_InjectsOneCondition(t *testing.T) { + patch, err := FabricHealthPatch([]string{"route_unhealthy"}) + require.NoError(t, err) + base := &engine.DeviceConfig{Fabric: &engine.FabricConfig{CliqueID: 4, State: "auto"}} + require.NoError(t, Validate(base, patch)) + merged, err := engine.MergeDeviceConfig(base, patch) + require.NoError(t, err) + + require.NotNil(t, merged.Fabric.Health) + require.True(t, merged.Fabric.Health.RouteUnhealthy, "route_unhealthy") + require.False(t, merged.Fabric.Health.RouteRecovery, "neighbouring conditions must stay healthy") + require.False(t, merged.Fabric.Health.DegradedBandwidth, "neighbouring conditions must stay healthy") + // Fabric identity is a separate concern and must survive the patch. + require.Equal(t, uint32(4), merged.Fabric.CliqueID, "clique") + require.Equal(t, "auto", merged.Fabric.State, "state") +} + +// TestFabricHealthPatch_Authoritative pins the same replace-not-accumulate +// contract the throttle command has: a second invocation must describe the +// whole health state, not add to it. +func TestFabricHealthPatch_Authoritative(t *testing.T) { + patch, err := FabricHealthPatch([]string{"degraded_bandwidth"}) + require.NoError(t, err) + base := &engine.DeviceConfig{Fabric: &engine.FabricConfig{ + Health: &engine.FabricHealthConfig{RouteUnhealthy: true, IncorrectConfiguration: "no_partition"}, + }} + merged, err := engine.MergeDeviceConfig(base, patch) + require.NoError(t, err) + require.True(t, merged.Fabric.Health.DegradedBandwidth, "degraded_bandwidth") + require.False(t, merged.Fabric.Health.RouteUnhealthy, "a stale condition must be cleared") + require.Equal(t, "none", merged.Fabric.Health.IncorrectConfiguration, "a stale misconfiguration must be cleared") +} + +func TestFabricHealthPatch_HealthyClears(t *testing.T) { + patch, err := FabricHealthPatch([]string{"healthy"}) + require.NoError(t, err) + merged, err := engine.MergeDeviceConfig(&engine.DeviceConfig{Fabric: &engine.FabricConfig{ + Health: &engine.FabricHealthConfig{RouteUnhealthy: true}, + }}, patch) + require.NoError(t, err) + require.False(t, merged.Fabric.Health.RouteUnhealthy, "healthy must clear every condition") +} + +// TestFabricHealthPatch_ReleasesPinnedSummary covers the interaction with a +// profile that pins fabric.health_summary: the injected conditions would +// otherwise have no effect on the summary nvidia-smi reports, so the patch +// hands the summary back to derivation. +func TestFabricHealthPatch_ReleasesPinnedSummary(t *testing.T) { + patch, err := FabricHealthPatch([]string{"route_unhealthy"}) + require.NoError(t, err) + merged, err := engine.MergeDeviceConfig(&engine.DeviceConfig{Fabric: &engine.FabricConfig{ + HealthSummary: "healthy", + }}, patch) + require.NoError(t, err) + require.Equal(t, "auto", merged.Fabric.HealthSummary, "summary must follow the injected conditions") +} + +func TestFabricHealthPatch_NamedMisconfiguration(t *testing.T) { + patch, err := FabricHealthPatch([]string{"no_partition"}) + require.NoError(t, err) + merged, err := engine.MergeDeviceConfig(&engine.DeviceConfig{}, patch) + require.NoError(t, err) + require.Equal(t, "no_partition", merged.Fabric.Health.IncorrectConfiguration) +} + +func TestFabricHealthPatch_Errors(t *testing.T) { + _, err := FabricHealthPatch(nil) + require.Error(t, err, "expected error for no conditions") + _, err = FabricHealthPatch([]string{"banana"}) + require.Error(t, err, "expected error for unknown condition") + _, err = FabricHealthPatch([]string{"healthy", "route_unhealthy"}) + require.Error(t, err, "expected error combining healthy with a fault") +} + func TestResolveTarget_UUID(t *testing.T) { cfg := &engine.Config{YAMLConfig: &engine.YAMLConfig{ Devices: []engine.DeviceOverride{{Index: 3, UUID: "GPU-abc"}}, diff --git a/pkg/gpu/mocknvml/bridge/fabric_tail_test.go b/pkg/gpu/mocknvml/bridge/fabric_tail_test.go new file mode 100644 index 000000000..139b85409 --- /dev/null +++ b/pkg/gpu/mocknvml/bridge/fabric_tail_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Struct-tail boundary tests for nvmlDeviceGetGpuFabricInfoV. +// +// A v2 caller reaches the bridge through a buffer that is only as big as the +// v2 struct: go-nvml's GpuFabricInfoHandler.V2() declares a GpuFabricInfo_v2 +// and casts it to *GpuFabricInfoV (the v3 alias) before calling. The bridge's +// v2 branch therefore may only write the fields the two layouts share — and +// that is only safe while healthSummary lives strictly past the end of a v2 +// struct, and while every shared field sits at the same offset in both. +// +// These invariants were free while the summary was always zero: writing a zero +// byte past a caller's buffer is still a corruption, but a benign-looking one. +// Now that a healthy fabric reports a non-zero summary (#677), a dispatch or +// layout regression writes a visibly wrong byte into a caller's neighbouring +// memory, so the boundary is pinned here. +// +// Like the sibling layout tests, these are pure Go: the package carries +// //export directives, so cgo is unavailable in its test files. They pin the +// go-nvml structs, which TestFabricStructLayouts pins byte-for-byte against +// the hand-written C layouts in nvml_types.h. + +package main + +import ( + "testing" + "unsafe" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + "github.com/stretchr/testify/require" +) + +// TestFabricHealthSummaryLiesOutsideV2 pins the tail boundary: the v3-only +// healthSummary byte must begin at or after the end of a v2 struct, so no +// write the bridge's v2 branch is allowed to make can reach it, and so the +// v3 branch running for a v2 caller would demonstrably overflow. +func TestFabricHealthSummaryLiesOutsideV2(t *testing.T) { + v2Size := unsafe.Sizeof(nvml.GpuFabricInfo_v2{}) + summaryOffset := unsafe.Offsetof(nvml.GpuFabricInfo_v3{}.HealthSummary) + + require.Equal(t, v2Size, summaryOffset, + "healthSummary sits at offset %d but a v2 caller's buffer ends at %d: the v2 branch's "+ + "shared-prefix assumption no longer holds and it must zero or skip the tail explicitly", + summaryOffset, v2Size) + require.Greater(t, unsafe.Sizeof(nvml.GpuFabricInfo_v3{}), v2Size, + "v3 must stay strictly larger than v2, or the version dispatch has nothing to protect") +} + +// TestFabricSharedFieldOffsetsMatch pins the precondition of the bridge's +// reinterpret cast in the v2 branch: every field the two versions share must +// live at the same offset. If go-nvml or nvml_types.h ever inserted a field +// into v3's prefix, the v2 branch would keep compiling and start writing each +// value into the wrong slot of the caller's struct. +func TestFabricSharedFieldOffsetsMatch(t *testing.T) { + var v2 nvml.GpuFabricInfo_v2 + var v3 nvml.GpuFabricInfo_v3 + cases := map[string]struct{ inV2, inV3 uintptr }{ + "version": {unsafe.Offsetof(v2.Version), unsafe.Offsetof(v3.Version)}, + "clusterUuid": {unsafe.Offsetof(v2.ClusterUuid), unsafe.Offsetof(v3.ClusterUuid)}, + "status": {unsafe.Offsetof(v2.Status), unsafe.Offsetof(v3.Status)}, + "cliqueId": {unsafe.Offsetof(v2.CliqueId), unsafe.Offsetof(v3.CliqueId)}, + "state": {unsafe.Offsetof(v2.State), unsafe.Offsetof(v3.State)}, + "healthMask": {unsafe.Offsetof(v2.HealthMask), unsafe.Offsetof(v3.HealthMask)}, + } + for name, tc := range cases { + require.Equal(t, tc.inV2, tc.inV3, + "%s is at offset %d in v2 but %d in v3 — the v2 branch's reinterpret cast writes the wrong bytes", + name, tc.inV2, tc.inV3) + } +} + +// TestClassifyFabricVersion_V2TagNeverTakesV3Path is the behavioural half of +// the boundary. The layout guarantees above only matter because the v2 tag +// must classify as v2: a v2 tag misclassified as v3 makes the bridge write +// healthSummary four bytes past a 36-byte caller buffer. +func TestClassifyFabricVersion_V2TagNeverTakesV3Path(t *testing.T) { + v2Tag := FabricStructVersion(unsafe.Sizeof(nvml.GpuFabricInfo_v2{}), 2) + v3Tag := FabricStructVersion(unsafe.Sizeof(nvml.GpuFabricInfo_v3{}), 3) + + require.Equal(t, FabricVersionV2, ClassifyFabricVersion(v2Tag, v2Tag, v3Tag), + "a go-nvml V2() caller must take the v2 branch") + require.NotEqual(t, v2Tag, v3Tag, "the two tags must be distinguishable") +} diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml index ca9bb033b..6535d141c 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-gb200.yaml @@ -46,7 +46,7 @@ device_defaults: cluster_uuid: "00000000-0000-0000-0000-000000000001" clique_id: 0 state: "auto" - health_mask: 0 + # Health defaults to a healthy fabric; see docs/configuration.md#fabric-health. # --------------------------------------------------------------------------- # InfoROM versions diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-gb300.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-gb300.yaml index 121d2d61a..5b7d0e549 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-gb300.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-gb300.yaml @@ -47,7 +47,7 @@ device_defaults: cluster_uuid: "00000000-0000-0000-0000-000000000001" clique_id: 0 state: "auto" - health_mask: 0 + # Health defaults to a healthy fabric; see docs/configuration.md#fabric-health. # --------------------------------------------------------------------------- # InfoROM versions diff --git a/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml b/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml index c6d7778b0..2ac4eb6db 100644 --- a/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml +++ b/pkg/gpu/mocknvml/configs/mock-nvml-config-h100.yaml @@ -46,7 +46,7 @@ device_defaults: cluster_uuid: "00000000-0000-0000-0000-000000000001" clique_id: 0 state: "auto" - health_mask: 0 + # Health defaults to a healthy fabric; see docs/configuration.md#fabric-health. # --------------------------------------------------------------------------- # InfoROM versions diff --git a/pkg/gpu/mocknvml/engine/config_types.go b/pkg/gpu/mocknvml/engine/config_types.go index 013fdfed6..f285f1b35 100644 --- a/pkg/gpu/mocknvml/engine/config_types.go +++ b/pkg/gpu/mocknvml/engine/config_types.go @@ -715,8 +715,46 @@ type FabricConfig struct { // readiness marker when fabricmanager is enabled, and resolves to // "completed" when it is disabled (see engine/fabric_readiness.go). State string `json:"state,omitempty"` - // HealthMask is the v2 health bitmask. Defaults to 0 (healthy). - HealthMask uint32 `json:"health_mask,omitempty"` + // HealthSummary pins the overall fabric health nvidia-smi reports as + // Fabric.Health.Summary: "healthy", "unhealthy", "limited_capacity" + // or "not_supported". Empty (or "auto") derives it from the conditions + // in Health, which is what makes an injected fault move the summary. + HealthSummary string `json:"health_summary,omitempty"` + // Health carries the individual fabric health conditions. Absent means + // an all-clear fabric. + Health *FabricHealthConfig `json:"health,omitempty"` + // HealthMask is an escape hatch for the raw v2/v3 health bitmask, + // for encodings Health cannot express. When set it replaces the mask + // derived from Health wholesale, and the summary is derived from it. + // A raw 0 means "the driver reported no health at all", which + // nvidia-smi renders as N/A for the whole Health block. + HealthMask *uint32 `json:"health_mask,omitempty"` +} + +// FabricHealthConfig models the individual NVLink fabric health conditions +// nvidia-smi renders under Fabric.Health, each mapping to one slot of the NVML +// health bitmask (see engine/fabric_health.go). Every condition defaults to +// its healthy value, so an absent block describes a healthy fabric rather than +// an unknown one. +// +// PartitionAssigned is deliberately absent: real hardware leaves that field +// unanswered (nvidia-smi renders it as N/A even on a healthy rack), so the +// mock reports the same. Use HealthMask to encode it. +type FabricHealthConfig struct { + // DegradedBandwidth reports the fabric attachment as running below + // full bandwidth (nvidia-smi Bandwidth: Degraded rather than Full). + // On its own it summarises as limited capacity, not unhealthy. + DegradedBandwidth bool `json:"degraded_bandwidth,omitempty"` + // RouteRecovery reports a route recovery in progress. + RouteRecovery bool `json:"route_recovery,omitempty"` + // RouteUnhealthy reports the GPU's fabric route as unhealthy. + RouteUnhealthy bool `json:"route_unhealthy,omitempty"` + // AccessTimeoutRecovery reports an access-timeout recovery in progress. + AccessTimeoutRecovery bool `json:"access_timeout_recovery,omitempty"` + // IncorrectConfiguration names a detected fabric misconfiguration + // ("no_partition", "insufficient_nvlinks", ...; see + // FabricIncorrectConfigNames). Empty means none detected. + IncorrectConfiguration string `json:"incorrect_configuration,omitempty"` } // TopologyDocument is the cluster-level ConfigMap that maps individual diff --git a/pkg/gpu/mocknvml/engine/fabric.go b/pkg/gpu/mocknvml/engine/fabric.go index 8cfaf3b70..fd6fc41b0 100644 --- a/pkg/gpu/mocknvml/engine/fabric.go +++ b/pkg/gpu/mocknvml/engine/fabric.go @@ -84,16 +84,18 @@ func (d *ConfigurableDevice) GetMockFabricInfoV() (FabricInfo, nvml.Return) { return FabricInfo{}, nvml.ERROR_NOT_SUPPORTED } info := buildFabricInfo(cfg.Fabric) - debugLog("[NVML] nvmlDeviceGetGpuFabricInfoV -> clique=%d state=%d healthMask=0x%x\n", - info.CliqueID, info.State, info.HealthMask) + debugLog("[NVML] nvmlDeviceGetGpuFabricInfoV -> clique=%d state=%d healthMask=0x%x healthSummary=%d\n", + info.CliqueID, info.State, info.HealthMask, info.HealthSummary) return info, nvml.SUCCESS } func buildFabricInfo(cfg *FabricConfig) FabricInfo { + mask := resolveFabricHealthMask(cfg) info := FabricInfo{ - CliqueID: cfg.CliqueID, - HealthMask: cfg.HealthMask, - State: resolveFabricState(cfg.State), + CliqueID: cfg.CliqueID, + HealthMask: mask, + HealthSummary: resolveFabricHealthSummary(cfg, mask), + State: resolveFabricState(cfg.State), } info.ClusterUUID = parseClusterUUID(cfg.ClusterUUID) return info diff --git a/pkg/gpu/mocknvml/engine/fabric_health.go b/pkg/gpu/mocknvml/engine/fabric_health.go new file mode 100644 index 000000000..d8f270f66 --- /dev/null +++ b/pkg/gpu/mocknvml/engine/fabric_health.go @@ -0,0 +1,186 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package engine + +import "strings" + +// FabricHealthSummary* values mirror NVML_GPU_FABRIC_HEALTH_SUMMARY_* from the +// public NVML header, re-declared here for the same reason as FabricState*: +// the engine layer stays free of CGo and callers compare against named +// constants. fabric_health_test.go pins them against go-nvml. +// +// A zero summary means "the driver did not report health", which nvidia-smi +// renders as N/A for every row of the Fabric.Health block — including the +// per-condition rows it would otherwise decode out of the health mask. That is +// why the mock defaults to Healthy rather than to the Go zero value. +const ( + FabricHealthSummaryNotSupported uint8 = 0 + FabricHealthSummaryHealthy uint8 = 1 + FabricHealthSummaryUnhealthy uint8 = 2 + FabricHealthSummaryLimitedCapacity uint8 = 3 +) + +// Tri-state values every boolean condition in the health mask takes, mirroring +// NVML_GPU_FABRIC_HEALTH_MASK__{NOT_SUPPORTED,TRUE,FALSE}. All four +// boolean conditions share this encoding. +const ( + fabricHealthNotSupported uint32 = 0 + fabricHealthTrue uint32 = 1 + fabricHealthFalse uint32 = 2 +) + +// fabricHealthField is one condition's slot in the health mask: a bit offset +// and a width mask, mirroring NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_* and +// _WIDTH_*. Consumers read a slot with the NVML_GPU_FABRIC_HEALTH_STATUS_GET +// macro, which is what get() reproduces. +type fabricHealthField struct { + shift uint32 + width uint32 +} + +func (f fabricHealthField) set(mask, value uint32) uint32 { + return (mask & ^(f.width << f.shift)) | ((value & f.width) << f.shift) +} + +func (f fabricHealthField) get(mask uint32) uint32 { + return (mask >> f.shift) & f.width +} + +var ( + fabricHealthDegradedBW = fabricHealthField{shift: 0, width: 0x3} + fabricHealthRouteRecovery = fabricHealthField{shift: 2, width: 0x3} + fabricHealthRouteUnhealthy = fabricHealthField{shift: 4, width: 0x3} + fabricHealthAccessTimeoutRecovery = fabricHealthField{shift: 6, width: 0x3} + fabricHealthIncorrectConfig = fabricHealthField{shift: 8, width: 0xf} +) + +// Incorrect-configuration values, mirroring +// NVML_GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_*. Unlike the boolean +// conditions this slot carries which misconfiguration was detected, so it is +// configured by name rather than as a bool. +const fabricIncorrectConfigNone uint32 = 1 + +var fabricIncorrectConfigValues = map[string]uint32{ + "not_supported": 0, + "none": fabricIncorrectConfigNone, + "incorrect_sysguid": 2, + "incorrect_chassis_sn": 3, + "no_partition": 4, + "insufficient_nvlinks": 5, + "incompatible_gpu_fw": 6, + "invalid_location": 7, + "gpu_state_invalid": 8, +} + +// FabricIncorrectConfigNames lists the accepted fabric.health +// incorrect_configuration values, so the CLI and its help text cannot drift +// from what the engine parses. +func FabricIncorrectConfigNames() []string { + out := make([]string, 0, len(fabricIncorrectConfigValues)) + for name := range fabricIncorrectConfigValues { + out = append(out, name) + } + return out +} + +// resolveFabricHealthMask builds the health mask a consumer decodes. A raw +// health_mask is an escape hatch for encodings the symbolic keys cannot +// express (a partition-assigned answer, a future condition) and wins outright; +// otherwise every condition is encoded from fabric.health, which defaults to +// an all-clear fabric. +func resolveFabricHealthMask(cfg *FabricConfig) uint32 { + if cfg.HealthMask != nil { + return *cfg.HealthMask + } + h := cfg.Health + if h == nil { + h = &FabricHealthConfig{} + } + var mask uint32 + mask = fabricHealthDegradedBW.set(mask, fabricHealthBool(h.DegradedBandwidth)) + mask = fabricHealthRouteRecovery.set(mask, fabricHealthBool(h.RouteRecovery)) + mask = fabricHealthRouteUnhealthy.set(mask, fabricHealthBool(h.RouteUnhealthy)) + mask = fabricHealthAccessTimeoutRecovery.set(mask, fabricHealthBool(h.AccessTimeoutRecovery)) + mask = fabricHealthIncorrectConfig.set(mask, parseFabricIncorrectConfig(h.IncorrectConfiguration)) + return mask +} + +func fabricHealthBool(faulted bool) uint32 { + if faulted { + return fabricHealthTrue + } + return fabricHealthFalse +} + +// parseFabricIncorrectConfig maps a configured name to its mask value. An +// empty or unrecognised name means "no misconfiguration detected", keeping +// misconfiguration from blocking startup (as parseFabricState does). +func parseFabricIncorrectConfig(s string) uint32 { + if v, ok := fabricIncorrectConfigValues[strings.ToLower(strings.TrimSpace(s))]; ok { + return v + } + return fabricIncorrectConfigNone +} + +// resolveFabricHealthSummary decides the overall summary. An explicitly +// configured summary always wins so a profile can pin an inconsistent pair +// (e.g. a driver that reports Unhealthy without saying why); otherwise the +// summary is derived from the mask, which keeps it honest whether the mask +// came from the symbolic keys or from the raw escape hatch. +func resolveFabricHealthSummary(cfg *FabricConfig, mask uint32) uint8 { + if summary, ok := parseFabricHealthSummary(cfg.HealthSummary); ok { + return summary + } + return deriveFabricHealthSummary(mask) +} + +// parseFabricHealthSummary reports the pinned summary and whether one was +// pinned at all. "" and "auto" both mean "derive from the conditions"; an +// unrecognised value is treated the same way rather than failing the device. +func parseFabricHealthSummary(s string) (uint8, bool) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "healthy": + return FabricHealthSummaryHealthy, true + case "unhealthy": + return FabricHealthSummaryUnhealthy, true + case "limited_capacity", "limitedcapacity": + return FabricHealthSummaryLimitedCapacity, true + case "not_supported", "notsupported": + return FabricHealthSummaryNotSupported, true + default: + return 0, false + } +} + +// deriveFabricHealthSummary summarises a health mask the way a fabric manager +// does: any routing or configuration fault makes the attachment unhealthy, +// degraded bandwidth on its own only limits capacity, and a mask that answers +// nothing at all reports nothing at all. +func deriveFabricHealthSummary(mask uint32) uint8 { + if mask == 0 { + return FabricHealthSummaryNotSupported + } + faulted := fabricHealthRouteRecovery.get(mask) == fabricHealthTrue || + fabricHealthRouteUnhealthy.get(mask) == fabricHealthTrue || + fabricHealthAccessTimeoutRecovery.get(mask) == fabricHealthTrue || + fabricHealthIncorrectConfig.get(mask) > fabricIncorrectConfigNone + switch { + case faulted: + return FabricHealthSummaryUnhealthy + case fabricHealthDegradedBW.get(mask) == fabricHealthTrue: + return FabricHealthSummaryLimitedCapacity + default: + return FabricHealthSummaryHealthy + } +} diff --git a/pkg/gpu/mocknvml/engine/fabric_health_test.go b/pkg/gpu/mocknvml/engine/fabric_health_test.go new file mode 100644 index 000000000..6585ed1bf --- /dev/null +++ b/pkg/gpu/mocknvml/engine/fabric_health_test.go @@ -0,0 +1,294 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package engine + +import ( + "testing" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + "github.com/stretchr/testify/require" +) + +// TestBuildFabricInfo_DefaultsToHealthyFabric pins the behaviour issue #677 +// exists for: a fabric block that says nothing about health must report a +// healthy fabric, not an unknown one. A zero summary makes nvidia-smi print +// N/A for the whole Fabric.Health block, so the default has to be non-zero. +func TestBuildFabricInfo_DefaultsToHealthyFabric(t *testing.T) { + info := buildFabricInfo(&FabricConfig{}) + + require.Equal(t, uint8(nvml.GPU_FABRIC_HEALTH_SUMMARY_HEALTHY), info.HealthSummary, "health summary") + require.Equal(t, uint32(nvml.GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_FALSE), + decodeFabricHealth(info.HealthMask, nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_DEGRADED_BW, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_DEGRADED_BW), "degraded bandwidth") + require.Equal(t, uint32(nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_FALSE), + decodeFabricHealth(info.HealthMask, nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_RECOVERY, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_RECOVERY), "route recovery") + require.Equal(t, uint32(nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_FALSE), + decodeFabricHealth(info.HealthMask, nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_UNHEALTHY, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_UNHEALTHY), "route unhealthy") + require.Equal(t, uint32(nvml.GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_FALSE), + decodeFabricHealth(info.HealthMask, nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_ACCESS_TIMEOUT_RECOVERY, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_ACCESS_TIMEOUT_RECOVERY), "access timeout recovery") + require.Equal(t, uint32(nvml.GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NONE), + decodeFabricHealth(info.HealthMask, nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_INCORRECT_CONFIGURATION, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_INCORRECT_CONFIGURATION), "incorrect configuration") + + // Real hardware leaves this field unanswered (nvidia-smi renders the row + // as N/A on a healthy GB300 tray), so the mock must not claim otherwise. + require.Equal(t, uint32(nvml.GPU_FABRIC_HEALTH_MASK_PARTITION_ASSIGNED_NOT_SUPPORTED), + decodeFabricHealth(info.HealthMask, nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_PARTITION_ASSIGNED, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_PARTITION_ASSIGNED), "partition assigned") +} + +// TestBuildFabricInfo_ConditionFlipsOnlyItsOwnField is the assertion a +// wholesale mask would pass and must not: injecting one condition has to leave +// every neighbouring condition reporting False, not NOT_SUPPORTED and not +// True. Each case also pins the summary the condition implies. +func TestBuildFabricInfo_ConditionFlipsOnlyItsOwnField(t *testing.T) { + type condition struct { + shift, width int + wantTrue uint32 + } + all := map[string]condition{ + "degraded_bandwidth": { + nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_DEGRADED_BW, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_DEGRADED_BW, + nvml.GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_TRUE, + }, + "route_recovery": { + nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_RECOVERY, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_RECOVERY, + nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_TRUE, + }, + "route_unhealthy": { + nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_UNHEALTHY, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_UNHEALTHY, + nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_TRUE, + }, + "access_timeout_recovery": { + nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_ACCESS_TIMEOUT_RECOVERY, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_ACCESS_TIMEOUT_RECOVERY, + nvml.GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_TRUE, + }, + } + // The False value is identical for all four conditions, so one constant + // covers the "neighbour unchanged" half of every case. + const wantFalse = uint32(nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_FALSE) + + cases := map[string]struct { + health FabricHealthConfig + wantSummary uint8 + }{ + "degraded_bandwidth": { + FabricHealthConfig{DegradedBandwidth: true}, + // Reduced bandwidth still routes, so the fabric is usable. + FabricHealthSummaryLimitedCapacity, + }, + "route_recovery": {FabricHealthConfig{RouteRecovery: true}, FabricHealthSummaryUnhealthy}, + "route_unhealthy": {FabricHealthConfig{RouteUnhealthy: true}, FabricHealthSummaryUnhealthy}, + "access_timeout_recovery": {FabricHealthConfig{AccessTimeoutRecovery: true}, FabricHealthSummaryUnhealthy}, + } + + for injected, tc := range cases { + t.Run(injected, func(t *testing.T) { + health := tc.health + info := buildFabricInfo(&FabricConfig{Health: &health}) + + require.Equal(t, tc.wantSummary, info.HealthSummary, "summary") + for name, c := range all { + got := decodeFabricHealth(info.HealthMask, c.shift, c.width) + if name == injected { + require.Equal(t, c.wantTrue, got, "%s should report the injected fault", name) + continue + } + require.Equal(t, wantFalse, got, "%s must stay healthy when only %s was injected", name, injected) + } + require.Equal(t, uint32(nvml.GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NONE), + decodeFabricHealth(info.HealthMask, nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_INCORRECT_CONFIGURATION, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_INCORRECT_CONFIGURATION), + "incorrect configuration must stay None when only %s was injected", injected) + }) + } +} + +func TestBuildFabricInfo_IncorrectConfiguration(t *testing.T) { + cases := map[string]struct { + configured string + wantValue uint32 + wantSummary uint8 + }{ + "named misconfiguration": { + "no_partition", + nvml.GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NO_PARTITION, + FabricHealthSummaryUnhealthy, + }, + "explicit none": { + "none", + nvml.GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NONE, + FabricHealthSummaryHealthy, + }, + // A typo must not silently downgrade a fabric to unhealthy, matching + // how parseFabricState treats an unknown state. + "unknown name falls back to none": { + "nonsense", + nvml.GPU_FABRIC_HEALTH_MASK_INCORRECT_CONFIGURATION_NONE, + FabricHealthSummaryHealthy, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + info := buildFabricInfo(&FabricConfig{ + Health: &FabricHealthConfig{IncorrectConfiguration: tc.configured}, + }) + require.Equal(t, tc.wantValue, + decodeFabricHealth(info.HealthMask, nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_INCORRECT_CONFIGURATION, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_INCORRECT_CONFIGURATION), "incorrect configuration") + require.Equal(t, tc.wantSummary, info.HealthSummary, "summary") + }) + } +} + +// TestBuildFabricInfo_ExplicitSummaryWins covers pinning a summary that the +// conditions do not imply — a driver that reports Unhealthy without saying +// which condition failed, which is a shape consumers have to tolerate. +func TestBuildFabricInfo_ExplicitSummaryWins(t *testing.T) { + info := buildFabricInfo(&FabricConfig{HealthSummary: "unhealthy"}) + require.Equal(t, FabricHealthSummaryUnhealthy, info.HealthSummary, "pinned summary") + require.Equal(t, uint32(nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_FALSE), + decodeFabricHealth(info.HealthMask, nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_UNHEALTHY, + nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_UNHEALTHY), + "pinning the summary must not invent a condition") + + // "auto" is the explicit spelling of the default and must derive. + derived := buildFabricInfo(&FabricConfig{ + HealthSummary: "auto", + Health: &FabricHealthConfig{RouteUnhealthy: true}, + }) + require.Equal(t, FabricHealthSummaryUnhealthy, derived.HealthSummary, "auto derives from the conditions") + + // A pinned summary of not_supported is how a profile asks for the old + // all-N/A rendering back. + off := buildFabricInfo(&FabricConfig{HealthSummary: "not_supported"}) + require.Equal(t, FabricHealthSummaryNotSupported, off.HealthSummary, "not_supported") +} + +// TestBuildFabricInfo_RawHealthMaskEscapeHatch covers a hand-assembled mask: +// it must reach the consumer verbatim (issue #677: setting it used to be +// silently dropped) and drive the summary, so the two can never disagree. +func TestBuildFabricInfo_RawHealthMaskEscapeHatch(t *testing.T) { + // Everything healthy except a partition-assigned answer, which the + // symbolic keys deliberately cannot express. + raw := uint32(nvml.GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_FALSE<> uint32(shift)) & uint32(width) +} diff --git a/pkg/gpu/mocknvml/engine/fabric_test.go b/pkg/gpu/mocknvml/engine/fabric_test.go index a22b5808d..88100b9cd 100644 --- a/pkg/gpu/mocknvml/engine/fabric_test.go +++ b/pkg/gpu/mocknvml/engine/fabric_test.go @@ -41,11 +41,12 @@ func TestGetMockFabricInfo_NotSupportedWhenNil(t *testing.T) { } func TestGetMockFabricInfo_PopulatesFields(t *testing.T) { + rawMask := uint32(0x42) dev := makeFabricDevice(t, &FabricConfig{ ClusterUUID: "00000000-0000-0000-0000-0000000000ab", CliqueID: 7, State: "completed", - HealthMask: 0x42, + HealthMask: &rawMask, }) info, ret := dev.GetMockFabricInfo() require.Equal(t, nvml.SUCCESS, ret, "v1") diff --git a/tests/e2e/go/assertions/nvidiasmi/exec.go b/tests/e2e/go/assertions/nvidiasmi/exec.go index 2aec88b2e..c6bb5ba19 100644 --- a/tests/e2e/go/assertions/nvidiasmi/exec.go +++ b/tests/e2e/go/assertions/nvidiasmi/exec.go @@ -166,6 +166,19 @@ func PlatformIdentity(ctx context.Context, k *kube.Client, pod kube.PodRef, p pr p.Name, strings.Join(problems, "\n")) } +// FabricHealth asserts nvidia-smi -q -x reports a healthy fabric on every GPU. +// Every element of the block read N/A while the mock reported no health +// summary, which says the driver answered nothing rather than that the fabric +// is well. See issue #677. +func FabricHealth(ctx context.Context, k *kube.Client, pod kube.PodRef) { + ginkgo.GinkgoHelper() + + ginkgo.By("nvidia-smi -q -x reports a healthy fabric health block") + problems := FabricHealthProblems(query(ctx, k, pod), HealthyFabricBlock()) + gomega.Expect(problems).To(gomega.BeEmpty(), "fabric health wrong:\n%s", + strings.Join(problems, "\n")) +} + // query execs `nvidia-smi -q -x` and asserts it succeeded, returning stdout. func query(ctx context.Context, k *kube.Client, pod kube.PodRef) string { ginkgo.GinkgoHelper() diff --git a/tests/e2e/go/assertions/nvidiasmi/fabric_health.go b/tests/e2e/go/assertions/nvidiasmi/fabric_health.go new file mode 100644 index 000000000..5ac1d0642 --- /dev/null +++ b/tests/e2e/go/assertions/nvidiasmi/fabric_health.go @@ -0,0 +1,107 @@ +// Copyright 2026 NVIDIA CORPORATION +// SPDX-License-Identifier: Apache-2.0 + +package nvidiasmi + +import ( + "fmt" + "strings" +) + +// The fabric health checks, kept out of checks.go because they compare a block +// of related readings per GPU and because the injection check compares two +// different expectations within one document. + +// FabricHealthBlock is the block as nvidia-smi renders it. +// Bodies are compared as text rather than decoded: what an operator and a +// fault-handling controller read is exactly this text, and "N/A" — the +// pre-#677 rendering of every element — is a meaningful value to assert +// against, not a missing one. +type FabricHealthBlock struct { + Summary string + Bandwidth string + RouteRecovery string + RouteUnhealthy string + AccessTimeoutRecovery string + IncorrectConfiguration string +} + +// HealthyFabricBlock is the block a fabric-attached GPU in a healthy rack reports, +// taken element for element from a real GB300 tray (driver 580.173.02). +func HealthyFabricBlock() FabricHealthBlock { + return FabricHealthBlock{ + Summary: "Healthy", + Bandwidth: "Full", + RouteRecovery: "False", + RouteUnhealthy: "False", + AccessTimeoutRecovery: "False", + IncorrectConfiguration: "None", + } +} + +// FabricHealthProblems checks every GPU's fabric health block against want. +func FabricHealthProblems(out string, want FabricHealthBlock) []string { + return fabricHealthProblems(out, func(int) FabricHealthBlock { return want }) +} + +// FabricHealthProblemsAt checks the GPU at index against want and every other +// GPU against others. Runtime injection is per device, so a fault that leaks +// onto the node's other GPUs — or one applied to the wrong device — is a +// defect this reports rather than one it passes over. +func FabricHealthProblemsAt(out string, index int, want, others FabricHealthBlock) []string { + return fabricHealthProblems(out, func(i int) FabricHealthBlock { + if i == index { + return want + } + return others + }) +} + +func fabricHealthProblems(out string, want func(int) FabricHealthBlock) []string { + snap, err := ParseSnapshot(out) + if err != nil { + return []string{err.Error()} + } + + var problems []string + for i, gpu := range snap.doc.GPUs { + problems = append(problems, fabricBlockProblems(gpu.label(i), gpu.Fabric, want(i))...) + } + return problems +} + +// fabricBlockProblems reports one GPU's differences, one element per problem, +// so a failure names the condition that regressed instead of dumping the block. +func fabricBlockProblems(name string, got fabricBlock, want FabricHealthBlock) []string { + // A GPU with no fabric at all emits no children; reporting that + // once is clearer than reporting six absent elements. + if !got.State.present() { + return []string{name + " emits no fabric state: the device reports no fabric attachment"} + } + + var problems []string + for _, element := range []struct { + name string + got reading + want string + }{ + {"summary", got.Health.Summary, want.Summary}, + {"bandwidth", got.Health.Bandwidth, want.Bandwidth}, + {"route_recovery_in_progress", got.Health.RouteRecovery, want.RouteRecovery}, + {"route_unhealthy", got.Health.RouteUnhealthy, want.RouteUnhealthy}, + {"access_timeout_recovery", got.Health.AccessTimeoutRecovery, want.AccessTimeoutRecovery}, + {"incorrect_configuration", got.Health.IncorrectConfiguration, want.IncorrectConfiguration}, + } { + body := strings.TrimSpace(string(element.got)) + switch { + case !element.got.present(): + problems = append(problems, fmt.Sprintf( + "%s fabric health emits no %s element, want %q; the driver may have renamed it", + name, element.name, element.want)) + case !strings.EqualFold(body, element.want): + problems = append(problems, fmt.Sprintf("%s fabric health %s = %q, want %q", + name, element.name, body, element.want)) + } + } + return problems +} diff --git a/tests/e2e/go/assertions/nvidiasmi/fabric_health_test.go b/tests/e2e/go/assertions/nvidiasmi/fabric_health_test.go new file mode 100644 index 000000000..d942812c3 --- /dev/null +++ b/tests/e2e/go/assertions/nvidiasmi/fabric_health_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 NVIDIA CORPORATION +// SPDX-License-Identifier: Apache-2.0 + +package nvidiasmi + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// routeUnhealthy is the block a GPU whose fabric route has failed reports: the +// one faulted condition plus the summary that follows from it, with every +// neighbouring condition inherited from the healthy reference. +func routeUnhealthy() FabricHealthBlock { + want := HealthyFabricBlock() + want.RouteUnhealthy = "True" + want.Summary = "Unhealthy" + return want +} + +// qx-gb200-healthy.xml was captured before #677, so it is the defect itself: +// nvidia-smi rendered every health element as N/A because the mock reported no +// health summary, leaving a consumer unable to tell a healthy fabric from an +// unknown one. +func TestFabricHealthProblems_RejectsUnreportedHealth(t *testing.T) { + problems := FabricHealthProblems(loadFixture(t, "qx-gb200-healthy.xml"), HealthyFabricBlock()) + require.Len(t, problems, 12, "six unreported elements on each of the two GPUs") + assert.Contains(t, strings.Join(problems, "; "), `summary = "N/A", want "Healthy"`) + assert.Contains(t, strings.Join(problems, "; "), `bandwidth = "N/A", want "Full"`) +} + +func TestFabricHealthProblems_AcceptsHealthyFabric(t *testing.T) { + // GPU 1 of this document carries the injected fault, so the healthy + // expectation is checked against GPU 0 by scoping the other GPU to it. + out := loadFixture(t, "qx-gb200-fabric-degraded.xml") + problems := FabricHealthProblemsAt(out, 1, routeUnhealthy(), HealthyFabricBlock()) + assert.Empty(t, problems, strings.Join(problems, "; ")) +} + +// The assertion that gives the feature its value: one injected condition moves +// its own element and the summary, and nothing else moves. A health mask +// applied wholesale, or a summary pinned without decoding the conditions, would +// satisfy a check that only looked at the faulted element. +func TestFabricHealthProblems_ReportsOnlyTheFaultedCondition(t *testing.T) { + problems := FabricHealthProblems(loadFixture(t, "qx-gb200-fabric-degraded.xml"), HealthyFabricBlock()) + require.Len(t, problems, 2, "only GPU 1's summary and route element may differ") + joined := strings.Join(problems, "; ") + assert.Contains(t, joined, `route_unhealthy = "True", want "False"`) + assert.Contains(t, joined, `summary = "Unhealthy", want "Healthy"`) + assert.NotContains(t, joined, "bandwidth") + assert.NotContains(t, joined, "access_timeout_recovery") +} + +// Injection is per device, so a fault reported on the wrong GPU is a defect and +// not a pass: the same document must fail when the fault is expected elsewhere. +func TestFabricHealthProblems_RejectsFaultOnTheWrongGPU(t *testing.T) { + problems := FabricHealthProblemsAt(loadFixture(t, "qx-gb200-fabric-degraded.xml"), + 0, routeUnhealthy(), HealthyFabricBlock()) + require.NotEmpty(t, problems, "GPU 0 is healthy and GPU 1 is faulted, so neither expectation holds") + assert.Len(t, problems, 4, "both GPUs report the opposite of what was expected") +} + +// A profile with no fabric attachment emits no children. Reporting +// that once is clearer than reporting six absent elements, and it must not be +// mistaken for a healthy fabric. +func TestFabricHealthProblems_ReportsAbsentFabric(t *testing.T) { + out := strings.ReplaceAll(loadFixture(t, "qx-gb200-fabric-degraded.xml"), + "Completed", "") + problems := FabricHealthProblems(out, HealthyFabricBlock()) + require.Len(t, problems, 2, "one problem per GPU") + assert.Contains(t, problems[0], "no fabric state") +} + +// An element the driver renamed is reported as absent rather than compared as +// an empty body, matching how the rest of the package reads the document. +func TestFabricHealthProblems_ReportsMissingElement(t *testing.T) { + out := strings.ReplaceAll(loadFixture(t, "qx-gb200-fabric-degraded.xml"), + "Full", "") + problems := FabricHealthProblems(out, HealthyFabricBlock()) + assert.Contains(t, strings.Join(problems, "; "), "emits no bandwidth element") +} + +func TestFabricHealthProblems_ReportsUnparseableDocument(t *testing.T) { + problems := FabricHealthProblems("not xml", HealthyFabricBlock()) + require.Len(t, problems, 1) + assert.Contains(t, problems[0], "parse nvidia-smi XML") +} diff --git a/tests/e2e/go/assertions/nvidiasmi/schema.go b/tests/e2e/go/assertions/nvidiasmi/schema.go index a464ec18d..8e70f4dda 100644 --- a/tests/e2e/go/assertions/nvidiasmi/schema.go +++ b/tests/e2e/go/assertions/nvidiasmi/schema.go @@ -124,6 +124,7 @@ type gpuElement struct { ECCErrors eccErrors `xml:"ecc_errors"` RemappedRows remappedRows `xml:"remapped_rows"` ClocksEventReasons eventReasons `xml:"clocks_event_reasons"` + Fabric fabricBlock `xml:"fabric"` Processes struct { Infos []processInfo `xml:"process_info"` } `xml:"processes"` @@ -303,6 +304,35 @@ type eventReasons struct { SWThermalSlowdown reading `xml:"clocks_event_reason_sw_thermal_slowdown"` } +// fabricBlock is : the GPU's NVLink fabric attachment. Its +// child is the NVML return code embedded in GpuFabricInfo, not a health +// verdict — the health verdict is the sub-block, whose every element +// read N/A until the mock reported a health summary (#677). +// +// The reference GB300 tray also renders a Partition Assigned row, but no +// element for it exists in the documents this package is pinned against, so +// none is decoded: nvidia-smi 580.65.06 does not know that row at all. +type fabricBlock struct { + State reading `xml:"state"` + Status reading `xml:"status"` + CliqueID reading `xml:"cliqueId"` + ClusterUUID reading `xml:"clusterUuid"` + Health fabricHealthBlock `xml:"health"` +} + +// fabricHealthBlock is . Each element carries one field of the +// NVML fabric health mask, except , which carries the overall +// healthSummary the mock has to report before nvidia-smi decodes any of the +// others. +type fabricHealthBlock struct { + Summary reading `xml:"summary"` + Bandwidth reading `xml:"bandwidth"` + RouteRecovery reading `xml:"route_recovery_in_progress"` + RouteUnhealthy reading `xml:"route_unhealthy"` + AccessTimeoutRecovery reading `xml:"access_timeout_recovery"` + IncorrectConfiguration reading `xml:"incorrect_configuration"` +} + type processInfo struct { PID int `xml:"pid"` Name string `xml:"process_name"` diff --git a/tests/e2e/go/assertions/nvidiasmi/testdata/README.md b/tests/e2e/go/assertions/nvidiasmi/testdata/README.md index 2defb5401..86f428858 100644 --- a/tests/e2e/go/assertions/nvidiasmi/testdata/README.md +++ b/tests/e2e/go/assertions/nvidiasmi/testdata/README.md @@ -12,6 +12,7 @@ per-GPU indexing and override scoping. | `qx-gb200-healthy.xml` | Blackwell. `*_tlimit_threshold` elements; the absolute ones are absent. | | `qx-gb200-lost.xml` | GPU 0 healthy, GPU 1 lost (`GPU is lost` bodies). | | `qx-gb200-ecc-injected.xml` | GPU 0 has a non-zero `ecc_errors/aggregate/dram_uncorrectable`. | +| `qx-gb200-fabric-degraded.xml` | GPU 0 fabric healthy, GPU 1 `route_unhealthy` (#677). Taken against the fixed library, so it is also the healthy-fabric reference. | Watch for two element names that repeat under different parents: `sm_clock` appears under both `clocks` (current) and `max_clocks`, and `average_power_draw` @@ -26,6 +27,7 @@ For the lost and ECC variants, inject first and wait out the 30 s override TTL: kubectl exec -n mokka -- nvml-mock-ctl fail --gpu 1 --mode lost kubectl exec -n mokka -- nvml-mock-ctl fail --gpu 0 --mode ecc_uncorrectable --after-calls 1 + kubectl exec -n mokka -- nvml-mock-ctl fabric-health --gpu 1 route_unhealthy Then trim to two GPUs, keeping the header and the first two `` blocks and rewriting `attached_gpus` to 2. diff --git a/tests/e2e/go/assertions/nvidiasmi/testdata/qx-gb200-fabric-degraded.xml b/tests/e2e/go/assertions/nvidiasmi/testdata/qx-gb200-fabric-degraded.xml new file mode 100644 index 000000000..61cc9e3fe --- /dev/null +++ b/tests/e2e/go/assertions/nvidiasmi/testdata/qx-gb200-fabric-degraded.xml @@ -0,0 +1,618 @@ + + + + Wed Aug 19 13:06:27 2026 + 580.65.06 + 13.0 + 2 + + NVIDIA GB200 + NVIDIA + Blackwell + Requested functionality has been deprecated + No + Disabled + Enabled + N/A + + Disabled + Disabled + + + None + + Disabled + 4000 + + N/A + N/A + + 1562849103700 + GPU-b200b200-0000-0000-0000-000000000000 + N/A + 0 + 96.00.9E.00.01 + No + 0xa00 + 699-2G530-0200-000 + N/A + N/A + + 1822725100200 + 21 + 11 + 1 + Switch Connected + 1 + 0x0000000000000000 + + + G530.0200.00.01 + 2.1 + 7.16 + 1.0 + + + N/A + N/A + + + All On + All On + + Enabled + + None + N/A + N/A + + N/A + 580.65.06 + + N/A + + + 0A + 00 + 0000 + 3 + 2 + 234110DE + 00000000:0A:00.0 + 181710DE + + + 6 + 6 + 6 + 6 + 6 + + + 16x + 16x + + + + N/A + N/A + + 0 + 0 + 0 KB/s + 0 KB/s + N/A + N/A + + N/A + P0 + + Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + + + N/A + N/A + N/A + N/A + N/A + + Disabled + + 196608 MiB + 1024 MiB + 0 MiB + 195584 MiB + + + 524288 MiB + 0 MiB + 524288 MiB + + + N/A + N/A + N/A + + Default + + 44 % + 17 % + 0 % + 0 % + 0 % + 0 % + + + 0 + 0 + 0 + + + 0 + 0 + 0 + + + N/A + N/A + + + Enabled + Enabled + + + + N/A + N/A + N/A + 0 + 0 + + + N/A + N/A + N/A + 0 + 0 + N/A + + + N/A + N/A + N/A + N/A + N/A + + N/A + N/A + + + + 0 + + + + + 0 + + + + No + No + + + 0 + 0 + No + No + N/A + + + 36 C + 54 C + -5 C + 0 C + 5 C + 85 C + 34 C + N/A + + + 85 C + 85 C + + + P0 + 145.00 W + 145.00 W + 1000.00 W + 1000.00 W + 1000.00 W + 400.00 W + 1200.00 W + + + N/A + N/A + + + P0 + N/A + N/A + N/A + N/A + N/A + N/A + N/A + + N/A + + N/A + N/A + + + 345 MHz + 345 MHz + 2500 MHz + 1200 MHz + + + 2100 MHz + 2500 MHz + + + 2100 MHz + 2500 MHz + + + N/A + + + 2100 MHz + 2100 MHz + 2500 MHz + 2100 MHz + + + N/A + + + N/A + N/A + + + Completed + Success + 0 + 00000000-0000-0000-0000-000000000001 + + Healthy + Full + False + False + False + None + + + N/A + + + N/A + + N/A + + + + NVIDIA GB200 + NVIDIA + Blackwell + Requested functionality has been deprecated + No + Disabled + Enabled + N/A + + Disabled + Disabled + + + None + + Disabled + 4000 + + N/A + N/A + + 1562849103701 + GPU-b200b200-0000-0000-0000-000000000001 + N/A + 1 + 96.00.9E.00.01 + No + 0xb00 + 699-2G530-0200-000 + N/A + N/A + + 1822725100200 + 21 + 11 + 1 + Switch Connected + 2 + 0x0000000000000000 + + + G530.0200.00.01 + 2.1 + 7.16 + 1.0 + + + N/A + N/A + + + All On + All On + + Enabled + + None + N/A + N/A + + N/A + 580.65.06 + + N/A + + + 0B + 00 + 0000 + 3 + 2 + 234110DE + 00000000:0B:00.0 + 181710DE + + + 6 + 6 + 6 + 6 + 6 + + + 16x + 16x + + + + N/A + N/A + + 0 + 0 + 0 KB/s + 0 KB/s + N/A + N/A + + N/A + P0 + + Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + + + N/A + N/A + N/A + N/A + N/A + + Disabled + + 196608 MiB + 1024 MiB + 0 MiB + 195584 MiB + + + 524288 MiB + 0 MiB + 524288 MiB + + + N/A + N/A + N/A + + Default + + 37 % + 13 % + 0 % + 0 % + 0 % + 0 % + + + 0 + 0 + 0 + + + 0 + 0 + 0 + + + N/A + N/A + + + Enabled + Enabled + + + + N/A + N/A + N/A + 0 + 0 + + + N/A + N/A + N/A + 0 + 0 + N/A + + + N/A + N/A + N/A + N/A + N/A + + N/A + N/A + + + + 0 + + + + + 0 + + + + No + No + + + 0 + 0 + No + No + N/A + + + 36 C + 54 C + -5 C + 0 C + 5 C + 85 C + 34 C + N/A + + + 85 C + 85 C + + + P0 + 145.00 W + 145.00 W + 1000.00 W + 1000.00 W + 1000.00 W + 400.00 W + 1200.00 W + + + N/A + N/A + + + P0 + N/A + N/A + N/A + N/A + N/A + N/A + N/A + + N/A + + N/A + N/A + + + 345 MHz + 345 MHz + 2500 MHz + 1200 MHz + + + 2100 MHz + 2500 MHz + + + 2100 MHz + 2500 MHz + + + N/A + + + 2100 MHz + 2100 MHz + 2500 MHz + 2100 MHz + + + N/A + + + N/A + N/A + + + Completed + Success + 0 + 00000000-0000-0000-0000-000000000001 + + Unhealthy + Full + False + True + False + None + + + N/A + + + N/A + + N/A + + + diff --git a/tests/e2e/go/scenario_fabric_health.go b/tests/e2e/go/scenario_fabric_health.go new file mode 100644 index 000000000..a2968291f --- /dev/null +++ b/tests/e2e/go/scenario_fabric_health.go @@ -0,0 +1,121 @@ +//go:build e2e + +// Copyright 2026 NVIDIA CORPORATION +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/assertions/nvidiasmi" + "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/harness" + "github.com/NVIDIA/k8s-test-infra/tests/e2e/go/framework/kube" +) + +// Runtime fabric health scenarios for issue #677. +// +// Before #677 every element of the `nvidia-smi -q -x` fabric health block read +// N/A, so a consumer could not tell a healthy fabric from an unknown one and +// degraded fabric handling could not be exercised at all. The deployed +// profile's healthy baseline is asserted by nvidiasmi.FabricHealth from the +// profile's own spec; what these scenarios add is the fabric degrading under a +// running workload, which is the case worth simulating: +// +// - one injected condition flips its own element and only its own element, on +// the targeted device and only that device — a mask applied wholesale, or a +// summary pinned without decoding the conditions, would pass a weaker +// assertion; +// - clearing the injection returns the block to healthy, without restarting +// the consumer. +// +// One `-q -x` document carries every GPU, so the scoping assertions compare the +// target and its neighbours from the same observation rather than racing +// separate execs against the override TTL. + +// assertRuntimeFabricHealthInjection covers acceptance criteria 2 and 3: +// degrade one specific condition through nvml-mock-ctl while the consumer keeps +// running, assert only that condition moved and only on the target GPU, then +// clear it and assert the fabric reports healthy again. +func assertRuntimeFabricHealthInjection(ctx SpecContext, h *harness.Harness, consumer kube.PodRef) { + GinkgoHelper() + resetRuntimeOverrides(ctx, h) + + healthy := nvidiasmi.HealthyFabricBlock() + target := gpuCount(ctx, h, consumer) - 1 // exercise a non-zero index where possible + + // Route Unhealthy specifically, not "some fault": its neighbours must stay + // False, which is what distinguishes decoding the conditions from applying + // a mask wholesale. + degraded := healthy + degraded.RouteUnhealthy = "True" + degraded.Summary = "Unhealthy" + + By("degrade the fabric route on GPU " + strconv.Itoa(target) + " via nvml-mock-ctl fabric-health") + nvmlMockCtl(ctx, h, "fabric-health", "--gpu", strconv.Itoa(target), "route_unhealthy") + + Eventually(func() []string { + return fabricHealthProblems(ctx, h, consumer, func(out string) []string { + return nvidiasmi.FabricHealthProblemsAt(out, target, degraded, healthy) + }) + }).WithContext(ctx).WithTimeout(runtimeTTLTimeout).WithPolling(runtimeTTLPoll). + Should(BeEmpty(), "GPU %d should report an unhealthy route while its neighbours stay healthy", target) + + // `fabric-health healthy` rather than `reset`: recovery must work without + // discarding the node's other overrides, the same contract + // `fail --mode healthy` has. + By("clear the degradation with fabric-health healthy (no reset)") + nvmlMockCtl(ctx, h, "fabric-health", "--gpu", strconv.Itoa(target), "healthy") + + Eventually(func() []string { + return fabricHealthProblems(ctx, h, consumer, func(out string) []string { + return nvidiasmi.FabricHealthProblems(out, healthy) + }) + }).WithContext(ctx).WithTimeout(runtimeTTLTimeout).WithPolling(runtimeTTLPoll). + Should(BeEmpty(), "every GPU should report a healthy fabric again") + + By("reset runtime overrides") + nvmlMockCtl(ctx, h, "reset", "--gpu", "all") +} + +// assertRuntimeFabricMisconfiguration covers the one condition that is not a +// boolean: the incorrect_configuration element carries which misconfiguration +// was detected, so a named value has to reach it rather than collapsing into a +// generic fault. +func assertRuntimeFabricMisconfiguration(ctx SpecContext, h *harness.Harness, consumer kube.PodRef) { + GinkgoHelper() + resetRuntimeOverrides(ctx, h) + + misconfigured := nvidiasmi.HealthyFabricBlock() + misconfigured.IncorrectConfiguration = "No Partition" + misconfigured.Summary = "Unhealthy" + + By("inject a fabric misconfiguration on GPU 0 via nvml-mock-ctl fabric-health") + nvmlMockCtl(ctx, h, "fabric-health", "--gpu", "0", "no_partition") + + Eventually(func() []string { + return fabricHealthProblems(ctx, h, consumer, func(out string) []string { + return nvidiasmi.FabricHealthProblemsAt(out, 0, misconfigured, nvidiasmi.HealthyFabricBlock()) + }) + }).WithContext(ctx).WithTimeout(runtimeTTLTimeout).WithPolling(runtimeTTLPoll). + Should(BeEmpty(), "GPU 0 should report the injected misconfiguration and nothing else") + + By("reset runtime overrides") + nvmlMockCtl(ctx, h, "reset", "--gpu", "all") +} + +// fabricHealthProblems reads a fresh `nvidia-smi -q -x` document from the +// consumer and hands it to check. A failed exec is returned as a problem rather +// than asserted, so a poll can ride out a consumer restart. +func fabricHealthProblems(ctx SpecContext, h *harness.Harness, consumer kube.PodRef, + check func(out string) []string) []string { + GinkgoHelper() + res, err := h.Kube.ExecQuiet(ctx, consumer, "nvidia-smi", "-q", "-x") + if err != nil { + return []string{"nvidia-smi -q -x failed: " + res.Combined()} + } + return check(res.Stdout) +} diff --git a/tests/e2e/go/scenario_standalone_test.go b/tests/e2e/go/scenario_standalone_test.go index 3dd049f3e..7258de81f 100644 --- a/tests/e2e/go/scenario_standalone_test.go +++ b/tests/e2e/go/scenario_standalone_test.go @@ -148,6 +148,17 @@ var _ = Describe("nvml-mock standalone", Ordered, func() { nvidiasmi.PlatformIdentity(ctx, h.Kube, pod, p) }) + It("reports a healthy fabric via nvidia-smi -q -x", Label("nvidia-smi"), func(ctx SpecContext) { + // Issue #677: every element of the fabric health block read + // N/A, which says the driver answered nothing rather than that + // the fabric is well, so a consumer could not tell a healthy + // fabric from an unknown one. + if !p.HasFabric() { + Skip("profile " + name + " reports no fabric (GpuFabricInfo NOT_SUPPORTED)") + } + nvidiasmi.FabricHealth(ctx, h.Kube, pod) + }) + It("exposes the NVLink topology (gated on fabricmanager)", Label("nvlink"), func(ctx SpecContext) { assertions.FabricManagerGate(ctx, h.Kube, nvmlMockNamespace, "nvml-mock", pod, config.ReadyTimeout(), config.PollInterval()) assertions.NVLink(ctx, h.Kube, pod, p) @@ -270,6 +281,17 @@ var _ = Describe("nvml-mock standalone", Ordered, func() { assertRuntimeNVLinkErrorInjection(ctx, h, pod) }) + It("degrades and restores fabric health via the nvml-mock-ctl fabric-health command", Label("runtime-control"), Label("nvidia-smi"), func(ctx SpecContext) { + // Issue #677: one named condition must flip its own row and + // only its own row, then clear — a fabric that degrades + // under a running workload is the case worth simulating. + if !p.HasFabric() { + Skip("profile " + name + " reports no fabric; fabric-health has no health to degrade") + } + assertRuntimeFabricHealthInjection(ctx, h, pod) + assertRuntimeFabricMisconfiguration(ctx, h, pod) + }) + It("injects ECC uncorrectable errors", func(ctx SpecContext) { assertECCUncorrectableFailure(ctx, h, p.ExpectedGPUs()) }) diff --git a/tests/mocknvml/bridge_tests.go b/tests/mocknvml/bridge_tests.go index a6ef1c033..9fdae603d 100644 --- a/tests/mocknvml/bridge_tests.go +++ b/tests/mocknvml/bridge_tests.go @@ -42,6 +42,126 @@ func bridgeTests(deviceCount int) []testResult { results = append(results, testEventSetWait(deviceCount)...) results = append(results, testInitShutdownCycles()...) results = append(results, testInternalExportTable()...) + results = append(results, testFabricHealth(deviceCount)...) + return results +} + +// --- Fabric health tests --- + +// fabricHealthField reads one condition out of a health mask the way a +// consumer does, via NVML_GPU_FABRIC_HEALTH_STATUS_GET. +func fabricHealthField(mask uint32, shift, width int) uint32 { + return (mask >> uint32(shift)) & uint32(width) +} + +// testFabricHealth drives nvmlDeviceGetGpuFabricInfo (v1) and +// nvmlDeviceGetGpuFabricInfoV (v2 and v3) through the built library, which is +// the only place the version dispatch runs against real caller-allocated +// buffers: go-nvml's V2() hands the bridge a 36-byte GpuFabricInfo_v2 cast to +// the 40-byte *GpuFabricInfoV, so a v2 caller must come back with its own +// version tag and no health summary, while a v3 caller gets both. +// +// The fixture's device_defaults declare a fabric with no health block (so it +// must report Healthy, not the N/A that #677 fixed) and device 1 overrides one +// condition (so a mask that ignored config, or applied a fault wholesale, +// fails here). +func testFabricHealth(deviceCount int) []testResult { + var results []testResult + + if deviceCount == 0 { + return results + } + + healthyMask := uint32(nvml.GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_FALSE<= deviceCount { + continue + } + name := "fabric/" + tc.label + device, ret := nvml.DeviceGetHandleByIndex(tc.index) + if ret != nvml.SUCCESS { + results = append(results, testResult{name, false, fmt.Sprintf("GetHandleByIndex(%d) failed: %v", tc.index, nvml.ErrorString(ret))}) + continue + } + + // v1 carries no health fields at all; it must keep working now that + // the engine reports a non-zero summary. + v1, ret := device.GetGpuFabricInfo() + switch { + case ret != nvml.SUCCESS: + results = append(results, testResult{name + "/v1", false, fmt.Sprintf("GetGpuFabricInfo failed: %v", nvml.ErrorString(ret))}) + case v1.CliqueId != 3: + results = append(results, testResult{name + "/v1", false, fmt.Sprintf("cliqueId = %d, want 3", v1.CliqueId)}) + default: + results = append(results, testResult{name + "/v1", true, ""}) + } + + v2, ret := device.GetGpuFabricInfoV().V2() + wantV2Version := nvml.STRUCT_VERSION(nvml.GpuFabricInfo_v2{}, 2) + switch { + case ret != nvml.SUCCESS: + results = append(results, testResult{name + "/v2", false, fmt.Sprintf("V2 failed: %v", nvml.ErrorString(ret))}) + case v2.Version != wantV2Version: + results = append(results, testResult{name + "/v2", false, fmt.Sprintf("version = 0x%x, want the v2 tag 0x%x", v2.Version, wantV2Version)}) + case v2.HealthMask != tc.wantMask: + results = append(results, testResult{name + "/v2", false, fmt.Sprintf("healthMask = 0x%x, want 0x%x", v2.HealthMask, tc.wantMask)}) + default: + results = append(results, testResult{name + "/v2", true, fmt.Sprintf("healthMask=0x%x", v2.HealthMask)}) + } + + v3, ret := device.GetGpuFabricInfoV().V3() + switch { + case ret != nvml.SUCCESS: + results = append(results, testResult{name + "/v3", false, fmt.Sprintf("V3 failed: %v", nvml.ErrorString(ret))}) + case v3.HealthMask != tc.wantMask: + results = append(results, testResult{name + "/v3", false, fmt.Sprintf("healthMask = 0x%x, want 0x%x", v3.HealthMask, tc.wantMask)}) + case v3.HealthSummary != tc.wantSummary: + results = append(results, testResult{name + "/v3", false, fmt.Sprintf("healthSummary = %d, want %d", v3.HealthSummary, tc.wantSummary)}) + default: + results = append(results, testResult{name + "/v3", true, fmt.Sprintf("summary=%d", v3.HealthSummary)}) + } + + // The per-condition rows nvidia-smi renders: the injected fault must + // be the only one that moved. + routeUnhealthy := fabricHealthField(v3.HealthMask, + nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_UNHEALTHY, nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_UNHEALTHY) + routeRecovery := fabricHealthField(v3.HealthMask, + nvml.GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_RECOVERY, nvml.GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_RECOVERY) + wantRouteUnhealthy := uint32(nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_FALSE) + if tc.label == "route_unhealthy" { + wantRouteUnhealthy = nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_TRUE + } + switch { + case routeUnhealthy != wantRouteUnhealthy: + results = append(results, testResult{name + "/conditions", false, fmt.Sprintf("route unhealthy = %d, want %d", routeUnhealthy, wantRouteUnhealthy)}) + case routeRecovery != nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_FALSE: + results = append(results, testResult{name + "/conditions", false, fmt.Sprintf("route recovery = %d, want False (%d)", routeRecovery, nvml.GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_FALSE)}) + default: + results = append(results, testResult{name + "/conditions", true, ""}) + } + } + return results } diff --git a/tests/mocknvml/util-test-config.yaml b/tests/mocknvml/util-test-config.yaml index feb692bd9..aac3c3eb7 100644 --- a/tests/mocknvml/util-test-config.yaml +++ b/tests/mocknvml/util-test-config.yaml @@ -38,6 +38,17 @@ device_defaults: minor: 0 num_gpu_cores: 6912 # CUDA cores + # --------------------------------------------------------------------------- + # NVLink fabric. Present so testFabricHealth exercises the v1/v2/v3 structs + # through the real C ABI: this block says nothing about health, so it must + # report a healthy fabric with a non-zero health summary. Device 1 overrides + # it with a fault (#677). + # --------------------------------------------------------------------------- + fabric: + cluster_uuid: "00000000-0000-0000-0000-0000000000ab" + clique_id: 3 + state: "completed" + # --------------------------------------------------------------------------- # InfoROM versions # --------------------------------------------------------------------------- @@ -333,6 +344,14 @@ devices: bus_id: "0000:0F:00.0" minor_number: 1 processes: [] # explicit clear: empty-path coverage + # One faulted condition, so the harness can tell a per-device health mask + # apart from a constant: only this row flips, and the summary follows. + fabric: + cluster_uuid: "00000000-0000-0000-0000-0000000000ab" + clique_id: 3 + state: "completed" + health: + route_unhealthy: true - index: 2 uuid: "GPU-12345678-1234-1234-1234-123456780002"