From 61f9bdee13ba1273967b7f8f1270d93375be0f98 Mon Sep 17 00:00:00 2001 From: Christopher Desiniotis Date: Thu, 30 Oct 2025 14:50:26 -0700 Subject: [PATCH] Redirect log message to stderr in nvidia runtime wrapper script This change is required to make our nvidia runtime wrapper compliant with the OCI runtime spec. All OCI-compliant runtimes must support the operations documented at https://github.com/opencontainers/runtime-spec/blob/v1.2.1/runtime.md#operations. Before this change, our nvidia runtime wrapper was not producing the expected output when the query state operation (`state `) was invoked AND the nvidia kernel modules happened to not be loaded. In this case, we were emitting an extra log message which caused the stdout of this command to not adhere to the schema defined in the OCI runtime spec. Redirecting the log message to stderr makes us compliant. This issue was discovered when deploying GPU Operator 25.10.0 on nodes using cri-o. GPU Operator 25.10.0 is the first release that installs nvidia runtime handlers with cri-o by default, as opposed to installing an OCI hook file. When performing a GPU driver upgrade, pods in the gpu-operator namespace would be in the `Init:RunContainerError` state for several minutes until the new driver finished installing -- note that no nvidia driver modules are loaded during this span of several minutes. When inspecting the cri-o logs, we observed the following error message: ``` level=warning msg="Error updating the container status \"16779f4cd2414a164aae56856b491f86fe0c6b803a3b4474ada2cc0864c8e028\": failed to decode container status for 16779f4cd2414a164aae56856b491f86fe0c6b803a3b4474ada2cc0864c8e028: skipThreeBytes: expect ull, error found in #2 byte of ...|nvidia drive|..., bigger context ...|nvidia driver modules are not yet loaded, invoking /|..." id=a4b48041-edc4-48c2-8d75-4ad03cb3d8e1 name=/runtime.v1.RuntimeService/CreateContainer ``` This error message indicates cri-o failed to get the status of the container because it could not decode the JSON returned by the runtime handler. Signed-off-by: Christopher Desiniotis --- cmd/nvidia-ctk-installer/toolkit/installer/executables.go | 2 +- .../toolkit/installer/executables_test.go | 2 +- .../toolkit/installer/installer_test.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/nvidia-ctk-installer/toolkit/installer/executables.go b/cmd/nvidia-ctk-installer/toolkit/installer/executables.go index 4f595bb90..0bedd8511 100644 --- a/cmd/nvidia-ctk-installer/toolkit/installer/executables.go +++ b/cmd/nvidia-ctk-installer/toolkit/installer/executables.go @@ -162,7 +162,7 @@ func (w *render) render() (io.Reader, error) { {{- if (.CheckModules) }} cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1 if [ "${?}" != "0" ]; then - echo "nvidia driver modules are not yet loaded, invoking {{ .DefaultRuntimeExecutablePath }} directly" + echo "nvidia driver modules are not yet loaded, invoking {{ .DefaultRuntimeExecutablePath }} directly" >&2 exec {{ .DefaultRuntimeExecutablePath }} "$@" fi {{- end }} diff --git a/cmd/nvidia-ctk-installer/toolkit/installer/executables_test.go b/cmd/nvidia-ctk-installer/toolkit/installer/executables_test.go index 28a5e0c09..7c8b0d83c 100644 --- a/cmd/nvidia-ctk-installer/toolkit/installer/executables_test.go +++ b/cmd/nvidia-ctk-installer/toolkit/installer/executables_test.go @@ -51,7 +51,7 @@ func TestWrapperRender(t *testing.T) { expected: `#! /bin/sh cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1 if [ "${?}" != "0" ]; then - echo "nvidia driver modules are not yet loaded, invoking runc directly" + echo "nvidia driver modules are not yet loaded, invoking runc directly" >&2 exec runc "$@" fi /dest-dir/some-runtime \ diff --git a/cmd/nvidia-ctk-installer/toolkit/installer/installer_test.go b/cmd/nvidia-ctk-installer/toolkit/installer/installer_test.go index 313f348fc..d308fb3c3 100644 --- a/cmd/nvidia-ctk-installer/toolkit/installer/installer_test.go +++ b/cmd/nvidia-ctk-installer/toolkit/installer/installer_test.go @@ -170,7 +170,7 @@ func TestToolkitInstaller(t *testing.T) { wrapper: `#! /bin/sh cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1 if [ "${?}" != "0" ]; then - echo "nvidia driver modules are not yet loaded, invoking runc directly" + echo "nvidia driver modules are not yet loaded, invoking runc directly" >&2 exec runc "$@" fi NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \ @@ -185,7 +185,7 @@ PATH=/foo/bar/baz:$PATH \ wrapper: `#! /bin/sh cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1 if [ "${?}" != "0" ]; then - echo "nvidia driver modules are not yet loaded, invoking runc directly" + echo "nvidia driver modules are not yet loaded, invoking runc directly" >&2 exec runc "$@" fi NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \ @@ -200,7 +200,7 @@ PATH=/foo/bar/baz:$PATH \ wrapper: `#! /bin/sh cat /proc/modules | grep -e "^nvidia " >/dev/null 2>&1 if [ "${?}" != "0" ]; then - echo "nvidia driver modules are not yet loaded, invoking runc directly" + echo "nvidia driver modules are not yet loaded, invoking runc directly" >&2 exec runc "$@" fi NVIDIA_CTK_CONFIG_FILE_PATH=/foo/bar/baz/.config/nvidia-container-runtime/config.toml \