From a633ac1dae284ea938331edb2390e86e936fc574 Mon Sep 17 00:00:00 2001 From: hksk Date: Sun, 2 Aug 2026 22:03:00 -0500 Subject: [PATCH] cuda: add GGML_CUDA_BLOCKING_SYNC env var to reduce CPU busy-wait --- ggml/src/ggml-cuda/ggml-cuda.cu | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 561ab7ac599f..cb9a077b337a 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -217,6 +217,22 @@ static int ggml_cuda_parse_id(char devName[]) { static ggml_cuda_device_info ggml_cuda_init() { ggml_cuda_device_info info = {}; + // Optionally use blocking synchronization so that the host thread sleeps + // instead of busy-waiting while the GPU is working. This greatly reduces + // CPU usage for fully offloaded inference at the cost of a small increase + // in synchronization latency. Must be set before any CUDA context is + // created on the device. + const char * blocking_sync_env = getenv("GGML_CUDA_BLOCKING_SYNC"); + if (blocking_sync_env != nullptr && std::atoi(blocking_sync_env) != 0) { + cudaError_t flags_err = cudaSetDeviceFlags(cudaDeviceScheduleBlockingSync); + if (flags_err != cudaSuccess) { + GGML_LOG_WARN("%s: failed to set cudaDeviceScheduleBlockingSync: %s\n", + __func__, cudaGetErrorString(flags_err)); + } else { + GGML_LOG_INFO("%s: using cudaDeviceScheduleBlockingSync\n", __func__); + } + } + cudaError_t err = cudaGetDeviceCount(&info.physical_device_count); if (err != cudaSuccess) { GGML_LOG_ERROR("%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err));