[NV] llm-d-vllm: optimize DSv4-Pro GB200 recipe configs

benchmarks/llm-d/Dockerfile

@@ -22,7 +22,7 @@
 #     --build-arg EPP_FROM_IMAGE="$EPP_FROM_IMAGE" \
 #     --build-arg ROUTING_SIDECAR_IMAGE="$ROUTING_SIDECAR_IMAGE" \
 #     --build-arg ENVOY_FROM_IMAGE="$ENVOY_FROM_IMAGE" ...
-ARG VLLM_BASE=vllm/vllm-openai:v0.23.0
+ARG VLLM_BASE=vllm/vllm-openai:v0.26.0
 ARG EPP_FROM_IMAGE=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0
 ARG ROUTING_SIDECAR_IMAGE=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0
 ARG ENVOY_FROM_IMAGE=envoyproxy/envoy:distroless-v1.33.2

benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml

@@ -18,7 +18,7 @@
 #      decode ep:1/dp-attn:false in the master key is what selects the
 #      pure-TP path in server.sh. Decode knobs are latency-sized: max-num-seqs
 #      256, max-num-batched-tokens 256, max-cudagraph-capture-size 256,
-#      FULL_DECODE_ONLY graphs, stream-interval 50, gpu-memory-utilization 0.9.
+#      FULL_DECODE_ONLY graphs, gpu-memory-utilization 0.9.
 #   2. PREFILL is IDENTICAL to the mid-curve/high-tpt prefill (copied verbatim,
 #      DEP8 with enable-ep-weight-filter + moe-backend deep_gemm_mega_moe). An
 #      earlier srt-slurm-style low-latency prefill (weight-offload, no filter,
@@ -51,7 +51,9 @@ plugins:
   - type: prefix-cache-scorer
   - type: queue-scorer
   - type: active-request-scorer
-  - type: max-score-picker
+  - type: weighted-random-picker
+    parameters:
+      threshold: "0.1"
 
 schedulingProfiles:
   - name: prefill
@@ -63,12 +65,13 @@ schedulingProfiles:
         weight: 2
       - pluginRef: active-request-scorer
         weight: 2
-      - pluginRef: max-score-picker
+      - pluginRef: weighted-random-picker
   - name: decode
     plugins:
       - pluginRef: decode-filter
       - pluginRef: active-request-scorer
-      - pluginRef: max-score-picker
+        weight: 2
+      - pluginRef: weighted-random-picker
 
 dataLayer:
   discovery:
@@ -89,7 +92,10 @@ prefill:
   extra-args: >-
     --kv-cache-dtype fp8
     --enforce-eager
-    --gpu-memory-utilization 0.9
+    --gpu-memory-utilization 0.95
+    --max-model-len 9280
+    --max-num-seqs 16
+    --max-num-batched-tokens 32768
     --enable-cumem-allocator
     --no-enable-prefix-caching
     --no-async-scheduling
@@ -98,6 +104,7 @@ prefill:
     --moe-backend deep_gemm_mega_moe
     --enable-ep-weight-filter
     --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
     --numa-bind
   env:
     NCCL_CUMEM_ENABLE: "1"
@@ -123,14 +130,16 @@ prefill:
     NVSHMEM_REMOTE_TRANSPORT: "none"
     NVSHMEM_ENABLE_NIC_PE_MAPPING: "1"
     NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1"
-    VLLM_USE_NCCL_SYMM_MEM: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "0"
     VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
     VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048"
     TILELANG_CLEANUP_TEMP_FILES: "1"
-    TORCH_SYMMMEM: "NVSHMEM"
+    NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC"
+    NVSHMEM_DISABLE_CUDA_VMM: "0"
     VLLM_SKIP_P2P_CHECK: "1"
-    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "0"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
     VLLM_USE_DEEP_GEMM: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
     NVIDIA_GDRCOPY: "enabled"
     VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120"
     # Harden distributed coordination against transient gloo/NCCL resets under
@@ -156,7 +165,6 @@ decode:
     --no-enable-flashinfer-autotune
     --block-size 256
     --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
-    --stream-interval 50
     --tokenizer-mode deepseek_v4
     --no-disable-hybrid-kv-cache-manager
     --enable-sleep-mode
@@ -171,9 +179,13 @@ decode:
     # rc added so KV falls back to InfiniBand, not TCP (see prefill note).
     UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp"
     UCX_CUDA_IPC_ENABLE_MNNVL: "y"
-    VLLM_USE_NCCL_SYMM_MEM: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "0"
     TILELANG_CLEANUP_TEMP_FILES: "1"
+    NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC"
+    NVSHMEM_DISABLE_CUDA_VMM: "0"
     VLLM_SKIP_P2P_CHECK: "1"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
     NVIDIA_GDRCOPY: "enabled"
     TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800"
 

benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml

@@ -45,7 +45,9 @@ plugins:
   - type: prefix-cache-scorer
   - type: queue-scorer
   - type: active-request-scorer
-  - type: max-score-picker
+  - type: weighted-random-picker
+    parameters:
+      threshold: "0.1"
 
 schedulingProfiles:
   - name: prefill
@@ -57,12 +59,13 @@ schedulingProfiles:
         weight: 2
       - pluginRef: active-request-scorer
         weight: 2
-      - pluginRef: max-score-picker
+      - pluginRef: weighted-random-picker
   - name: decode
     plugins:
       - pluginRef: decode-filter
       - pluginRef: active-request-scorer
-      - pluginRef: max-score-picker
+        weight: 2
+      - pluginRef: weighted-random-picker
 
 dataLayer:
   discovery:
@@ -87,7 +90,10 @@ prefill:
   extra-args: >-
     --kv-cache-dtype fp8
     --enforce-eager
-    --gpu-memory-utilization 0.9
+    --gpu-memory-utilization 0.95
+    --max-model-len 9280
+    --max-num-seqs 16
+    --max-num-batched-tokens 32768
     --enable-cumem-allocator
     --no-enable-prefix-caching
     --no-async-scheduling
@@ -96,6 +102,7 @@ prefill:
     --moe-backend deep_gemm_mega_moe
     --enable-ep-weight-filter
     --no-disable-hybrid-kv-cache-manager
+    --no-enable-flashinfer-autotune
     --numa-bind
   env:
     NCCL_CUMEM_ENABLE: "1"
@@ -121,14 +128,16 @@ prefill:
     NVSHMEM_REMOTE_TRANSPORT: "none"
     NVSHMEM_ENABLE_NIC_PE_MAPPING: "1"
     NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1"
-    VLLM_USE_NCCL_SYMM_MEM: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "0"
     VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
     VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048"
     TILELANG_CLEANUP_TEMP_FILES: "1"
-    TORCH_SYMMMEM: "NVSHMEM"
+    NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC"
+    NVSHMEM_DISABLE_CUDA_VMM: "0"
     VLLM_SKIP_P2P_CHECK: "1"
-    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "0"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
     VLLM_USE_DEEP_GEMM: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
     NVIDIA_GDRCOPY: "enabled"
     VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120"
     # Harden distributed coordination against transient gloo/NCCL connection
@@ -146,12 +155,13 @@ decode:
     --max-num-seqs 512
     --max-num-batched-tokens 512
     --max-cudagraph-capture-size 512
-    --gpu-memory-utilization 0.85
+    --gpu-memory-utilization 0.9
+    --max-model-len 9280
     --enable-cumem-allocator
     --no-enable-prefix-caching
+    --no-enable-flashinfer-autotune
     --block-size 256
     --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}
-    --stream-interval 50
     --tokenizer-mode deepseek_v4
     --moe-backend deep_gemm_mega_moe
     --enable-ep-weight-filter
@@ -170,12 +180,14 @@ decode:
     NVSHMEM_REMOTE_TRANSPORT: "none"
     NVSHMEM_ENABLE_NIC_PE_MAPPING: "1"
     NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1"
-    VLLM_USE_NCCL_SYMM_MEM: "1"
+    VLLM_USE_NCCL_SYMM_MEM: "0"
     TILELANG_CLEANUP_TEMP_FILES: "1"
-    TORCH_SYMMMEM: "NVSHMEM"
+    NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC"
+    NVSHMEM_DISABLE_CUDA_VMM: "0"
     VLLM_SKIP_P2P_CHECK: "1"
-    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "0"
+    VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
     VLLM_USE_DEEP_GEMM: "1"
+    VLLM_USE_RUST_FRONTEND: "1"
     NVIDIA_GDRCOPY: "enabled"
     # See prefill note: harden the decode's cross-node DP8/EP8 coordination
     # against transient gloo resets under the conc4096 warmup burst.

configs/nvidia-master.yaml

@@ -5272,7 +5272,7 @@ dsv4-fp4-gb200-dynamo-vllm:
 # TODO: change image to official llmd image.
 # Build source: benchmarks/llm-d/Dockerfile.
 dsv4-fp4-gb200-llmd-vllm:
-  image: ghcr.io/ezrasilvera/llm-d-nokube-vllm:vllm0.23
+  image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:vllm0.26
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: gb200

perf-changelog.yaml

@@ -5356,6 +5356,16 @@
     - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213
 
+- config-keys:
+    - dsv4-fp4-gb200-llmd-vllm
+  description:
+    - "Decode: gpu-memory-utilization 0.85->0.9, add max-model-len 9280, max-num-seqs 512->1024, disable NCCL symmetric memory, add no-enable-flashinfer-autotune, drop stream-interval (unsupported by rust frontend)"
+    - "Prefill: gpu-memory-utilization 0.9->0.95, add max-model-len 9280, add max-num-seqs 16 + max-num-batched-tokens 32768, enable rust frontend, enable randomize-dp-dummy-inputs"
+    - "EPP: switch from max-score-picker to weighted-random-picker (threshold=0.1) for better load distribution under high concurrency"
+    - "Low-latency recipe: same prefill optimizations, decode NCCL symmetric memory disabled + rust frontend enabled, drop stream-interval"
+    - "Bump llmd-vllm image vllm0.23 -> vllm0.26 (v0.26 strips data_parallel_hybrid_lb from rust args-json; required for VLLM_USE_RUST_FRONTEND=1 on multi-node DEP8)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2242
+
 - config-keys:
     - qwen3.5-fp4-b300-sglang-agentic-mtp
   description:
@@ -5373,7 +5383,6 @@
     - "Use the shared AIPerf watchdog to cap whole-trajectory runtime idle gaps at 300 seconds"
     - "Image: lmsysorg/sglang:v0.5.16-cu130"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420
-
 
 - config-keys:
     - minimaxm3-fp4-b200-dynamo-vllm