From 7bf8ecf993a190eb0b251215da172e7221de5320 Mon Sep 17 00:00:00 2001 From: khluu Date: Wed, 22 Jul 2026 01:39:34 -0700 Subject: [PATCH 1/4] Add DeepSeek V4 Pro B200 workload Follow the official NVFP4 Blackwell recipe and retain the existing accuracy, BFCL, and serving benchmark coverage. Co-Authored-By: Codex Signed-off-by: khluu --- workloads/deepseek_v4_pro_b200.yaml | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 workloads/deepseek_v4_pro_b200.yaml diff --git a/workloads/deepseek_v4_pro_b200.yaml b/workloads/deepseek_v4_pro_b200.yaml new file mode 100644 index 0000000..ef86ca3 --- /dev/null +++ b/workloads/deepseek_v4_pro_b200.yaml @@ -0,0 +1,61 @@ +# DeepSeek-V4-Pro NVFP4 on B200 (8xB200, TP=8 + EP) +# Recipe: https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro +name: deepseek_v4_pro-b200 +gpu: B200 +num_gpus: 8 +nightly: true + +vllm: + model: nvidia/DeepSeek-V4-Pro-NVFP4 + env: + TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas + serve_args: >- + --tensor-parallel-size 8 + --enable-expert-parallel + --max-model-len 32768 + --max-num-seqs 512 + --max-num-batched-tokens 512 + --kv-cache-dtype fp8 + --block-size 256 + --attention_config.use_fp4_indexer_cache=True + --tokenizer-mode deepseek_v4 + --gpu-memory-utilization 0.95 + --tool-call-parser deepseek_v4 + --reasoning-parser deepseek_v4 + --enable-auto-tool-choice + --trust-remote-code + --compilation-config {"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]} + +lm_eval: + model_args: + tokenized_requests: false + tokenizer_backend: null + timeout: 6000 + tasks: + - name: gsm8k + num_fewshot: 5 + model_args: + num_concurrent: 64 + max_length: 32768 + max_gen_toks: 8192 + +bfcl: + test_categories: + - simple_python + - multiple + - parallel + - parallel_multiple + - multi_turn + num_threads: 16 + max_test_cases: + multi_turn: 240 + +vllm_bench: + configs: + - name: 8k-in-1k-out-conc-128 + backend: openai + dataset: random + input_len: 8192 + output_len: 1024 + num_prompts: 512 + max_concurrency: 128 From 7e9eb94c5b4b98d070c767b7fa3abf1970ff9c5e Mon Sep 17 00:00:00 2001 From: khluu Date: Wed, 22 Jul 2026 02:23:03 -0700 Subject: [PATCH 2/4] Use system CUDA toolkit for DeepSeek kernels Point TileLang at the CUDA 13.0 toolkit installed in the vLLM image so its NVCC compiler and CCCL headers remain version-matched.\n\nCo-Authored-By: Codex Signed-off-by: khluu --- workloads/deepseek_v4_pro_b200.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/workloads/deepseek_v4_pro_b200.yaml b/workloads/deepseek_v4_pro_b200.yaml index ef86ca3..cbd292c 100644 --- a/workloads/deepseek_v4_pro_b200.yaml +++ b/workloads/deepseek_v4_pro_b200.yaml @@ -8,6 +8,7 @@ nightly: true vllm: model: nvidia/DeepSeek-V4-Pro-NVFP4 env: + CUDA_HOME: /usr/local/cuda TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas serve_args: >- --tensor-parallel-size 8 From aaab28e3025572643d67073fa17005f486ad0e22 Mon Sep 17 00:00:00 2001 From: khluu Date: Wed, 22 Jul 2026 02:56:03 -0700 Subject: [PATCH 3/4] Use DeepSeek data parallel topology Co-Authored-By: Codex Signed-off-by: khluu --- workloads/deepseek_v4_pro_b200.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workloads/deepseek_v4_pro_b200.yaml b/workloads/deepseek_v4_pro_b200.yaml index cbd292c..54507cd 100644 --- a/workloads/deepseek_v4_pro_b200.yaml +++ b/workloads/deepseek_v4_pro_b200.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V4-Pro NVFP4 on B200 (8xB200, TP=8 + EP) +# DeepSeek-V4-Pro NVFP4 on B200 (8xB200, DP=8 + EP) # Recipe: https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro name: deepseek_v4_pro-b200 gpu: B200 @@ -11,7 +11,7 @@ vllm: CUDA_HOME: /usr/local/cuda TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas serve_args: >- - --tensor-parallel-size 8 + --data-parallel-size 8 --enable-expert-parallel --max-model-len 32768 --max-num-seqs 512 From eb441cccfcb23b636d33767e1391e85a4cf22f80 Mon Sep 17 00:00:00 2001 From: khluu Date: Wed, 22 Jul 2026 03:11:52 -0700 Subject: [PATCH 4/4] Skip incompatible DeepSeek MLA warmup Co-Authored-By: Codex Signed-off-by: khluu --- workloads/deepseek_v4_pro_b200.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workloads/deepseek_v4_pro_b200.yaml b/workloads/deepseek_v4_pro_b200.yaml index 54507cd..b6a8a7d 100644 --- a/workloads/deepseek_v4_pro_b200.yaml +++ b/workloads/deepseek_v4_pro_b200.yaml @@ -1,4 +1,4 @@ -# DeepSeek-V4-Pro NVFP4 on B200 (8xB200, DP=8 + EP) +# DeepSeek-V4-Pro NVFP4 on B200 (8xB200, TP=8 + EP) # Recipe: https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro name: deepseek_v4_pro-b200 gpu: B200 @@ -11,7 +11,7 @@ vllm: CUDA_HOME: /usr/local/cuda TRITON_PTXAS_PATH: /usr/local/cuda/bin/ptxas serve_args: >- - --data-parallel-size 8 + --tensor-parallel-size 8 --enable-expert-parallel --max-model-len 32768 --max-num-seqs 512 @@ -25,6 +25,7 @@ vllm: --reasoning-parser deepseek_v4 --enable-auto-tool-choice --trust-remote-code + --kernel-config {"enable_jit_warmup":false} --compilation-config {"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]} lm_eval: