diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh index 011f9dedb..718b11b8b 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh @@ -37,16 +37,16 @@ if [ "${DP_ATTENTION}" = "true" ]; then fi EP_ARGS=() +MOE_ARGS=() if [ "${EP_SIZE:-1}" -gt 1 ]; then EP_ARGS=(--enable-expert-parallel) + MOE_ARGS=(--moe-backend deep_gemm_mega_moe) fi GMU_ARGS=() -MOE_ARGS=() EPLB_ARGS=() PREFILL_SCHEDULE_ARGS=() if [ "${DP_ATTENTION}" = "true" ]; then - MOE_ARGS=(--moe-backend deep_gemm_mega_moe) EPLB_ARGS=(--enable-eplb --eplb-config '{"communicator":"torch_nccl", "use_async": false}') PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) fi @@ -60,6 +60,10 @@ fi MAX_CUDAGRAPH_CAPTURE_SIZE=2048 BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" +# Cap MAX_MODEL_LEN to free CUDA-graph memory and give more headroom for KV blocks +if [ "${BENCHMARK_MAX_MODEL_LEN}" -gt 12288 ]; then + BENCHMARK_MAX_MODEL_LEN=12288 +fi if [ "${EVAL_ONLY}" = "true" ]; then EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") @@ -91,6 +95,7 @@ vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ --enable-auto-tool-choice \ --reasoning-parser deepseek_v4 \ --max-cudagraph-capture-size "$MAX_CUDAGRAPH_CAPTURE_SIZE" \ + --gpu-memory-utilization 0.95 \ --max-model-len "$SERVE_MAX_MODEL_LEN" \ --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c752b04f6..661a05b99 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -929,7 +929,23 @@ dsv4-fp4-b200-sglang: - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } dsv4-fp4-b200-vllm: - image: vllm/vllm-openai:v0.25.0 + image: vllm/vllm-openai:nightly-821717118fc26667dd474b9b0ab81d29259dfc5c + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 1, conc-end: 256 } + - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } + - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 } +m: + image: vllm/vllm-openai:v0.26.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b200-dgxc diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0bce8d894..e54ceaf39 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5818,3 +5818,9 @@ description: - "Extend the SimpleCPUOffloadConnector grid to c8/c12/c16/c20/c24/c28/c32/c48/c64 to locate its crossover against the resident curve" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 + +- config-keys: + - dsv4-fp4-b200-vllm + description: + - "B200 DSv4 FP4 vLLM: nightly image, new sweep (TP8, TEP8, DEP8), megamoe when ep>1, gmu 0.95, cap max-model-len 12288" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2534