diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200.sh index 966448b0c9..95a53a8297 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200.sh @@ -33,7 +33,6 @@ SERVER_LOG=/workspace/server.log export VLLM_ENGINE_READY_TIMEOUT_S=3600 export VLLM_FLOAT32_MATMUL_PRECISION=high -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm if [ "${DP_ATTENTION}" = "true" ]; then PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" @@ -52,6 +51,7 @@ start_gpu_monitor set -x vllm serve $MODEL --port $PORT \ $PARALLEL_ARGS \ +--attention_config.indexer_kv_dtype fp8 \ --gpu-memory-utilization 0.95 \ --max-model-len $MAX_MODEL_LEN \ --kv-cache-dtype fp8 \ diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200_mtp.sh index 3d00f4e75d..8f767954bf 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200_mtp.sh @@ -47,7 +47,6 @@ SERVER_LOG=/workspace/server.log export VLLM_ENGINE_READY_TIMEOUT_S=3600 export VLLM_FLOAT32_MATMUL_PRECISION=high -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm if [ "${DP_ATTENTION}" = "true" ]; then PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh index f91419edb7..5df5bc1d69 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh @@ -38,7 +38,6 @@ SERVER_LOG=/workspace/server.log export VLLM_ENGINE_READY_TIMEOUT_S=3600 export VLLM_FLOAT32_MATMUL_PRECISION=high -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm if [ "${DP_ATTENTION}" = "true" ]; then PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" @@ -57,6 +56,7 @@ start_gpu_monitor set -x vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ $PARALLEL_ARGS \ +--attention_config.indexer_kv_dtype fp8 \ --gpu-memory-utilization 0.95 \ --max-model-len $MAX_MODEL_LEN \ --kv-cache-dtype fp8 \ diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300_mtp.sh index 4ed72bc022..2c515917c1 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300_mtp.sh @@ -52,7 +52,6 @@ SERVER_LOG=/workspace/server.log export VLLM_ENGINE_READY_TIMEOUT_S=3600 export VLLM_FLOAT32_MATMUL_PRECISION=high -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm if [ "${DP_ATTENTION}" = "true" ]; then PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index be87b7aa29..4cf6ce1a66 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7527,7 +7527,7 @@ minimaxm3-fp8-b300-vllm: # weights are pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added to # the STAGED_MODELS allow-list in launch_b300-nv.sh). minimaxm3-fp4-b300-vllm: - image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226 + image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: b300 @@ -7553,7 +7553,7 @@ minimaxm3-fp4-b300-vllm: # /scratch/models/MiniMax-M3-NVFP4 (added to the STAGED_MODELS allow-list in # launch_b300-nv.sh); the EAGLE3 draft is downloaded to the writable models dir. minimaxm3-fp4-b300-vllm-mtp: - image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226 + image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: b300 @@ -7608,7 +7608,7 @@ minimaxm3-fp8-b200-vllm: # (MSA sparse/index cache); weights are pre-staged at /scratch/fsw/models/MiniMax-M3-NVFP4 # (launch_b200-dgxc.sh resolves MODEL_PATH for minimaxm3-fp4). minimaxm3-fp4-b200-vllm: - image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226 + image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: b200-dgxc @@ -7663,7 +7663,7 @@ minimaxm3-fp8-b200-vllm-mtp: # resolves MODEL_PATH for minimaxm3-fp4); the EAGLE3 draft is fetched next to # the target weights. minimaxm3-fp4-b200-vllm-mtp: - image: vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 + image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: b200-dgxc diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ac1de53c7d..f732503249 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5191,3 +5191,13 @@ - "Agentic experiment Variant K (probe, of the #2374 Variant G): REMOVES the kimi-k3-container-deps.sh in-container patch (setup_script, script file, and launcher copy) to verify whether the mamba_hybrid index_fill_ dtype patch is still required by the current vllm/vllm-openai:kimi-k3 image — expected to fail at the first decode step if it is" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2391 + +- config-keys: + - minimaxm3-fp4-b200-vllm + - minimaxm3-fp4-b200-vllm-mtp + - minimaxm3-fp4-b300-vllm + - minimaxm3-fp4-b300-vllm-mtp + description: + - "Update vllm image, remove setting trtllm allreduce backend" + - "Use fp8 indexer cache dtype for the B200/B300 non-eagle configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2120