diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200_mtp.sh index 51147129d..9bb1dd777 100644 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200_mtp.sh @@ -3,15 +3,15 @@ # MiniMax-M3 MXFP8 B200 single-node vLLM recipe with EAGLE3 speculative # decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_b200.sh # (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3 draft head via --speculative-config with 3 +# Inferact/MiniMax-M3-EAGLE3-GQA draft head via --speculative-config with 3 # speculative tokens. Everything else keeps the non-MTP serve shape: # --block-size 128 is mandatory (MSA sparse_block_size is 128; the default 16 # misaligns sparse indexing), and --language-model-only skips the vision # encoder for the text-only benchmark. dp-attn=true maps to DP×EP (DEP); # ep>1 maps to TP+EP (TEP). # -# The target uses the FlashInfer TRT-LLM attention path. The EAGLE3 drafter is -# pinned separately to TRITON_ATTN. +# The target uses the FlashInfer TRT-LLM attention path. The EAGLE3-GQA drafter +# is pinned separately to FLASH_ATTN. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -27,40 +27,7 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME -# The 0618 image keeps MiniMax M3 top-k indices in a persistent -# [head_kv, max_tokens, topK] buffer for CUDA graphs. Slicing that buffer to -# the actual prefill length is non-contiguous when TP leaves multiple local KV -# heads, and the MSA CSR builder rejects it. Materialize the slice until the -# image includes this fix. -python3 - <<'PYEOF' || { echo "MiniMax M3 MSA contiguity patch failed" >&2; exit 1; } -import importlib.util -import pathlib - -spec = importlib.util.find_spec("vllm") -if spec is None or not spec.submodule_search_locations: - raise RuntimeError("Could not locate the installed vllm package") - -target = ( - pathlib.Path(next(iter(spec.submodule_search_locations))) - / "models" - / "minimax_m3" - / "nvidia" - / "sparse_attention_msa.py" -) -src = target.read_text() -old = " prefill_topk = topk[:, nd:num_tokens, :]\n" -new = " prefill_topk = topk[:, nd:num_tokens, :].contiguous()\n" - -if new in src: - print(f"[minimax-m3-msa-patch] already applied: {target}") -elif src.count(old) == 1: - target.write_text(src.replace(old, new, 1)) - print(f"[minimax-m3-msa-patch] patched: {target}") -else: - raise RuntimeError(f"Expected exactly one patch anchor in {target}") -PYEOF - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" @@ -116,14 +83,13 @@ $PARALLEL_ARGS \ --gpu-memory-utilization 0.90 \ --max-model-len $MAX_MODEL_LEN \ --block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true}' \ ---attention-config.indexer_kv_dtype "fp8" \ +--attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' \ --kv-cache-dtype fp8 \ --language-model-only \ --max-cudagraph-capture-size 2048 \ --max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ +--speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ +--stream-interval 32 --no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300_mtp.sh index 01bf23eb6..e5df40e5f 100644 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300_mtp.sh @@ -3,13 +3,13 @@ # MiniMax-M3 MXFP8 B300 single-node vLLM recipe with EAGLE3 speculative # decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_b300.sh # (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3 draft head via --speculative-config with 3 +# Inferact/MiniMax-M3-EAGLE3-GQA draft head via --speculative-config with 3 # speculative tokens. Everything else keeps the non-MTP serve shape: # --block-size 128 is mandatory (MSA sparse/index cache); the benchmark is # text-only, so --language-model-only frees the vision encoder's VRAM. # -# The target uses the FlashInfer TRT-LLM attention path. The EAGLE3 drafter is -# pinned separately to TRITON_ATTN. +# The target uses the FlashInfer TRT-LLM attention path. The EAGLE3-GQA drafter +# is pinned separately to FLASH_ATTN. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -25,40 +25,7 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME -# The 0618 image keeps MiniMax M3 top-k indices in a persistent -# [head_kv, max_tokens, topK] buffer for CUDA graphs. Slicing that buffer to -# the actual prefill length is non-contiguous when TP leaves multiple local KV -# heads, and the MSA CSR builder rejects it. Materialize the slice until the -# image includes this fix. -python3 - <<'PYEOF' || { echo "MiniMax M3 MSA contiguity patch failed" >&2; exit 1; } -import importlib.util -import pathlib - -spec = importlib.util.find_spec("vllm") -if spec is None or not spec.submodule_search_locations: - raise RuntimeError("Could not locate the installed vllm package") - -target = ( - pathlib.Path(next(iter(spec.submodule_search_locations))) - / "models" - / "minimax_m3" - / "nvidia" - / "sparse_attention_msa.py" -) -src = target.read_text() -old = " prefill_topk = topk[:, nd:num_tokens, :]\n" -new = " prefill_topk = topk[:, nd:num_tokens, :].contiguous()\n" - -if new in src: - print(f"[minimax-m3-msa-patch] already applied: {target}") -elif src.count(old) == 1: - target.write_text(src.replace(old, new, 1)) - print(f"[minimax-m3-msa-patch] patched: {target}") -else: - raise RuntimeError(f"Expected exactly one patch anchor in {target}") -PYEOF - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" # `hf download` creates the target dir if missing and is itself idempotent. # When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. @@ -118,14 +85,13 @@ $PARALLEL_ARGS \ --gpu-memory-utilization 0.90 \ --max-model-len $MAX_MODEL_LEN \ --block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true}' \ ---attention-config.indexer_kv_dtype "fp8" \ +--attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' \ --kv-cache-dtype fp8 \ --language-model-only \ --max-cudagraph-capture-size 2048 \ --max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ +--speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ +--stream-interval 32 --no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6e4fc8d34..9b1596fe8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7478,7 +7478,7 @@ minimaxm3-fp4-b200-vllm: # precedent: spec decode pays off at low/mid concurrency while acceptance # dilutes in big batches, and the draft weights + draft KV shave headroom. minimaxm3-fp8-b200-vllm-mtp: - image: vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 + image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 runner: b200-dgxc @@ -7494,7 +7494,6 @@ minimaxm3-fp8-b200-vllm-mtp: - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128, spec-decoding: mtp } - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256, spec-decoding: mtp } # EAGLE3 speculative-decoding (spec-decoding: mtp) variant of MiniMax-M3 NVFP4 @@ -7534,7 +7533,7 @@ minimaxm3-fp4-b200-vllm-mtp: # big batches, and the draft weights + draft KV shave headroom — tp2-ep2 is # dropped entirely since its KV headroom was already thin without a draft. minimaxm3-fp8-b300-vllm-mtp: - image: vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 + image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 runner: b300 @@ -7550,7 +7549,6 @@ minimaxm3-fp8-b300-vllm-mtp: - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128, spec-decoding: mtp } - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256, spec-decoding: mtp } # EAGLE3 speculative-decoding (spec-decoding: mtp) variant of diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c3602ff74..7064abe5c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5085,3 +5085,11 @@ description: - "Bump SGLang container image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.15.post1-cu130 (https://github.com/sgl-project/sglang/releases/tag/v0.5.15.post1)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2313 + +- config-keys: + - minimaxm3-fp8-b200-vllm-mtp + - minimaxm3-fp8-b300-vllm-mtp + description: + - "Bump image to vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9; set --stream-interval 32; consolidate --attention-config with indexer_kv_dtype fp8; drop the sparse_attention_msa contiguity patch (shipped in the nightly image)" + - "EAGLE3 draft head Inferact/MiniMax-M3-EAGLE3 -> Inferact/MiniMax-M3-EAGLE3-GQA; speculative attention_backend TRITON_ATTN -> FLASH_ATTN; num_speculative_tokens kept at 3" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2337