diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh index e46c19126..55ffc501b 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../benchmark_lib.sh" +source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars \ MODEL \ @@ -26,12 +26,41 @@ PARALLEL_ARGS=(-tp "$TP") #TP if [ "$DP_ATTENTION" = "true" ]; then if [ "$EP_SIZE" -gt 1 ]; then #DP+EP PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP + else #DPA+TP PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) fi -fi +fi + +# MTP speculative decoding (ATOM self-draft). +SPEC_ARGS=(--method mtp --num-speculative-tokens 3) -SPEC_ARGS=(--method mtp --num-speculative-tokens 3 ) +# max_req=conc for dp-on cells (dp-attention keeps a full KV pool per rank, and MTP +# reserves q=num_speculative_tokens+1 per request, so the large default max_num_seqs +# OOMs) and for conc>=64. dp-off low conc uses the ATOM default. +if [ "$DP_ATTENTION" = "true" ] || [ "$CONC" -ge 64 ]; then + PARALLEL_ARGS+=(--max-num-seqs "$CONC") +fi + +# prefill-only TBO (--enable-tbo -> argparse const=prefill -> enable_tbo_decode=False): +# MTP-compatible because it never touches decode. (--enable-tbo all / decode-TBO WOULD +# drop MTP's spec_decode_metadata in UBatchWrapper -- that is the real incompatibility; +# prefill-only does not.) Threshold is conc>=256 for MTP (NOT the non-MTP #2327 conc>=64): +# measured crossover (TBO run 30257759947 vs non-TBO run 30238071409, same image/harness) +# shows TBO HURTS at c64/c128 (-14%/-10% output tput) but HELPS at c256+ (+11/+8/+14%). +# MTP already removes low-conc latency, so TBO overlap there is pure overhead; the win +# only appears once high conc turns compute-bound. Matches ATOM models.json non-MTP +# DPA-TBO 256-2048 band. Together with GPU_MAX_HW_QUEUES=5. +if [ "$DP_ATTENTION" = "true" ] && [ "$CONC" -ge 256 ]; then + PARALLEL_ARGS+=(--enable-tbo) + export GPU_MAX_HW_QUEUES=5 +fi + +BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" + +if [ "${EVAL_ONLY}" = "true" ]; then + EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") + export EVAL_MAX_MODEL_LEN +fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -40,6 +69,7 @@ set -x export ATOM_DISABLE_MMAP=true export AITER_BF16_FP8_MOE_BOUND=0 export ATOM_MOE_GU_ITLV=1 + python3 -m atom.entrypoints.openai_server \ --model $MODEL \ --server-port $PORT \ @@ -47,19 +77,20 @@ python3 -m atom.entrypoints.openai_server \ "${SPEC_ARGS[@]}" \ --kv_cache_dtype fp8 \ --trust-remote-code \ - > $SERVER_LOG 2>&1 & + --no-enable_prefix_caching \ + > "$SERVER_LOG" 2>&1 & SERVER_PID=$! # Wait for server to be ready wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -# --dsv4 routes prompts through encoding_dsv4.py (PR #1153), which emits the -# ... framing DeepSeek-V4-Pro expects. The DSv4-Pro -# tokenizer ships without a jinja chat_template, so plain --use-chat-template -# would crash; --dsv4 sidesteps that and satisfies the AGENTS.md rule that all -# MTP scripts must benchmark against chat-formatted inputs (EAGLE acceptance -# silently regresses on raw random tokens). +# --dsv4: InferenceX's bench (utils/bench_serving/benchmark_serving.py) uses the +# DeepSeek-V4 chat encoder (encoding_dsv4.py) instead of the tokenizer's jinja +# template. DSv4-Pro has NO jinja chat_template, so a plain --use-chat-template +# produces empty/broken prompts (bench yields no result). --dsv4 implies +# --use-chat-template. Chat-formatted inputs are required for MTP: EAGLE/MTP +# acceptance silently regresses on raw random tokens (AGENTS.md). run_benchmark_serving \ --model "$MODEL" \ --port "$PORT" \ diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d3ca241c4..8e6bee505 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1427,7 +1427,7 @@ dsv4-fp4-mi355x-atom: - { tp: 8, ep: 1, dp-attn: true, conc-start: 64, conc-end: 2048 } dsv4-fp4-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 + image: rocm/atom-dev:nightly_202607231538 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x @@ -1439,7 +1439,16 @@ dsv4-fp4-mi355x-atom-mtp: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1024, spec-decoding: mtp } + # Aligns ATOM official DSv4 MTP (models.json): MTP3 dp-off at low-mid conc, + # DPA MTP3 (dp-attn) at mid-high conc. dp-on starts at c32 (dp-attn helps from c32). + # prefill-only TBO on dp-attn cells at conc>=256 (measured crossover: TBO hurts + # c64/c128 -10~14%, helps c256+ +8~14%; MTP removes low-conc latency so TBO there is + # pure overhead). MTP-safe — only decode-TBO (--enable-tbo all) drops spec metadata. + - { tp: 8, ep: 1, dp-attn: false, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: true, conc-start: 32, conc-end: 1024, spec-decoding: mtp } + # tp4 removed: tp4+MTP+dp-on OOMs — 806G/4=201G/GPU weight leaves KV budget + # NEGATIVE (-10.98GB) even at gpu-mem-util 0.90 (run 30277221626). #2327 tp4 + # works only because it's non-MTP (no draft KV layer). MTP stays tp8-only. qwen3.5-bf16-mi325x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1942c0026..d969b6ce4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,3 +5122,10 @@ description: - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349 + +- config-keys: + - dsv4-fp4-mi355x-atom-mtp + description: + - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on; tp4 tried but removed — tp4+MTP+dp-on OOMs, KV budget negative). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=256 (measured crossover run 30257759947 vs non-TBO 30238071409: TBO -10~14% output tput at c64/c128 but +8~14% at c256+; MTP removes low-conc latency so TBO overlap there is pure overhead); MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345 +