[AMD] dsv4-fp4-mi355x-atom: latest atom-dev image + opt recipe / DSv4-Pro FP4 MI355X ATOM：升级最新 atom-dev 镜像 + 优化配方 + MTP

benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-source "$(dirname "$0")/../benchmark_lib.sh"
+source "$(dirname "$0")/../../benchmark_lib.sh"
 
 check_env_vars \
     MODEL \
@@ -26,12 +26,41 @@ PARALLEL_ARGS=(-tp "$TP") #TP
 if [ "$DP_ATTENTION" = "true" ]; then
     if [ "$EP_SIZE" -gt 1 ]; then #DP+EP
         PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention )
-    else #DP+TP
+    else #DPA+TP
         PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention )
     fi
-fi 
+fi
+
+# MTP speculative decoding (ATOM self-draft).
+SPEC_ARGS=(--method mtp --num-speculative-tokens 3)
 
-SPEC_ARGS=(--method mtp --num-speculative-tokens 3 )
+# max_req=conc for dp-on cells (dp-attention keeps a full KV pool per rank, and MTP
+# reserves q=num_speculative_tokens+1 per request, so the large default max_num_seqs
+# OOMs) and for conc>=64. dp-off low conc uses the ATOM default.
+if [ "$DP_ATTENTION" = "true" ] || [ "$CONC" -ge 64 ]; then
+    PARALLEL_ARGS+=(--max-num-seqs "$CONC")
+fi
+
+# prefill-only TBO (--enable-tbo -> argparse const=prefill -> enable_tbo_decode=False):
+# MTP-compatible because it never touches decode. (--enable-tbo all / decode-TBO WOULD
+# drop MTP's spec_decode_metadata in UBatchWrapper -- that is the real incompatibility;
+# prefill-only does not.) Threshold is conc>=256 for MTP (NOT the non-MTP #2327 conc>=64):
+# measured crossover (TBO run 30257759947 vs non-TBO run 30238071409, same image/harness)
+# shows TBO HURTS at c64/c128 (-14%/-10% output tput) but HELPS at c256+ (+11/+8/+14%).
+# MTP already removes low-conc latency, so TBO overlap there is pure overhead; the win
+# only appears once high conc turns compute-bound. Matches ATOM models.json non-MTP
+# DPA-TBO 256-2048 band. Together with GPU_MAX_HW_QUEUES=5.
+if [ "$DP_ATTENTION" = "true" ] && [ "$CONC" -ge 256 ]; then
+    PARALLEL_ARGS+=(--enable-tbo)
+    export GPU_MAX_HW_QUEUES=5
+fi
+
+BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN"
+
+if [ "${EVAL_ONLY}" = "true" ]; then
+    EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN")
+    export EVAL_MAX_MODEL_LEN
+fi
 
 # Start GPU monitoring (power, temperature, clocks every second)
 start_gpu_monitor
@@ -40,26 +69,28 @@ set -x
 export ATOM_DISABLE_MMAP=true
 export AITER_BF16_FP8_MOE_BOUND=0
 export ATOM_MOE_GU_ITLV=1
+
 python3 -m atom.entrypoints.openai_server \
     --model $MODEL \
     --server-port $PORT \
     "${PARALLEL_ARGS[@]}" \
     "${SPEC_ARGS[@]}" \
     --kv_cache_dtype fp8 \
     --trust-remote-code \
-    > $SERVER_LOG 2>&1 &
+    --no-enable_prefix_caching \
+    > "$SERVER_LOG" 2>&1 &
 
 SERVER_PID=$!
 
 # Wait for server to be ready
 wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
 
-# --dsv4 routes prompts through encoding_dsv4.py (PR #1153), which emits the
-# <bos><User>...<Assistant><think> framing DeepSeek-V4-Pro expects. The DSv4-Pro
-# tokenizer ships without a jinja chat_template, so plain --use-chat-template
-# would crash; --dsv4 sidesteps that and satisfies the AGENTS.md rule that all
-# MTP scripts must benchmark against chat-formatted inputs (EAGLE acceptance
-# silently regresses on raw random tokens).
+# --dsv4: InferenceX's bench (utils/bench_serving/benchmark_serving.py) uses the
+# DeepSeek-V4 chat encoder (encoding_dsv4.py) instead of the tokenizer's jinja
+# template. DSv4-Pro has NO jinja chat_template, so a plain --use-chat-template
+# produces empty/broken prompts (bench yields no result). --dsv4 implies
+# --use-chat-template. Chat-formatted inputs are required for MTP: EAGLE/MTP
+# acceptance silently regresses on raw random tokens (AGENTS.md).
 run_benchmark_serving \
     --model "$MODEL" \
     --port "$PORT" \

configs/amd-master.yaml

@@ -1427,7 +1427,7 @@ dsv4-fp4-mi355x-atom:
       - { tp: 8, ep: 1, dp-attn: true,  conc-start: 64, conc-end: 2048 }
 
 dsv4-fp4-mi355x-atom-mtp:
-  image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3
+  image: rocm/atom-dev:nightly_202607231538
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: mi355x
@@ -1439,7 +1439,16 @@ dsv4-fp4-mi355x-atom-mtp:
     - isl: 8192
       osl: 1024
       search-space:
-      - { tp: 8, ep: 1, conc-start: 1, conc-end: 1024, spec-decoding: mtp }
+      # Aligns ATOM official DSv4 MTP (models.json): MTP3 dp-off at low-mid conc,
+      # DPA MTP3 (dp-attn) at mid-high conc. dp-on starts at c32 (dp-attn helps from c32).
+      # prefill-only TBO on dp-attn cells at conc>=256 (measured crossover: TBO hurts
+      # c64/c128 -10~14%, helps c256+ +8~14%; MTP removes low-conc latency so TBO there is
+      # pure overhead). MTP-safe — only decode-TBO (--enable-tbo all) drops spec metadata.
+      - { tp: 8, ep: 1, dp-attn: false, conc-start: 4,  conc-end: 256,  spec-decoding: mtp }
+      - { tp: 8, ep: 1, dp-attn: true,  conc-start: 32, conc-end: 1024, spec-decoding: mtp }
+      # tp4 removed: tp4+MTP+dp-on OOMs — 806G/4=201G/GPU weight leaves KV budget
+      # NEGATIVE (-10.98GB) even at gpu-mem-util 0.90 (run 30277221626). #2327 tp4
+      # works only because it's non-MTP (no draft KV layer). MTP stays tp8-only.
 
 qwen3.5-bf16-mi325x-sglang-mtp:
   image: lmsysorg/sglang:v0.5.12-rocm720-mi30x

perf-changelog.yaml

@@ -5122,3 +5122,10 @@
   description:
     - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349
+
+- config-keys:
+    - dsv4-fp4-mi355x-atom-mtp
+  description:
+    - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on; tp4 tried but removed — tp4+MTP+dp-on OOMs, KV budget negative). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=256 (measured crossover run 30257759947 vs non-TBO 30238071409: TBO -10~14% output tput at c64/c128 but +8~14% at c256+; MTP removes low-conc latency so TBO overlap there is pure overhead); MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345
+