From 2f6015bdde7a54f2248ba6d32f763488e27cb655 Mon Sep 17 00:00:00 2001 From: zufayu Date: Mon, 27 Jul 2026 03:31:21 +0000 Subject: [PATCH 1/8] dsv4-fp4-mi355x-atom-mtp: latest atom-dev image + align ATOM official MTP recipe - image: atom0.1.3 release -> rocm/atom-dev:nightly_202607231538 - grid: align ATOM models.json (MTP3 dp-off conc 4-256 + DPA MTP3 dp-on conc 64-1024); no TBO (incompatible with MTP) - recipe: add max_num_seqs=conc (avoid OOM on dp-on/conc>=64), --no-enable_prefix_caching, eval-context handling - bench: --dsv4 -> --use-chat-template (--dsv4 removed upstream; DSv4 encoder ships with model dir) - fix benchmark_lib source path (../ -> ../../) --- .../fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh | 42 +++++++++++++------ configs/amd-master.yaml | 8 +++- perf-changelog.yaml | 6 +++ 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh index e46c19126b..bbe68a413a 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../benchmark_lib.sh" +source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars \ MODEL \ @@ -26,12 +26,29 @@ PARALLEL_ARGS=(-tp "$TP") #TP if [ "$DP_ATTENTION" = "true" ]; then if [ "$EP_SIZE" -gt 1 ]; then #DP+EP PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP + else #DPA+TP PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) fi -fi +fi + +# MTP speculative decoding (ATOM self-draft). NO --enable-tbo: TBO is incompatible +# with MTP in ATOM (UBatchWrapper sets spec_decode_metadata=None; official +# models.json DPA MTP3 has no --enable-tbo). +SPEC_ARGS=(--method mtp --num-speculative-tokens 3) + +# max_req=conc for dp-on cells (dp-attention keeps a full KV pool per rank, and MTP +# reserves q=num_speculative_tokens+1 per request, so the large default max_num_seqs +# OOMs) and for conc>=64. dp-off low conc uses the ATOM default. +if [ "$DP_ATTENTION" = "true" ] || [ "$CONC" -ge 64 ]; then + PARALLEL_ARGS+=(--max-num-seqs "$CONC") +fi -SPEC_ARGS=(--method mtp --num-speculative-tokens 3 ) +BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" + +if [ "${EVAL_ONLY}" = "true" ]; then + EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") + export EVAL_MAX_MODEL_LEN +fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -40,6 +57,7 @@ set -x export ATOM_DISABLE_MMAP=true export AITER_BF16_FP8_MOE_BOUND=0 export ATOM_MOE_GU_ITLV=1 + python3 -m atom.entrypoints.openai_server \ --model $MODEL \ --server-port $PORT \ @@ -47,19 +65,19 @@ python3 -m atom.entrypoints.openai_server \ "${SPEC_ARGS[@]}" \ --kv_cache_dtype fp8 \ --trust-remote-code \ - > $SERVER_LOG 2>&1 & + --no-enable_prefix_caching \ + > "$SERVER_LOG" 2>&1 & SERVER_PID=$! # Wait for server to be ready wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -# --dsv4 routes prompts through encoding_dsv4.py (PR #1153), which emits the -# ... framing DeepSeek-V4-Pro expects. The DSv4-Pro -# tokenizer ships without a jinja chat_template, so plain --use-chat-template -# would crash; --dsv4 sidesteps that and satisfies the AGENTS.md rule that all -# MTP scripts must benchmark against chat-formatted inputs (EAGLE acceptance -# silently regresses on raw random tokens). +# --use-chat-template: DSv4-Pro's dsv4 message encoder ships with the model dir +# (encoding/encoding_*.py); ATOM auto-discovers it and applies the same chat framing +# on both server and bench, so AL stays aligned. (Replaces the removed --dsv4 flag.) +# Chat-formatted inputs are required for MTP: EAGLE/MTP acceptance silently regresses +# on raw random tokens (AGENTS.md). run_benchmark_serving \ --model "$MODEL" \ --port "$PORT" \ @@ -72,7 +90,7 @@ run_benchmark_serving \ --result-filename "$RESULT_FILENAME" \ --result-dir /workspace/ \ --trust-remote-code \ - --dsv4 + --use-chat-template # After throughput, run evaluation only if RUN_EVAL is true if [ "${RUN_EVAL}" = "true" ]; then diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0f547cba86..8097a2965c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1342,7 +1342,7 @@ dsv4-fp4-mi355x-atom: - { tp: 8, ep: 1, dp-attn: true, conc-start: 128, conc-end: 2048 } dsv4-fp4-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 + image: rocm/atom-dev:nightly_202607231538 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x @@ -1354,7 +1354,11 @@ dsv4-fp4-mi355x-atom-mtp: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1024, spec-decoding: mtp } + # Aligns ATOM official DSv4 MTP (models.json): MTP3 dp-off at low-mid conc, + # DPA MTP3 (dp-attn) at mid-high conc. No TBO — TBO is incompatible with MTP + # (ATOM UBatchWrapper drops spec_decode_metadata; DPA MTP3 has no --enable-tbo). + - { tp: 8, ep: 1, dp-attn: false, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: true, conc-start: 64, conc-end: 1024, spec-decoding: mtp } qwen3.5-bf16-mi325x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c3602ff749..1687954c27 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5085,3 +5085,9 @@ description: - "Bump SGLang container image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.15.post1-cu130 (https://github.com/sgl-project/sglang/releases/tag/v0.5.15.post1)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2313 + +- config-keys: + - dsv4-fp4-mi355x-atom-mtp + description: + - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 64-1024; no TBO (incompatible with MTP in ATOM). max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench --dsv4 -> --use-chat-template (--dsv4 removed upstream; DSv4 chat encoder now ships with the model dir). Fixed benchmark_lib source path (../ -> ../../)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/0 From 9548e5780e096ea85c0a10a568a9bce334999ccc Mon Sep 17 00:00:00 2001 From: zufayu Date: Mon, 27 Jul 2026 03:32:51 +0000 Subject: [PATCH 2/8] perf-changelog: fill atom-mtp pr-link #2345 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1687954c27..1d745c61b8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5090,4 +5090,4 @@ - dsv4-fp4-mi355x-atom-mtp description: - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 64-1024; no TBO (incompatible with MTP in ATOM). max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench --dsv4 -> --use-chat-template (--dsv4 removed upstream; DSv4 chat encoder now ships with the model dir). Fixed benchmark_lib source path (../ -> ../../)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/0 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345 From 3a73a41957e49ce4b4d35f6aaf68cfa45213dd68 Mon Sep 17 00:00:00 2001 From: zufayu Date: Mon, 27 Jul 2026 04:48:12 +0000 Subject: [PATCH 3/8] atom-mtp: bench --use-chat-template -> --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja template so plain --use-chat-template yields no result) --- .../fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh | 13 +++++++------ perf-changelog.yaml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh index bbe68a413a..007357c52c 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh @@ -73,11 +73,12 @@ SERVER_PID=$! # Wait for server to be ready wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -# --use-chat-template: DSv4-Pro's dsv4 message encoder ships with the model dir -# (encoding/encoding_*.py); ATOM auto-discovers it and applies the same chat framing -# on both server and bench, so AL stays aligned. (Replaces the removed --dsv4 flag.) -# Chat-formatted inputs are required for MTP: EAGLE/MTP acceptance silently regresses -# on raw random tokens (AGENTS.md). +# --dsv4: InferenceX's bench (utils/bench_serving/benchmark_serving.py) uses the +# DeepSeek-V4 chat encoder (encoding_dsv4.py) instead of the tokenizer's jinja +# template. DSv4-Pro has NO jinja chat_template, so a plain --use-chat-template +# produces empty/broken prompts (bench yields no result). --dsv4 implies +# --use-chat-template. Chat-formatted inputs are required for MTP: EAGLE/MTP +# acceptance silently regresses on raw random tokens (AGENTS.md). run_benchmark_serving \ --model "$MODEL" \ --port "$PORT" \ @@ -90,7 +91,7 @@ run_benchmark_serving \ --result-filename "$RESULT_FILENAME" \ --result-dir /workspace/ \ --trust-remote-code \ - --use-chat-template + --dsv4 # After throughput, run evaluation only if RUN_EVAL is true if [ "${RUN_EVAL}" = "true" ]; then diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1d745c61b8..a5ac40efe7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5089,5 +5089,5 @@ - config-keys: - dsv4-fp4-mi355x-atom-mtp description: - - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 64-1024; no TBO (incompatible with MTP in ATOM). max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench --dsv4 -> --use-chat-template (--dsv4 removed upstream; DSv4 chat encoder now ships with the model dir). Fixed benchmark_lib source path (../ -> ../../)." + - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 64-1024; no TBO (incompatible with MTP in ATOM). max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345 From 1996de2c3aa2964f5952598f42b7ae6b4247947b Mon Sep 17 00:00:00 2001 From: zufayu Date: Mon, 27 Jul 2026 10:12:29 +0000 Subject: [PATCH 4/8] atom-mtp: add c32 dp-on + prefill-only TBO on dp-attn conc>=64 (align #2327) - grid: DPA MTP3 conc-start 64->32 (dp-attn helps from c32) - recipe: --enable-tbo (prefill-only; argparse const=prefill => enable_tbo_decode=False) on dp-attn cells at conc>=64 + GPU_MAX_HW_QUEUES=5, mirrors non-MTP #2327. MTP-safe: only decode-TBO (--enable-tbo all) drops spec_decode_metadata. - changelog updated (triggers re-sweep) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh | 14 +++++++++++--- configs/amd-master.yaml | 7 ++++--- perf-changelog.yaml | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh index 007357c52c..1eec524484 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh @@ -31,9 +31,7 @@ if [ "$DP_ATTENTION" = "true" ]; then fi fi -# MTP speculative decoding (ATOM self-draft). NO --enable-tbo: TBO is incompatible -# with MTP in ATOM (UBatchWrapper sets spec_decode_metadata=None; official -# models.json DPA MTP3 has no --enable-tbo). +# MTP speculative decoding (ATOM self-draft). SPEC_ARGS=(--method mtp --num-speculative-tokens 3) # max_req=conc for dp-on cells (dp-attention keeps a full KV pool per rank, and MTP @@ -43,6 +41,16 @@ if [ "$DP_ATTENTION" = "true" ] || [ "$CONC" -ge 64 ]; then PARALLEL_ARGS+=(--max-num-seqs "$CONC") fi +# prefill-only TBO (--enable-tbo -> argparse const=prefill -> enable_tbo_decode=False): +# MTP-compatible because it never touches decode. (--enable-tbo all / decode-TBO WOULD +# drop MTP's spec_decode_metadata in UBatchWrapper -- that is the real incompatibility; +# prefill-only does not.) Mirrors non-MTP dsv4_fp4_mi355x_atom.sh (#2327): TBO on +# dp-attn cells at conc>=64, together with GPU_MAX_HW_QUEUES=5. +if [ "$DP_ATTENTION" = "true" ] && [ "$CONC" -ge 64 ]; then + PARALLEL_ARGS+=(--enable-tbo) + export GPU_MAX_HW_QUEUES=5 +fi + BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" if [ "${EVAL_ONLY}" = "true" ]; then diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index baa6a9e249..b462055f13 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1438,10 +1438,11 @@ dsv4-fp4-mi355x-atom-mtp: osl: 1024 search-space: # Aligns ATOM official DSv4 MTP (models.json): MTP3 dp-off at low-mid conc, - # DPA MTP3 (dp-attn) at mid-high conc. No TBO — TBO is incompatible with MTP - # (ATOM UBatchWrapper drops spec_decode_metadata; DPA MTP3 has no --enable-tbo). + # DPA MTP3 (dp-attn) at mid-high conc. dp-on starts at c32 (dp-attn helps from c32). + # prefill-only TBO on dp-attn cells at conc>=64 (mirrors non-MTP #2327): MTP-safe — + # only --enable-tbo all (decode-TBO) drops spec_decode_metadata, prefill-only doesn't. - { tp: 8, ep: 1, dp-attn: false, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 1, dp-attn: true, conc-start: 64, conc-end: 1024, spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: true, conc-start: 32, conc-end: 1024, spec-decoding: mtp } qwen3.5-bf16-mi325x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 240cc24230..fda7ad7481 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5089,7 +5089,7 @@ - config-keys: - dsv4-fp4-mi355x-atom-mtp description: - - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 64-1024; no TBO (incompatible with MTP in ATOM). max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." + - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=64, mirroring non-MTP #2327; MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345 - config-keys: From 0bc4ba82f10298381c7ee0b4f9085e5d4e2fcfec Mon Sep 17 00:00:00 2001 From: zufayu Date: Mon, 27 Jul 2026 14:20:48 +0000 Subject: [PATCH 5/8] atom-mtp: raise TBO threshold conc>=64 -> conc>=256 (measured crossover) TBO run 30257759947 vs non-TBO 30238071409 (same image/harness/axis): c64 -14.1% / c128 -9.8% output tput (TBO hurts) c256 +11.0% / c512 +8.3% / c1024 +13.9% (TBO helps) MTP already removes low-conc latency, so prefill-TBO overlap at c64/c128 is pure overhead; win only appears once high conc turns compute-bound. Matches ATOM models.json non-MTP DPA-TBO 256-2048 band. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh | 10 +++++++--- configs/amd-master.yaml | 5 +++-- perf-changelog.yaml | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh index 1eec524484..55ffc501b7 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh @@ -44,9 +44,13 @@ fi # prefill-only TBO (--enable-tbo -> argparse const=prefill -> enable_tbo_decode=False): # MTP-compatible because it never touches decode. (--enable-tbo all / decode-TBO WOULD # drop MTP's spec_decode_metadata in UBatchWrapper -- that is the real incompatibility; -# prefill-only does not.) Mirrors non-MTP dsv4_fp4_mi355x_atom.sh (#2327): TBO on -# dp-attn cells at conc>=64, together with GPU_MAX_HW_QUEUES=5. -if [ "$DP_ATTENTION" = "true" ] && [ "$CONC" -ge 64 ]; then +# prefill-only does not.) Threshold is conc>=256 for MTP (NOT the non-MTP #2327 conc>=64): +# measured crossover (TBO run 30257759947 vs non-TBO run 30238071409, same image/harness) +# shows TBO HURTS at c64/c128 (-14%/-10% output tput) but HELPS at c256+ (+11/+8/+14%). +# MTP already removes low-conc latency, so TBO overlap there is pure overhead; the win +# only appears once high conc turns compute-bound. Matches ATOM models.json non-MTP +# DPA-TBO 256-2048 band. Together with GPU_MAX_HW_QUEUES=5. +if [ "$DP_ATTENTION" = "true" ] && [ "$CONC" -ge 256 ]; then PARALLEL_ARGS+=(--enable-tbo) export GPU_MAX_HW_QUEUES=5 fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b462055f13..608983f15b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1439,8 +1439,9 @@ dsv4-fp4-mi355x-atom-mtp: search-space: # Aligns ATOM official DSv4 MTP (models.json): MTP3 dp-off at low-mid conc, # DPA MTP3 (dp-attn) at mid-high conc. dp-on starts at c32 (dp-attn helps from c32). - # prefill-only TBO on dp-attn cells at conc>=64 (mirrors non-MTP #2327): MTP-safe — - # only --enable-tbo all (decode-TBO) drops spec_decode_metadata, prefill-only doesn't. + # prefill-only TBO on dp-attn cells at conc>=256 (measured crossover: TBO hurts + # c64/c128 -10~14%, helps c256+ +8~14%; MTP removes low-conc latency so TBO there is + # pure overhead). MTP-safe — only decode-TBO (--enable-tbo all) drops spec metadata. - { tp: 8, ep: 1, dp-attn: false, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: true, conc-start: 32, conc-end: 1024, spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fda7ad7481..fb375ccac3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5089,7 +5089,7 @@ - config-keys: - dsv4-fp4-mi355x-atom-mtp description: - - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=64, mirroring non-MTP #2327; MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." + - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=256 (measured crossover run 30257759947 vs non-TBO 30238071409: TBO -10~14% output tput at c64/c128 but +8~14% at c256+; MTP removes low-conc latency so TBO overlap there is pure overhead); MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345 - config-keys: From 53f0be6e198893e1cd4339314a663d758f3af8ae Mon Sep 17 00:00:00 2001 From: zufayu Date: Mon, 27 Jul 2026 14:43:42 +0000 Subject: [PATCH 6/8] atom-mtp: add tp4 low-conc exploration (dp-off c1-64 + dp-on c16-128) Mirrors non-MTP #2327 tp4 grid. Fewer TP ranks -> less TP comm overhead may win at low conc (c16-128); tp4 dp-on is the point of interest, tp4 dp-off is the intra-tp4 control. No TBO (all tp4 cells conc<256). Compare vs tp8 at same conc for the tp4-vs-tp8 low-conc crossover. Co-Authored-By: Claude Opus 4.8 (1M context) --- configs/amd-master.yaml | 4 ++++ perf-changelog.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 608983f15b..a66e92d5bc 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1444,6 +1444,10 @@ dsv4-fp4-mi355x-atom-mtp: # pure overhead). MTP-safe — only decode-TBO (--enable-tbo all) drops spec metadata. - { tp: 8, ep: 1, dp-attn: false, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: true, conc-start: 32, conc-end: 1024, spec-decoding: mtp } + # tp4 low-conc exploration (mirrors non-MTP #2327): fewer TP ranks -> less TP comm, + # may win at low conc. dp-off c1-64 + dp-on c16-128. No TBO (conc<256). + - { tp: 4, ep: 1, dp-attn: false, conc-start: 1, conc-end: 64, spec-decoding: mtp } + - { tp: 4, ep: 1, dp-attn: true, conc-start: 16, conc-end: 128, spec-decoding: mtp } qwen3.5-bf16-mi325x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5634d66ba0..755104aeeb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5089,7 +5089,7 @@ - config-keys: - dsv4-fp4-mi355x-atom-mtp description: - - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=256 (measured crossover run 30257759947 vs non-TBO 30238071409: TBO -10~14% output tput at c64/c128 but +8~14% at c256+; MTP removes low-conc latency so TBO overlap there is pure overhead); MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." + - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on) + tp4 low-conc exploration (dp-off c1-64, dp-on c16-128, mirroring non-MTP #2327). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=256 (measured crossover run 30257759947 vs non-TBO 30238071409: TBO -10~14% output tput at c64/c128 but +8~14% at c256+; MTP removes low-conc latency so TBO overlap there is pure overhead); MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345 - config-keys: From 1afecb38986d763e9f67f7225726c6e113c110c6 Mon Sep 17 00:00:00 2001 From: zufayu Date: Mon, 27 Jul 2026 15:18:04 +0000 Subject: [PATCH 7/8] =?UTF-8?q?atom-mtp:=20remove=20tp4=20=E2=80=94=20tp4+?= =?UTF-8?q?MTP+dp-on=20OOMs=20(KV=20budget=20-10.98GB);=20tp8-only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tp4 gives 806G/4=201G/GPU weight, leaving negative KV budget even at gpu-mem-util 0.90 (run 30277221626 tp4 dp-on c64/c128 failed at engine init). #2327 tp4 works only because non-MTP (no draft KV). MTP stays tp8. Co-Authored-By: Claude Opus 4.8 (1M context) --- configs/amd-master.yaml | 7 +++---- perf-changelog.yaml | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index a66e92d5bc..f57a9505c1 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1444,10 +1444,9 @@ dsv4-fp4-mi355x-atom-mtp: # pure overhead). MTP-safe — only decode-TBO (--enable-tbo all) drops spec metadata. - { tp: 8, ep: 1, dp-attn: false, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: true, conc-start: 32, conc-end: 1024, spec-decoding: mtp } - # tp4 low-conc exploration (mirrors non-MTP #2327): fewer TP ranks -> less TP comm, - # may win at low conc. dp-off c1-64 + dp-on c16-128. No TBO (conc<256). - - { tp: 4, ep: 1, dp-attn: false, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 1, dp-attn: true, conc-start: 16, conc-end: 128, spec-decoding: mtp } + # tp4 removed: tp4+MTP+dp-on OOMs — 806G/4=201G/GPU weight leaves KV budget + # NEGATIVE (-10.98GB) even at gpu-mem-util 0.90 (run 30277221626). #2327 tp4 + # works only because it's non-MTP (no draft KV layer). MTP stays tp8-only. qwen3.5-bf16-mi325x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 12f668687f..9af057e8b4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5089,7 +5089,7 @@ - config-keys: - dsv4-fp4-mi355x-atom-mtp description: - - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on) + tp4 low-conc exploration (dp-off c1-64, dp-on c16-128, mirroring non-MTP #2327). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=256 (measured crossover run 30257759947 vs non-TBO 30238071409: TBO -10~14% output tput at c64/c128 but +8~14% at c256+; MTP removes low-conc latency so TBO overlap there is pure overhead); MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." + - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on; tp4 tried but removed — tp4+MTP+dp-on OOMs, KV budget negative). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=256 (measured crossover run 30257759947 vs non-TBO 30238071409: TBO -10~14% output tput at c64/c128 but +8~14% at c256+; MTP removes low-conc latency so TBO overlap there is pure overhead); MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345 - config-keys: From bf096b442625ac7536aa443b908df37bfab791d6 Mon Sep 17 00:00:00 2001 From: Cameron Quilici Date: Mon, 27 Jul 2026 22:30:49 -0500 Subject: [PATCH 8/8] Update perf-changelog.yaml --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 086c94372b..d969b6ce47 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5128,4 +5128,4 @@ description: - "Re-sweep dsv4-fp4-mi355x-atom-mtp on latest atom-dev nightly (nightly_202607231538, was atom0.1.3 release). Align ATOM official MTP config (models.json): MTP3 dp-off conc 4-256 + DPA MTP3 (dp-attn) conc 32-1024 (added c32 dp-on; tp4 tried but removed — tp4+MTP+dp-on OOMs, KV budget negative). prefill-only TBO (--enable-tbo, argparse const=prefill so enable_tbo_decode=False) on dp-attn cells at conc>=256 (measured crossover run 30257759947 vs non-TBO 30238071409: TBO -10~14% output tput at c64/c128 but +8~14% at c256+; MTP removes low-conc latency so TBO overlap there is pure overhead); MTP-safe because only decode-TBO (--enable-tbo all) drops spec_decode_metadata. max_num_seqs=conc on dp-on cells and conc>=64 to avoid OOM. bench keeps --dsv4 (InferenceX bench uses encoding_dsv4.py; DSv4-Pro has no jinja chat_template). Fixed benchmark_lib source path (../ -> ../../)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2345 - \ No newline at end of file +