From 2e2b5ab47ac80da79b4135babc8f2a3e55dd7611 Mon Sep 17 00:00:00 2001 From: zufayu Date: Fri, 24 Jul 2026 05:35:38 +0000 Subject: [PATCH 1/6] [AMD] dsv4-fp4-mi355x-atom: latest atom-dev image + opt recipe + 1k1k - image: nightly_202606161823 -> nightly_202607231538 (latest atom-dev) - recipe (dsv4_fp4_mi355x_atom.sh): max_num_seqs=conc + TBO on dp-attn cells at conc>=64 (was ATOM-default max_num_seqs + TBO per-scenario gate 256/1024) - add 1k1k scenario (was 8k1k only; mirrors prior grid) - 8k1k grid: align AMD dev sweep (run 20260723) + tp8 dp-off low-conc redundancy Co-Authored-By: Claude Opus 4.8 (1M context) --- .../fixed_seq_len/dsv4_fp4_mi355x_atom.sh | 8 +++----- configs/amd-master.yaml | 19 ++++++++++++------- perf-changelog.yaml | 6 ++++++ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh index 66be04a524..3b3dbd781d 100644 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh @@ -27,11 +27,8 @@ if [ "$DP_ATTENTION" = "true" ]; then if [ "$EP_SIZE" -gt 1 ]; then #DP+EP PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) else #DPA+TP - #DPA+TP+TBO - if [ "$ISL" -eq 1024 ] && [ "$OSL" -eq 1024 ] && [ "$CONC" -ge 1024 ]; then - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention --enable-tbo) - export GPU_MAX_HW_QUEUES=5 - elif [ "$ISL" -eq 8192 ] && [ "$OSL" -eq 1024 ] && [ "$CONC" -ge 256 ]; then + #DPA+TP+TBO (opt: TBO on for dp-attn cells at conc>=64, no per-scenario gate) + if [ "$CONC" -ge 64 ]; then PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention --enable-tbo) export GPU_MAX_HW_QUEUES=5 else @@ -63,6 +60,7 @@ python3 -m atom.entrypoints.openai_server \ --trust-remote-code \ --gpu-memory-utilization $MEM_FRAC_STATIC \ --no-enable_prefix_caching \ + --max-num-seqs "$CONC" \ --cudagraph-capture-sizes "${CUDAGRAPH_SIZES}" \ > "$SERVER_LOG" 2>&1 & diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0f547cba86..ae37dbad1f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1322,7 +1322,7 @@ dsv4-fp4-mi355x-vllm-mtp: - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp } dsv4-fp4-mi355x-atom: - image: rocm/atom-dev:nightly_202606161823 + image: rocm/atom-dev:nightly_202607231538 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x @@ -1331,15 +1331,20 @@ dsv4-fp4-mi355x-atom: multinode: false scenarios: fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 1, conc-end: 64 } + - { tp: 8, ep: 1, dp-attn: true, conc-start: 64, conc-end: 2048 } - isl: 8192 osl: 1024 search-space: - # conc4-64, TP8 - # conc128, DPA - # conc256-2048, DPA TBO - - { tp: 4, ep: 1, conc-list: [8, 16, 32, 64] } - - { tp: 8, ep: 1, conc-list: [1, 2, 4, 8, 16, 32, 64] } - - { tp: 8, ep: 1, dp-attn: true, conc-start: 128, conc-end: 2048 } + # conc<64: tp4 dp-off (best at low conc); conc>=64: dp-on + TBO. + # tp8 dp-off low-conc kept as redundancy to recheck the crossover on the new image. + - { tp: 4, ep: 1, dp-attn: false, conc-start: 1, conc-end: 64 } + - { tp: 4, ep: 1, dp-attn: true, conc-start: 16, conc-end: 128 } + - { tp: 8, ep: 1, dp-attn: false, conc-start: 1, conc-end: 64 } + - { tp: 8, ep: 1, dp-attn: true, conc-start: 64, conc-end: 2048 } dsv4-fp4-mi355x-atom-mtp: image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 335245a6c3..8f645bdba9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1,3 +1,9 @@ +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Re-sweep dsv4-fp4-mi355x-atom on latest atom-dev nightly (nightly_202607231538, was 202606161823). Recipe: max_num_seqs=conc + TBO on dp-attn cells at conc>=64 (was per-scenario gate 256/1024). Add 1k1k scenario; 8k1k grid adds tp8 dp-off low-conc redundancy to recheck the crossover on the new image." + pr-link: PLACEHOLDER + - config-keys: - 70b-fp8-*-vllm description: From bc813452958b3b26bab65543ef1f08d499723050 Mon Sep 17 00:00:00 2001 From: zufayu Date: Fri, 24 Jul 2026 05:36:08 +0000 Subject: [PATCH 2/6] chore: fill pr-link for dsv4-fp4-mi355x-atom changelog (#2325) --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8f645bdba9..103515a094 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -2,7 +2,7 @@ - dsv4-fp4-mi355x-atom description: - "Re-sweep dsv4-fp4-mi355x-atom on latest atom-dev nightly (nightly_202607231538, was 202606161823). Recipe: max_num_seqs=conc + TBO on dp-attn cells at conc>=64 (was per-scenario gate 256/1024). Add 1k1k scenario; 8k1k grid adds tp8 dp-off low-conc redundancy to recheck the crossover on the new image." - pr-link: PLACEHOLDER + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2325 - config-keys: - 70b-fp8-*-vllm From c8af0668ce9fa6cc5e16db87e87cc377fe2affe4 Mon Sep 17 00:00:00 2001 From: zufayu Date: Fri, 24 Jul 2026 05:49:51 +0000 Subject: [PATCH 3/6] fix(recipe): gate max_num_seqs=conc at conc>=64 (low conc uses default) max_req=conc is a mid/high-concurrency optimization. At very low conc (e.g. c2) the ATOM default is on-par or ~4% better, so only set --max-num-seqs=conc for conc>=64 (matches the TBO gate). High conc still gets it (removes the cap). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh index 3b3dbd781d..53585147cf 100644 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh @@ -37,6 +37,12 @@ if [ "$DP_ATTENTION" = "true" ]; then fi fi +# max_req=conc only for mid/high concurrency (conc>=64). Low conc uses the ATOM +# default: dev shows default is on-par or ~4% better at very low conc (e.g. c2). +if [ "$CONC" -ge 64 ]; then + PARALLEL_ARGS+=(--max-num-seqs "$CONC") +fi + BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" if [ "${EVAL_ONLY}" = "true" ]; then @@ -60,7 +66,6 @@ python3 -m atom.entrypoints.openai_server \ --trust-remote-code \ --gpu-memory-utilization $MEM_FRAC_STATIC \ --no-enable_prefix_caching \ - --max-num-seqs "$CONC" \ --cudagraph-capture-sizes "${CUDAGRAPH_SIZES}" \ > "$SERVER_LOG" 2>&1 & From a0f62e804d34331d3d7cc8006f164abaa5be014a Mon Sep 17 00:00:00 2001 From: zufayu Date: Fri, 24 Jul 2026 06:04:25 +0000 Subject: [PATCH 4/6] =?UTF-8?q?drop=201k1k=20scenario=20=E2=80=94=208k1k?= =?UTF-8?q?=20only=20for=20this=20sweep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- configs/amd-master.yaml | 5 ----- perf-changelog.yaml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ae37dbad1f..c986e6902c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1331,11 +1331,6 @@ dsv4-fp4-mi355x-atom: multinode: false scenarios: fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 1, dp-attn: true, conc-start: 64, conc-end: 2048 } - isl: 8192 osl: 1024 search-space: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 103515a094..4005a11565 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1,7 +1,7 @@ - config-keys: - dsv4-fp4-mi355x-atom description: - - "Re-sweep dsv4-fp4-mi355x-atom on latest atom-dev nightly (nightly_202607231538, was 202606161823). Recipe: max_num_seqs=conc + TBO on dp-attn cells at conc>=64 (was per-scenario gate 256/1024). Add 1k1k scenario; 8k1k grid adds tp8 dp-off low-conc redundancy to recheck the crossover on the new image." + - "Re-sweep dsv4-fp4-mi355x-atom on latest atom-dev nightly (nightly_202607231538, was 202606161823). Recipe: max_num_seqs=conc + TBO on dp-attn cells at conc>=64 (was per-scenario gate 256/1024). 8k1k grid adds tp8 dp-off low-conc redundancy to recheck the crossover on the new image." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2325 - config-keys: From 40a31d83fa5c9fc5723c0083363dfaba1f83e80d Mon Sep 17 00:00:00 2001 From: zufayu Date: Fri, 24 Jul 2026 07:54:36 +0000 Subject: [PATCH 5/6] dsv4-atom: gate max_req=conc on dp-on to avoid OOM dp-attention keeps a full KV pool per rank, so the large default max_num_seqs OOMs even at low conc (c16/c32). Set max_req=conc for every dp-on cell; dp-off low conc still uses the ATOM default. TBO gate unchanged (conc>=64). --- .../single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh | 8 +++++--- configs/amd-master.yaml | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh index 53585147cf..2548978b15 100644 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh @@ -37,9 +37,11 @@ if [ "$DP_ATTENTION" = "true" ]; then fi fi -# max_req=conc only for mid/high concurrency (conc>=64). Low conc uses the ATOM -# default: dev shows default is on-par or ~4% better at very low conc (e.g. c2). -if [ "$CONC" -ge 64 ]; then +# max_req=conc for every dp-on cell (mandatory: dp-attention keeps a full KV pool +# per rank, so the large default max_num_seqs OOMs even at low conc like c16/c32) +# and for mid/high conc (conc>=64). dp-off low conc uses the ATOM default +# (dev: default is on-par or ~4% better at very low conc, e.g. c2). +if [ "$DP_ATTENTION" = "true" ] || [ "$CONC" -ge 64 ]; then PARALLEL_ARGS+=(--max-num-seqs "$CONC") fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c986e6902c..423feb6248 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1336,6 +1336,8 @@ dsv4-fp4-mi355x-atom: search-space: # conc<64: tp4 dp-off (best at low conc); conc>=64: dp-on + TBO. # tp8 dp-off low-conc kept as redundancy to recheck the crossover on the new image. + # dp-on cells always set max_req=conc (recipe .sh) to avoid OOM from the large + # default max_num_seqs (dp-attn keeps a full KV pool per rank); TBO stays conc>=64. - { tp: 4, ep: 1, dp-attn: false, conc-start: 1, conc-end: 64 } - { tp: 4, ep: 1, dp-attn: true, conc-start: 16, conc-end: 128 } - { tp: 8, ep: 1, dp-attn: false, conc-start: 1, conc-end: 64 } From b656b7b99434cc8a348f86e29affb30595e61816 Mon Sep 17 00:00:00 2001 From: zufayu Date: Fri, 24 Jul 2026 09:08:41 +0000 Subject: [PATCH 6/6] perf-changelog: move dsv4-atom entry to end (append-only) + pr-link -> #2327 --- perf-changelog.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4005a11565..6f1ce33685 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1,9 +1,3 @@ -- config-keys: - - dsv4-fp4-mi355x-atom - description: - - "Re-sweep dsv4-fp4-mi355x-atom on latest atom-dev nightly (nightly_202607231538, was 202606161823). Recipe: max_num_seqs=conc + TBO on dp-attn cells at conc>=64 (was per-scenario gate 256/1024). 8k1k grid adds tp8 dp-off low-conc redundancy to recheck the crossover on the new image." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2325 - - config-keys: - 70b-fp8-*-vllm description: @@ -5066,3 +5060,9 @@ - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 (previous nightly-20260629-e667ebb was garbage-collected from Docker Hub)" - "Exclude known-bad nodes mia1-p01-g09,g14 from the disagg node pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2301 + +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Re-sweep dsv4-fp4-mi355x-atom on latest atom-dev nightly (nightly_202607231538, was 202606161823). Recipe: max_num_seqs=conc for all dp-on cells (avoids dp-attn OOM) and conc>=64; TBO on dp-attn cells at conc>=64 (was per-scenario gate 256/1024). 8k1k grid adds tp8 dp-off low-conc redundancy to recheck the crossover on the new image." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2327