diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh index 66be04a52..2548978b1 100644 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh @@ -27,11 +27,8 @@ if [ "$DP_ATTENTION" = "true" ]; then if [ "$EP_SIZE" -gt 1 ]; then #DP+EP PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) else #DPA+TP - #DPA+TP+TBO - if [ "$ISL" -eq 1024 ] && [ "$OSL" -eq 1024 ] && [ "$CONC" -ge 1024 ]; then - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention --enable-tbo) - export GPU_MAX_HW_QUEUES=5 - elif [ "$ISL" -eq 8192 ] && [ "$OSL" -eq 1024 ] && [ "$CONC" -ge 256 ]; then + #DPA+TP+TBO (opt: TBO on for dp-attn cells at conc>=64, no per-scenario gate) + if [ "$CONC" -ge 64 ]; then PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention --enable-tbo) export GPU_MAX_HW_QUEUES=5 else @@ -40,6 +37,14 @@ if [ "$DP_ATTENTION" = "true" ]; then fi fi +# max_req=conc for every dp-on cell (mandatory: dp-attention keeps a full KV pool +# per rank, so the large default max_num_seqs OOMs even at low conc like c16/c32) +# and for mid/high conc (conc>=64). dp-off low conc uses the ATOM default +# (dev: default is on-par or ~4% better at very low conc, e.g. c2). +if [ "$DP_ATTENTION" = "true" ] || [ "$CONC" -ge 64 ]; then + PARALLEL_ARGS+=(--max-num-seqs "$CONC") +fi + BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" if [ "${EVAL_ONLY}" = "true" ]; then diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 68f0e25fa..d3ca241c4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1405,7 +1405,7 @@ dsv4-fp4-mi355x-vllm-mtp: - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp } dsv4-fp4-mi355x-atom: - image: rocm/atom-dev:nightly_202606161823 + image: rocm/atom-dev:nightly_202607231538 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x @@ -1417,12 +1417,14 @@ dsv4-fp4-mi355x-atom: - isl: 8192 osl: 1024 search-space: - # conc4-64, TP8 - # conc128, DPA - # conc256-2048, DPA TBO - - { tp: 4, ep: 1, conc-list: [8, 16, 32, 64] } - - { tp: 8, ep: 1, conc-list: [1, 2, 4, 8, 16, 32, 64] } - - { tp: 8, ep: 1, dp-attn: true, conc-start: 128, conc-end: 2048 } + # conc<64: tp4 dp-off (best at low conc); conc>=64: dp-on + TBO. + # tp8 dp-off low-conc kept as redundancy to recheck the crossover on the new image. + # dp-on cells always set max_req=conc (recipe .sh) to avoid OOM from the large + # default max_num_seqs (dp-attn keeps a full KV pool per rank); TBO stays conc>=64. + - { tp: 4, ep: 1, dp-attn: false, conc-start: 1, conc-end: 64 } + - { tp: 4, ep: 1, dp-attn: true, conc-start: 16, conc-end: 128 } + - { tp: 8, ep: 1, dp-attn: false, conc-start: 1, conc-end: 64 } + - { tp: 8, ep: 1, dp-attn: true, conc-start: 64, conc-end: 2048 } dsv4-fp4-mi355x-atom-mtp: image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0524d0c0c..1942c0026 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5086,6 +5086,12 @@ - "Bump SGLang container image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.15.post1-cu130 (https://github.com/sgl-project/sglang/releases/tag/v0.5.15.post1)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2313 +- config-keys: + - dsv4-fp4-mi355x-atom + description: + - "Re-sweep dsv4-fp4-mi355x-atom on latest atom-dev nightly (nightly_202607231538, was 202606161823). Recipe: max_num_seqs=conc for all dp-on cells (avoids dp-attn OOM) and conc>=64; TBO on dp-attn cells at conc>=64 (was per-scenario gate 256/1024). 8k1k grid adds tp8 dp-off low-conc redundancy to recheck the crossover on the new image." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2327 + - config-keys: - minimaxm3-fp8-b200-vllm - minimaxm3-fp8-b300-vllm