From becdac09ab0bc74da22f84b29066519019b17e8b Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Wed, 5 Aug 2026 08:03:12 +0000 Subject: [PATCH 01/10] [AMD] job.slurm: replace duplicate GPU sanity check with wait_for_amd_gpu_clean Removes helpers/gpu_sanity.sh, whose VRAM-drain check duplicated the wait_for_amd_gpu_clean() gate added to benchmark_lib.sh in #2490. Wires that shared function into job.slurm's pre-flight gate instead, guarding against benchmark_lib.sh's unrelated agentic KV_OFFLOADING validation by unsetting IS_AGENTIC/SCENARIO_TYPE before sourcing it standalone. Co-authored-by: Cursor --- .../amd_utils/helpers/gpu_sanity.sh | 51 ------------------- benchmarks/multi_node/amd_utils/job.slurm | 16 +++--- 2 files changed, 8 insertions(+), 59 deletions(-) delete mode 100755 benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh diff --git a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh deleted file mode 100755 index 4ebf16aea8..0000000000 --- a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# GPU sanity gate, run on each node AFTER all containers are stopped. -# -# At that point VRAM should fall back to driver baseline. Any large remaining -# usage means a bare (non-containerized) process is hogging the GPU -- which -# would otherwise surface as an opaque "memory capacity is unbalanced" OOM in -# the middle of model load ~30 min into the job. We fail fast and name the -# offender instead. We only report; we never kill another user's process. -# -# Ported from https://github.com/ROCm/InferenceY/blob/main/benchmarks/multi_node/amd_utils/gpu_sanity.sh -set -uo pipefail - -# Fail the node if any single GPU still has >= this many GB used. -GPU_BUSY_GB="${GPU_BUSY_GB:-8}" -# Poll a few times to give docker stop time to release VRAM before judging. -GPU_DRAIN_TRIES="${GPU_DRAIN_TRIES:-48}" - -if ! command -v rocm-smi >/dev/null 2>&1; then - echo "[gpu-sanity] rocm-smi not found on $(hostname); skipping GPU check" >&2 - exit 0 -fi - -busy_gb=0 -busy_gpus="" -for _ in $(seq "$GPU_DRAIN_TRIES"); do - if ! mapfile -t _used_bytes < <(rocm-smi --showmeminfo vram --json 2>/dev/null \ - | grep -oE '"VRAM Total Used Memory \(B\)": "[0-9]+"' \ - | grep -oE '[0-9]+'); then - _used_bytes=() - fi - busy_gb=0 - busy_gpus="" - idx=0 - for used in "${_used_bytes[@]}"; do - gb=$(( used / 1024 / 1024 / 1024 )) - if (( gb >= GPU_BUSY_GB )); then - busy_gpus+=" gpu${idx}=${gb}GB" - (( busy_gb < gb )) && busy_gb=$gb - fi - idx=$((idx + 1)) - done - if [ "$busy_gb" -lt "$GPU_BUSY_GB" ]; then - exit 0 - fi - sleep 30 -done - -echo "FATAL: $(hostname) GPU still has ${busy_gb}GB used after stopping all containers" >&2 -echo " (threshold ${GPU_BUSY_GB}GB; busy:${busy_gpus}) -- a bare (non-containerized) process is hogging the GPU:" >&2 -rocm-smi --showpids >&2 || true -exit 1 diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 3c8a6f16a8..edf83015a9 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -676,21 +676,21 @@ else fi fi # end: if ENGINE == atom-disagg -# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released before -# the sanity gate. stop-only left containers in Created/Exited state on some nodes. +# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released +# before the drain gate. stop-only left containers in Created/Exited state +# on some nodes. \$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop -t 15 || true \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD rm -f || true sleep 2 -# GPU sanity gate: containers are stopped, so any remaining VRAM use is a bare -# (non-containerized) process hogging the GPU -- fail fast (and name it) -# instead of OOMing in model load ~30 min later. set -e + --kill-on-bad-exit -# tears down the whole job on non-zero exit. +# GPU drain gate: fail fast on leftover VRAM use instead of OOMing in model +# load ~15 min later. Reuses wait_for_amd_gpu_clean from benchmark_lib.sh. if [[ \"${SKIP_GPU_SANITY:-0}\" == \"1\" ]]; then - echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU sanity pre-flight check\" + echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU pre-flight drain check\" else - bash \"$DI_REPO_DIR/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh\" + # Unset so benchmark_lib.sh's unrelated agentic KV_OFFLOADING check doesn't exit 1 here. + bash -c \"unset IS_AGENTIC SCENARIO_TYPE; source $DI_REPO_DIR/benchmarks/benchmark_lib.sh && wait_for_amd_gpu_clean\" fi # Start vLLM external router container on node 0 From 2fd1ce5070b02b6a6f5b20125b3ee1c4547e0472 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Wed, 5 Aug 2026 09:06:15 +0000 Subject: [PATCH 02/10] [AMD] dsv4-agentx: tune HiCache ratio/memory settings, bump MTP hicache image - models.yaml (DeepSeek-V4-Pro-AgentX): move swa-full-tokens-ratio from a global base_flag to per-role prefill/decode fields, add --enable-two-batch-overlap to dp_flags, raise prefill mem_fraction_static to 0.85 and no_dp max_running_requests to 128, enable decode CUDA graphs. - dsv4_fp4_mi355x_sglang-disagg.sh: lower default HICACHE_RATIO to 3 and trim the FORCE_HICACHE_RATIO rationale comment. - amd-master.yaml: bump the dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp image, extend the MTP sweep conc-list to include 64, and comment out the superseded TP8/EP1 single-worker search-space arm. Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 17 +++---- benchmarks/multi_node/amd_utils/models.yaml | 11 +++-- configs/amd-master.yaml | 44 +++++++++---------- 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh index f43c81e1c6..95653207b6 100755 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -84,18 +84,11 @@ if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; t # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" # HiCache ratio (host pool = ratio * GPU KV pool). - export HICACHE_RATIO="${HICACHE_RATIO:-4}" - # server_sglang.sh prefers an absolute --hicache-size (derived from - # TOTAL_CPU_DRAM_GB, the sweep generator's per-node DRAM budget) over - # --hicache-ratio whenever TOTAL_CPU_DRAM_GB is set. DSv4 wants the - # ratio-based pool instead. Use FORCE_HICACHE_RATIO to opt out of the - # --hicache-size path rather than unsetting TOTAL_CPU_DRAM_GB itself: - # that var is also the shared client-side gate (benchmark_lib.sh requires - # it to be a positive integer whenever KV_OFFLOADING=dram) and gets - # forwarded into the aiperf sibling container's client.env, so unsetting - # it here made the client fail its own env validation before benchmarking - # ("DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB - # capacity") even though the servers came up fine. + export HICACHE_RATIO="${HICACHE_RATIO:-3}" + # DSv4 wants the ratio-based pool, but server_sglang.sh prefers + # --hicache-size over --hicache-ratio when TOTAL_CPU_DRAM_GB is set. + # Opt out via FORCE_HICACHE_RATIO instead of unsetting TOTAL_CPU_DRAM_GB + # (also required client-side by benchmark_lib.sh when KV_OFFLOADING=dram). export FORCE_HICACHE_RATIO=1 # ── HiCache layout/backend by tier ── diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 2231e6fe60..926176de59 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -364,12 +364,13 @@ DeepSeek-R1-0528-MXFP4-v2: cuda_graph_bs_range: "1-128" DeepSeek-V4-Pro-AgentX: - base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" - dp_flags: "--enable-dp-attention --enable-prefill-delayer" + base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" + dp_flags: "--enable-dp-attention --enable-prefill-delayer --enable-two-batch-overlap" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" prefill: - mem_fraction_static: 0.72 + mem_fraction_static: 0.85 + swa_full_tokens_ratio: 0.1 disable_radix_cache: false disable_cuda_graph: true dp: @@ -378,7 +379,7 @@ DeepSeek-V4-Pro-AgentX: context_length: 1048576 # max_total_tokens: 1048576 no_dp: - max_running_requests: 64 + max_running_requests: 128 # Small prefill chunks interleave long-context agentic prefills across # requests instead of letting one ~100K-token prefill monopolize the # engine (the conc>=16 queue-saturation / decode-stall failure mode). @@ -389,6 +390,8 @@ DeepSeek-V4-Pro-AgentX: context_length: 1048576 decode: mem_fraction_static: 0.85 + swa_full_tokens_ratio: 0.1 + disable_cuda_graph: false prefill_round_robin_balance: true disagg_decode_enable_radix_cache: false dp: diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 3329f74b89..3544519d50 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1515,7 +1515,7 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 + image: aigmkt/pytorch-private:sglang-0.5.16-rocm720-mi35x-20260803-pr32746-moria0afee14-20260804 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1528,28 +1528,28 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.80 search-space: + # - spec-decoding: "mtp" + # conc-list: [ 1,2,4,8,16,32 ] + # kv-offloading: dram + # kv-offload-backend: { name: hicache } + # prefill: + # num-worker: 1 + # tp: 8 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "PREFILL_NODES=1" + # - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + # decode: + # num-worker: 1 + # tp: 8 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "DECODE_NODES=1" + # - "DECODE_MTP_SIZE=3" - spec-decoding: "mtp" - conc-list: [ 1,2,4,8,16,32 ] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - spec-decoding: "mtp" - conc-list: [ 96, 128 ] + conc-list: [ 64, 96, 128 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: From 3c2ebae7b8d5893a7dc385ad9f22582a7ba3eb3e Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Wed, 5 Aug 2026 09:38:13 +0000 Subject: [PATCH 03/10] [AMD] submit.sh: exclude nodes per (framework, model) instead of a fixed global list Known-bad nodes only affect Kimi-K2.5-MXFP4 on vllm-disagg; other framework/model combos no longer get any nodes excluded by default. Mapping lives in node_excludes.yaml; SLURM_EXCLUDE_NODES still overrides it. Co-authored-by: Cursor --- .../multi_node/amd_utils/node_excludes.yaml | 14 +++++++++ benchmarks/multi_node/amd_utils/submit.sh | 29 ++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 benchmarks/multi_node/amd_utils/node_excludes.yaml diff --git a/benchmarks/multi_node/amd_utils/node_excludes.yaml b/benchmarks/multi_node/amd_utils/node_excludes.yaml new file mode 100644 index 0000000000..6ba840328e --- /dev/null +++ b/benchmarks/multi_node/amd_utils/node_excludes.yaml @@ -0,0 +1,14 @@ +# Per (framework, model) SLURM node exclusions, consumed by submit.sh. +# +# A rule applies when FRAMEWORK matches "framework" exactly and MODEL_NAME is +# in "models"; its exclude_nodes are then passed to sbatch --exclude. Any +# (framework, model) combination with no matching rule gets an empty exclude +# list (i.e. no nodes excluded). +# +# SLURM_EXCLUDE_NODES env var, if set, always overrides this file. + +rules: + - framework: vllm-disagg + models: + - Kimi-K2.5-MXFP4 + exclude_nodes: mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12 diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 6a1598d81d..144465327c 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -183,12 +183,33 @@ if [[ -n "${NODE_LIST//[[:space:]]/}" ]]; then NODELIST_OPT=(--nodelist "$NODELIST_CSV") fi -# Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). -# Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. +# Optional: exclude specific nodes for known-bad (FRAMEWORK, MODEL_NAME) +# combos (e.g. nodes with broken Docker sockets), looked up from +# node_excludes.yaml. Set SLURM_EXCLUDE_NODES to override with an explicit +# comma-separated hostname list (takes precedence over the file). EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12}" +NODE_EXCLUDES_YAML="$(dirname "$0")/node_excludes.yaml" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then - EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") + RESOLVED_EXCLUDE_NODES="$SLURM_EXCLUDE_NODES" +elif [[ -f "$NODE_EXCLUDES_YAML" ]]; then + RESOLVED_EXCLUDE_NODES=$(python3 -c " +import yaml + +with open('${NODE_EXCLUDES_YAML}') as f: + cfg = yaml.safe_load(f) or {} + +framework = '${FRAMEWORK}' +model = '${MODEL_NAME}' +for rule in cfg.get('rules', []): + if rule.get('framework') == framework and model in (rule.get('models') or []): + print(rule.get('exclude_nodes', '')) + break +") +else + RESOLVED_EXCLUDE_NODES="" +fi +if [[ -n "$RESOLVED_EXCLUDE_NODES" ]]; then + EXCLUDE_OPT=(--exclude "$RESOLVED_EXCLUDE_NODES") fi # ============================================================================= From fc97ce908ab293bbd11850a4ccc46244afef2d1c Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Fri, 14 Aug 2026 02:04:24 -0500 Subject: [PATCH 04/10] bump image to rocm/sgl-dev:v0.5.17-rocm720-mi35x-20260813 Signed-off-by: Theresa Shan bump image to aigmkt/mori-dev:sglang-0.5.17-rocm720-mi35x-mori-0814 Signed-off-by: Theresa Shan add one more sweep dot Signed-off-by: Theresa Shan Update amd-master.yaml --- benchmarks/multi_node/amd_utils/models.yaml | 6 +-- configs/amd-master.yaml | 44 ++++++++++----------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 926176de59..68a0993ce6 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -364,13 +364,12 @@ DeepSeek-R1-0528-MXFP4-v2: cuda_graph_bs_range: "1-128" DeepSeek-V4-Pro-AgentX: - base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" - dp_flags: "--enable-dp-attention --enable-prefill-delayer --enable-two-batch-overlap" + base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" + dp_flags: "--enable-dp-attention --enable-prefill-delayer" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" prefill: mem_fraction_static: 0.85 - swa_full_tokens_ratio: 0.1 disable_radix_cache: false disable_cuda_graph: true dp: @@ -390,7 +389,6 @@ DeepSeek-V4-Pro-AgentX: context_length: 1048576 decode: mem_fraction_static: 0.85 - swa_full_tokens_ratio: 0.1 disable_cuda_graph: false prefill_round_robin_balance: true disagg_decode_enable_radix_cache: false diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 3544519d50..db1e937972 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1515,7 +1515,7 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - image: aigmkt/pytorch-private:sglang-0.5.16-rocm720-mi35x-20260803-pr32746-moria0afee14-20260804 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1528,28 +1528,28 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - # - spec-decoding: "mtp" - # conc-list: [ 1,2,4,8,16,32 ] - # kv-offloading: dram - # kv-offload-backend: { name: hicache } - # prefill: - # num-worker: 1 - # tp: 8 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "PREFILL_NODES=1" - # - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" - # decode: - # num-worker: 1 - # tp: 8 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "DECODE_NODES=1" - # - "DECODE_MTP_SIZE=3" - spec-decoding: "mtp" - conc-list: [ 64, 96, 128 ] + conc-list: [ 1,2,4,8,16,32,48,64 ] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=3" + - spec-decoding: "mtp" + conc-list: [ 64, 96, 128, 192 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: From 5b2d9195d18e202372a4548b85f9cfd4c5e59701 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 18 Aug 2026 04:18:17 +0000 Subject: [PATCH 05/10] [AMD] models.yaml: revert DeepSeek-V4-Pro-AgentX no_dp max_running_requests to 64 Restores the pre-2fd1ce507 value; 128 was too aggressive for the no_dp prefill role. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/models.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 68a0993ce6..8901b1514e 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -378,7 +378,7 @@ DeepSeek-V4-Pro-AgentX: context_length: 1048576 # max_total_tokens: 1048576 no_dp: - max_running_requests: 128 + max_running_requests: 64 # Small prefill chunks interleave long-context agentic prefills across # requests instead of letting one ~100K-token prefill monopolize the # engine (the conc>=16 queue-saturation / decode-stall failure mode). From bab55d661d4ab0f06319472c2bc8b78745cdbcc0 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 18 Aug 2026 04:32:30 +0000 Subject: [PATCH 06/10] [AMD] submit.sh: fail loudly if node-exclude YAML resolution can't be trusted Checks python3/PyYAML availability upfront and the exit status of the python3 invocation, falling back to an awk parser (mirroring job.slurm's style) when python3/PyYAML are unavailable. Previously a missing python3 or yaml module would silently yield an empty exclude list, reintroducing the known-bad-node scheduling issue with zero diagnostic. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/submit.sh | 49 ++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 144465327c..e7ecc8d913 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -187,12 +187,18 @@ fi # combos (e.g. nodes with broken Docker sockets), looked up from # node_excludes.yaml. Set SLURM_EXCLUDE_NODES to override with an explicit # comma-separated hostname list (takes precedence over the file). +# +# Resolution must fail loudly (not silently yield an empty exclude list) if +# it can't be trusted: a submit host missing python3/PyYAML, or a genuine +# parse error, must not silently reintroduce the known-bad-node issue this +# exclusion mechanism exists to prevent. EXCLUDE_OPT=() NODE_EXCLUDES_YAML="$(dirname "$0")/node_excludes.yaml" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then RESOLVED_EXCLUDE_NODES="$SLURM_EXCLUDE_NODES" elif [[ -f "$NODE_EXCLUDES_YAML" ]]; then - RESOLVED_EXCLUDE_NODES=$(python3 -c " + if command -v python3 >/dev/null 2>&1 && python3 -c "import yaml" >/dev/null 2>&1; then + RESOLVED_EXCLUDE_NODES=$(python3 -c " import yaml with open('${NODE_EXCLUDES_YAML}') as f: @@ -205,6 +211,47 @@ for rule in cfg.get('rules', []): print(rule.get('exclude_nodes', '')) break ") + PYTHON_EXCLUDE_RC=$? + if [[ $PYTHON_EXCLUDE_RC -ne 0 ]]; then + echo "Error: python3 failed (exit ${PYTHON_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 + echo "Error: fix the YAML, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 + exit 1 + fi + else + # Fall back to an awk parser (mirrors job.slurm's awk-based models.yaml + # parsing) matched to node_excludes.yaml's fixed rule/models/exclude_nodes + # shape. Only exercised when python3 or its yaml module is unavailable. + echo "Warning: python3/PyYAML unavailable on submit host; falling back to awk parsing of ${NODE_EXCLUDES_YAML}" >&2 + RESOLVED_EXCLUDE_NODES=$(awk -v fw="$FRAMEWORK" -v model="$MODEL_NAME" ' + /^ - framework:/ { + line = $0 + sub(/^ - framework: */, "", line) + fw_match = (line == fw) + model_match = 0 + next + } + fw_match && /^ - / { + m = $0 + sub(/^ - */, "", m) + gsub(/^"|"$/, "", m) + if (m == model) model_match = 1 + next + } + fw_match && model_match && /^ exclude_nodes:/ { + val = $0 + sub(/^ *exclude_nodes: */, "", val) + gsub(/^"|"$/, "", val) + print val + exit + } + ' "$NODE_EXCLUDES_YAML") + AWK_EXCLUDE_RC=$? + if [[ $AWK_EXCLUDE_RC -ne 0 ]]; then + echo "Error: awk fallback failed (exit ${AWK_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 + echo "Error: fix the YAML/parser, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 + exit 1 + fi + fi else RESOLVED_EXCLUDE_NODES="" fi From 9d4c2d4ee0b710999a3347ca29c1c1724a7617f8 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 18 Aug 2026 04:33:48 +0000 Subject: [PATCH 07/10] [AMD] models.yaml: drop redundant disable_cuda_graph: false on DeepSeek-V4-Pro-AgentX decode Decode CUDA graphs are enabled by default; the explicit false was a no-op alongside the existing dp/no_dp cuda_graph_bs_range settings. Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/models.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 8901b1514e..88a61ccc10 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -389,7 +389,6 @@ DeepSeek-V4-Pro-AgentX: context_length: 1048576 decode: mem_fraction_static: 0.85 - disable_cuda_graph: false prefill_round_robin_balance: true disagg_decode_enable_radix_cache: false dp: From 7297cdfcb1c97204a5a99f78cb15dfe07ace69d6 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Wed, 19 Aug 2026 15:15:58 +0000 Subject: [PATCH 08/10] Update per-changelog --- perf-changelog.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 52c74bf168..e8500b6450 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6195,3 +6195,15 @@ - "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs." - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Bump the DSv4 FP4 MI355X SGLang disagg AgentX HiCache MTP image from lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 to lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817." + - "Extend the MTP concurrency sweep: the TP8/EP1 no-DP arm to 1, 2, 4, 8, 16, 32, 48, and 64, and the TP8/EP8 DP-attention arm to 64, 96, 128, and 192, so concurrency 64 is measured on both topologies at the crossover." + - "models.yaml (DeepSeek-V4-Pro-AgentX): raise prefill mem_fraction_static from 0.72 to 0.85." + - "dsv4_fp4_mi355x_sglang-disagg.sh: lower the default HICACHE_RATIO from 4 to 3 (host pool = ratio * GPU KV pool), still pinned to the --hicache-ratio path through FORCE_HICACHE_RATIO." + - "AMD multi-node infrastructure, expected to be performance-neutral: job.slurm's pre-flight GPU drain gate now reuses wait_for_amd_gpu_clean from benchmark_lib.sh instead of the removed helpers/gpu_sanity.sh, and submit.sh resolves SLURM node exclusions per (framework, model) from node_excludes.yaml (Kimi-K2.5-MXFP4 on vllm-disagg only) with SLURM_EXCLUDE_NODES still overriding." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2643 From dee12b02a56b495e5a7adc0cbee76fdc290c650b Mon Sep 17 00:00:00 2001 From: Cameron Quilici Date: Thu, 20 Aug 2026 22:53:41 -0500 Subject: [PATCH 09/10] Update perf-changelog.yaml --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3430760e95..7a7e524d5a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6289,4 +6289,4 @@ - "dsv4_fp4_mi355x_sglang-disagg.sh: lower the default HICACHE_RATIO from 4 to 3 (host pool = ratio * GPU KV pool), still pinned to the --hicache-ratio path through FORCE_HICACHE_RATIO." - "AMD multi-node infrastructure, expected to be performance-neutral: job.slurm's pre-flight GPU drain gate now reuses wait_for_amd_gpu_clean from benchmark_lib.sh instead of the removed helpers/gpu_sanity.sh, and submit.sh resolves SLURM node exclusions per (framework, model) from node_excludes.yaml (Kimi-K2.5-MXFP4 on vllm-disagg only) with SLURM_EXCLUDE_NODES still overriding." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2643 - \ No newline at end of file + From 8c4c6b57eac223fb9ce53b5f3250e411ad60d292 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 20 Aug 2026 23:05:25 -0500 Subject: [PATCH 10/10] chore: tighten PR 2643 performance changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep only the image bump, added concurrency points, and performance tuning deltas. 中文：精简 PR 2643 的性能变更日志，仅保留镜像更新、新增并发点和性能参数调整。 --- perf-changelog.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7a7e524d5a..d85a0f87d2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6283,10 +6283,8 @@ scenario-type: - agentic-coding description: - - "Bump the DSv4 FP4 MI355X SGLang disagg AgentX HiCache MTP image from lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 to lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817." - - "Extend the MTP concurrency sweep: the TP8/EP1 no-DP arm to 1, 2, 4, 8, 16, 32, 48, and 64, and the TP8/EP8 DP-attention arm to 64, 96, 128, and 192, so concurrency 64 is measured on both topologies at the crossover." - - "models.yaml (DeepSeek-V4-Pro-AgentX): raise prefill mem_fraction_static from 0.72 to 0.85." - - "dsv4_fp4_mi355x_sglang-disagg.sh: lower the default HICACHE_RATIO from 4 to 3 (host pool = ratio * GPU KV pool), still pinned to the --hicache-ratio path through FORCE_HICACHE_RATIO." - - "AMD multi-node infrastructure, expected to be performance-neutral: job.slurm's pre-flight GPU drain gate now reuses wait_for_amd_gpu_clean from benchmark_lib.sh instead of the removed helpers/gpu_sanity.sh, and submit.sh resolves SLURM node exclusions per (framework, model) from node_excludes.yaml (Kimi-K2.5-MXFP4 on vllm-disagg only) with SLURM_EXCLUDE_NODES still overriding." + - "Bump the SGLang image from lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 to lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817." + - "Add concurrency 48 and 64 to the TP8/EP1 sweep, and 64 and 192 to the TP8/EP8 DP-attention sweep." + - "Increase prefill mem_fraction_static from 0.72 to 0.85 and reduce HICACHE_RATIO from 4 to 3." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2643