diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh index f43c81e1c6..95653207b6 100755 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -84,18 +84,11 @@ if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; t # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" # HiCache ratio (host pool = ratio * GPU KV pool). - export HICACHE_RATIO="${HICACHE_RATIO:-4}" - # server_sglang.sh prefers an absolute --hicache-size (derived from - # TOTAL_CPU_DRAM_GB, the sweep generator's per-node DRAM budget) over - # --hicache-ratio whenever TOTAL_CPU_DRAM_GB is set. DSv4 wants the - # ratio-based pool instead. Use FORCE_HICACHE_RATIO to opt out of the - # --hicache-size path rather than unsetting TOTAL_CPU_DRAM_GB itself: - # that var is also the shared client-side gate (benchmark_lib.sh requires - # it to be a positive integer whenever KV_OFFLOADING=dram) and gets - # forwarded into the aiperf sibling container's client.env, so unsetting - # it here made the client fail its own env validation before benchmarking - # ("DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB - # capacity") even though the servers came up fine. + export HICACHE_RATIO="${HICACHE_RATIO:-3}" + # DSv4 wants the ratio-based pool, but server_sglang.sh prefers + # --hicache-size over --hicache-ratio when TOTAL_CPU_DRAM_GB is set. + # Opt out via FORCE_HICACHE_RATIO instead of unsetting TOTAL_CPU_DRAM_GB + # (also required client-side by benchmark_lib.sh when KV_OFFLOADING=dram). export FORCE_HICACHE_RATIO=1 # ── HiCache layout/backend by tier ── diff --git a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh deleted file mode 100755 index 4ebf16aea8..0000000000 --- a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# GPU sanity gate, run on each node AFTER all containers are stopped. -# -# At that point VRAM should fall back to driver baseline. Any large remaining -# usage means a bare (non-containerized) process is hogging the GPU -- which -# would otherwise surface as an opaque "memory capacity is unbalanced" OOM in -# the middle of model load ~30 min into the job. We fail fast and name the -# offender instead. We only report; we never kill another user's process. -# -# Ported from https://github.com/ROCm/InferenceY/blob/main/benchmarks/multi_node/amd_utils/gpu_sanity.sh -set -uo pipefail - -# Fail the node if any single GPU still has >= this many GB used. -GPU_BUSY_GB="${GPU_BUSY_GB:-8}" -# Poll a few times to give docker stop time to release VRAM before judging. -GPU_DRAIN_TRIES="${GPU_DRAIN_TRIES:-48}" - -if ! command -v rocm-smi >/dev/null 2>&1; then - echo "[gpu-sanity] rocm-smi not found on $(hostname); skipping GPU check" >&2 - exit 0 -fi - -busy_gb=0 -busy_gpus="" -for _ in $(seq "$GPU_DRAIN_TRIES"); do - if ! mapfile -t _used_bytes < <(rocm-smi --showmeminfo vram --json 2>/dev/null \ - | grep -oE '"VRAM Total Used Memory \(B\)": "[0-9]+"' \ - | grep -oE '[0-9]+'); then - _used_bytes=() - fi - busy_gb=0 - busy_gpus="" - idx=0 - for used in "${_used_bytes[@]}"; do - gb=$(( used / 1024 / 1024 / 1024 )) - if (( gb >= GPU_BUSY_GB )); then - busy_gpus+=" gpu${idx}=${gb}GB" - (( busy_gb < gb )) && busy_gb=$gb - fi - idx=$((idx + 1)) - done - if [ "$busy_gb" -lt "$GPU_BUSY_GB" ]; then - exit 0 - fi - sleep 30 -done - -echo "FATAL: $(hostname) GPU still has ${busy_gb}GB used after stopping all containers" >&2 -echo " (threshold ${GPU_BUSY_GB}GB; busy:${busy_gpus}) -- a bare (non-containerized) process is hogging the GPU:" >&2 -rocm-smi --showpids >&2 || true -exit 1 diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 3c8a6f16a8..edf83015a9 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -676,21 +676,21 @@ else fi fi # end: if ENGINE == atom-disagg -# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released before -# the sanity gate. stop-only left containers in Created/Exited state on some nodes. +# Pre-clean (idempotent): stop then force-remove so GPU VRAM is released +# before the drain gate. stop-only left containers in Created/Exited state +# on some nodes. \$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop -t 15 || true \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD rm -f || true sleep 2 -# GPU sanity gate: containers are stopped, so any remaining VRAM use is a bare -# (non-containerized) process hogging the GPU -- fail fast (and name it) -# instead of OOMing in model load ~30 min later. set -e + --kill-on-bad-exit -# tears down the whole job on non-zero exit. +# GPU drain gate: fail fast on leftover VRAM use instead of OOMing in model +# load ~15 min later. Reuses wait_for_amd_gpu_clean from benchmark_lib.sh. if [[ \"${SKIP_GPU_SANITY:-0}\" == \"1\" ]]; then - echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU sanity pre-flight check\" + echo \"[INFO] SKIP_GPU_SANITY=1 set; skipping GPU pre-flight drain check\" else - bash \"$DI_REPO_DIR/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh\" + # Unset so benchmark_lib.sh's unrelated agentic KV_OFFLOADING check doesn't exit 1 here. + bash -c \"unset IS_AGENTIC SCENARIO_TYPE; source $DI_REPO_DIR/benchmarks/benchmark_lib.sh && wait_for_amd_gpu_clean\" fi # Start vLLM external router container on node 0 diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 2231e6fe60..88a61ccc10 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -369,7 +369,7 @@ DeepSeek-V4-Pro-AgentX: ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" prefill: - mem_fraction_static: 0.72 + mem_fraction_static: 0.85 disable_radix_cache: false disable_cuda_graph: true dp: diff --git a/benchmarks/multi_node/amd_utils/node_excludes.yaml b/benchmarks/multi_node/amd_utils/node_excludes.yaml new file mode 100644 index 0000000000..6ba840328e --- /dev/null +++ b/benchmarks/multi_node/amd_utils/node_excludes.yaml @@ -0,0 +1,14 @@ +# Per (framework, model) SLURM node exclusions, consumed by submit.sh. +# +# A rule applies when FRAMEWORK matches "framework" exactly and MODEL_NAME is +# in "models"; its exclude_nodes are then passed to sbatch --exclude. Any +# (framework, model) combination with no matching rule gets an empty exclude +# list (i.e. no nodes excluded). +# +# SLURM_EXCLUDE_NODES env var, if set, always overrides this file. + +rules: + - framework: vllm-disagg + models: + - Kimi-K2.5-MXFP4 + exclude_nodes: mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12 diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 6a1598d81d..e7ecc8d913 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -183,12 +183,80 @@ if [[ -n "${NODE_LIST//[[:space:]]/}" ]]; then NODELIST_OPT=(--nodelist "$NODELIST_CSV") fi -# Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). -# Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. +# Optional: exclude specific nodes for known-bad (FRAMEWORK, MODEL_NAME) +# combos (e.g. nodes with broken Docker sockets), looked up from +# node_excludes.yaml. Set SLURM_EXCLUDE_NODES to override with an explicit +# comma-separated hostname list (takes precedence over the file). +# +# Resolution must fail loudly (not silently yield an empty exclude list) if +# it can't be trusted: a submit host missing python3/PyYAML, or a genuine +# parse error, must not silently reintroduce the known-bad-node issue this +# exclusion mechanism exists to prevent. EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12}" +NODE_EXCLUDES_YAML="$(dirname "$0")/node_excludes.yaml" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then - EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") + RESOLVED_EXCLUDE_NODES="$SLURM_EXCLUDE_NODES" +elif [[ -f "$NODE_EXCLUDES_YAML" ]]; then + if command -v python3 >/dev/null 2>&1 && python3 -c "import yaml" >/dev/null 2>&1; then + RESOLVED_EXCLUDE_NODES=$(python3 -c " +import yaml + +with open('${NODE_EXCLUDES_YAML}') as f: + cfg = yaml.safe_load(f) or {} + +framework = '${FRAMEWORK}' +model = '${MODEL_NAME}' +for rule in cfg.get('rules', []): + if rule.get('framework') == framework and model in (rule.get('models') or []): + print(rule.get('exclude_nodes', '')) + break +") + PYTHON_EXCLUDE_RC=$? + if [[ $PYTHON_EXCLUDE_RC -ne 0 ]]; then + echo "Error: python3 failed (exit ${PYTHON_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 + echo "Error: fix the YAML, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 + exit 1 + fi + else + # Fall back to an awk parser (mirrors job.slurm's awk-based models.yaml + # parsing) matched to node_excludes.yaml's fixed rule/models/exclude_nodes + # shape. Only exercised when python3 or its yaml module is unavailable. + echo "Warning: python3/PyYAML unavailable on submit host; falling back to awk parsing of ${NODE_EXCLUDES_YAML}" >&2 + RESOLVED_EXCLUDE_NODES=$(awk -v fw="$FRAMEWORK" -v model="$MODEL_NAME" ' + /^ - framework:/ { + line = $0 + sub(/^ - framework: */, "", line) + fw_match = (line == fw) + model_match = 0 + next + } + fw_match && /^ - / { + m = $0 + sub(/^ - */, "", m) + gsub(/^"|"$/, "", m) + if (m == model) model_match = 1 + next + } + fw_match && model_match && /^ exclude_nodes:/ { + val = $0 + sub(/^ *exclude_nodes: */, "", val) + gsub(/^"|"$/, "", val) + print val + exit + } + ' "$NODE_EXCLUDES_YAML") + AWK_EXCLUDE_RC=$? + if [[ $AWK_EXCLUDE_RC -ne 0 ]]; then + echo "Error: awk fallback failed (exit ${AWK_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 + echo "Error: fix the YAML/parser, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 + exit 1 + fi + fi +else + RESOLVED_EXCLUDE_NODES="" +fi +if [[ -n "$RESOLVED_EXCLUDE_NODES" ]]; then + EXCLUDE_OPT=(--exclude "$RESOLVED_EXCLUDE_NODES") fi # ============================================================================= diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 83762e375b..67a0605398 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1559,7 +1559,7 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1573,7 +1573,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - dram-utilization: 0.80 search-space: - spec-decoding: "mtp" - conc-list: [ 1,2,4,8,16,32 ] + conc-list: [ 1,2,4,8,16,32,48,64 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -1593,7 +1593,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=3" - spec-decoding: "mtp" - conc-list: [ 96, 128 ] + conc-list: [ 64, 96, 128, 192 ] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f2c945ffc0..d85a0f87d2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6277,3 +6277,14 @@ description: - "Refresh with lower stream interval to collect correct client metrics" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2686 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Bump the SGLang image from lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 to lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260817." + - "Add concurrency 48 and 64 to the TP8/EP1 sweep, and 64 and 192 to the TP8/EP8 DP-attention sweep." + - "Increase prefill mem_fraction_static from 0.72 to 0.85 and reduce HICACHE_RATIO from 4 to 3." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2643 +