diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh new file mode 100644 index 000000000..9016d651f --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using ATOM. +# +# Serving flags follow the validated MI355X recipe from +# https://github.com/ROCm/ATOM/blob/main/recipes/DeepSeek-V4-Agentic-Benchmark.md +# Image is configured in amd-master.yaml. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# KV_OFFLOADING=dram is not enabled. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# Nightly ROCm image may be missing runtime deps; ensure they are present. +agentic_pip_install --quiet Pillow fastapi uvicorn + +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 + +# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. +export VLLM_ENGINE_READY_TIMEOUT_S=3600 + +# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while +# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's +# store mask. 32k matches the trace-replay tuning validated for this workload. +export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 + +# VLLM_PREFIX_CACHE_RETENTION_INTERVAL only applies to sliding-window/Mamba +# models; this vLLM build raises ValueError if it is set for DSv4. + +# ---- LLM server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +VLLM_BACKEND_PORT="$PORT" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +ROUTER_PID="" +MOONCAKE_MASTER_PID="" + +PARALLEL_ARGS=(-tp "$TP") +SPEC_ARGS=(--method mtp --num-speculative-tokens 3 ) +OFFLOAD_ARGS=() +if [ "$DP_ATTENTION" = "true" ]; then + PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention) +fi +if [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--enable-expert-parallel) +fi + +# AgentX concurrency counts live session trees, not individual requests. +# Subagent fan-out can push instantaneous request concurrency above CONC, so +# leave 2x headroom rather than clipping those bursts at the scheduler. +MAX_NUM_SEQS=$((2 * CONC)) + +echo "Starting atom server..." +set -x +export AITER_BF16_FP8_MOE_BOUND=0 +export ATOM_MOE_GU_ITLV=1 +export AITER_LOG_LEVEL=WARNING +export ATOM_DISABLE_MMAP=true +export MEM_FRAC_STATIC=0.9 + +{ set +x; } 2>/dev/null +ATOM_CMD=( + python3 -m atom.entrypoints.openai_server + --model "$MODEL_PATH" + --server-port "$VLLM_BACKEND_PORT" + "${PARALLEL_ARGS[@]}" + "${SPEC_ARGS[@]}" + --kv_cache_dtype fp8 + --trust-remote-code + --enable_prefix_caching + --max-num-seqs "$MAX_NUM_SEQS" + --gpu-memory-utilization "$MEM_FRAC_STATIC" + "${OFFLOAD_ARGS[@]}" +) + +printf '%q ' "${ATOM_CMD[@]}" | tee "$RESULT_DIR/atom_command.txt" +printf '\n' | tee -a "$RESULT_DIR/atom_command.txt" +"${ATOM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0f547cba8..7f093fc09 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1356,6 +1356,20 @@ dsv4-fp4-mi355x-atom-mtp: search-space: - { tp: 8, ep: 1, conc-start: 1, conc-end: 1024, spec-decoding: mtp } +dsv4-fp4-mi355x-atom-agentic-mtp: + image: rocm/atom-dev:nightly_202607211523 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.60 + search-space: + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 48], spec-decoding: mtp } + qwen3.5-bf16-mi325x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x model: Qwen/Qwen3.5-397B-A17B diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c3602ff74..e8517f5bf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5085,3 +5085,10 @@ description: - "Bump SGLang container image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.15.post1-cu130 (https://github.com/sgl-project/sglang/releases/tag/v0.5.15.post1)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2313 + +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + description: + - "Add DeepSeek-V4-Pro FP4 MI355X ATOM agentic-coding MTP benchmark serving via the ATOM openai_server entrypoint with speculative decoding (--method mtp --num-speculative-tokens 3), following the ROCm ATOM DeepSeek-V4-Agentic recipe (https://github.com/ROCm/ATOM/blob/main/recipes/DeepSeek-V4-Agentic-Benchmark.md)" + - "Register the dsv4-fp4-mi355x-atom-agentic-mtp config in configs/amd-master.yaml on rocm/atom-dev:nightly_202607211523 (TP8/EP1, conc list [1,2,4,8,16,32,48], spec-decoding mtp, no KV offload)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2346