Skip to content

Refresh DeepSeek-V4 B300 SGLang AgentX MTP #2577

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cquil11 merged 6 commits into from
Aug 13, 2026
+305 −0
Merged

Refresh DeepSeek-V4 B300 SGLang AgentX MTP #2577

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f42a66a
perf(agentx): refresh DSV4 B300 SGLang MTP sweep
cquil11 Aug 12, 2026
38171d3
fix: allow B300 eval result metadata
cquil11 Aug 12, 2026
b3f7b49
fix: stabilize B300 DeepSeek V4 AgentX sweep
cquil11 Aug 13, 2026
34fd133
Merge remote-tracking branch 'origin/main' into agent/refresh-dsv4-b3…
cquil11 Aug 13, 2026
d8fd38a
Merge branch 'main' into agent/refresh-dsv4-b300-sglang-agentx
cquil11 Aug 13, 2026
0aac240
Update perf-changelog.yaml
cquil11 Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
276 changes: 276 additions & 0 deletions benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,276 @@
#!/usr/bin/env bash
set -eo pipefail
set -x

# Agentic trace replay for DeepSeek-V4-Pro FP4 on B300 with native EAGLE MTP.
# Throughput uses the committed golden synthetic AL; eval retains real target
# verification.
#
# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache.

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
INFERENCEX_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
export INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/workspace}"

# The B200 DeepSeek-V4 Blackwell image installs SGLang editable under
# /workspace, so its launcher mounts InferenceX at /ix instead. Resolve the
# agentic tooling and results against the actual repository mount so the image
# can keep its /workspace install and GitHub Actions can collect the outputs.
if [[ ! -d "$INFMAX_CONTAINER_WORKSPACE/utils/aiperf" ]]; then
export INFMAX_CONTAINER_WORKSPACE="$INFERENCEX_ROOT"
fi
if [[ "${RESULT_DIR:-}" == /workspace/* && "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then
export RESULT_DIR="$INFMAX_CONTAINER_WORKSPACE/${RESULT_DIR#/workspace/}"
fi
source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh"

export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:"

check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION

if [[ -n "${SLURM_JOB_ID:-}" ]]; then
echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
fi

if [[ -n "${MODEL_PATH:-}" ]]; then
if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
hf download "$MODEL" --local-dir "$MODEL_PATH"
fi
else
hf download "$MODEL"
export MODEL_PATH="$MODEL"
fi
nvidia-smi

resolve_trace_source

# Keep AIPerf's Transformers-main dependency from replacing the older
# Transformers build pinned by the B200-specialized SGLang image. The server
# always launches with the image's original interpreter; AIPerf and result
# processing use the isolated environment when InferenceX is mounted at /ix.
SGLANG_PYTHON="$(command -v python3)"
if [[ "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then
AGENTIC_VENV="${AGENTIC_VENV:-/tmp/inferencex-agentic-venv}"
"$SGLANG_PYTHON" -m venv "$AGENTIC_VENV"
export PATH="$AGENTIC_VENV/bin:$PATH"
fi
install_agentic_deps

SERVER_LOG="$RESULT_DIR/server.log"
mkdir -p "$RESULT_DIR"

export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1
export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1

CACHE_ARGS=()
WARMUP_ARGS=()
if require_agentic_kv_offload_backend hicache; then
# DeepSeek V4 HiCache currently rejects --hicache-size and supports
# capacity control only through a host/device token-capacity ratio.
# DSv4 exposes capacity as a host/device token ratio rather than bytes.
# Measurements put TP8 ratio=2 near 950 GB and TP4 ratio=8 near 1 TB,
# both below their configured capacities. The old TP4 ratio=16
# used roughly 2 TB and violated the half-node allocation rule.
if [ "$TP" -ge 8 ]; then
DEFAULT_HICACHE_RATIO=2
else
DEFAULT_HICACHE_RATIO=8
fi
HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}"
if [ "$HICACHE_RATIO" -gt "$DEFAULT_HICACHE_RATIO" ]; then
echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2
exit 1
fi
HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}"
HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}"
CACHE_ARGS=(
--enable-hierarchical-cache
--hicache-ratio "$HICACHE_RATIO"
--hicache-write-policy "$HICACHE_WRITE_POLICY"
--hicache-io-backend "$HICACHE_IO_BACKEND"
--hicache-mem-layout "$HICACHE_MEM_LAYOUT"
)
# AIPerf owns the representative warmup for AgentX. Avoid SGLang's
# redundant per-DP warmup timing out after the API is already healthy.
WARMUP_ARGS=(--skip-server-warmup)
echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT"
fi

USE_SGLANG_ROUTER=false
SGLANG_BACKEND_PORT="$PORT"
ROUTER_LOG="$RESULT_DIR/router.log"
if [ "$DP_ATTENTION" = "true" ]; then
USE_SGLANG_ROUTER=true
export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true
SGLANG_BACKEND_PORT=$((PORT + 1))
SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000))
SGLANG_ROUTER_CMD=("$SGLANG_PYTHON" -m sglang_router.launch_router)
fi

PARALLEL_ARGS=(--tp "$TP")
METRICS_ARGS=(--enable-metrics --enable-cache-report)
MEM_FRACTION_STATIC=0.88
CHUNKED_PREFILL_SIZE=8192
if [ "$DP_ATTENTION" = "true" ]; then
PARALLEL_ARGS+=(
--dp "$TP"
--tokenizer-worker-num "$TP"
--enable-dp-attention
--enable-dp-attention-local-control-broadcast
--incremental-streaming-output
--stream-interval 20
--dist-init-addr "127.0.0.1:$((PORT + 2000))"
--ep-size "$EP_SIZE"
--moe-runner-backend flashinfer_mxfp4
--disable-flashinfer-autotune
)
MEM_FRACTION_STATIC=0.95
if [ "$CONC" -ge 512 ]; then
# Leave room for FlashInfer's transient MoE workspace at the DEP8 tail.
MEM_FRACTION_STATIC=0.94
fi
CHUNKED_PREFILL_SIZE=16384
else
PARALLEL_ARGS+=(
--moe-runner-backend flashinfer_mxfp4
--disable-flashinfer-autotune
)
fi

MODEL_ARGS=(
--attention-backend compressed
--page-size 256
--disable-shared-experts-fusion
)

# AgentX concurrency counts live session trees, not individual requests.
# Allow subagent fan-out to exceed CONC without clipping request bursts.
MAX_RUNNING_REQUESTS=$((2 * CONC))
CUDA_GRAPH_MAX_BS=$CONC
[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64

export PYTHONNOUSERSITE=1
export TORCH_CUDA_ARCH_LIST=10.0
# Agentic warmup dispatches hundreds of large prompts at once. SGLang's
# tokenizer process can leave request bytes unacknowledged for longer than
# AIPerf's 30-second TCP_USER_TIMEOUT while it admits that initial burst,
# causing Linux to abort otherwise-live localhost connections. Keep the
# six-hour request timeout unchanged, but allow up to 15 minutes for TCP
# progress before declaring the connection dead.
export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race
# Uvicorn's five-second keep-alive closure.
export SGLANG_TIMEOUT_KEEP_ALIVE=900
export SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1
export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1
Comment on lines +161 to +166

@claude claude Bot Aug 12, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 This new B300 DeepSeek-V4 SGLang AgentX script never sets SGLANG_TIMEOUT_KEEP_ALIVE, leaving uvicorn's keep-alive at its ~5s default for the whole run, unlike every sibling SGLang AgentX MTP recipe in this directory (glm5.2_fp4_b300/b200/mi355x set 900; qwen3.5 variants set 1800). Add export SGLANG_TIMEOUT_KEEP_ALIVE=900 right after the AIPERF_HTTP_TCP_USER_TIMEOUT=900000 line (~153) to avoid ECONNRESET/terminal warmup failures on inter-turn idle gaps, especially on the DP-attention/high-concurrency arms (conc up to 512).

Extended reasoning...

The bug: dsv4_fp4_b300_sglang_mtp.sh never exports SGLANG_TIMEOUT_KEEP_ALIVE. Since benchmarks/benchmark_lib.sh sets no fallback for this variable either, SGLang's uvicorn server runs the entire sweep with its built-in ~5-second keep-alive timeout.

Why this matters here specifically: AIPerf's agentic replay harness (build_replay_cmd/run_agentic_replay_and_write_outputs, used by this script) pins one pooled HTTP keep-alive connection per session, with a client-side keep-alive window far longer than 5 seconds. During the long inter-turn idle gaps that are inherent to agentic multi-turn replay, a request can arrive on a pooled socket right after uvicorn has already closed it server-side (since the server's 5s timer fired first), producing an ECONNRESET instead of a normal request. Sibling recipes describe this exact failure mode as a terminal warmup failure — not a retryable blip, but one that can kill the whole sweep run.

The strongest evidence this is an accidental omission, not a deliberate choice: every other SGLang AgentX MTP recipe in benchmarks/single_node/agentic/ sets this variable — glm5.2_fp4_b300_sglang_mtp.sh, glm5.2_fp4_b200, and glm5.2_fp4_mi355x all set SGLANG_TIMEOUT_KEEP_ALIVE=900, and the qwen3.5 SGLang variants set 1800. In glm5.2_fp4_b300_sglang_mtp.sh specifically, SGLANG_TIMEOUT_KEEP_ALIVE=900 (line 184) sits immediately after AIPERF_HTTP_TCP_USER_TIMEOUT=900000 (line 178) — the two lines form a matched client/server pair fixing two sides of the same idle-connection problem (client TCP-level timeout vs. server HTTP keep-alive timeout). This DSv4 script copies the client-side half of that pair verbatim (AIPERF_HTTP_TCP_USER_TIMEOUT=900000 at line 153, with an analogous comment about long admission bursts) but drops the server-side half that follows it in every sibling. That is a strong signature of a partial copy-paste, not an intentional decision to leave the default alone.

Step-by-step proof of the failure path:

  1. Server starts via sglang.launch_server with no SGLANG_TIMEOUT_KEEP_ALIVE env set → uvicorn keep-alive timeout defaults to ~5s.
  2. AIPerf's agentic replay client opens a pooled connection to the server (or, on the DP-attention arms, through the sglang-router) and reuses it across a session's multi-turn trace.
  3. Between two turns of a session, the client is idle (e.g. waiting on tool-call/subagent completion) for more than 5 seconds — routine for agentic traces.
  4. Because the server's keep-alive timer fires before the client's, uvicorn closes the idle socket while the client still considers it open.
  5. The client's next write on that socket happens after the server already closed it, and Linux answers with RST, producing a client-side ECONNRESET.
  6. Since this happens during warmup dispatch of the initial large-prompt burst on high-concurrency/DP-attention arms (conc up to 512), the failure is described by the sibling scripts as a terminal warmup failure — the run doesn't just log a retried request, it aborts.

Fix: add export SGLANG_TIMEOUT_KEEP_ALIVE=900 (matching the B300 glm5.2 sibling, since this script shares the same 900000ms AIPERF_HTTP_TCP_USER_TIMEOUT) immediately after the AIPERF_HTTP_TCP_USER_TIMEOUT=900000 export at line 153.

Why nothing else in the diff prevents this: benchmark_lib.sh has no global default for this variable, the router configuration (--connect-timeout-secs 900, --request-timeout-secs 14400) governs router-side timeouts, not the SGLang backend's own uvicorn keep-alive, and the TP-only arms (kv-offloading none, TP4/TP8) connect AIPerf directly to the SGLang backend with no router in between, so nothing shields those arms from the default 5s keep-alive either.

export SGLANG_OPT_USE_JIT_NORM=1
export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1
export SGLANG_OPT_USE_TOPK_V2=1
export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1
if [ "${EVAL_ONLY}" != "true" ]; then
export SGLANG_SIMULATE_ACC_LEN=2.49
export SGLANG_SIMULATE_ACC_METHOD=match-expected
export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token
fi
TRITON_PTXAS_PATH=$(find \
/usr/local/cuda* \
/usr/local/lib/python*/dist-packages/nvidia \
/usr/local/lib/python*/site-packages/nvidia \
-type f -name ptxas -perm -u+x -print -quit 2>/dev/null || true)
if [ -n "$TRITON_PTXAS_PATH" ]; then
export TRITON_PTXAS_PATH
echo "Using ptxas for Triton: $TRITON_PTXAS_PATH"
fi
SGLANG_CMD=(
"$SGLANG_PYTHON" -m sglang.launch_server
--model-path "$MODEL_PATH"
--served-model-name "$MODEL"
--host 0.0.0.0
--port "$SGLANG_BACKEND_PORT"
--trust-remote-code
"${PARALLEL_ARGS[@]}"
--mem-fraction-static "$MEM_FRACTION_STATIC"
--swa-full-tokens-ratio 0.1
--max-running-requests "$MAX_RUNNING_REQUESTS"
--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
--allow-auto-truncate
--chunked-prefill-size "$CHUNKED_PREFILL_SIZE"
--tool-call-parser deepseekv4
--reasoning-parser deepseek-v4
--chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja"
--watchdog-timeout 1800
--speculative-algorithm EAGLE
--speculative-num-steps 3
--speculative-eagle-topk 1
--speculative-num-draft-tokens 4
"${MODEL_ARGS[@]}"
"${METRICS_ARGS[@]}"
"${CACHE_ARGS[@]}"
"${WARMUP_ARGS[@]}"
)

write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}"

{
echo "=== SGLANG_* env vars at launch ==="
env | grep -E '^SGLANG_' | sort
echo "==================================="
} | tee "$SERVER_LOG"

echo "Starting SGLang server for B300..."
"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 &
SERVER_PID=$!
echo "Server PID: $SERVER_PID"

capture_cache_metrics() {
{
echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ==="
curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \
| grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \
|| true
echo "============================================================"
} >> "$SERVER_LOG"
}

wait_for_ready \
--endpoint "http://localhost:$SGLANG_BACKEND_PORT/health" \
--log "$SERVER_LOG" \
--pid "$SERVER_PID"

if [ "$USE_SGLANG_ROUTER" = "true" ]; then
echo "Starting SGLang router on port $PORT for $TP DP ranks..."
"${SGLANG_ROUTER_CMD[@]}" \
--worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \
--policy consistent_hashing \
--request-id-headers x-correlation-id \
--dp-aware \
--host 0.0.0.0 \
--port "$PORT" \
--prometheus-host 127.0.0.1 \
--prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \
--connect-timeout-secs 900 \
--request-timeout-secs 14400 \
--disable-health-check \
--disable-retries > "$ROUTER_LOG" 2>&1 &
ROUTER_PID=$!
echo "Router PID: $ROUTER_PID"
wait_for_ready \
--endpoint "http://localhost:$PORT/health" \
--log "$ROUTER_LOG" \
--pid "$ROUTER_PID"
fi

if [ "${#METRICS_ARGS[@]}" -gt 0 ]; then
capture_cache_metrics
trap capture_cache_metrics EXIT
fi
Comment on lines +264 to +267

@claude claude Bot Aug 12, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 The guard if [ "${#METRICS_ARGS[@]}" -gt 0 ] at lines 253-256 in dsv4_fp4_b300_sglang_mtp.sh is dead code: METRICS_ARGS is assigned once at line 108 as a fixed two-element array and is never modified, so the condition is always true. This is a minor readability nit — recommend removing the guard and running capture_cache_metrics and trap capture_cache_metrics EXIT unconditionally, matching the sibling recipe qwen3.5_fp4_b300_sglang_mtp.sh, which does exactly this with no guard.

Extended reasoning...

The bug: In benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh, line 108 assigns METRICS_ARGS=(--enable-metrics --enable-cache-report) once, and this array is never reassigned, appended to, or conditionally cleared anywhere else in the script. Later, lines 253-256 guard the cache-metrics capture behind this array's length:

if [ "${#METRICS_ARGS[@]}" -gt 0 ]; then
    capture_cache_metrics
    trap capture_cache_metrics EXIT
fi

Because METRICS_ARGS always has exactly 2 elements, ${#METRICS_ARGS[@]} is always 2, so the condition 2 -gt 0 is always true. The if block can never be skipped.

Code path: METRICS_ARGS is referenced in exactly three places — the assignment (line 108), its expansion into SGLANG_CMD (line 198, "${METRICS_ARGS[@]}"), and the guard (line 253). Unlike CACHE_ARGS (which starts empty and is populated only inside the require_agentic_kv_offload_backend hicache branch) or PARALLEL_ARGS (which branches on DP_ATTENTION), there is no code path in this script that leaves METRICS_ARGS empty or shrinks it. The DP_ATTENTION branch only touches PARALLEL_ARGS.

Why nothing catches this today: The script is a shell recipe with no linter enforcing dead-conditional detection, and the guard reads as plausible defensive code — a reviewer skimming it would reasonably assume METRICS_ARGS can be empty in some arm of the recipe, since that's the pattern used elsewhere in the same file for CACHE_ARGS. It isn't until you trace all three references that the always-true nature becomes apparent.

Impact: None functionally — capture_cache_metrics and the EXIT trap always run today regardless of the guard, so behavior is correct. The only cost is misleading readability: a future maintainer might assume there's a code path where metrics capture is skipped, and either preserve the guard when refactoring (carrying the confusion forward) or spend time looking for the non-existent empty-array case.

Proof by exhaustive trace:

  1. Line 108: METRICS_ARGS=(--enable-metrics --enable-cache-report) — this is the only assignment in the file.
  2. grep -n METRICS_ARGS benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh returns exactly 3 lines: 108 (assignment), 198 (expansion in SGLANG_CMD), 253 (the guard).
  3. No +=, no reassignment, no conditional that could zero it out — contrast with CACHE_ARGS=() at line 34, which starts empty and is only populated inside if require_agentic_kv_offload_backend hicache; then ... fi.
  4. Therefore for every possible combination of TP, DP_ATTENTION, KV_OFFLOADING, etc., ${#METRICS_ARGS[@]} evaluates to 2 at line 253, and [ "2" -gt 0 ] is always true.
  5. Corroborating evidence: the sibling recipe qwen3.5_fp4_b300_sglang_mtp.sh calls capture_cache_metrics and installs trap capture_cache_metrics EXIT unconditionally (no guard at all), confirming the intended/idiomatic form in this codebase is to run these unconditionally.

Fix: Remove the if guard and run the two statements unconditionally:

capture_cache_metrics
trap capture_cache_metrics EXIT

This is a strictly simplifying, behavior-preserving cleanup — a pure readability nit, not a functional bug, so it should not block merging.


if [ "${EVAL_ONLY}" = "true" ]; then
git config --global --add safe.directory "$INFMAX_CONTAINER_WORKSPACE"
run_eval --port "$PORT"
else
build_replay_cmd "$RESULT_DIR"
REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics"
run_agentic_replay_and_write_outputs "$RESULT_DIR"
fi
18 changes: 18 additions & 0 deletions configs/nvidia-master.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1142,6 +1142,24 @@ dsv4-fp4-b300-sglang:
- { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 }
- { tp: 8, ep: 8, dp-attn: true, conc-start: 4096, conc-end: 4096 }

dsv4-fp4-b300-sglang-agentic-hicache-mtp:
image: lmsysorg/sglang:v0.5.17-cu130
model: deepseek-ai/DeepSeek-V4-Pro
model-prefix: dsv4
runner: cluster:b300-nv
precision: fp4
framework: sglang
multinode: false
scenarios:
agentic-coding:
- dram-utilization: 0.80
search-space:
- { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 20, 24, 32] }
- { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] }
- { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 24, 32, 40, 64], router: { name: sglang-router, version: "0.3.2" } }
- { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128], router: { name: sglang-router, version: "0.3.2" } }
- { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [52, 72, 100, 128, 144, 196, 512], router: { name: sglang-router, version: "0.3.2" } }

# DeepSeek-V4-Pro on B300 with EAGLE/MTP speculative decoding. Recipe is
# selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by
# DP_ATTENTION:
Expand Down
11 changes: 11 additions & 0 deletions perf-changelog.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5918,3 +5918,14 @@
- "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64"
- "Rides on the NVFP4-V2 checkpoint switch from #2205"
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550

- config-keys:
- dsv4-fp4-b300-sglang-agentic-hicache-mtp
scenario-type:
- agentic-coding
description:
- "Refresh the purged B300 DeepSeek-V4-Pro SGLang AgentX TP4, TP8, DP-attention, and HiCache grid on SGLang v0.5.17."
- "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification."
- "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, keep pooled AgentX connections alive, let AIPerf own HiCache warmup, and reserve transient MoE workspace at DEP8 c512."
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2577

Loading