diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index a9256a48ab..1212309a21 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -2,71 +2,59 @@ set -eo pipefail set -x -# Agentic trace replay benchmark for Kimi-K3 (MXFP4) on B300 using vLLM with -# DSpark speculative decoding at level 2, probabilistic drafting, synthetic -# acceptance pinned to the committed golden AL (real verification for evals). -# -# MEASURED, for anyone reading the throughput numbers: this draft head's REAL -# acceptance collapses on the agentic corpus. Its native window is 32k -# (rope_parameters yarn, factor 32, original_max_position_embeddings 32768, -# stretched to the declared 1M), and at the corpus's 100k-330k ISL the measured -# acceptance length is 1.16-2.01 with 13-41% position-1 acceptance -- versus 4.18 -# AL and 0.826 position-1 on short-context work under the identical image, draft -# and spec config. Synthetic acceptance is the AgentX-prescribed way to measure -# system performance at a fixed acceptance target (see the SPEC_CONFIG block), so -# these throughput numbers are NOT evidence that the draft predicts well at -# agentic context lengths. It does not. -# -# MTP sibling of kimik3_fp4_b300_vllm.sh. Everything about the target server is -# identical; the deltas are: -# - --speculative-config method "dspark", draft Inferact/Kimi-K3-DSpark, -# attention_backend FLASHINFER_MLA, -# draft_sample_method probabilistic, -# rejection_sample_method block -# - cudagraph capture sizes are expressed in TOKENS, not sequences +# Agentic trace replay for Kimi-K3 (MXFP4) on B300: TP8 x DCP8, TokenspeedMLA, +# Mooncake as the external KV tier. Concurrency selects the arm: +# conc <= 8 DSpark level 7, golden AL 3.84 +# conc 16 DSpark level 3, golden AL 3.00 +# conc > 16 no drafting +# Keep the arms disjoint in concurrency: exp-name carries conc and spec but not +# the arm, so a shared concurrency would collide. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION +# Optional: +# DCP_SIZE (default 8), KV_OFFLOAD_BACKEND (mooncake, or empty for resident) # -# TP8 is the only single-node layout. The MXFP4 checkpoint is ~1.5 TB on disk; -# at TP4 that is ~375 GB of weights per GPU against B300's 288 GB of HBM, so -# only the full 8-GPU shard (~188 GB/GPU) fits. Do not add TP4/TP2 arms. +# TP8 is the only single-node layout: the MXFP4 checkpoint is ~1.5 TB, so TP4 +# would need ~375 GB/GPU against B300's 288 GB. # -# Chat formatting: AGENTS.md requires MTP scripts to pass --use-chat-template to -# run_benchmark_serving. Agentic recipes never call it -- the replay drives -# AIPerf against /v1/chat/completions, so prompts are already chat-formatted and -# spec-decode acceptance is measured on chat-shaped inputs. Nothing to add here. +# The draft's real acceptance on this corpus is 1.16-2.01 (13-41% position-1); +# its native window is 32k YaRN-stretched to 1M. Synthetic acceptance measures +# the system at a prescribed acceptance, not the draft's fitness at 100k+ ISL. source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -# The 2.8T MXFP4 checkpoint only fits across all 8 B300s (see header). if [ "$TP" -ne 8 ]; then - echo "Error: Kimi-K3 on B300 requires TP=8 (a ~1.5 TB MXFP4 checkpoint does not fit at TP<8), got TP='$TP'" >&2 + echo "Error: Kimi-K3 on B300 requires TP=8, got TP='$TP'" >&2 exit 1 fi if [[ -n "${EP_SIZE:-}" && "${EP_SIZE}" -gt 1 ]]; then - echo "Error: this recipe ships the pure-TP8 profile; EP_SIZE='$EP_SIZE' is not wired yet" >&2 + echo "Error: this recipe ships the pure-TP8 profile; EP_SIZE='$EP_SIZE' is not wired" >&2 exit 1 fi +# DCP shards decode KV across the TP ranks, so it must divide TP. +DCP_SIZE="${DCP_SIZE:-8}" +if [ $((TP % DCP_SIZE)) -ne 0 ]; then + echo "Error: TP='$TP' must be divisible by DCP_SIZE='$DCP_SIZE'" >&2 + exit 1 +fi +CP_ARGS=() +if [ "$DCP_SIZE" -gt 1 ]; then + CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a) +fi + if [[ -n "${SLURM_JOB_ID:-}" ]]; then echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" fi DRAFT_MODEL="${DRAFT_MODEL:-Inferact/Kimi-K3-DSpark}" -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. -# Either way, MODEL_PATH is what the server is launched with. -# -# The draft must NOT land next to a pre-staged target: Kimi-K3 is in the -# b300-nv launcher's STAGED_MODELS, so dirname(MODEL_PATH) is the read-only -# /scratch/models mount and writing there fails with PermissionError. Use the -# launcher's writable models dir instead, which also caches the draft across -# the jobs in a sweep (same placement as DeepSeek-V4-Pro-DSpark). +# The draft must not land next to a pre-staged target: dirname(MODEL_PATH) is a +# read-only mount, so use the launcher's writable models dir. if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" @@ -83,135 +71,189 @@ else fi nvidia-smi -# ---- Resolve traces and install deps ---------------------------------------- resolve_trace_source install_agentic_deps -# ---- Kimi-K3 production serving environment --------------------------------- -export NCCL_DMABUF_ENABLE=0 +# ---- Serving environment ---------------------------------------------------- export VLLM_ALLREDUCE_USE_FLASHINFER=1 -# Required by the upstream recipe on BOTH its blackwell and nvidia paths -# (recipes.vllm.ai/moonshotai/Kimi-K3), and never set here before. It defaults to -# 0, and it is threaded into LatentMoERunner as -# runner_args={"enable_k3_latent_moe_tail_fusion": ...} at -# vllm/models/kimi_k3/nvidia/model.py:549 -- the same runner whose shared-experts -# output buffer asserted (fused_moe/runner/shared_experts.py:165, all 8 TP ranks) -# when the wider flag alignment was tried. With this unset we have been running a -# MoE tail path upstream never exercises, so enable it on its own before -# re-attempting any of the other upstream flags. export VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 -export VLLM_USE_RUST_FRONTEND=1 -# Loading ~1.5 TB of MXFP4 shards off the staged mount takes well past the -# default readiness window even with fastsafetensors. +export VLLM_USE_V2_MODEL_RUNNER=1 +# These default to auto, which self-enables on B300; name them so the measured +# DCP a2a path is the one that runs. +export VLLM_USE_DIRECT_DCP_A2A=1 +export VLLM_USE_DIRECT_DCP_Q_GATHER=1 +export VLLM_USE_DIRECT_DCP_KV_GATHER=1 +# ~1.5 TB of MXFP4 shards loads well past the default readiness window. export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_RPC_TIMEOUT=600000 +export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0 +export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 export PYTHONNOUSERSITE=1 -# AIPerf pins one pooled keep-alive connection per agentic session and reuses it -# across turns, while the Rust frontend's default VLLM_HTTP_TIMEOUT_KEEP_ALIVE is -# 5s. An inter-turn idle gap longer than that lets the client reuse a socket at -# the moment the server closes it -> aiohttp ServerDisconnectedError -> AIPerf -# treats it as a terminal warmup failure and aborts the whole job. Outlast the -# client pool so the race cannot occur. +export TORCH_CUDA_ARCH_LIST=10.0 +# Identical prefixes must hash to identical block keys run-to-run. +export PYTHONHASHSEED=42 +# AIPerf reuses one pooled connection per session; outlast its idle gaps or a +# socket race aborts warmup. export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=900 -# Agentic warmup dispatches large prompts at once; allow up to 15 minutes of TCP -# progress before AIPerf declares a connection dead. export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" mkdir -p "$RESULT_DIR" +MOONCAKE_MASTER_PID="" -# ---- KV offloading ---------------------------------------------------------- -# The generated TOTAL_CPU_DRAM_GB budget is the aggregate host-DRAM pool for the -# node; SimpleCPUOffloadConnector is sized per rank. At dram-utilization 0.63 on -# cluster:b300-nv this resolves to ~220 GiB per rank across the 8 TP ranks. +cleanup() { + if [[ -n "$MOONCAKE_MASTER_PID" ]]; then + kill "$MOONCAKE_MASTER_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +# ---- External KV tier ------------------------------------------------------- OFFLOAD_ARGS=() case "${KV_OFFLOAD_BACKEND:-}" in "") require_agentic_kv_offload_none ;; - vllm-simple) - require_agentic_kv_offload_backend vllm-simple - CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / TP )) - # Identical prefixes must hash to identical block keys run-to-run. - export PYTHONHASHSEED=42 - # lazy_offload must be a JSON boolean, not a quoted string: the - # connector does bool(extra_config.get("lazy_offload", False)), and - # bool("false") is True in Python. + mooncake) + require_agentic_kv_offload_backend mooncake + PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) + MOONCAKE_VERSION=0.3.11.post1 + agentic_pip_install --quiet --no-cache-dir --no-deps \ + --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" + python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + # One rail for every rank. These nodes are rail-isolated, so two + # different RNICs cannot reach each other even within a node, and the + # embedded store's ranks are eight processes on one host. + # + # Chosen at runtime: mlx5_0 is down on some nodes (b300-016, b300-017), + # and a hardcoded rail has no fallback -- topology discovery finds 0 + # HCAs and every rank dies in a 20-retry loop that reads like a store + # problem. Order starts at mlx5_0 so a healthy node is unchanged. + MOONCAKE_RAIL="" + for _d in mlx5_0 mlx5_1 mlx5_2 mlx5_3 mlx5_4 mlx5_5 mlx5_8 mlx5_9 \ + mlx5_10 mlx5_11 mlx5_16 mlx5_17 mlx5_20 mlx5_21 mlx5_22 mlx5_23; do + if grep -q ACTIVE "/sys/class/infiniband/$_d/ports/1/state" 2>/dev/null; then + MOONCAKE_RAIL="$_d" + break + fi + done + if [ -z "$MOONCAKE_RAIL" ]; then + echo "Error: no active RDMA rail on $(hostname); Mooncake cannot initialise" >&2 + exit 1 + fi + echo "Mooncake rail: $MOONCAKE_RAIL" + + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + MOONCAKE_MASTER_PID=$! + sleep 2 + if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + echo "Mooncake master died during startup." >&2 + cat "$MOONCAKE_MASTER_LOG" >&2 + exit 1 + fi + OFFLOAD_ARGS=( --kv-transfer-config - "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":${CPU_BYTES_PER_RANK},\"lazy_offload\":false}}" + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' ) ;; *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or vllm-simple)" >&2 + echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or mooncake)" >&2 exit 1 ;; esac -# ---- DSpark speculative decoding ------------------------------------------- -# Probabilistic drafting at DSpark level 2, with synthetic acceptance pinned to -# the committed golden AL, per the AgentX policy in -# golden_al_distribution/README.md: "a submission may choose any supported draft -# length, but it may not substitute a different acceptance target", because -# "InferenceX is evaluating inference-system performance, not the ability to -# fine-tune a benchmark-specific speculative head". AL is workload-dependent by -# the README's own admission, so the prescribed acceptance -- not the acceptance -# this corpus happens to produce -- is what makes submissions comparable. -# -# Level 2 rather than 7. Draft length IS ours to choose, and the verify width is -# a real system cost that synthetic acceptance does not remove: at K=7 the target -# verifies 8 positions per step, which loses to no-speculation once the GPU -# saturates even at the forced golden AL of 3.84. Using cost ~ 1 + m(K-1) + f, -# K=2 clears break-even at both ends of the m range while K=3 is a wash at -# saturation and K=7 is negative: -# K=2 golden AL 2.51 -> ~1.7x low conc, ~1.2x saturated -# K=3 golden AL 3.00 -> ~1.7x low conc, ~0.95x saturated -# K=7 golden AL 3.84 -> ~1.3x low conc, ~0.54x saturated -# The drafter's KV footprint (~25% of GPU KV) is unaffected by K and remains a -# genuine cost of running spec decoding at all on 100k+ prompts. -NUM_SPEC_TOKENS=2 -TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) -# Committed golden AL at K=2 on the probabilistic curve we run -# (golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml: -# thinking_on 2 -> 2.51). The greedy/standard curve's K=2 value is 2.45 -- do not -# mix curves. -SYNTHETIC_ACCEPT_LEN=2.51 - -# Throughput runs pin synthetic acceptance; the EVAL_ONLY accuracy run must use -# real target verification instead. Synthetic acceptance commits drafted tokens -# regardless of the target's logits, so the generated text is wrong and the -# SWE-bench eval scores 0.0000 -- the same split dsv4_fp4_b300_vllm_mtp.sh makes -# (follow the DSV4 MTP precedent for this split). -# rejection_sample_method=block does real verification, so it is what EVAL_ONLY -# uses. vLLM rejects synthetic_acceptance_length unless the method is 'synthetic'. -if [ "${EVAL_ONLY:-false}" = "true" ]; then - SPEC_CONFIG="{\"method\": \"dspark\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASHINFER_MLA\", \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"block\"}" +# ---- Speculative decoding --------------------------------------------------- +if [ "${SPEC_DECODING:-none}" != "mtp" ]; then + echo "Error: this recipe expects spec-decoding=mtp for every arm, got '${SPEC_DECODING:-}'" >&2 + exit 1 +fi +# Draft length by concurrency. The golden AL must track it, from the +# probabilistic curve in golden_al_distribution/. +if [ "$CONC" -le 8 ]; then + NUM_SPEC_TOKENS=7 + SYNTHETIC_ACCEPT_LEN=3.84 +elif [ "$CONC" -le 16 ]; then + NUM_SPEC_TOKENS=3 + SYNTHETIC_ACCEPT_LEN=3.00 else - SPEC_CONFIG="{\"method\": \"dspark\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASHINFER_MLA\", \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" + NUM_SPEC_TOKENS=0 +fi + +SPEC_ARGS=() +if [ "$NUM_SPEC_TOKENS" -gt 0 ]; then + # EVAL_ONLY needs real verification: synthetic acceptance commits drafts + # regardless of target logits and would zero the eval score. + if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_CONFIG="{\"method\": \"dspark\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TOKENSPEED_MLA\", \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"block\"}" + else + SPEC_CONFIG="{\"method\": \"dspark\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TOKENSPEED_MLA\", \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" + fi + SPEC_ARGS=(--speculative-config "$SPEC_CONFIG") fi -# Agentic fan-out: the scheduler headroom convention shared by the other agentic -# recipes. Upstream pins 32 instead, but the aligned flag set crashed the engine -# core on every concurrency (see the serve-flag note below), so this stays on the -# value that ran green. Capture decode graphs only to this bound — a 93-layer -# 2.8T model makes capturing vLLM's full 2048-wide ladder prohibitively slow. MAX_NUM_SEQS=$((2 * CONC)) -# With spec decoding, cudagraph capture sizes are in TOKENS. vLLM rounds -# configured sizes up to multiples of (1 + num_speculative_tokens) and dedups -# them (adjust_cudagraph_sizes_for_spec_decode), so a plain 1..MAX_NUM_SEQS list -# would collapse to covering only MAX_NUM_SEQS/(1+N) sequences and drop the -# largest decode batches to eager. Enumerate the multiples explicitly so there -# is one graph per decode batch of 1..MAX_NUM_SEQS sequences. The graph mode is -# left at the recipe default that the non-MTP K3 arm already validated. -CUDA_GRAPH_CAPTURE_SIZES="" -for ((num_seqs = 1; num_seqs <= MAX_NUM_SEQS; num_seqs++)); do - if [ -n "$CUDA_GRAPH_CAPTURE_SIZES" ]; then - CUDA_GRAPH_CAPTURE_SIZES+="," +# 1 - this is the buffer for what is not sized against the budget: the cudagraph +# pool, the FlashInfer MoE workspace and fragmentation. Only c56 and c70 ran out +# of it (2.78 GiB wanted, 1.60 GiB free); the pool grows with concurrency, so +# lower points keep the default and their full KV cache. +if [ "$CONC" -ge 56 ]; then + GPU_MEM_UTIL=0.90 +else + GPU_MEM_UTIL=0.92 +fi + +# Capture sizes: step * 1..min(max-num-seqs, 128), then the fixed powers of two +# above that. Sizes are tokens when drafting, sequences when not; the 128 caps +# the number of dense entries, not their value. +CAPTURE_STEP=$((1 + NUM_SPEC_TOKENS)) +DENSE_COUNT=$MAX_NUM_SEQS +if [ "$DENSE_COUNT" -gt 128 ]; then + DENSE_COUNT=128 +fi +CAPTURE_SIZES="" +for ((n = 1; n <= DENSE_COUNT; n++)); do + CAPTURE_SIZES+="${CAPTURE_SIZES:+,}$((n * CAPTURE_STEP))" +done +DENSE_MAX=$((DENSE_COUNT * CAPTURE_STEP)) +for t in 64 128 256 512 1024 2048 4096 8192; do + if [ "$t" -gt "$DENSE_MAX" ]; then + CAPTURE_SIZES+=",$t" fi - CUDA_GRAPH_CAPTURE_SIZES+="$((num_seqs * TOKENS_PER_SEQ))" done -COMPILATION_CONFIG="{\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}]}" +COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[${CAPTURE_SIZES}]}" echo "Starting vllm server..." @@ -221,29 +263,23 @@ VLLM_CMD=( --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP" - # Adopting the full upstream flag set (gpu-memory-utilization 0.95, - # max-num-seqs 32, max-model-len 1000000, max-num-batched-tokens 32768, - # --no-enable-flashinfer-autotune, VLLM_USE_V2_MODEL_RUNNER=1) killed the - # engine core on every concurrency with an assertion in the FlashInfer MoE - # runner's shared-experts output buffer - # (fused_moe/runner/shared_experts.py:165, all 8 TP ranks at once). Only - # mla_prefill_backend=TRTLLM_RAGGED is retained from that set; the rest stay - # on the values that ran 12/12 green in run 30326393603. - --gpu-memory-utilization 0.90 + "${CP_ARGS[@]}" --max-num-seqs "$MAX_NUM_SEQS" - --max-model-len 1048576 + --gpu-memory-utilization "$GPU_MEM_UTIL" + --max-num-batched-tokens 16384 --trust-remote-code + --language-model-only --load-format fastsafetensors --moe-backend auto + --no-enable-flashinfer-autotune + --enable-cumem-allocator --enable-prefix-caching + --prefix-match-unit 128 --kv-cache-dtype fp8 - --reasoning-parser kimi_k3 - --tool-call-parser kimi_k3 - --enable-auto-tool-choice - # FP8 KV cache requires prefill query quantization. Use FlashInfer MLA - # prefill, matching the current published upstream Kimi-K3 recipe. - --attention-config '{"mla_prefill_backend":"flashinfer","use_prefill_query_quantization":true}' - --speculative-config "$SPEC_CONFIG" + --stream-interval 10 + --attention-backend TOKENSPEED_MLA + --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + "${SPEC_ARGS[@]}" --compilation-config "$COMPILATION_CONFIG" --disable-uvicorn-access-log "${OFFLOAD_ARGS[@]}" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 62ba9cff33..9d55b30631 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1496,24 +1496,15 @@ dsr1-fp8-b300-sglang-mtp: - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } kimik3-fp4-b300-vllm-agentic-dspark: - # DSpark speculative decoding on the Inferact/Kimi-K3-DSpark draft head, at - # level 2 with probabilistic drafting and synthetic acceptance pinned to the - # committed golden AL 2.51 (golden_al_distribution/ - # kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml, - # thinking_on 2 -> 2.51), per the AgentX policy in that directory's README: - # draft length is the submission's choice, the acceptance target is not. - # EVAL_ONLY runs switch to real block verification, since synthetic acceptance - # commits drafts regardless of target logits and would zero the SWE-bench score. + # TP8 x DCP8 with Mooncake as the external KV tier. The recipe drafts with + # DSpark level 7 at conc <= 8 and stops above it; synthetic acceptance is + # pinned to the committed golden AL 3.84 (golden_al_distribution/ + # kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml), + # and EVAL_ONLY switches to real block verification. # - # Level 2 not 7: verify width is a real system cost synthetic acceptance does - # not remove, and at K=7 the 8-position verify loses to no-speculation once the - # GPU saturates even at the forced golden AL of 3.84. - # - # The draft's REAL acceptance on this corpus is 1.16-2.01 (13-41% position-1) -- - # its native window is 32k YaRN-stretched to 1M, and the same image/draft/config - # reaches AL 4.18 on short-context work. These throughput numbers measure the - # system at a prescribed acceptance, not the draft's fitness at 100k+ context. - image: vllm/vllm-openai:nightly-b22afe45ac797ae58e67a7a3ad79ee5714024420@sha256:144356af876edbb3a4bfee23e1444b196cc3fdadd0a0c1a7f11f721756972a21 + # The image carries the Kimi-K3 DCP, DSpark-under-DCP and Mooncake-hybrid + # changes (vllm-project/vllm agentx-k3 @ 5894fdf). + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-5894fdf model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:b300-nv @@ -1523,19 +1514,15 @@ kimik3-fp4-b300-vllm-agentic-dspark: scenarios: # Agentic-coding only: no fixed-seq-len (1k1k / 8k1k) arms for this recipe. agentic-coding: - - dram-utilization: 0.63 + # Sizes the Mooncake segment: this resolves to total-cpu-dram-gb, which + # the recipe divides across the 8 TP ranks. + - dram-utilization: 0.75 search-space: - # Keep equal resident/offload points through conc 8. The digest-pinned - # bring-up sweep (run 31404943911) confirmed that resident conc 16 - # remained in warmup after 1,770 seconds (96/177 requests), while conc 8 - # entered profiling. The offload conc 16 arm completed warmup in 804 - # seconds and exercised the external KV tier, so retain it as the - # capacity endpoint. # TP8-only: a ~1.5 TB MXFP4 checkpoint does not fit below 8 GPUs. - # TP8 GPU-resident - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8] } - # TP8 SimpleCPUOffload (host DRAM) - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16] } + # One entry for every arm: the recipe drafts at DSpark 7 up to conc 8, + # DSpark 3 at conc 16, and not at all above. Keep the concurrencies + # disjoint across arms so exp-names stay unique. + - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70] } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index da3f8d6dd5..57e53d9acb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6018,3 +6018,13 @@ description: - "Refresh the MiniMax-M3 B200 AgentX submission: no-offload TP8 C1 and TP4 C1/C5/C10/C15/C20, plus a TP4 SimpleCPU KV-offload sweep at C15–C40 using the full-capacity DRAM budget." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2611 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Move the B300 Kimi K3 recipe to TP8 x DCP8 with Mooncake as the external KV tier" + - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 40, 48, 56 and 70, drafting with DSpark level 7 at concurrency 8 and below, DSpark level 3 at concurrency 16" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2616 +