diff --git a/configs/agentx_aiperf.sh b/configs/agentx_aiperf.sh new file mode 100755 index 000000000..4c032d9d9 --- /dev/null +++ b/configs/agentx_aiperf.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Self-contained AIPerf driver for AgentX trace sweeps. The dataset is selected +# through the recipe environment, and the virtual environment must already be +# mounted. + +AIPERF_DIR="${AIPERF_DIR:-/workspace/srt-slurm-sa/aiperf}" +AIPERF_VENV="${AIPERF_VENV:-$AIPERF_DIR/.venv}" +AIPERF_PYTHON="${AIPERF_VENV}/bin/python" +AIPERF_CLI="${AIPERF_VENV}/bin/aiperf" +AIPERF_FAILED_REQUEST_THRESHOLD="${AIPERF_FAILED_REQUEST_THRESHOLD:-0.10}" +AIPERF_MAX_OSL="${AIPERF_MAX_OSL:-8192}" +AIPERF_TRAJECTORY_START_MIN_RATIO="${AIPERF_TRAJECTORY_START_MIN_RATIO:-0.25}" +AIPERF_TRAJECTORY_START_MAX_RATIO="${AIPERF_TRAJECTORY_START_MAX_RATIO:-0.75}" +AIPERF_AGENTIC_CACHE_WARMUP_DURATION="${AIPERF_AGENTIC_CACHE_WARMUP_DURATION:-600}" +AIPERF_WARMUP_GRACE_PERIOD="${AIPERF_WARMUP_GRACE_PERIOD:-1800}" +AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING="${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-0}" +AIPERF_USE_LEGACY_DYNAMO_SESSION_CONTROL="${AIPERF_USE_LEGACY_DYNAMO_SESSION_CONTROL:-0}" +AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS="${AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS:-}" + +check_env_vars() { + local missing=0 + for name in "$@"; do + if [ -z "${!name:-}" ]; then + echo "ERROR: required environment variable is missing: $name" >&2 + missing=1 + fi + done + if [ "$missing" -ne 0 ]; then + exit 1 + fi +} + +run_agentic_replay_and_write_outputs() { + local result_dir="$1" + mkdir -p "$result_dir" + env | sort > "$result_dir/env.txt" + printf '%s\n' "$REPLAY_CMD" > "$result_dir/aiperf_command.txt" + set +e + bash -lc "$REPLAY_CMD" 2>&1 | tee "$result_dir/aiperf.log" + local replay_rc="${PIPESTATUS[0]}" + set -e + return "$replay_rc" +} + +check_env_vars MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION + +if [ ! -x "$AIPERF_PYTHON" ]; then + echo "ERROR: AIPerf Python is missing or not executable: $AIPERF_PYTHON" >&2 + exit 1 +fi +if [ ! -x "$AIPERF_CLI" ]; then + echo "ERROR: AIPerf CLI is missing or not executable: $AIPERF_CLI" >&2 + exit 1 +fi + +"$AIPERF_PYTHON" - <<'PY' +import aiperf +import huggingface_hub + +print(f"Using aiperf {getattr(aiperf, '__version__', 'unknown')} from {aiperf.__file__}") +print(f"Using huggingface_hub from {huggingface_hub.__file__}") +PY + +AIPERF_PROFILE_HELP="$(COLUMNS=240 "$AIPERF_CLI" profile --help 2>&1)" + +RESULT_DIR="${RESULT_DIR:-/logs/agentic}" +AGENTIC_OUTPUT_DIR="${AGENTIC_OUTPUT_DIR:-$RESULT_DIR}" +PORT="${PORT:-8000}" +AIPERF_BASE_URL="${AIPERF_BASE_URL:-http://localhost:$PORT}" +MAX_CONTEXT_LENGTH="${MAX_CONTEXT_LENGTH:-1000000}" +NUM_DATASET_ENTRIES="${NUM_DATASET_ENTRIES:-472}" +PUBLIC_DATASET="${PUBLIC_DATASET:-semianalysis_cc_traces_weka_with_subagents_060826}" +HF_WEKA_DATASET="${HF_WEKA_DATASET:-}" +SLICE_DURATION="${SLICE_DURATION:-1.0}" +VLLM_START_PROFILE_AFTER_SECONDS="${VLLM_START_PROFILE_AFTER_SECONDS:-900}" + +mkdir -p "$RESULT_DIR" + +export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES="${AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES:-0}" +export AIPERF_DATASET_CONFIGURATION_TIMEOUT="${AIPERF_DATASET_CONFIGURATION_TIMEOUT:-1800}" +export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT="${AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT:-1800}" + +REPLAY_CMD="$AIPERF_CLI profile" +REPLAY_CMD+=" --scenario inferencex-agentx-mvp" +REPLAY_CMD+=" --url $AIPERF_BASE_URL" +REPLAY_CMD+=" --endpoint /v1/chat/completions" +REPLAY_CMD+=" --endpoint-type chat" +REPLAY_CMD+=" --streaming" +REPLAY_CMD+=" --model $MODEL" +REPLAY_CMD+=" --concurrency $CONC" +REPLAY_CMD+=" --benchmark-duration $DURATION" +REPLAY_CMD+=" --random-seed 42" +REPLAY_CMD+=" --failed-request-threshold $AIPERF_FAILED_REQUEST_THRESHOLD" +REPLAY_CMD+=" --trajectory-start-min-ratio $AIPERF_TRAJECTORY_START_MIN_RATIO" +REPLAY_CMD+=" --trajectory-start-max-ratio $AIPERF_TRAJECTORY_START_MAX_RATIO" +if [ -n "$AIPERF_AGENTIC_CACHE_WARMUP_DURATION" ] && [ "$AIPERF_AGENTIC_CACHE_WARMUP_DURATION" != "none" ]; then + REPLAY_CMD+=" --agentic-cache-warmup-duration $AIPERF_AGENTIC_CACHE_WARMUP_DURATION" +fi +if [ -n "$AIPERF_WARMUP_GRACE_PERIOD" ] && [ "$AIPERF_WARMUP_GRACE_PERIOD" != "none" ]; then + if grep -q -- "--warmup-grace-period" <<<"$AIPERF_PROFILE_HELP"; then + REPLAY_CMD+=" --warmup-grace-period $AIPERF_WARMUP_GRACE_PERIOD" + else + echo "NOTE: current aiperf does not support --warmup-grace-period; skipping AIPERF_WARMUP_GRACE_PERIOD=$AIPERF_WARMUP_GRACE_PERIOD." \ + | tee "$RESULT_DIR/aiperf_warmup_grace_flag_note.txt" + fi +fi +REPLAY_CMD+=" --use-server-token-count" +REPLAY_CMD+=" --no-gpu-telemetry" +REPLAY_CMD+=" --tokenizer-trust-remote-code" +if [ "$AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING" = "1" ]; then + if grep -q -- "--use-dynamo-conv-aware-routing" <<<"$AIPERF_PROFILE_HELP"; then + REPLAY_CMD+=" --use-dynamo-conv-aware-routing" + else + echo "ERROR: current aiperf does not support --use-dynamo-conv-aware-routing." >&2 + exit 1 + fi + if [ -n "$AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS" ]; then + if ! [[ "$AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS must be a positive integer." >&2 + exit 1 + fi + if grep -q -- "--dynamo-session-timeout-seconds" <<<"$AIPERF_PROFILE_HELP"; then + REPLAY_CMD+=" --dynamo-session-timeout-seconds $AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS" + else + echo "ERROR: current aiperf does not support --dynamo-session-timeout-seconds." >&2 + exit 1 + fi + fi + if [ "$AIPERF_USE_LEGACY_DYNAMO_SESSION_CONTROL" = "1" ]; then + if grep -q -- "--use-legacy-dynamo-session-control" <<<"$AIPERF_PROFILE_HELP"; then + REPLAY_CMD+=" --use-legacy-dynamo-session-control" + else + echo "ERROR: current aiperf does not support --use-legacy-dynamo-session-control." >&2 + exit 1 + fi + fi +elif [ "$AIPERF_USE_LEGACY_DYNAMO_SESSION_CONTROL" = "1" ]; then + echo "ERROR: legacy Dynamo session control requires AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=1." >&2 + exit 1 +fi +if grep -q -- "--vllm-start-profile-after-seconds" <<<"$AIPERF_PROFILE_HELP"; then + REPLAY_CMD+=" --vllm-start-profile-after-seconds $VLLM_START_PROFILE_AFTER_SECONDS" +else + echo "NOTE: current aiperf does not support --vllm-start-profile-after-seconds; skipping it." \ + | tee "$RESULT_DIR/aiperf_profile_flag_note.txt" +fi +if [ -n "$AIPERF_MAX_OSL" ] && [ "$AIPERF_MAX_OSL" != "none" ]; then + if grep -q -- "--trace-max-osl" <<<"$AIPERF_PROFILE_HELP"; then + REPLAY_CMD+=" --trace-max-osl $AIPERF_MAX_OSL" + elif grep -q -- "--synthesis-max-osl" <<<"$AIPERF_PROFILE_HELP"; then + REPLAY_CMD+=" --synthesis-max-osl $AIPERF_MAX_OSL" + else + echo "NOTE: current aiperf does not support max OSL capping; skipping AIPERF_MAX_OSL=$AIPERF_MAX_OSL." \ + | tee "$RESULT_DIR/aiperf_max_osl_flag_note.txt" + fi +fi +if grep -q -- "--max-context-length" <<<"$AIPERF_PROFILE_HELP"; then + REPLAY_CMD+=" --max-context-length $MAX_CONTEXT_LENGTH" +else + echo "NOTE: current aiperf does not support --max-context-length; skipping MAX_CONTEXT_LENGTH=$MAX_CONTEXT_LENGTH." \ + | tee "$RESULT_DIR/aiperf_max_context_flag_note.txt" +fi +REPLAY_CMD+=" --num-dataset-entries $NUM_DATASET_ENTRIES" +REPLAY_CMD+=" --slice-duration $SLICE_DURATION" +REPLAY_CMD+=" --output-artifact-dir $RESULT_DIR/aiperf_artifacts" +if [ "$DURATION" -lt 900 ] || [ "${AIPERF_UNSAFE_OVERRIDE:-false}" = "true" ]; then + REPLAY_CMD+=" --unsafe-override" +fi +if [ -n "$HF_WEKA_DATASET" ]; then + REPLAY_CMD+=" --hf-weka-dataset $HF_WEKA_DATASET" +else + REPLAY_CMD+=" --public-dataset $PUBLIC_DATASET" +fi + +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/patches/vllm-mooncake-mtp-eagle-group-split-patch.sh b/configs/patches/vllm-mooncake-mtp-eagle-group-split-patch.sh new file mode 100644 index 000000000..ffce6591b --- /dev/null +++ b/configs/patches/vllm-mooncake-mtp-eagle-group-split-patch.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Keep same-spec eagle and non-eagle physical KV groups separate in Mooncake's +# coordinator (see vllm_mooncake_mtp_eagle_group_split.py). +# Transitional c188b96 workaround only. vllm-internal e5827fabc1 + +# 89510b05dc are the official fixes; do not carry this patch into newer images. +set -euo pipefail + +python3 /configs/patches/vllm_mooncake_mtp_eagle_group_split.py diff --git a/configs/patches/vllm_mooncake_mtp_eagle_group_split.py b/configs/patches/vllm_mooncake_mtp_eagle_group_split.py new file mode 100644 index 000000000..8334cfd5a --- /dev/null +++ b/configs/patches/vllm_mooncake_mtp_eagle_group_split.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Split mixed eagle/non-eagle Mooncake attention groups. + +DeepSeek V4 + MTP marks the physical KV group containing the MTP layer as an +eagle group. In c188b96, Mooncake's coordinator merges physical groups solely +by KVCacheSpec. That coalesces the eagle SWA group with three non-eagle SWA +groups, even though their reachable-block proofs differ by one peek block. +The merged pool requires every physical group to contain every candidate hash, +so the non-eagle groups remove the eagle peek block before eagle verification +can consume it. The resulting cross-attention-group intersection is always +empty and the Mooncake master reports Get=0. + +Keep all upstream eagle semantics, but make eagle membership part of the +coordinator's grouping key. This produces separate same-spec eagle and +non-eagle attention groups whose proofs can converge correctly. + +Runs inside the worker container after vLLM is installed (post_install_script). +It is deliberately anchored to the c188b96 source shape and fails closed when +that shape changes. + +Transition only: vllm-internal PRs #275 (e5827fabc1) and #287 (89510b05dc) +supersede this workaround. Do not upstream this group-split patch. Remove it +when the runtime image contains both official fixes (89510b05dc or a +descendant). +""" + +from __future__ import annotations + +import argparse +import importlib +from pathlib import Path + +MARKER = "SRT patch(mtp-mooncake-eagle-group-split)" + +MODULE = "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator" + +OLD_GROUPING = """ for existing_spec, group_ids, existing_cls in attention_groups: + if existing_spec == spec: + assert manager_cls is existing_cls + group_ids.append(i) + break""" + +NEW_GROUPING = """ # SRT patch(mtp-mooncake-eagle-group-split): groups with + # identical specs can still require different reachable-block + # proofs. Do not merge an eagle physical group with non-eagle + # groups, or the shared block-pool intersection drops its peek. + is_eagle_group = i in self.eagle_group_ids + for existing_spec, group_ids, existing_cls in attention_groups: + existing_is_eagle_group = group_ids[0] in self.eagle_group_ids + if ( + existing_spec == spec + and existing_is_eagle_group == is_eagle_group + ): + assert manager_cls is existing_cls + group_ids.append(i) + break""" + + +def patch_text(text: str, source: str) -> str: + """Return patched coordinator source, failing on an unexpected shape.""" + if MARKER in text: + return text + + anchor_count = text.count(OLD_GROUPING) + if anchor_count != 1: + raise RuntimeError( + f"expected exactly one Mooncake grouping anchor in {source}, " + f"found {anchor_count}" + ) + + patched = text.replace(OLD_GROUPING, NEW_GROUPING, 1) + compile(patched, source, "exec") + + required = ( + MARKER, + "is_eagle_group = i in self.eagle_group_ids", + "existing_is_eagle_group = group_ids[0] in self.eagle_group_ids", + "existing_is_eagle_group == is_eagle_group", + ) + missing = [needle for needle in required if needle not in patched] + if missing: + raise RuntimeError(f"patched source validation failed: missing {missing}") + return patched + + +def resolve_installed_path() -> Path: + module = importlib.import_module(MODULE) + if module.__file__ is None: + raise RuntimeError(f"module {MODULE} has no __file__") + return Path(module.__file__) + + +def run_self_test() -> None: + fixture = ( + "def split(self, attention_groups, spec, manager_cls, i):\n" + " if True:\n" + " if True:\n" + f"{OLD_GROUPING}\n" + ) + patched = patch_text(fixture, "") + assert patched.count(MARKER) == 1 + assert patched == patch_text(patched, "") + print("Mooncake eagle-group split patch self-test passed") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--path", + type=Path, + help="coordinator.py to validate/patch instead of importing installed vLLM", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="validate transformed syntax without writing the target", + ) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + run_self_test() + return + + path = args.path if args.path is not None else resolve_installed_path() + text = path.read_text() + patched = patch_text(text, str(path)) + + if text == patched: + print(f"Mooncake eagle-group split patch already applied -> {path}") + return + if args.dry_run: + print(f"Mooncake eagle-group split patch dry-run passed -> {path}") + return + + path.write_text(patched) + if path.read_text() != patched: + raise RuntimeError(f"failed to verify patched contents in {path}") + print(f"Patched Mooncake coordinator eagle/non-eagle grouping -> {path}") + + +if __name__ == "__main__": + main() diff --git a/recipes/vllm/deepseek-v4-pro/GB200/agentic/2p1d-pdep8-ddep12-c1024-062126.yaml b/recipes/vllm/deepseek-v4-pro/GB200/agentic/2p1d-pdep8-ddep12-c1024-062126.yaml new file mode 100644 index 000000000..6770545e2 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/GB200/agentic/2p1d-pdep8-ddep12-c1024-062126.yaml @@ -0,0 +1,232 @@ +# DeepSeek V4 Pro AgentX 062126 configuration derived from GB200 job 9723. +# Default per-GPU launch uses NVSHMEM for P/D MegaMoE symmetric memory. +# Connector prefetch and the DeepGEMM EP12 fix are built into the image. +name: "2p1d-pdep8-ddep12-c1024-062126" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "e487326ec2cf66cd7ab52cf6c3675b84df23e42e" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + hash: "e487326ec2cf66cd7ab52cf6c3675b84df23e42e" + install: true + +setup_script: "vllm-container-deps.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 3 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 12 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +# Host paths below are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf-mmap-cache: /aiperf_mmap_cache + /path/to/hf-models/hub: /hf_hub_cache + +backend: + type: vllm + connector: null + # Omit launch-mode overrides to use srt-slurm's default per-GPU layout. + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_1,mlx5_3,mlx5_4" # Set to the local RDMA devices. + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_1:1,mlx5_3:1,mlx5_4:1" + NCCL_IB_HCA: "mlx5_1,mlx5_3,mlx5_4" + UCX_TLS: "rc,cuda_copy" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_1:1,mlx5_3:1,mlx5_4:1" + NCCL_IB_HCA: "mlx5_1,mlx5_3,mlx5_4" + UCX_TLS: "rc,cuda_copy" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + numa-bind: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 120 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + numa-bind: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + +sbatch_directives: + cpus-per-task: "72" + +benchmark: + type: custom + command: bash /configs/agentx_aiperf.sh + env: + AIPERF_DIR: /workspace/srt-slurm-sa/aiperf + AIPERF_VENV: /workspace/srt-slurm-sa/aiperf/.venv + RESULT_DIR: /logs/agentic + AGENTIC_OUTPUT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "1024" + DURATION: "3600" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_2xpdep8_1xddep12_allpergpu_fastokens_numabind_megamoe_fp8c_nvshmem_fp4idx_agentx062126_c1024" + RUNNER_TYPE: "gb200" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "none" + DISAGG: "true" + OFFLOADING: "none" + TP: "8" + EP_SIZE: "12" + DP_ATTENTION: "false" + PREFILL_NUM_WORKERS: "2" + PREFILL_TP: "1" + PREFILL_EP: "8" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "12" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_MAX_OSL: "none" + SLICE_DURATION: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/recipes/vllm/deepseek-v4-pro/GB200/agentic/2p1d-pdep8-ddep12-c640-062126.yaml b/recipes/vllm/deepseek-v4-pro/GB200/agentic/2p1d-pdep8-ddep12-c640-062126.yaml new file mode 100644 index 000000000..3ebcee241 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/GB200/agentic/2p1d-pdep8-ddep12-c640-062126.yaml @@ -0,0 +1,233 @@ +# DeepSeek V4 Pro AgentX 062126 P/D MegaMoE reference derived from job 9637. +# The requested FP4 indexer-cache attention update is not validated by that run. +# Requires NVIDIA/srt-slurm#90 for per-role DP launch modes. +# Connector-prefetch support is built into the selected vLLM image. +name: "2p1d-pdep8-ddep12-c640-062126" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + frameworks: + dynamo: "e487326ec2cf66cd7ab52cf6c3675b84df23e42e" + vllm: "0.17.2rc1.dev3669+g515d6e98c" + +dynamo: + hash: "e487326ec2cf66cd7ab52cf6c3675b84df23e42e" + install: true + +setup_script: "vllm-container-deps.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 3 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 12 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +# Host paths below are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf-mmap-cache: /aiperf_mmap_cache + /path/to/hf-models/hub: /hf_hub_cache + +backend: + type: vllm + connector: null + # Keep MegaMoE prefill collectives in per-node processes while exposing each + # decode DP rank as an independent Dynamo worker. + dp_launch_mode: per_node + prefill_dp_launch_mode: per_node + decode_dp_launch_mode: per_gpu + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_1,mlx5_3,mlx5_4" # Set to the local RDMA devices. + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_1:1,mlx5_3:1,mlx5_4:1" + NCCL_IB_HCA: "mlx5_1,mlx5_3,mlx5_4" + UCX_TLS: "rc,cuda_copy" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_1:1,mlx5_3:1,mlx5_4:1" + NCCL_IB_HCA: "mlx5_1,mlx5_3,mlx5_4" + UCX_TLS: "rc,cuda_copy" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 120 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + +sbatch_directives: + cpus-per-task: "72" + +benchmark: + type: custom + command: bash /configs/agentx_aiperf.sh + env: + AIPERF_DIR: /workspace/srt-slurm-sa/aiperf + AIPERF_VENV: /workspace/srt-slurm-sa/aiperf/.venv + RESULT_DIR: /logs/agentic + AGENTIC_OUTPUT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "640" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_2xpdep8_1xddep12_megamoe_agentx062126_c640" + AIPERF_MAX_OSL: "none" + RUNNER_TYPE: "gb200" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + SPEC_DECODING: "none" + DISAGG: "true" + OFFLOADING: "none" + TP: "8" + EP_SIZE: "12" + DP_ATTENTION: "false" + PREFILL_NUM_WORKERS: "2" + PREFILL_TP: "1" + PREFILL_EP: "8" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "12" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + SLICE_DURATION: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/recipes/vllm/deepseek-v4-pro/GB200/agentic/3p1d-pdep8-ddep16-c1280-062126.yaml b/recipes/vllm/deepseek-v4-pro/GB200/agentic/3p1d-pdep8-ddep16-c1280-062126.yaml new file mode 100644 index 000000000..275a1a4be --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/GB200/agentic/3p1d-pdep8-ddep16-c1280-062126.yaml @@ -0,0 +1,230 @@ +# DeepSeek V4 Pro AgentX 062126 throughput configuration derived from job 9700. +# The forward configuration retains FP4 indexer cache on both roles. +# Requires NVIDIA/srt-slurm#90 for explicit per-role DP launch modes. +name: "3p1d-pdep8-ddep16-c1280-062126" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + frameworks: + dynamo: "e487326ec2cf66cd7ab52cf6c3675b84df23e42e" + vllm: "0.17.2rc1.dev3669+g515d6e98c" + +dynamo: + hash: "e487326ec2cf66cd7ab52cf6c3675b84df23e42e" + install: true + +setup_script: "vllm-container-deps.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +# Host paths below are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf-mmap-cache: /aiperf_mmap_cache + /path/to/hf-models/hub: /hf_hub_cache + +backend: + type: vllm + connector: null + # Keep both roles in shared per-node CUDA namespaces for MegaMoE. + dp_launch_mode: per_node + prefill_dp_launch_mode: per_node + decode_dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_1,mlx5_3,mlx5_4" # Set to the local RDMA devices. + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_1:1,mlx5_3:1,mlx5_4:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_1,mlx5_3,mlx5_4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_1:1,mlx5_3:1,mlx5_4:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_1,mlx5_3,mlx5_4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + numa-bind: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 120 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + numa-bind: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + +sbatch_directives: + cpus-per-task: "72" + +benchmark: + type: custom + command: bash /configs/agentx_aiperf.sh + env: + AIPERF_DIR: /workspace/srt-slurm-sa/aiperf + AIPERF_VENV: /workspace/srt-slurm-sa/aiperf/.venv + RESULT_DIR: /logs/agentic + AGENTIC_OUTPUT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "1280" + DURATION: "3600" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_3xpdep8_1xddep16_pnode-dpnode_fastokens_numabind_megamoe_nofp8c_fp4idx_agentx062126_c1280" + RUNNER_TYPE: "gb200" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + SPEC_DECODING: "none" + DISAGG: "true" + OFFLOADING: "none" + TP: "8" + EP_SIZE: "16" + DP_ATTENTION: "false" + PREFILL_NUM_WORKERS: "3" + PREFILL_TP: "1" + PREFILL_EP: "8" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "16" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_MAX_OSL: "none" + SLICE_DURATION: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/recipes/vllm/deepseek-v4-pro/GB200/agentic/3p1d-pdep8-ddep16-c1280-megamoe-pergpu-062126.yaml b/recipes/vllm/deepseek-v4-pro/GB200/agentic/3p1d-pdep8-ddep16-c1280-megamoe-pergpu-062126.yaml new file mode 100644 index 000000000..9e16fefe4 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/GB200/agentic/3p1d-pdep8-ddep16-c1280-megamoe-pergpu-062126.yaml @@ -0,0 +1,231 @@ +# DeepSeek V4 Pro AgentX 062126 all-per-GPU P/D MegaMoE control. +# This keeps the job 9700 service and frontend configuration, but uses the +# default per-GPU launcher with NVSHMEM. FP4 indexer cache remains enabled. +name: "3p1d-pdep8-ddep16-c1280-megamoe-pergpu-062126" + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + frameworks: + dynamo: "e487326ec2cf66cd7ab52cf6c3675b84df23e42e" + vllm: "0.17.2rc1.dev3669+g515d6e98c" + +dynamo: + hash: "e487326ec2cf66cd7ab52cf6c3675b84df23e42e" + install: true + +setup_script: "vllm-container-deps.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +# Host paths below are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf-mmap-cache: /aiperf_mmap_cache + /path/to/hf-models/hub: /hf_hub_cache + +backend: + type: vllm + connector: null + # Omit launch-mode overrides to use srt-slurm's default per-GPU layout. + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_1,mlx5_3,mlx5_4" # Set to the local RDMA devices. + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + TORCH_SYMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_1:1,mlx5_3:1,mlx5_4:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_1,mlx5_3,mlx5_4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + TORCH_SYMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_1:1,mlx5_3:1,mlx5_4:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_1,mlx5_3,mlx5_4" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + numa-bind: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 120 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + numa-bind: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + +sbatch_directives: + cpus-per-task: "72" + +benchmark: + type: custom + command: bash /configs/agentx_aiperf.sh + env: + AIPERF_DIR: /workspace/srt-slurm-sa/aiperf + AIPERF_VENV: /workspace/srt-slurm-sa/aiperf/.venv + RESULT_DIR: /logs/agentic + AGENTIC_OUTPUT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "1280" + DURATION: "3600" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_3xpdep8_1xddep16_allpergpu_fastokens_numabind_megamoe_nofp8c_nvshmem_fp4idx_agentx062126_c1280" + RUNNER_TYPE: "gb200" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + SPEC_DECODING: "none" + DISAGG: "true" + OFFLOADING: "none" + TP: "8" + EP_SIZE: "16" + DP_ATTENTION: "false" + PREFILL_NUM_WORKERS: "3" + PREFILL_TP: "1" + PREFILL_EP: "8" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "16" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_MAX_OSL: "none" + SLICE_DURATION: "1.0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/recipes/vllm/deepseek-v4-pro/GB300/agentic/1p1d-pdep4-ddep8-c256-062126.yaml b/recipes/vllm/deepseek-v4-pro/GB300/agentic/1p1d-pdep4-ddep8-c256-062126.yaml new file mode 100644 index 000000000..a6d583994 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/GB300/agentic/1p1d-pdep4-ddep8-c256-062126.yaml @@ -0,0 +1,272 @@ +# DeepSeek V4 Pro AgentX 062126 GB300 scaling reference derived from completed +# job 534. It preserves the single fastokens frontend, per-node P/D launch, +# P/D MegaMoE, NUMA binding, and tuned memory/scheduler limits used for +# 1xPDEP4 + 1xDDEP8 at c256. +name: "1p1d-pdep4-ddep8-c256-062126" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + # Commit pointed to by the public v1.2.1 tag used by job 534's wheel. + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: "vllm-container-deps.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/aiperf-mmap-cache: /aiperf_mmap_cache + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # Match job 534's one process per physical node for both DP roles. + # Requires NVIDIA/srt-slurm#90. + dp_launch_mode: per_node + prefill_dp_launch_mode: per_node + decode_dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "256" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_1xpdep4_1xddep8_1fe_fastok_pernode_megamoe_numa_agentx062126_c256" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "none" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "8" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_MAX_OSL: "none" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "1" + PREFILL_TP: "1" + PREFILL_EP: "4" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "8" + DECODE_DP_ATTN: "false" diff --git a/recipes/vllm/deepseek-v4-pro/GB300/agentic/1p1d-pdep4-ddep8-c256-job564-dynamo130dev20260720-regression.yaml b/recipes/vllm/deepseek-v4-pro/GB300/agentic/1p1d-pdep4-ddep8-c256-job564-dynamo130dev20260720-regression.yaml new file mode 100644 index 000000000..5865f2b73 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/GB300/agentic/1p1d-pdep4-ddep8-c256-job564-dynamo130dev20260720-regression.yaml @@ -0,0 +1,267 @@ +# Completed GB300 job 564 regression of the job 534 topology on Dynamo +# 1.3.0.dev20260720. This portable reference preserves the single fastokens +# frontend, uniform per-node P/D launch, P/D MegaMoE, NUMA binding, and the +# tuned memory/scheduler limits used for 1xPDEP4 + 1xDDEP8 at c256. +name: "1p1d-pdep4-ddep8-c256-job564-dynamo130dev20260720-regression" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.3.0.dev20260720" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + version: "1.3.0.dev20260720" + install: true + +setup_script: "vllm-container-deps.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/aiperf-mmap-cache: /aiperf_mmap_cache + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # Match job 564's one process per physical node for both DP roles. + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "256" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_1xpdep4_1xddep8_1fe_fastok_pernode_megamoe_numa_agentx062126_c256_job564_dyn130dev20260720_regression" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "none" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "8" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_MAX_OSL: "none" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "1" + PREFILL_TP: "1" + PREFILL_EP: "4" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "8" + DECODE_DP_ATTN: "false" diff --git a/recipes/vllm/deepseek-v4-pro/GB300/agentic/1p1d-pdep8-ddep8-c512-062126.yaml b/recipes/vllm/deepseek-v4-pro/GB300/agentic/1p1d-pdep8-ddep8-c512-062126.yaml new file mode 100644 index 000000000..281419a1c --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/GB300/agentic/1p1d-pdep8-ddep8-c512-062126.yaml @@ -0,0 +1,274 @@ +# DeepSeek V4 Pro AgentX 062126 GB300 scaling reference derived from job 522. +# It preserves job 522's single fastokens frontend, per-node P/D launch, +# P/D MegaMoE, and NUMA binding. This exact topology and concurrency have not +# yet been performance-validated. +name: "1p1d-pdep8-ddep8-c512-062126" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3669+g515d6e98c" + +dynamo: + # Commit pointed to by the public v1.2.1 tag used by job 522's wheel. + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: "vllm-container-deps.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/aiperf-mmap-cache: /aiperf_mmap_cache + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # Match job 522's one process per physical node for both DP roles. + # Requires NVIDIA/srt-slurm#90. + dp_launch_mode: per_node + prefill_dp_launch_mode: per_node + decode_dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + TORCH_SYMMMEM: "NVSHMEM" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + TORCH_SYMMMEM: "NVSHMEM" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "512" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_1xpdep8_1xddep8_1fe_fastok_pernode_megamoe_numa_agentx062126_c512" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-515d6e9" + SPEC_DECODING: "none" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "8" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_MAX_OSL: "none" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "1" + PREFILL_TP: "1" + PREFILL_EP: "8" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "8" + DECODE_DP_ATTN: "false" diff --git a/recipes/vllm/deepseek-v4-pro/GB300/agentic/2p1d-pdep8-ddep8-c1280-062126.yaml b/recipes/vllm/deepseek-v4-pro/GB300/agentic/2p1d-pdep8-ddep8-c1280-062126.yaml new file mode 100644 index 000000000..d1be2e755 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/GB300/agentic/2p1d-pdep8-ddep8-c1280-062126.yaml @@ -0,0 +1,272 @@ +# DeepSeek V4 Pro AgentX 062126 GB300 scaling reference derived from completed +# job 529. It preserves the single fastokens frontend, per-node P/D launch, +# P/D MegaMoE, NUMA binding, and tuned memory/scheduler limits used for +# 2xPDEP8 + 1xDDEP8 at c1280. +name: "2p1d-pdep8-ddep8-c1280-062126" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + # Commit pointed to by the public v1.2.1 tag used by job 529's wheel. + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: "vllm-container-deps.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 4 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/aiperf-mmap-cache: /aiperf_mmap_cache + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # Match job 529's one process per physical node for both DP roles. + # Requires NVIDIA/srt-slurm#90. + dp_launch_mode: per_node + prefill_dp_launch_mode: per_node + decode_dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "1280" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_2xpdep8_1xddep8_1fe_fastok_pernode_megamoe_numa_agentx062126_c1280" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "none" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "8" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_MAX_OSL: "none" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "2" + PREFILL_TP: "1" + PREFILL_EP: "8" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "8" + DECODE_DP_ATTN: "false" diff --git a/recipes/vllm/deepseek-v4-pro/GB300/agentic/3p1d-pdep8-ddep16-c1536-062126.yaml b/recipes/vllm/deepseek-v4-pro/GB300/agentic/3p1d-pdep8-ddep16-c1536-062126.yaml new file mode 100644 index 000000000..6b231594d --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/GB300/agentic/3p1d-pdep8-ddep16-c1536-062126.yaml @@ -0,0 +1,272 @@ +# DeepSeek V4 Pro AgentX 062126 GB300 scaling reference derived from completed +# job 527. It preserves the single fastokens frontend, per-node P/D launch, +# P/D MegaMoE, NUMA binding, and the patch-free c188b96 runtime used for +# 3xPDEP8 + 1xDDEP16 at c1536. +name: "3p1d-pdep8-ddep16-c1536-062126" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + # Commit pointed to by the public v1.2.1 tag used by job 527's wheel. + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: "vllm-container-deps.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/aiperf-mmap-cache: /aiperf_mmap_cache + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # Match job 527's one process per physical node for both DP roles. + # Requires NVIDIA/srt-slurm#90. + dp_launch_mode: per_node + prefill_dp_launch_mode: per_node + decode_dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "1536" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_3xpdep8_1xddep16_1fe_fastok_pernode_megamoe_numa_agentx062126_c1536" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "none" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "16" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + AIPERF_MAX_OSL: "none" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "3" + PREFILL_TP: "1" + PREFILL_EP: "8" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "16" + DECODE_DP_ATTN: "false" diff --git a/recipes/vllm/deepseek-v4-pro/gb300-mtp/1p1d-pdep4-ddep8-mtp-c128-062126.yaml b/recipes/vllm/deepseek-v4-pro/gb300-mtp/1p1d-pdep4-ddep8-mtp-c128-062126.yaml new file mode 100644 index 000000000..6edcdd138 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/gb300-mtp/1p1d-pdep4-ddep8-mtp-c128-062126.yaml @@ -0,0 +1,291 @@ +# DeepSeek V4 Pro AgentX 062126 GB300 MTP low-latency reference (source job 149, +# 12 serving GPUs). Validated: 72.05k active total tok/s per GPU, 6.43 req/s, +# p50/p99 TTFT 5.91s/38.9s, p50/p90 ITL 17.65ms/18.67ms, actual prefix cache +# hit 96.5% vs 97.1% theoretical, 11,769 requests at 0.14% errors. +# +# MTP requires three additions over the non-MTP references, all present below: +# 1. On the pinned c188b96 image, DeepSeek V4 + MTP zeroes all MooncakeStore +# external reads (the eagle-flagged KV group is merged with same-spec +# non-eagle groups and the cross-group hit intersection collapses). +# post_install_script applies the transitional eagle-group-split patch. +# Images containing vllm-internal e5827fabc1 + 89510b05dc have the +# official fixes; drop post_install_script there. +# 2. PYTHONHASHSEED=0 keeps prefix-block hashes (Mooncake keys) consistent +# across processes and nodes. +# 3. DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS=3600 keeps long AgentX requests +# alive past the kv-router's 300s default active-request expiry. +name: "1p1d-pdep4-ddep8-mtp-c128-062126" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.3.0.dev20260720" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + version: "1.3.0.dev20260720" + install: true + +setup_script: "vllm-container-deps.sh" +# Transitional MTP Mooncake read-path fix for c188b96; remove on images that +# contain vllm-internal e5827fabc1 + 89510b05dc. +post_install_script: "vllm-mooncake-mtp-eagle-group-split-patch.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +# Cross-process prefix-block hash (Mooncake key) consistency. +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + env: + # AgentX requests can stay in flight far longer than the kv-router's + # 300s default active-request expiry. + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # One process per physical node for both DP roles. + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "128" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_1xpdep4_1xddep8_mtp_agentx062126_c128" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "mtp" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "8" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_MAX_OSL: "none" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "1" + PREFILL_TP: "1" + PREFILL_EP: "4" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "8" + DECODE_DP_ATTN: "false" diff --git a/recipes/vllm/deepseek-v4-pro/gb300-mtp/1p1d-pdep8-ddep8-mtp-c384-062126.yaml b/recipes/vllm/deepseek-v4-pro/gb300-mtp/1p1d-pdep8-ddep8-mtp-c384-062126.yaml new file mode 100644 index 000000000..74785d207 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/gb300-mtp/1p1d-pdep8-ddep8-mtp-c384-062126.yaml @@ -0,0 +1,291 @@ +# DeepSeek V4 Pro AgentX 062126 GB300 MTP efficiency reference (source job 133, +# 16 serving GPUs). Validated: 110.08k active total tok/s per GPU, 12.91 req/s, +# p50/p99 TTFT 17.09s/65.3s, p90 ITL 24.70ms, actual prefix cache hit 96.0% +# vs 96.7% theoretical, 0.088% errors. +# +# MTP requires three additions over the non-MTP references, all present below: +# 1. On the pinned c188b96 image, DeepSeek V4 + MTP zeroes all MooncakeStore +# external reads (the eagle-flagged KV group is merged with same-spec +# non-eagle groups and the cross-group hit intersection collapses). +# post_install_script applies the transitional eagle-group-split patch. +# Images containing vllm-internal e5827fabc1 + 89510b05dc have the +# official fixes; drop post_install_script there. +# 2. PYTHONHASHSEED=0 keeps prefix-block hashes (Mooncake keys) consistent +# across processes and nodes. +# 3. DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS=3600 keeps long AgentX requests +# alive past the kv-router's 300s default active-request expiry. +name: "1p1d-pdep8-ddep8-mtp-c384-062126" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.3.0.dev20260720" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + version: "1.3.0.dev20260720" + install: true + +setup_script: "vllm-container-deps.sh" +# Transitional MTP Mooncake read-path fix for c188b96; remove on images that +# contain vllm-internal e5827fabc1 + 89510b05dc. +post_install_script: "vllm-mooncake-mtp-eagle-group-split-patch.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +# Cross-process prefix-block hash (Mooncake key) consistency. +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + env: + # AgentX requests can stay in flight far longer than the kv-router's + # 300s default active-request expiry. + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # One process per physical node for both DP roles. + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "384" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_1xpdep8_1xddep8_mtp_agentx062126_c384" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "mtp" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "8" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_MAX_OSL: "none" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "1" + PREFILL_TP: "1" + PREFILL_EP: "8" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "8" + DECODE_DP_ATTN: "false" diff --git a/recipes/vllm/deepseek-v4-pro/gb300-mtp/2p1d-pdep8-ddep12-mtp-c1024-062126.yaml b/recipes/vllm/deepseek-v4-pro/gb300-mtp/2p1d-pdep8-ddep12-mtp-c1024-062126.yaml new file mode 100644 index 000000000..55372fa92 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/gb300-mtp/2p1d-pdep8-ddep12-mtp-c1024-062126.yaml @@ -0,0 +1,291 @@ +# DeepSeek V4 Pro AgentX 062126 GB300 MTP throughput-ceiling reference (source +# job 151, 28 serving GPUs). Validated: 125.53k active total tok/s per GPU, +# 25.70 req/s, p50/p99 TTFT 28.75s/94.4s, p90 ITL 32.31ms, actual prefix cache +# hit 95.7% vs 96.8% theoretical, 0.108% errors. +# +# MTP requires three additions over the non-MTP references, all present below: +# 1. On the pinned c188b96 image, DeepSeek V4 + MTP zeroes all MooncakeStore +# external reads (the eagle-flagged KV group is merged with same-spec +# non-eagle groups and the cross-group hit intersection collapses). +# post_install_script applies the transitional eagle-group-split patch. +# Images containing vllm-internal e5827fabc1 + 89510b05dc have the +# official fixes; drop post_install_script there. +# 2. PYTHONHASHSEED=0 keeps prefix-block hashes (Mooncake keys) consistent +# across processes and nodes. +# 3. DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS=3600 keeps long AgentX requests +# alive past the kv-router's 300s default active-request expiry. +name: "2p1d-pdep8-ddep12-mtp-c1024-062126" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.3.0.dev20260720" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + version: "1.3.0.dev20260720" + install: true + +setup_script: "vllm-container-deps.sh" +# Transitional MTP Mooncake read-path fix for c188b96; remove on images that +# contain vllm-internal e5827fabc1 + 89510b05dc. +post_install_script: "vllm-mooncake-mtp-eagle-group-split-patch.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 4 + decode_nodes: 3 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 12 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +# Cross-process prefix-block hash (Mooncake key) consistency. +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + env: + # AgentX requests can stay in flight far longer than the kv-router's + # 300s default active-request expiry. + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # One process per physical node for both DP roles. + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + max-cudagraph-capture-size: 512 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "1024" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_2xpdep8_1xddep12_mtp_agentx062126_c1024" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "mtp" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "12" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_MAX_OSL: "none" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "2" + PREFILL_TP: "1" + PREFILL_EP: "8" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "12" + DECODE_DP_ATTN: "false" diff --git a/recipes/vllm/deepseek-v4-pro/gb300-mtp/2p1d-pdep8-ddep16-mtp-c256-random-session-062126.yaml b/recipes/vllm/deepseek-v4-pro/gb300-mtp/2p1d-pdep8-ddep16-mtp-c256-random-session-062126.yaml new file mode 100644 index 000000000..edeac0e7d --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/gb300-mtp/2p1d-pdep8-ddep16-mtp-c256-random-session-062126.yaml @@ -0,0 +1,294 @@ +# DeepSeek V4 Pro AgentX 062126 GB300 MTP random-router reference (source +# job 200, 32 serving GPUs). Validated: 1.943M active total tok/s +# (60.70k tok/s/GPU), 12.35 req/s, p50/p99 TTFT 5.67s/44.30s, p50/p90/p99 +# ITL 16.58ms/18.32ms/21.72ms, actual prefix cache hit 96.71% vs 97.29% +# theoretical, 0.127% errors. +# +# The first request in each session is distributed randomly; subsequent +# requests carrying X-Dynamo-Session-ID stay on the selected worker for 900s. +# AIPerf derives this header from correlation_id below. Random routing does not +# consume vLLM KV events, so backend.kv_events_config is intentionally absent. +# +# MTP requires three additions over the non-MTP references, all present below: +# 1. On the pinned c188b96 image, DeepSeek V4 + MTP zeroes all MooncakeStore +# external reads (the eagle-flagged KV group is merged with same-spec +# non-eagle groups and the cross-group hit intersection collapses). +# post_install_script applies the transitional eagle-group-split patch. +# Images containing vllm-internal e5827fabc1 + 89510b05dc have the +# official fixes; drop post_install_script there. +# 2. PYTHONHASHSEED=0 keeps prefix-block hashes (Mooncake keys) consistent +# across processes and nodes. +# 3. DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS=3600 keeps long AgentX requests +# alive past the router's 300s default active-request expiry. +name: "2p1d-pdep8-ddep16-mtp-c256-random-session-062126" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.3.0.dev20260720" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + version: "1.3.0.dev20260720" + install: true + +setup_script: "vllm-container-deps.sh" +# Transitional MTP Mooncake read-path fix for c188b96; remove on images that +# contain vllm-internal e5827fabc1 + 89510b05dc. +post_install_script: "vllm-mooncake-mtp-eagle-group-split-patch.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 4 + decode_nodes: 4 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +# Cross-process prefix-block hash (Mooncake key) consistency. +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + # AgentX requests can stay in flight far longer than the router's + # 300s default active-request expiry. + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # One process per physical node for both DP roles. + dp_launch_mode: per_node + # Random + session-affinity routing does not require KV event publication. + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + # Capture sizes are tokens under MTP: 32 seqs * (1 target + 3 drafts). + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + # Capture sizes are tokens under MTP: 256 seqs * (1 target + 3 drafts). + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "256" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_2xpdep8_1xddep16_mtp_random_session_agentx062126_c256" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "mtp" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "16" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_MAX_OSL: "none" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "2" + PREFILL_TP: "1" + PREFILL_EP: "8" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "16" + DECODE_DP_ATTN: "false" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" diff --git a/recipes/vllm/deepseek-v4-pro/gb300-mtp/2p1d-pdep8-ddep8-mtp-c512-062126.yaml b/recipes/vllm/deepseek-v4-pro/gb300-mtp/2p1d-pdep8-ddep8-mtp-c512-062126.yaml new file mode 100644 index 000000000..8eff48bf8 --- /dev/null +++ b/recipes/vllm/deepseek-v4-pro/gb300-mtp/2p1d-pdep8-ddep8-mtp-c512-062126.yaml @@ -0,0 +1,291 @@ +# DeepSeek V4 Pro AgentX 062126 GB300 MTP balanced reference (source job 139, +# 24 serving GPUs). Validated: 104.32k active total tok/s per GPU, 18.35 req/s, +# p50/p99 TTFT 8.16s/49.4s, p90 ITL 30.81ms, actual prefix cache hit 96.2% +# vs 96.8% theoretical, 0.113% errors. +# +# MTP requires three additions over the non-MTP references, all present below: +# 1. On the pinned c188b96 image, DeepSeek V4 + MTP zeroes all MooncakeStore +# external reads (the eagle-flagged KV group is merged with same-spec +# non-eagle groups and the cross-group hit intersection collapses). +# post_install_script applies the transitional eagle-group-split patch. +# Images containing vllm-internal e5827fabc1 + 89510b05dc have the +# official fixes; drop post_install_script there. +# 2. PYTHONHASHSEED=0 keeps prefix-block hashes (Mooncake keys) consistent +# across processes and nodes. +# 3. DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS=3600 keeps long AgentX requests +# alive past the kv-router's 300s default active-request expiry. +name: "2p1d-pdep8-ddep8-mtp-c512-062126" + +model: + path: "deepseekv4-fp4" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.3.0.dev20260720" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + version: "1.3.0.dev20260720" + install: true + +setup_script: "vllm-container-deps.sh" +# Transitional MTP Mooncake read-path fix for c188b96; remove on images that +# contain vllm-internal e5827fabc1 + 89510b05dc. +post_install_script: "vllm-mooncake-mtp-eagle-group-split-patch.sh" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 4 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +# Cross-process prefix-block hash (Mooncake key) consistency. +environment: + PYTHONHASHSEED: "0" + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + env: + # AgentX requests can stay in flight far longer than the kv-router's + # 300s default active-request expiry. + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + +# Host paths are placeholders; point them at local checkouts and caches. +container_mounts: + /path/to/srt-slurm-sa: /workspace/srt-slurm-sa + /path/to/aiperf: /workspace/aiperf + /path/to/hf-models: /hf_models + +backend: + type: vllm + connector: null + # One process per physical node for both DP roles. + dp_launch_mode: per_node + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set to local RDMA devices. + mode: "embedded" + enable_offload: false + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method": "mtp", "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + + prefill_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-vllm-{node_id}/hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + TRANSFORMERS_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TMPDIR: "/tmp/srt-vllm-{node_id}" + TMP: "/tmp/srt-vllm-{node_id}" + TEMP: "/tmp/srt-vllm-{node_id}" + TRITON_CACHE_DIR: "/tmp/srt-vllm-{node_id}/triton-cache" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-inf-yasong-{job_id}" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" # Set locally. + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" # Set locally. + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: "bash /configs/agentx_aiperf.sh" + env: + AIPERF_DIR: "/workspace/aiperf" + AIPERF_VENV: "/workspace/aiperf/.venv" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "512" + DURATION: "1800" + RESULT_FILENAME: "dsv4_fp4_dynamo-vllm_gb300_2xpdep8_1xddep8_mtp_agentx062126_c512" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + SPEC_DECODING: "mtp" + DISAGG: "true" + OFFLOADING: "none" + TP: "1" + EP_SIZE: "8" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" + AIPERF_MAX_OSL: "none" + AIPERF_AGENTIC_CACHE_WARMUP_DURATION: "600" + SLICE_DURATION: "1.0" + VLLM_START_PROFILE_AFTER_SECONDS: "900" + HF_HOME: "/hf_models" + HF_TOKEN_PATH: "/tmp/srt-aiperf-hf-token" + HF_HUB_CACHE: "/hf_models/hub" + HUGGINGFACE_HUB_CACHE: "/hf_models/hub" + HF_HUB_DISABLE_XET: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + PREFILL_NUM_WORKERS: "2" + PREFILL_TP: "1" + PREFILL_EP: "8" + PREFILL_DP_ATTN: "false" + DECODE_NUM_WORKERS: "1" + DECODE_TP: "1" + DECODE_EP: "8" + DECODE_DP_ATTN: "false" diff --git a/tests/test_agentx_aiperf_script.py b/tests/test_agentx_aiperf_script.py new file mode 100644 index 000000000..82cee614c --- /dev/null +++ b/tests/test_agentx_aiperf_script.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + + +def _write_executable(path: Path, contents: str) -> None: + path.write_text(contents) + path.chmod(0o755) + + +def test_agentx_aiperf_driver_is_self_contained_and_applies_defaults(tmp_path: Path) -> None: + venv_bin = tmp_path / "venv" / "bin" + venv_bin.mkdir(parents=True) + capture = tmp_path / "aiperf-args.txt" + result_dir = tmp_path / "results" + + _write_executable( + venv_bin / "python", + """#!/usr/bin/env bash +cat >/dev/null +echo "Using fake aiperf" +""", + ) + _write_executable( + venv_bin / "aiperf", + """#!/usr/bin/env bash +if [ "${1:-}" = "profile" ] && [ "${2:-}" = "--help" ]; then + printf '%s\n' \ + '--warmup-grace-period' \ + '--agentic-cache-warmup-duration' \ + '--trace-max-osl' \ + '--max-context-length' \ + '--vllm-start-profile-after-seconds' + exit 0 +fi +printf '%s\n' "$@" > "$AIPERF_TEST_CAPTURE" +""", + ) + + env = os.environ.copy() + env.update( + { + "AIPERF_VENV": str(venv_bin.parent), + "AIPERF_TEST_CAPTURE": str(capture), + "RESULT_DIR": str(result_dir), + "MODEL": "deepseek-ai/DeepSeek-V4-Pro", + "MODEL_PREFIX": "dsv4", + "FRAMEWORK": "dynamo-vllm", + "PRECISION": "fp4", + "CONC": "8", + "RESULT_FILENAME": "agentx-test", + "DURATION": "1", + } + ) + env.pop("INFMAX_CONTAINER_WORKSPACE", None) + env.pop("AGENTIC_DIR", None) + + script = Path(__file__).parents[1] / "configs" / "agentx_aiperf.sh" + subprocess.run(["bash", str(script)], check=True, env=env, capture_output=True, text=True) + + args = capture.read_text().splitlines() + assert "--agentic-cache-warmup-duration" in args + assert args[args.index("--agentic-cache-warmup-duration") + 1] == "600" + assert "--warmup-grace-period" in args + assert args[args.index("--warmup-grace-period") + 1] == "1800" + assert "--trace-max-osl" in args + assert args[args.index("--trace-max-osl") + 1] == "8192" + assert "--max-context-length" in args + assert args[args.index("--max-context-length") + 1] == "1000000" + assert (result_dir / "env.txt").is_file() + assert (result_dir / "aiperf_command.txt").is_file() + assert (result_dir / "aiperf.log").is_file()