diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 2ba3d8923d..5fd8b94601 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1797,7 +1797,11 @@ build_replay_cmd() { # X-Correlation-ID is useful tracing metadata but does not establish that # binding by itself. AIPerf emits nvext.session_control bind/close actions # keyed by the stable conversation correlation ID when this flag is set. - if [[ "${FRAMEWORK:-}" == dynamo-* ]]; then + # Opt-out: recipes set AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 to skip this. + # aiperf's conv-aware routing emits nvext.session_control, a removed POC field + # (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400 + # (they moved to router/routing_constraints/agent_context). Default stays on. + if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then REPLAY_CMD+=" --use-dynamo-conv-aware-routing" # The upstream 300s affinity TTL is shorter than an overloaded # high-concurrency agentic request. Keep bindings alive across long diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml new file mode 100644 index 0000000000..8912ea8bf0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -0,0 +1,152 @@ +name: "svf-vllm-agg-gb200-tp8-agentic" + +# AgentX low-concurrency aggregate recipe for DeepSeek-V4-Pro on GB200. One +# TP8 vLLM worker spans two four-GPU nodes and serves both prefill and decode. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + aggregated_environment: + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + # Off: the c188b96 container's CUDA-graph capture path enters vLLM's + # nccl_symm_mem_context without set_graph_pool_id, so capture asserts + # "graph_pool_id is not set under graph capture". Re-enable once this recipe + # moves to a container built from a vLLM with the cuda_graph.py fix. + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-cumem-allocator: true + # numa-bind disabled: srt-slurm's cpus-per-task/cgroup already constrains + # CPU affinity, so vLLM's auto-NUMA detection bails and --numa-bind hard- + # errors ("could not detect the GPU-to-NUMA topology"). Re-enable only with + # an explicit --numa-bind-nodes mapping for this cluster. + # numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + # AgentX session fan-out can exceed the configured live-session + # concurrency; retain scheduler/graph headroom for subagent bursts. + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.95 + kv-cache-memory: 37580963840 # 35 GiB + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # One aggregate worker owns all eight GPUs. Use the aggregate accounting + # path instead of summing independent prefill and decode allocations. + # Aggregate (IS_MULTINODE=false) uses the single-node topology labels below + # directly (num_gpus = TP), so they are set here rather than CI-injected. + IS_MULTINODE: "false" + TP: "8" + EP_SIZE: "1" + DP_ATTENTION: "false" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml new file mode 100644 index 0000000000..8d09942af6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -0,0 +1,192 @@ +name: "svf-vllm-disagg-gb200-1p1d-dep8-dep12-agentic" + +# One DEP8 prefill worker feeding one DEP12 decode worker, with attention / +# expert parallelism matching each role and hybrid load balancing. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 12 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: "null" + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml new file mode 100644 index 0000000000..e367b81949 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -0,0 +1,192 @@ +name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-agentic" + +# One DEP8 prefill worker feeding one DEP8 decode worker, with attention DP8 / +# expert EP8 and hybrid load balancing on both roles. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: "null" + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 64 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml new file mode 100644 index 0000000000..4fb9fa7e2a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -0,0 +1,192 @@ +name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-agentic" + +# Two DEP8 prefill workers feeding one DEP12 decode worker, with attention / +# expert parallelism matching each role and hybrid load balancing. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 3 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 12 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: "null" + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 192 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml index b647276179..3504cb99c8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml @@ -20,7 +20,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 1440 + max_attempts: 720 interval_seconds: 10 resources: @@ -47,7 +47,6 @@ frontend: ETCD_LEASE_TTL: "120" args: router-mode: "kv" - router-reset-states: true backend: type: vllm diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml new file mode 100644 index 0000000000..503f3d3ea3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -0,0 +1,192 @@ +name: "svf-vllm-disagg-gb200-3p1d-dep8-dep16-agentic" + +# Three DEP8 prefill workers feeding one DEP16 decode worker, with attention / +# expert parallelism matching each role and hybrid load balancing. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: "null" + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 214 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 160 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 160 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml index 9e2b665843..5be157f273 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml @@ -21,7 +21,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 1440 + max_attempts: 720 interval_seconds: 10 resources: @@ -46,7 +46,6 @@ frontend: ETCD_LEASE_TTL: "120" args: router-mode: "kv" - router-reset-states: true backend: type: vllm diff --git a/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch b/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch new file mode 100644 index 0000000000..d42553fb61 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch @@ -0,0 +1,63 @@ +diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py +index fe6cbf1..f282fe6 100644 +--- a/src/srtctl/cli/mixins/benchmark_stage.py ++++ b/src/srtctl/cli/mixins/benchmark_stage.py +@@ -46,12 +46,32 @@ def _vllm_data_parallel_size(config: "SrtConfig", mode: str) -> int: + return int(mode_config.get("data-parallel-size") or mode_config.get("data_parallel_size") or 1) + + +-def _get_health_expectations(config: "SrtConfig") -> tuple[int, int, str, int]: ++def _vllm_health_entries( ++ config: "SrtConfig", ++ mode: str, ++ logical_workers: int, ++ backend_processes: list["Process"] | None, ++) -> int: ++ """Return expected Dynamo generate registrations for a vLLM worker mode.""" ++ dp_size = _vllm_data_parallel_size(config, mode) ++ if dp_size > 1 and getattr(config.backend, "dp_launch_mode", "per_gpu") == "per_node": ++ if backend_processes is None: ++ raise ValueError("backend_processes are required for per-node DP health expectations") ++ endpoint_mode = "agg" if mode == "aggregated" else mode ++ return sum(process.endpoint_mode == endpoint_mode for process in backend_processes) ++ ++ return logical_workers * dp_size ++ ++ ++def _get_health_expectations( ++ config: "SrtConfig", backend_processes: list["Process"] | None = None ++) -> tuple[int, int, str, int]: + """Compute expected health counts in the units reported by the frontend. + + Dynamo's /health endpoint reports registered generate instances. For vLLM +- DP workers, that means one entry per DP rank, not one entry per logical +- srt-slurm worker. Other frontends keep using logical worker counts. ++ DP workers, per-GPU launch registers one entry per DP rank, while per-node ++ launch registers one entry per node-local process. Other frontends keep ++ using logical worker counts. + """ + r = config.resources + +@@ -67,10 +87,10 @@ def _get_health_expectations(config: "SrtConfig") -> tuple[int, int, str, int]: + if config.frontend.type == "dynamo" and getattr(config.backend, "type", None) == "vllm": + if r.num_agg > 0: + n_prefill = 0 +- n_decode = logical_decode * _vllm_data_parallel_size(config, "aggregated") ++ n_decode = _vllm_health_entries(config, "aggregated", logical_decode, backend_processes) + else: +- n_prefill = logical_prefill * _vllm_data_parallel_size(config, "prefill") +- n_decode = logical_decode * _vllm_data_parallel_size(config, "decode") ++ n_prefill = _vllm_health_entries(config, "prefill", logical_prefill, backend_processes) ++ n_decode = _vllm_health_entries(config, "decode", logical_decode, backend_processes) + + count_desc = f"{n_prefill}P + {n_decode}D Dynamo generate instances; logical workers: {worker_desc}" + return n_prefill, n_decode, count_desc, n_prefill + n_decode +@@ -131,7 +151,7 @@ class BenchmarkStageMixin: + """Run the benchmark.""" + logger.info("Waiting for workers to be ready...") + +- n_prefill, n_decode, count_desc, num_workers = _get_health_expectations(self.config) ++ n_prefill, n_decode, count_desc, num_workers = _get_health_expectations(self.config, self.backend_processes) + logger.info("Waiting for server health (expecting %d health entries: %s)...", num_workers, count_desc) + + hc = self.config.health_check diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3aac1b8378..95ce1fe838 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7663,6 +7663,320 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true +dsv4-fp4-gb200-dynamo-vllm-agentic-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [1, 4, 8, 16] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + +# dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8: +# image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 +# model: deepseek-ai/DeepSeek-V4-Pro +# model-prefix: dsv4 +# runner: cluster:gb200-nv +# precision: fp4 +# framework: dynamo-vllm +# router: { name: dynamo-router, version: "1.3.0.dev20260618" } +# kv-p2p-transfer: nixl +# multinode: true +# disagg: true +# scenarios: +# agentic-coding: +# - search-space: +# - spec-decoding: none +# conc-list: [64, 128, 192, 256] +# prefill: +# num-worker: 1 +# tp: 8 +# ep: 8 +# dp-attn: true +# additional-settings: +# - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml" +# decode: +# num-worker: 1 +# tp: 8 +# ep: 8 +# dp-attn: true + +# dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep12: +# image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 +# model: deepseek-ai/DeepSeek-V4-Pro +# model-prefix: dsv4 +# runner: cluster:gb200-nv +# precision: fp4 +# framework: dynamo-vllm +# router: { name: dynamo-router, version: "1.3.0.dev20260618" } +# kv-p2p-transfer: nixl +# multinode: true +# disagg: true +# scenarios: +# agentic-coding: +# - search-space: +# - spec-decoding: none +# conc-list: [384, 512] +# prefill: +# num-worker: 1 +# tp: 8 +# ep: 8 +# dp-attn: true +# additional-settings: +# - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml" +# decode: +# num-worker: 1 +# tp: 12 +# ep: 12 +# dp-attn: true + +# dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep12: +# image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 +# model: deepseek-ai/DeepSeek-V4-Pro +# model-prefix: dsv4 +# runner: cluster:gb200-nv +# precision: fp4 +# framework: dynamo-vllm +# router: { name: dynamo-router, version: "1.3.0.dev20260618" } +# kv-p2p-transfer: nixl +# multinode: true +# disagg: true +# scenarios: +# agentic-coding: +# - search-space: +# - spec-decoding: none +# conc-list: [640, 720, 768] +# prefill: +# num-worker: 2 +# tp: 8 +# ep: 8 +# dp-attn: true +# additional-settings: +# - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml" +# decode: +# num-worker: 1 +# tp: 12 +# ep: 12 +# dp-attn: true + +# dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16: +# image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 +# model: deepseek-ai/DeepSeek-V4-Pro +# model-prefix: dsv4 +# runner: cluster:gb200-nv +# precision: fp4 +# framework: dynamo-vllm +# router: { name: dynamo-router, version: "1.3.0.dev20260618" } +# kv-p2p-transfer: nixl +# multinode: true +# disagg: true +# scenarios: +# agentic-coding: +# - search-space: +# - spec-decoding: none +# conc-list: [800, 960, 1024, 1280] +# prefill: +# num-worker: 3 +# tp: 8 +# ep: 8 +# dp-attn: true +# additional-settings: +# - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml" +# decode: +# num-worker: 1 +# tp: 16 +# ep: 16 +# dp-attn: true + +glm5-fp4-gb300-dynamo-trt-mtp: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # MTP configurations + - spec-decoding: "mtp" + conc-list: [ 5 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 15 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 30 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 84 ] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 180 ] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 333 ] + prefill: + num-worker: 6 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 615 ] + prefill: + num-worker: 10 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 1229 ] + prefill: + num-worker: 14 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 615 ] + prefill: + num-worker: 11 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 2253 ] + prefill: + num-worker: 21 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 005967fc36..c4a96059eb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4950,7 +4950,7 @@ description: - "Bump vLLM image to v0.25.0" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2256 - + - config-keys: - glm5.2-fp4-b300-sglang-agentic description: @@ -4966,3 +4966,14 @@ - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-agg +# - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8 +# - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep12 +# - dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep12 +# - dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16 + description: + - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 at conc [1,4,8,16] and disaggregated topologies: 1P/1D DEP8/DEP8 at [64,128,256,320], 1P/1D DEP8/DEP12 at [128,256,320], 2P/1D DEP8/DEP12 at [480,640,768], and 3P/1D DEP8/DEP16 at [640,800,960,1280]." + - "Image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2282 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index a5f917da09..10a0ce6a2e 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -156,11 +156,15 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" export SRT_SLURM_MODEL_PREFIX="kimi-k2.5-nvfp4" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - # The FP4 checkpoint is staged on compute-visible Lustre. The former - # /mnt/numa1 path is no longer present on watchtower compute nodes; - # the lowercase Lustre sibling is the FP8 checkpoint, so keep the - # NVFP4 path explicit here. - export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/" + if [[ "${IS_AGENTIC:-0}" == "1" ]]; then + # AgentX was tuned against the canonical DeepSeek-V4-Pro + # checkpoint, which is staged on compute-visible Lustre. + export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro/" + else + # Existing fixed-sequence GB200 recipes use the NVIDIA ModelOpt + # NVFP4 checkpoint. + export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/" + fi export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/mnt/lustre01/models/MiniMax-M2.5-NVFP4" @@ -293,21 +297,58 @@ fi # TODO(CJQ): make first class upon srt-slurm upstream refactor if [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node uses the same pinned cquil11/srt-slurm-nv commit as - # launch_gb300-nv.sh — everything the agentic recipes need is there: - # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env - # (the hook that hands off to benchmarks/multi_node/agentic_srt.sh) - # - DynamoConfig.wheel (recipes pin the ai-dynamo wheel) - # - srtctl apply --no-preflight (model path /mnt/numa1 is compute-node - # local NVMe, invisible to the login-node runner) - # - benchmark_stage srun_options propagation (container-remap-root - # must reach the agentic_srt.sh srun) - git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout de59739b172e507e15ebf145bfe305f606e82fbf - mkdir -p recipes/vllm/deepseek-v4/agentic + # v1.0.27 is the last release with the compatible mooncake_master command; + # v1.0.28 introduced the unsupported --nof_* flag. + # The pinned release also provides the vLLM mooncake_kv_store + SRT_SLURM_AGENTIC_SHA="f6eb42aee4664207dcf2ec601e3bd57bd527efd6" + git clone --branch v1.0.27 --depth 1 https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + if [[ "$(git rev-parse HEAD)" != "$SRT_SLURM_AGENTIC_SHA" ]]; then + echo "Error: NVIDIA/srt-slurm v1.0.27 did not resolve to $SRT_SLURM_AGENTIC_SHA" >&2 + exit 1 + fi + + # Backport NVIDIA/srt-slurm#90 without taking the post-v1.0.27 + # mooncake_master changes. The feature launches one vLLM process per node + # and lets that process manage all node-local data-parallel ranks. + SRT_SLURM_PER_NODE_DP_SHA="1a0f9e3633318ab1ee9428d2129161b583786b18" + git fetch --depth 2 origin refs/pull/90/head || exit 1 + if [[ "$(git rev-parse FETCH_HEAD)" != "$SRT_SLURM_PER_NODE_DP_SHA" ]]; then + echo "Error: NVIDIA/srt-slurm PR #90 commit did not resolve to $SRT_SLURM_PER_NODE_DP_SHA" >&2 + exit 1 + fi + git cherry-pick --no-commit "$SRT_SLURM_PER_NODE_DP_SHA" || exit 1 + + # Pin VLLM_PORT only for single-GPU processes. Multi-GPU worker processes + # (multi-node tensor parallel, e.g. the TP8 aggregate spanning two 4-GPU + # nodes) run vLLM's internal multiproc executor whose same-node subprocesses + # otherwise all read one VLLM_PORT and race to bind the shm-broadcast port + # (EADDRINUSE crash). Not yet in an srt-slurm release, so apply it here as a + # patch until it lands upstream. + git apply "$GITHUB_WORKSPACE/runners/patches/srt-slurm-vllm-port-single-gpu.patch" || exit 1 + + # Fail fast when a worker logs a fatal engine error but its OS process stays + # alive. dynamo.vllm keeps its runtime up after the vLLM EngineCore dies, so + # the srun step never exits and srt-slurm's exit-code monitor never trips -- + # the readiness gate then polls until the health timeout (hours). The patch + # makes the process monitor also scan running workers' logs for terminal + # markers. Not yet in an srt-slurm release, so apply it here until upstream. + git apply "$GITHUB_WORKSPACE/runners/patches/srt-slurm-fail-fast-log-scan.patch" || exit 1 + + # Per-node DP launches one Dynamo generate endpoint per node-local process, + # not one per DP rank. Backport the health-count fix from + # ivanium/srt-slurm@ca0880138fa606130ae4acbb8d0afddfb84c69fa. + SRT_SLURM_PER_NODE_HEALTH_PATCH="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch" + git apply --check "$SRT_SLURM_PER_NODE_HEALTH_PATCH" || exit 1 + git apply "$SRT_SLURM_PER_NODE_HEALTH_PATCH" || exit 1 + + # ai-dynamo/dynamo#11303 is merged into ai-dynamo/dynamo main, so the + # recipe-pinned dynamo hash resolves against upstream directly -- no + # esmeetu/dynamo fork redirect of srt-slurm's schema.py needed anymore. + + mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic + recipes/vllm/deepseek-v4/agentic || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" diff --git a/runners/patches/srt-slurm-fail-fast-log-scan.patch b/runners/patches/srt-slurm-fail-fast-log-scan.patch new file mode 100644 index 0000000000..560d5526ae --- /dev/null +++ b/runners/patches/srt-slurm-fail-fast-log-scan.patch @@ -0,0 +1,111 @@ +diff --git a/src/srtctl/core/processes.py b/src/srtctl/core/processes.py +index 46e6c69..8df0e63 100644 +--- a/src/srtctl/core/processes.py ++++ b/src/srtctl/core/processes.py +@@ -22,6 +22,20 @@ from pathlib import Path + logger = logging.getLogger(__name__) + + ++# Terminal markers a worker may log while its OS process stays alive. e.g. ++# dynamo.vllm keeps its runtime (and etcd leases) running after the vLLM ++# EngineCore fails to initialize, so the srun step never exits and the plain ++# exit-code check never trips -- the readiness gate then polls until the health ++# timeout (hours). Scanning worker logs for these lets the monitor fail fast. ++# Keep these unambiguous and non-recoverable to avoid false positives. ++FATAL_LOG_MARKERS: tuple[str, ...] = ( ++ "EngineCore failed to start", ++ "Engine core initialization failed", ++ "Worker failed with error", ++ "torch.OutOfMemoryError", ++) ++ ++ + @dataclass + class ManagedProcess: + """A process managed by the registry. +@@ -95,6 +109,9 @@ class ProcessRegistry: + self._processes: dict[str, ManagedProcess] = {} + self._lock = threading.Lock() + self._failed_processes: list[str] = [] ++ # Byte offset scanned so far per process log, for incremental fatal- ++ # marker detection (see _scan_log_for_fatal_marker). ++ self._log_scan_pos: dict[str, int] = {} + + def add_process(self, process: ManagedProcess) -> None: + """Add a process to the registry. +@@ -129,23 +146,72 @@ class ProcessRegistry: + def check_failures(self) -> bool: + """Check if any critical process has failed. + ++ A process counts as failed if it either exited with a non-zero code, or ++ is still running but logged a terminal engine-failure marker (a hung ++ worker -- see FATAL_LOG_MARKERS). The latter catches dynamo.vllm workers ++ that keep their runtime alive after the vLLM EngineCore dies, which would ++ otherwise stall the readiness gate until the health-check timeout. ++ + Returns: +- True if any critical process has exited with non-zero code ++ True if any critical process has failed. + """ + with self._lock: + for name, proc in self._processes.items(): +- if proc.critical and not proc.is_running: ++ if not proc.critical or name in self._failed_processes: ++ continue ++ if not proc.is_running: + exit_code = proc.exit_code +- if exit_code != 0 and name not in self._failed_processes: ++ if exit_code != 0: + self._failed_processes.append(name) + logger.error( + "Critical process '%s' exited with code %d", + name, + exit_code, + ) ++ else: ++ marker = self._scan_log_for_fatal_marker(name, proc.log_file) ++ if marker is not None: ++ self._failed_processes.append(name) ++ logger.error( ++ "Critical process '%s' logged a fatal error and is not " ++ "recovering (marker: %r)", ++ name, ++ marker, ++ ) + + return len(self._failed_processes) > 0 + ++ def _scan_log_for_fatal_marker(self, name: str, log_file: Path | None) -> str | None: ++ """Scan a process log's newly-appended bytes for a fatal marker. ++ ++ Only content written since the previous scan is read, so a marker is ++ caught when it first appears regardless of later log spam, and repeated ++ scans stay cheap. Returns the matched marker, or None. Never raises -- ++ log I/O problems must not take down the monitor. ++ """ ++ if log_file is None: ++ return None ++ try: ++ if not log_file.exists(): ++ return None ++ size = log_file.stat().st_size ++ pos = self._log_scan_pos.get(name, 0) ++ if size < pos: # log rotated or truncated ++ pos = 0 ++ if size <= pos: ++ return None ++ with log_file.open("rb") as fh: ++ fh.seek(pos) ++ chunk = fh.read() ++ self._log_scan_pos[name] = size ++ except OSError: ++ return None ++ text = chunk.decode("utf-8", errors="replace") ++ for marker in FATAL_LOG_MARKERS: ++ if marker in text: ++ return marker ++ return None ++ + def cleanup(self) -> None: + """Terminate all registered processes.""" + with self._lock: diff --git a/runners/patches/srt-slurm-vllm-port-single-gpu.patch b/runners/patches/srt-slurm-vllm-port-single-gpu.patch new file mode 100644 index 0000000000..39a879c14a --- /dev/null +++ b/runners/patches/srt-slurm-vllm-port-single-gpu.patch @@ -0,0 +1,36 @@ +diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py +index 0484d70..f3a95ca 100644 +--- a/src/srtctl/backends/vllm.py ++++ b/src/srtctl/backends/vllm.py +@@ -284,7 +284,8 @@ class VLLMProtocol: + multi-node NIXL handshake) + - VLLM_PORT: private base for vLLM's get_open_port() scans, unique per + process so co-located workers don't race for the same rendezvous port +- (see the notes on VLLM_PORT_BASE in srtctl.ports) ++ (see the notes on VLLM_PORT_BASE in srtctl.ports). Set ONLY for ++ single-GPU processes -- see below. + """ + from srtctl.core.slurm import get_hostname_ip + +@@ -299,8 +300,19 @@ class VLLMProtocol: + # 4xGB200 nodes: each prefill endpoint is DEP2 (uses 2 of the 4 GPUs), so + # two endpoints share one physical node and would otherwise scan + # overlapping get_open_port() ranges. +- proc_index = max(process.sys_port - DYN_SYSTEM_PORT_BASE, 0) +- env["VLLM_PORT"] = str(VLLM_PORT_BASE + proc_index * VLLM_PORT_STRIDE) ++ # ++ # Only pin it for single-GPU processes (the per-GPU DP layout). A ++ # multi-GPU process runs vLLM's internal multiproc executor, whose N ++ # same-node worker subprocesses all read this one VLLM_PORT; in a ++ # multi-node TP group the remote-node subprocesses then race to bind the ++ # shm-broadcast port from that shared base and crash with EADDRINUSE ++ # (e.g. the TP8 aggregate spanning two 4-GPU nodes). Leaving VLLM_PORT ++ # unset lets those subprocesses fall back to OS-assigned ephemeral ports, ++ # which are unique per bind. Single-node multi-GPU TP never hit this ++ # (same-node reader -> IPC, no TCP bind), so this is safe there too. ++ if len(process.gpu_indices) == 1: ++ proc_index = max(process.sys_port - DYN_SYSTEM_PORT_BASE, 0) ++ env["VLLM_PORT"] = str(VLLM_PORT_BASE + proc_index * VLLM_PORT_STRIDE) + return env + + def get_mooncake_worker_env(self, infra_node_ip: str, local_hostname: str) -> dict[str, str]: diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index e14c50c66b..5272fe78e6 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -24,7 +24,9 @@ MIN_EVAL_CONC = 16 # Bound how many multinode agentic conc points share one server allocation. -MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 4 +# One task/Slurm allocation per concurrency gives each benchmark point a +# fresh server deployment, matching single-node agentic sweep isolation. +MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 1 BYTES_PER_MIB = 1024 * 1024 BYTES_PER_GB = 1_000_000_000 # 3 TB decimal DRAM cap, expressed in MiB, before utilization scaling. diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 9a1c67d86e..8d23b83a18 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2253,8 +2253,8 @@ def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_conf with pytest.raises(ValueError, match="exceeds gpus-per-node"): generate_test_config_sweep(args, config, runner_config) - def test_multinode_agentic_groups_concurrencies_per_search_entry(self): - """One server allocation should run the selected concurrency batch.""" + def test_multinode_agentic_uses_one_allocation_per_concurrency(self): + """Each concurrency should get its own server allocation.""" config = { "dsv4-agentic-2p1d": { "image": "vllm/vllm-openai:v0.23.0", @@ -2292,17 +2292,27 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): result = generate_test_config_sweep(args, config) - assert len(result) == 2 - assert result[0]["conc"] == [16, 32, 64, 128] - assert result[0]["exp-name"] == "dsv4_p2x4_d1x4_conc16x32x64x128" + assert len(result) == 5 + assert [entry["conc"] for entry in result] == [ + [16], + [32], + [64], + [128], + [256], + ] + assert [entry["exp-name"] for entry in result] == [ + "dsv4_p2x4_d1x4_conc16", + "dsv4_p2x4_d1x4_conc32", + "dsv4_p2x4_d1x4_conc64", + "dsv4_p2x4_d1x4_conc128", + "dsv4_p2x4_d1x4_conc256", + ] assert result[0]["prefill"]["pp"] == 2 assert result[0]["prefill"]["dcp-size"] == 2 assert result[0]["prefill"]["pcp-size"] == 2 assert result[0]["decode"]["pp"] == 2 assert result[0]["decode"]["dcp-size"] == 2 assert result[0]["decode"]["pcp-size"] == 1 - assert result[1]["conc"] == [256] - assert result[1]["exp-name"] == "dsv4_p2x4_d1x4_conc256" assert all(entry["router"] == {"name": "dynamo-router", "version": "1.3.0"} for entry in result) assert all(entry["kv-p2p-transfer"] == "nixl" for entry in result) @@ -2493,9 +2503,10 @@ def test_node_type_filters_apply_to_agentic_configs( assert "prefill" not in single_result[0] assert single_result[0]["runner"] == "cluster:b300-nv" assert single_result[0]["pp"] == 2 - assert len(multi_result) == 1 - assert "prefill" in multi_result[0] - assert multi_result[0]["runner"] == "cluster:gb200-nv" + assert len(multi_result) == 2 + assert [entry["conc"] for entry in multi_result] == [[16], [32]] + assert all("prefill" in entry for entry in multi_result) + assert all(entry["runner"] == "cluster:gb200-nv" for entry in multi_result) assert ( multi_result[0]["prefill"]["pp"], multi_result[0]["prefill"]["dcp-size"],