From d94c8505136b1e021ade99f1e7cd3359fc491cb9 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 27 Jul 2026 14:02:16 -0700 Subject: [PATCH 1/3] recipes(vllm): add Kimi K2.6 B200 configurations Signed-off-by: Rohit Pujar Nagraj --- .../disagg-b200-1p1d-dep4-dep8-c1024.yaml | 113 ++++++++++++++++++ .../disagg-b200-1p1d-dep8-dep8-c2048.yaml | 113 ++++++++++++++++++ .../8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml | 111 +++++++++++++++++ .../8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml | 110 +++++++++++++++++ .../8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml | 110 +++++++++++++++++ .../8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml | 110 +++++++++++++++++ .../disagg-b200-2p1d-dep8-dep8-c8192.yaml | 113 ++++++++++++++++++ 7 files changed, 780 insertions(+) create mode 100644 recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml create mode 100644 recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml create mode 100644 recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml create mode 100644 recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml create mode 100644 recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml create mode 100644 recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml create mode 100644 recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml new file mode 100644 index 00000000..7b1c6286 --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml @@ -0,0 +1,113 @@ +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep4-dep8-c1024 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1024' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml new file mode 100644 index 00000000..ec9a9665 --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml @@ -0,0 +1,113 @@ +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-dep8-c2048 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '2048' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml new file mode 100644 index 00000000..3d979d7c --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml @@ -0,0 +1,111 @@ +name: kimi-k2.6-vllm-disagg-b200-1p1d-dep8-tp8-c1 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '0' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + max-model-len: 9216 + max-num-seqs: 1024 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 256 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml new file mode 100644 index 00000000..67f30448 --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml @@ -0,0 +1,110 @@ +name: kimi-k2.6-vllm-disagg-b200-1p4d-dep4-tp4-c512 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 4 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '512' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml new file mode 100644 index 00000000..d7ba901e --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml @@ -0,0 +1,110 @@ +name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c128 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 8 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.94 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '128' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml new file mode 100644 index 00000000..106bdd64 --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml @@ -0,0 +1,110 @@ +name: kimi-k2.6-vllm-disagg-b200-1p8d-dep4-tp4-c32 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 8 + gpus_per_prefill: 4 + gpus_per_decode: 4 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep4"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.93 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + max-model-len: 9216 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.93 + stream-interval: 50 + max-cudagraph-capture-size: 2048 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '32' + req_rate: inf + use_chat_template: true diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml new file mode 100644 index 00000000..ae5819ca --- /dev/null +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml @@ -0,0 +1,113 @@ +name: kimi-k2.6-vllm-disagg-b200-2p1d-dep8-dep8-c8192 +model: + path: kimi-k2.6-nvfp4 + container: vllm/vllm-openai:v0.25.1 + precision: fp4 +dynamo: + wheel: 1.3.0.dev20260721 + install: true +setup_script: vllm-container-deps.sh +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 +infra: + etcd_nats_dedicated_node: true +frontend: + type: dynamo + enable_multiple_frontends: false +backend: + type: vllm + connector: null + dp_launch_mode: per_gpu + prefill_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '900' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + decode_environment: + PYTHONUNBUFFERED: '1' + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_WATCHDOG_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_CUDA_IPC_ENABLE_MNNVL: n + UCX_MAX_RMA_RAILS: '1' + UCX_MAX_RNDV_RAILS: '1' + UCX_RNDV_SCHEME: put_zcopy + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13346 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + enforce-eager: true + compilation-config: '{"custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + max-num-batched-tokens: 16384 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + attention-backend: FLASHINFER_MLA + block-size: 128 + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED"}' + gpu-memory-utilization: 0.9 + decode: + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + served-model-name: nvidia/Kimi-K2.6-NVFP4 + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + max-model-len: 10240 + max-num-seqs: 2048 + max-num-batched-tokens: 8192 + safetensors-load-strategy: prefetch + trust-remote-code: true + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + async-scheduling: true + attention-backend: FLASHINFER_MLA + block-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+quant_fp8","+rms_norm","+rotary_embedding"],"pass_config":{"fuse_attn_quant":true,"fuse_allreduce_rms":true}}' + gpu-memory-utilization: 0.9 + stream-interval: 50 + max-cudagraph-capture-size: 1024 +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '8192' + req_rate: inf + use_chat_template: true From 003cc5d5f350fbe8838e950ad197736fb85a5559 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 28 Jul 2026 20:01:23 -0700 Subject: [PATCH 2/3] fix(vllm): let Kimi K2.6 prefill workers use unique IDs Signed-off-by: Rohit Pujar Nagraj --- .../b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml index ae5819ca..d3ac8b7e 100644 --- a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml @@ -60,7 +60,7 @@ backend: UCX_RNDV_SCHEME: put_zcopy vllm_config: prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both", "engine_id": "kimi-k26-prefill-dep8"}' + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' served-model-name: nvidia/Kimi-K2.6-NVFP4 kv-cache-dtype: fp8 tensor-parallel-size: 1 From b7414ddf34dd36b5cf79b4151f01e50b4708d042 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Wed, 29 Jul 2026 15:22:08 -0700 Subject: [PATCH 3/3] fix(vllm): run Kimi K2.6 without setup hooks Use the pinned vLLM 0.25.1 image as shipped by removing the generic setup-script hook from all Kimi K2.6 recipes. Signed-off-by: Rohit Pujar Nagraj --- .../b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml | 1 - .../b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml | 1 - .../kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml | 1 - .../kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml | 1 - .../kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml | 1 - .../kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml | 1 - .../b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml | 1 - 7 files changed, 7 deletions(-) diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml index 7b1c6286..5cb6c798 100644 --- a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep4-dep8-c1024.yaml @@ -6,7 +6,6 @@ model: dynamo: wheel: 1.3.0.dev20260721 install: true -setup_script: vllm-container-deps.sh resources: gpu_type: b200 gpus_per_node: 8 diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml index ec9a9665..e3c08dc4 100644 --- a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-dep8-c2048.yaml @@ -6,7 +6,6 @@ model: dynamo: wheel: 1.3.0.dev20260721 install: true -setup_script: vllm-container-deps.sh resources: gpu_type: b200 gpus_per_node: 8 diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml index 3d979d7c..1c0abe2f 100644 --- a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p1d-dep8-tp8-c1.yaml @@ -6,7 +6,6 @@ model: dynamo: wheel: 1.3.0.dev20260721 install: true -setup_script: vllm-container-deps.sh resources: gpu_type: b200 gpus_per_node: 8 diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml index 67f30448..1ca87ad9 100644 --- a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p4d-dep4-tp4-c512.yaml @@ -6,7 +6,6 @@ model: dynamo: wheel: 1.3.0.dev20260721 install: true -setup_script: vllm-container-deps.sh resources: gpu_type: b200 gpus_per_node: 8 diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml index d7ba901e..44f8ff78 100644 --- a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c128.yaml @@ -6,7 +6,6 @@ model: dynamo: wheel: 1.3.0.dev20260721 install: true -setup_script: vllm-container-deps.sh resources: gpu_type: b200 gpus_per_node: 8 diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml index 106bdd64..c748107f 100644 --- a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-1p8d-dep4-tp4-c32.yaml @@ -6,7 +6,6 @@ model: dynamo: wheel: 1.3.0.dev20260721 install: true -setup_script: vllm-container-deps.sh resources: gpu_type: b200 gpus_per_node: 8 diff --git a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml index d3ac8b7e..54e623d7 100644 --- a/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml +++ b/recipes/vllm/kimi-k2.6/b200-fp4/8k1k/disagg-b200-2p1d-dep8-dep8-c8192.yaml @@ -6,7 +6,6 @@ model: dynamo: wheel: 1.3.0.dev20260721 install: true -setup_script: vllm-container-deps.sh resources: gpu_type: b200 gpus_per_node: 8