[KimiK3][AgentX]: H200 KimiK3 Day 0 support / [KimiK3][AgentX]：H200 KimiK3 Day 0 支持

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml

@@ -0,0 +1,89 @@
+name: "kimik3-vllm-agg-h200-tp16dp2ep32-latency-agentic"
+
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:kimi-k3"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+    revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569"
+  container:
+    image: "vllm/vllm-openai:kimi-k3"
+  frameworks:
+    vllm: "0.1.dev19262+gb6bbf29dd.d20260727"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "h200"
+  gpus_per_node: 8
+  agg_nodes: 4
+  agg_workers: 1
+  gpus_per_agg: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    GLOO_SOCKET_IFNAME: "eth0"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+    NCCL_SOCKET_IFNAME: "eth0"
+    NCCL_CUMEM_ENABLE: "1"
+    PYTHONNOUSERSITE: "1"
+    TRANSFORMERS_CACHE: "/hf_hub_cache"
+    VLLM_ENGINE_READY_TIMEOUT_S: "7200"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 16
+      data-parallel-size: 2
+      enable-expert-parallel: true
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: marlin
+      attention-backend: FLASHMLA
+      gpu-memory-utilization: 0.95
+      max-num-seqs: 2
+      max-num-batched-tokens: 1024
+      speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
+      enable-prefix-caching: true
+      compilation-config: '{"cudagraph_capture_sizes":[3,6],"pass_config":{"fuse_allreduce_rms":false}}'
+      no-enable-flashinfer-autotune: true
+      disable-custom-all-reduce: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml

@@ -0,0 +1,89 @@
+name: "kimik3-vllm-agg-h200-tp8dp4ep32-balanced-agentic"
+
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:kimi-k3"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+    revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569"
+  container:
+    image: "vllm/vllm-openai:kimi-k3"
+  frameworks:
+    vllm: "0.1.dev19262+gb6bbf29dd.d20260727"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "h200"
+  gpus_per_node: 8
+  agg_nodes: 4
+  agg_workers: 1
+  gpus_per_agg: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    GLOO_SOCKET_IFNAME: "eth0"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+    NCCL_SOCKET_IFNAME: "eth0"
+    NCCL_CUMEM_ENABLE: "1"
+    PYTHONNOUSERSITE: "1"
+    TRANSFORMERS_CACHE: "/hf_hub_cache"
+    VLLM_ENGINE_READY_TIMEOUT_S: "7200"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 8
+      data-parallel-size: 4
+      enable-expert-parallel: true
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: marlin
+      attention-backend: FLASHMLA
+      gpu-memory-utilization: 0.95
+      max-num-seqs: 8
+      max-num-batched-tokens: 2048
+      speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
+      compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12]}'
+      enable-prefix-caching: true
+      no-enable-flashinfer-autotune: true
+      disable-custom-all-reduce: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml

@@ -0,0 +1,91 @@
+name: "kimik3-vllm-agg-h200-tp8dp4ep32-vllm-simple-agentic"
+
+model:
+  path: "kimik3"
+  container: "vllm/vllm-openai:kimi-k3"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+    revision: "9f62e4e9fffbd0a83ddd60e1c209d828994b3569"
+  container:
+    image: "vllm/vllm-openai:kimi-k3"
+  frameworks:
+    vllm: "0.1.dev19262+gb6bbf29dd.d20260727"
+
+dynamo:
+  install: false
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "h200"
+  gpus_per_node: 8
+  agg_nodes: 4
+  agg_workers: 1
+  gpus_per_agg: 32
+
+frontend:
+  type: vllm
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    GLOO_SOCKET_IFNAME: "eth0"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
+    NCCL_SOCKET_IFNAME: "eth0"
+    NCCL_CUMEM_ENABLE: "1"
+    PYTHONHASHSEED: "42"
+    PYTHONNOUSERSITE: "1"
+    TRANSFORMERS_CACHE: "/hf_hub_cache"
+    VLLM_ENGINE_READY_TIMEOUT_S: "7200"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 8
+      data-parallel-size: 4
+      enable-expert-parallel: true
+      trust-remote-code: true
+      load-format: fastsafetensors
+      moe-backend: marlin
+      attention-backend: FLASHMLA
+      gpu-memory-utilization: 0.95
+      max-num-seqs: 16
+      max-num-batched-tokens: 4096
+      speculative-config: '{"method":"dspark","model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
+      compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33]}'
+      enable-prefix-caching: true
+      kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":154250000000,"lazy_offload":false}}'
+      no-enable-flashinfer-autotune: true
+      disable-custom-all-reduce: true
+      enable-auto-tool-choice: true
+      tool-call-parser: kimi_k3
+      reasoning-parser: kimi_k3
+      language-model-only: true
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

configs/nvidia-master.yaml

@@ -1467,6 +1467,80 @@ kimik2.5-int4-h200-vllm-agentic:
       - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7] }
       - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] }
 
+# Kimi K3 is a 2.8T MXFP4 MoE served on H200 nodes.
+# These are the three aggregated strategy classes from the official H200
+# recipe (https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=h200):
+# Wider TP is latency-oriented, while progressively wider DP is balanced or
+# throughput-oriented. EP32 keeps the model shards within HBM and preserves
+# the native 1M context on four H200 nodes. All profiles use the official
+# Kimi K3 DSpark K7 probabilistic/block recipe.
+# The existing `mtp` matrix label covers speculative-decoding methods; the
+# checked-in recipes select DSpark explicitly. The H200 runner supplies
+# cluster paths while srt-slurm owns allocation, direct vLLM startup,
+# readiness, and log capture. P/D disaggregation is intentionally deferred.
+kimik3-fp4-h200-vllm-agentic:
+  image: vllm/vllm-openai:kimi-k3
+  model: moonshotai/Kimi-K3
+  model-prefix: kimik3
+  runner: cluster:h200-dgxc
+  precision: fp4
+  framework: vllm
+  multinode: true
+  disagg: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.80
+      search-space:
+      # Latency-oriented: TP16 x DP2 with EP32 across four H200 nodes.
+      - spec-decoding: mtp
+        kv-offloading: none
+        conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12]
+        prefill:
+          num-worker: 2
+          tp: 16
+          ep: 32
+          dp-attn: true
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 16
+          ep: 32
+          dp-attn: true
+      # Balanced: TP8 x DP4 with EP32 across four H200 nodes.
+      - spec-decoding: mtp
+        kv-offloading: none
+        conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16]
+        prefill:
+          num-worker: 4
+          tp: 8
+          ep: 32
+          dp-attn: true
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 8
+          ep: 32
+          dp-attn: true
+      # Throughput-oriented hybrid TP/DEP/EP with vLLM's host-DRAM KV tier.
+      - spec-decoding: mtp
+        kv-offloading: dram
+        kv-offload-backend: { name: vllm-simple }
+        conc-list: [8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32]
+        prefill:
+          num-worker: 4
+          tp: 8
+          ep: 32
+          dp-attn: true
+          additional-settings:
+          - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml"
+        decode:
+          num-worker: 0
+          tp: 8
+          ep: 32
+          dp-attn: true
+
 # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html
 # does not have a B300-specific recipe, so this config reuses the existing
 # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available.

perf-changelog.yaml

@@ -5230,3 +5230,9 @@
     - "Bump image to vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 and dynamo to 1.3.0.dev20260710; drop the minimax-m3-gb300-vllm-fixes.sh setup patch (shipped in the nightly image)"
     - "Collapse the 2xDEP8 decode into a single DEP16 worker (decode_workers 2->1, gpus_per_decode 8->16, data-parallel-size 8->16); recipe renamed 3p2d-dep2-dep8 -> 3p1d-dep2-dep16 (conc 512). The remaining decodes are 1xDEP8 or TEP and are unchanged."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2340
+
+- config-keys:
+    - kimik3-fp4-h200-vllm-agentic
+  description:
+    - "Add Day-0 Kimi K3 FP4 AgentX H200 performance recipes from the official vLLM guidance, using fit-safe TP/DP/EP topologies with optional SimpleCPU KV offload"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2353