From b4fd077b70279324617f6c180dc11393f529ba06 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:34:29 -0400 Subject: [PATCH 01/21] feat: add Kimi-K3 MXFP4 B200 aggregated TP8xPP2 Dynamo-vLLM agentic recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (no expert parallelism) for the agentic-coding trace replay. Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1, fastsafetensors load format, kimi_k3 tool-call/reasoning parsers. Model pre-staged at /lustre/fsw/models/Kimi-K3; launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay, and the agentic cache default_mounts used by the GB200/GB300 agentic paths. 中文：新增 Kimi-K3 MXFP4 B200 聚合式 TP8xPP2 Dynamo-vLLM 智能体编码基准测试配方 （2 节点 / 16 GPU，纯 TP，不启用专家并行(EP)）。使用专用 bring-up 镜像 vllm/vllm-openai:kimi-k3（VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1、 fastsafetensors 加载格式、kimi_k3 工具调用/推理解析器）。模型已预置于 /lustre/fsw/models/Kimi-K3；启动器 launch_b200-dgxc.sh 增加 kimik3/fp4 模型路径映射、智能体配方覆盖及智能体缓存挂载。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 109 ++++++++++++++++++ configs/nvidia-master.yaml | 43 +++++++ perf-changelog.yaml | 11 ++ runners/launch_b200-dgxc.sh | 28 +++++ 4 files changed, 191 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml new file mode 100644 index 0000000000..2ded17c2fd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -0,0 +1,109 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-agentic" + +# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). +# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not +# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the +# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is +# deliberately off, so the 896 routed experts are TP-sharded inside each +# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. +# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in +# the kimi-k3 bring-up image. +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0.dev20260618" + +dynamo: + install: true + wheel: "1.3.0.dev20260618" + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_SERVER_DEV_MODE: "1" + # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window + # generous, and let one long AgentX request hold a PP stage beyond vLLM's + # 300-second model-execution default. + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + gpu-memory-utilization: 0.95 + no-enable-flashinfer-autotune: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + # Full native 1M context: the AgentX trace carries multi-hundred-k-token + # requests, and K3's KDA layers keep per-token KV small (only the 24 + # gated-MLA layers hold cache). Cap prefill chunks so a single long + # request cannot OOM a pipeline stage; let vLLM pick max-num-seqs. + max-model-len: 1048576 + max-num-batched-tokens: 8192 + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + aiperf_server_metrics: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep the aggregate worker in the multinode result schema so ingestion + # uses the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 36ae816809..bf1cb716d5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8325,3 +8325,46 @@ qwen3.5-fp8-gb200-dynamo-sglang-mtp: tp: 16 ep: 16 dp-attn: true + +# Kimi-K3 MXFP4 B200 aggregated vLLM via Dynamo (TP8 x PP2, 2 nodes / 16 +# GPUs), agentic bring-up. The native MXFP4 checkpoint (2.8T total params, +# ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense +# and PP2 splits layers. Plain TP (NOT TEP): ep 1, no expert parallelism — +# the 896 routed experts are TP-sharded within each pipeline stage. Node +# count = tp*pp/gpus_per_node = 8*2/8 = 2. Aggregated (prefill num-worker 1 + +# decode num-worker 0, RECIPES.md section 5) — the single worker serves both +# phases, so no P/D KV transfer. Dedicated kimi-k3 vLLM bring-up image with +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 and the kimi_k3 tool-call/reasoning +# parsers. +# Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +kimik3-fp4-b200-dynamo-vllm-agentic: + image: vllm/vllm-openai:kimi-k3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b200-dgxc + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [8, 16, 32] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts the 16 GPUs only once. + decode: + num-worker: 0 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0524d0c0c9..0a4e40426b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5116,3 +5116,14 @@ description: - "Bump image from lmsysorg/sglang:v0.5.14-rocm720-mi35x to lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260726" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2349 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic + description: + - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" + - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code" + - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 8/16/32" + - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay into the srt-slurm-nv clone, and the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index a276644575..e166dbb595 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -72,6 +72,10 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then # NVFP4 checkpoint, pre-staged on the b200-dgxc scratch tree. export MODEL_PATH="/scratch/fsw/models/MiniMax-M3-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m3-nvfp4" +elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then + # Native MXFP4 checkpoint, pre-staged on the SRE-managed Lustre tree. + export MODEL_PATH="/lustre/fsw/models/Kimi-K3" + export SRT_SLURM_MODEL_PREFIX="kimik3" else echo "Unsupported model prefix/precision: $MODEL_PREFIX/$PRECISION" echo "Available models under /lustre/fsw/models:" @@ -107,6 +111,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 + # Overlay InferenceX-staged agentic recipes onto the clone (cp -rT so + # an upstream stub directory is merged rather than nested). + if [[ $MODEL_PREFIX == "kimik3" ]]; then + mkdir -p recipes/vllm/kimi-k3/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic + fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 @@ -207,6 +218,22 @@ if [[ "$IS_MULTINODE" == "true" ]]; then export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" + # Agentic runs bind-mount two persistent caches into every worker + # container (Lustre, shared across nodes): aiperf's content-addressed + # dataset mmap cache and the HF hub cache holding the trace dataset + # download. The container-side paths are referenced by the agentic + # recipes' benchmark.env (AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache, + # HF_HUB_CACHE=/hf_hub_cache). + DEFAULT_MOUNTS_BLOCK="" + if [[ "$IS_AGENTIC" == "1" ]]; then + HF_HUB_CACHE_HOST_PATH="/lustre/fsw/gharunners/hf-hub-cache" + mkdir -p "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" + chmod 777 "$AIPERF_MMAP_CACHE_HOST_PATH" "$HF_HUB_CACHE_HOST_PATH" 2>/dev/null || true + DEFAULT_MOUNTS_BLOCK="default_mounts: + ${AIPERF_MMAP_CACHE_HOST_PATH}: /aiperf_mmap_cache + ${HF_HUB_CACHE_HOST_PATH}: /hf_hub_cache" + fi + # Create srtslurm.yaml for srtctl (used by both frameworks) SRTCTL_ROOT="${GITHUB_WORKSPACE}/${SRT_REPO_DIR}" echo "Creating srtslurm.yaml configuration..." @@ -234,6 +261,7 @@ containers: "${IMAGE}": "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" use_exclusive_sbatch_directive: true +${DEFAULT_MOUNTS_BLOCK} EOF echo "Generated srtslurm.yaml:" From 4dbbdc8a5908a6d6177674be46a226b721be80f3 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:35:35 -0400 Subject: [PATCH 02/21] docs: link PR #2355 in changelog entry and MODELS rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：在更新日志条目与 MODELS 表格行中补充 PR #2355 链接。 Co-Authored-By: Claude Fable 5 --- MODELS.md | 2 +- MODELS_zh.md | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODELS.md b/MODELS.md index 3fc822420e..31cb9df252 100644 --- a/MODELS.md +++ b/MODELS.md @@ -18,7 +18,7 @@ This document tracks every model benchmarked by InferenceX-e2e: when it was adde | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | TBD | Agentic coding | | -| Kimi-K3 | `kimik3` | 2026-07-27 | Agentic coding | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | Agentic coding | | | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding | | | MiniMax-M3 | `minimaxm3` | 2026-06-12 ([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24 ([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | diff --git a/MODELS_zh.md b/MODELS_zh.md index 177a4b4ba3..e71067c13b 100644 --- a/MODELS_zh.md +++ b/MODELS_zh.md @@ -18,7 +18,7 @@ | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | 待定 | 智能体编码 | | -| Kimi-K3 | `kimik3` | 2026-07-27 | 智能体编码 | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | 智能体编码 | | | GLM-5.2 | `glm5.2` | 2026-07-18（[#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)） | 智能体编码 | | | MiniMax-M3 | `minimaxm3` | 2026-06-12（[#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24（[#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0a4e40426b..5a7800bd48 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5126,4 +5126,4 @@ - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 8/16/32" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay into the srt-slurm-nv clone, and the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From e8d42a73cdc7126fae0a3c1a30b8e54132dff1f1 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:48:01 -0400 Subject: [PATCH 03/21] fix: pin agentic srt-slurm to NVIDIA v1.0.36, dynamo 1.2.1, conc-8 smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cquil11/srt-slurm-nv cam/sa-submission-q2-2026 fork rejected the recipe (benchmark.aiperf_server_metrics: Unknown field). Switch the b200-dgxc agentic clone to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341), drop the aiperf_server_metrics field, pin dynamo wheel/router to 1.2.1 (the combination validated with v1.0.36), and reduce the bring-up to a single conc-8 smoke test. 中文：cquil11/srt-slurm-nv 分支的 srtctl 校验拒绝了配方字段 benchmark.aiperf_server_metrics（Unknown field）。将 b200-dgxc 智能体路径改用 上游 NVIDIA/srt-slurm v1.0.36（已在 #2302/#2341 验证），移除该字段，dynamo wheel/router 固定为 1.2.1，并将 bring-up 缩减为单并发（conc 8）冒烟测试。 Co-Authored-By: Claude Fable 5 --- .../kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 5 ++--- configs/nvidia-master.yaml | 6 ++++-- perf-changelog.yaml | 5 +++-- runners/launch_b200-dgxc.sh | 13 ++++++++++--- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 2ded17c2fd..cbee00a2b5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -20,11 +20,11 @@ identity: container: image: "vllm/vllm-openai:kimi-k3" frameworks: - dynamo: "1.3.0.dev20260618" + dynamo: "1.2.1" dynamo: + wheel: "1.2.1" install: true - wheel: "1.3.0.dev20260618" slurm: time_limit: "8:00:00" @@ -94,7 +94,6 @@ srun_options: benchmark: type: custom - aiperf_server_metrics: true command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bf1cb716d5..82c69638c8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8344,14 +8344,16 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260618" } + router: { name: dynamo-router, version: "1.2.1" } multinode: true disagg: false scenarios: agentic-coding: - search-space: + # Single-concurrency smoke test for the bring-up; widen the conc curve + # once the topology is proven green. - spec-decoding: none - conc-list: [8, 16, 32] + conc-list: [8] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5a7800bd48..b8b11fa508 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5123,7 +5123,8 @@ - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code" - - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 8/16/32" - - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, the agentic recipe overlay into the srt-slurm-nv clone, and the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" + - "Dynamo 1.2.1 (ai-dynamo wheel + dynamo-router), the version validated with srt-slurm v1.0.36 in #2341's GB300 aggregate AgentX recipes" + - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index e166dbb595..1dd4e19d14 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -109,14 +109,21 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # TODO(CJQ): make first class upon srt-slurm upstream refactor if [[ "$IS_AGENTIC" == "1" ]]; then - git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" + # Agentic recipes use NVIDIA/srt-slurm v1.0.36, the upstream version + # validated in InferenceX PR #2302/#2341 for the vLLM agentic path + # (BenchmarkType.CUSTOM + benchmark.command/env, DynamoConfig.wheel, + # srun_options propagation, per-node DP, matching Dynamo health + # counts). Keep it pinned so sweeps are reproducible. Note the older + # cquil11/srt-slurm-nv cam/sa-submission-q2-2026 fork previously + # cloned here rejects newer recipe schema fields. + git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 # Overlay InferenceX-staged agentic recipes onto the clone (cp -rT so # an upstream stub directory is merged rather than nested). if [[ $MODEL_PREFIX == "kimik3" ]]; then - mkdir -p recipes/vllm/kimi-k3/agentic + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic + recipes/vllm/kimi-k3/agentic || exit 1 fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" From c1e2a568101d37556da96f9fd80c262f30d2dabb Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:54:05 -0400 Subject: [PATCH 04/21] fix: drop OpenAI-frontend tool-choice flags from dynamo-vllm worker args MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dynamo-vllm worker entrypoint rejected --enable-auto-tool-choice --tool-call-parser kimi_k3 (unrecognized arguments; different arg parser than vllm serve). Chat parsing happens at the dynamo frontend — same convention as the DSv4 GB300 agentic recipes. Keep --reasoning-parser kimi_k3 (accepted by the worker). Also drop the explicit max-model-len and let vLLM derive the native 1M window from the model config, mirroring the agentic recipe convention. 中文：dynamo-vllm worker 入口不接受 --enable-auto-tool-choice 与 --tool-call-parser kimi_k3（unrecognized arguments，与 vllm serve 的参数解析器 不同），聊天解析由 dynamo 前端处理，与 DSv4 GB300 智能体配方约定一致；保留 worker 可接受的 --reasoning-parser kimi_k3。同时移除显式 max-model-len， 由 vLLM 从模型配置推导原生 1M 上下文窗口。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 19 ++++++++++++------- perf-changelog.yaml | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index cbee00a2b5..dcb1ec7afa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -76,14 +76,19 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 + # Note: --enable-auto-tool-choice / --tool-call-parser are NOT set on + # the worker. The dynamo-vllm worker entrypoint doesn't accept them + # (different arg parser than `vllm serve`; first sweep attempt died with + # "unrecognized arguments"). Chat parsing happens at the dynamo + # frontend, not at the worker — same convention as the DSv4 GB300 + # agentic recipes. --reasoning-parser IS accepted by the worker. reasoning-parser: kimi_k3 - # Full native 1M context: the AgentX trace carries multi-hundred-k-token - # requests, and K3's KDA layers keep per-token KV small (only the 24 - # gated-MLA layers hold cache). Cap prefill chunks so a single long - # request cannot OOM a pipeline stage; let vLLM pick max-num-seqs. - max-model-len: 1048576 + # No explicit max-model-len: let vLLM derive the native 1M window from + # the model config (agentic trajectories blow past any small cap, and + # K3's KDA layers keep per-token KV small — only the 24 gated-MLA + # layers hold cache). Prefix caching stays on (default) for trajectory + # reuse. Cap prefill chunks so a single long request cannot OOM a + # pipeline stage; let vLLM pick max-num-seqs. max-num-batched-tokens: 8192 sbatch_directives: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b8b11fa508..61b86f3db3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,8 +5122,8 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code" - - "Full native 1M max-model-len for the AgentX trace (K3's KDA layers keep per-token KV small; only the 24 gated-MLA layers hold cache), max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --reasoning-parser kimi_k3, --trust-remote-code. --enable-auto-tool-choice/--tool-call-parser are NOT set: the dynamo-vllm worker entrypoint rejects them (unrecognized arguments; different arg parser than vllm serve) — chat parsing happens at the dynamo frontend, same convention as the DSv4 GB300 agentic recipes" + - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "Dynamo 1.2.1 (ai-dynamo wheel + dynamo-router), the version validated with srt-slurm v1.0.36 in #2341's GB300 aggregate AgentX recipes" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" From ef35fd1f030fd821fa31436d9a26f425e77aaeec Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:20:42 -0400 Subject: [PATCH 05/21] fix: try dynamo wheel 1.2.0.dev20260426 for Kimi-K3 frontend tokenizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third sweep attempt: the engine loaded and served (TP8xPP2 healthy in ~14 min), but dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type 'kimi_k3' (supported: kimi, kimi_k2, kimi_k25, deepseek_v3), so the model never registered and all chat completions returned 404, aborting the AgentX warmup. Switch to the 1.2.0.dev20260426 wheel used by the DSv4 GB300/B200 Dynamo-vLLM recipes. Upstream published v1.4.0-kimi-k3-dev.1 (2026-07-27) as the day-zero K3 build if this wheel also lacks support. 中文：第三次扫描中引擎已成功加载并提供服务（TP8xPP2 约 14 分钟就绪），但 dynamo 1.2.1 的 rust 前端分词器不支持 Kimi-K3 的 tiktoken model_type 'kimi_k3'，模型未能注册，所有请求返回 404，AgentX 预热中止。改用 DSv4 GB300/B200 Dynamo-vLLM 配方所用的 1.2.0.dev20260426 wheel；如仍不支持， 上游已于 2026-07-27 发布 day-zero 构建 v1.4.0-kimi-k3-dev.1。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index dcb1ec7afa..d36158073f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -20,11 +20,11 @@ identity: container: image: "vllm/vllm-openai:kimi-k3" frameworks: - dynamo: "1.2.1" + dynamo: "1.2.0.dev20260426" dynamo: - wheel: "1.2.1" install: true + wheel: "1.2.0.dev20260426" slurm: time_limit: "8:00:00" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 82c69638c8..3dd5261b27 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8344,7 +8344,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.1" } + router: { name: dynamo-router, version: "1.2.0.dev20260426" } multinode: true disagg: false scenarios: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 61b86f3db3..2001751894 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5124,7 +5124,7 @@ - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --reasoning-parser kimi_k3, --trust-remote-code. --enable-auto-tool-choice/--tool-call-parser are NOT set: the dynamo-vllm worker entrypoint rejects them (unrecognized arguments; different arg parser than vllm serve) — chat parsing happens at the dynamo frontend, same convention as the DSv4 GB300 agentic recipes" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - - "Dynamo 1.2.1 (ai-dynamo wheel + dynamo-router), the version validated with srt-slurm v1.0.36 in #2341's GB300 aggregate AgentX recipes" + - "Dynamo 1.2.0.dev20260426 (ai-dynamo wheel + dynamo-router), the wheel used by the DSv4 GB300/B200 Dynamo-vLLM recipes. Note: dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registers and all chat completions 404 (third sweep attempt); upstream published v1.4.0-kimi-k3-dev.1 on 2026-07-27 as the day-zero K3 build if this wheel also lacks kimi_k3 tiktoken support" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From be6c56e6a1a6717d3fc018570e21471f496c7138 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:26:38 -0400 Subject: [PATCH 06/21] fix: pin dynamo to day-zero Kimi-K3 commit, restore kimi_k3 parser flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin dynamo to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ("feat: Added support for Kimi-K3", tag v1.4.0-kimi-k3-dev.1) via srt-slurm's hash-cached source install: it adds the kimi_k3 tiktoken tokenizer to the rust frontend (dynamo <=1.2.1 404s every request because the model never registers) and accepts the kimi_k3 tool-call/reasoning parser worker args, so restore --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3. 中文：将 dynamo 固定到 day-zero Kimi-K3 提交 ba83080（"feat: Added support for Kimi-K3"，标签 v1.4.0-kimi-k3-dev.1），通过 srt-slurm 的哈希缓存源码 安装：该提交为 rust 前端新增 kimi_k3 tiktoken 分词器（dynamo <=1.2.1 因模型 无法注册而全部返回 404），worker 亦支持 kimi_k3 解析器参数，故恢复 --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 21 ++++++++++++------- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 4 ++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index d36158073f..22362d730c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -20,11 +20,16 @@ identity: container: image: "vllm/vllm-openai:kimi-k3" frameworks: - dynamo: "1.2.0.dev20260426" + dynamo: "1.4.0-kimi-k3-dev.1" dynamo: install: true - wheel: "1.2.0.dev20260426" + # Day-zero Kimi-K3 dynamo ("feat: Added support for Kimi-K3", tag + # v1.4.0-kimi-k3-dev.1 == this commit): adds the kimi_k3 tiktoken tokenizer + # to the rust frontend (dynamo <=1.2.1 only knows kimi/kimi_k2/kimi_k25/ + # deepseek_v3, so the model never registers and every request 404s) and the + # kimi_k3 tool-call/reasoning parser worker args. + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" slurm: time_limit: "8:00:00" @@ -76,12 +81,12 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - # Note: --enable-auto-tool-choice / --tool-call-parser are NOT set on - # the worker. The dynamo-vllm worker entrypoint doesn't accept them - # (different arg parser than `vllm serve`; first sweep attempt died with - # "unrecognized arguments"). Chat parsing happens at the dynamo - # frontend, not at the worker — same convention as the DSv4 GB300 - # agentic recipes. --reasoning-parser IS accepted by the worker. + # kimi_k3 parsers: the day-zero K3 dynamo build pinned above accepts + # these on the worker. Dynamo <=1.2.1 rejected --enable-auto-tool-choice + # / --tool-call-parser as unrecognized arguments (different arg parser + # than `vllm serve`). + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3dd5261b27..89bf2fad2b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8344,7 +8344,7 @@ kimik3-fp4-b200-dynamo-vllm-agentic: runner: cluster:b200-dgxc precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.0.dev20260426" } + router: { name: dynamo-router, version: "1.4.0-kimi-k3-dev.1" } multinode: true disagg: false scenarios: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2001751894..c687f6da00 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,9 +5122,9 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --reasoning-parser kimi_k3, --trust-remote-code. --enable-auto-tool-choice/--tool-call-parser are NOT set: the dynamo-vllm worker entrypoint rejects them (unrecognized arguments; different arg parser than vllm serve) — chat parsing happens at the dynamo frontend, same convention as the DSv4 GB300 agentic recipes" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code (the kimi_k3 parser worker args require the day-zero K3 dynamo below; dynamo <=1.2.1 rejects the tool-choice flags as unrecognized arguments)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - - "Dynamo 1.2.0.dev20260426 (ai-dynamo wheel + dynamo-router), the wheel used by the DSv4 GB300/B200 Dynamo-vLLM recipes. Note: dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registers and all chat completions 404 (third sweep attempt); upstream published v1.4.0-kimi-k3-dev.1 on 2026-07-27 as the day-zero K3 build if this wheel also lacks kimi_k3 tiktoken support" + - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From c6917e640fb34327cd684189f5741ae475ada231 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:32:17 -0400 Subject: [PATCH 07/21] fix: use dynamo namespaced --dyn-* kimi_k3 parser args on the worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the vLLM OpenAI-frontend spellings (--enable-auto-tool-choice / --tool-call-parser) with dynamo's namespaced worker args: --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3. 中文：将 vLLM OpenAI 前端风格参数（--enable-auto-tool-choice / --tool-call-parser）替换为 dynamo 命名空间的 worker 参数： --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3。 Co-Authored-By: Claude Fable 5 --- .../kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 13 +++++++------ perf-changelog.yaml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 22362d730c..ab4c5314b0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -81,13 +81,14 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - # kimi_k3 parsers: the day-zero K3 dynamo build pinned above accepts - # these on the worker. Dynamo <=1.2.1 rejected --enable-auto-tool-choice - # / --tool-call-parser as unrecognized arguments (different arg parser - # than `vllm serve`). - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 + # kimi_k3 parsers via dynamo's namespaced worker args (--dyn-*), which + # the day-zero K3 dynamo build pinned above provides. The vLLM + # OpenAI-frontend spellings (--enable-auto-tool-choice / + # --tool-call-parser) are rejected by the dynamo worker entrypoint as + # unrecognized arguments (different arg parser than `vllm serve`). + dyn-tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and # K3's KDA layers keep per-token KV small — only the 24 gated-MLA diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c687f6da00..43fd4df391 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,7 +5122,7 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3, --trust-remote-code (the kimi_k3 parser worker args require the day-zero K3 dynamo below; dynamo <=1.2.1 rejects the tool-choice flags as unrecognized arguments)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, and kimi_k3 parsers via dynamo's namespaced worker args --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3 (the vLLM OpenAI-frontend spellings --enable-auto-tool-choice/--tool-call-parser are rejected by the dynamo worker entrypoint as unrecognized arguments)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" From a26853aa844da823eaddb96fd2138038dc7cc666 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:59:49 -0400 Subject: [PATCH 08/21] fix: disable aiperf conv-aware routing (session_control 400-rejected) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth sweep attempt: the day-zero dynamo registered the kimi_k3 tiktoken tokenizer and the engine served, but all warmup requests got 400 — aiperf's conv-aware routing emits nvext.session_control, a removed POC field this dynamo build rejects (schema moved to router/routing_constraints/ agent_hints). Opt out via AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, matching the GB300 aggregate AgentX recipes; a single aggregate worker has no P/D routing to bind anyway. 中文：第五次扫描中 day-zero dynamo 已成功注册 kimi_k3 tiktoken 分词器并正常 服务，但全部预热请求返回 400——aiperf 的会话感知路由会发送 nvext.session_control（已被移除的 POC 字段，schema 已迁移至 router/routing_constraints/agent_hints）。通过 AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 关闭，与 GB300 聚合式 AgentX 配方 一致；单聚合 worker 本无需 P/D 路由绑定。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 7 ++++++- perf-changelog.yaml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index ab4c5314b0..ba6da59153 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -113,7 +113,12 @@ benchmark: # Keep the aggregate worker in the multinode result schema so ingestion # uses the zero decode-worker count instead of duplicating TP into P and D. IS_MULTINODE: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + # aiperf's conv-aware routing emits nvext.session_control, a removed POC + # field this dynamo build 400-rejects at warmup (schema moved to + # router/routing_constraints/agent_hints). Same opt-out as the GB300 + # aggregate AgentX recipes — and with a single aggregate worker there is + # no P/D routing to bind anyway. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 43fd4df391..b94f744c83 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5126,5 +5126,6 @@ - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 From f61eafbe4bf13c25828715f2c6488c549708f77a Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:52:22 -0400 Subject: [PATCH 09/21] fix: patch kimi-k3 image mamba_hybrid index_fill_ dtype via setup_script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sixth sweep attempt (both A and C variants): warmup requests 500 then the model 503s — the image's first decode step crashes in the KDA hybrid-state postprocess (mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index; torch requires an int64 index but the runner passes the int32 idx_mapping). Ship an in-container patch through srt-slurm's setup_script hook (same pattern as configs/patches/ vllm_numa_bind_hash_fix.py): coerce the index with .long(), idempotent, refuses to run if the image layout changed. 中文：第六次扫描（A、C 两个变体一致）：预热请求先 500、随后模型 503——镜像 首个解码步在 KDA 混合状态后处理中崩溃（mamba_hybrid.py postprocess_state， IndexError: index_fill_() 需要 int64 索引，但 runner 传入 int32 idx_mapping）。 通过 srt-slurm 的 setup_script 钩子在容器内打补丁：将索引用 .long() 转换， 幂等，且镜像布局变化时拒绝执行。 Co-Authored-By: Claude Fable 5 --- .../configs/kimi-k3-container-deps.sh | 39 +++++++++++++++++++ .../agentic/agg-b200-tp8pp2-agentic.yaml | 5 +++ perf-changelog.yaml | 1 + runners/launch_b200-dgxc.sh | 4 ++ 4 files changed, 49 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh new file mode 100644 index 0000000000..abddf4d9c3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Setup script for the Kimi-K3 vLLM bring-up image (vllm/vllm-openai:kimi-k3). +# srt-slurm runs this in every worker container before dynamo install and +# worker startup (recipe field: setup_script). + +set -euo pipefail + +# The image's first decode step crashes in the KDA hybrid-state postprocess: +# vllm/v1/worker/gpu/model_states/mamba_hybrid.py, postprocess_state: +# IndexError: index_fill_(): Expected dtype int64 for index. +# torch's index_fill_ requires an int64 index tensor, but the runner passes +# the int32 idx_mapping (hit by moonshotai/Kimi-K3 agentic bring-up, first +# decode step, engine v0.1.dev19262+gb6bbf29dd). Coerce the index to int64. +# Idempotent: exits 0 if the patch is already applied. +python3 - <<'PY' +import pathlib +import re + +import vllm.v1.worker.gpu.model_states.mamba_hybrid as mh + +path = pathlib.Path(mh.__file__) +src = path.read_text() +if "idx_mapping.long()" in src: + print(f"mamba_hybrid index_fill_ patch already applied: {path}") + raise SystemExit(0) + +new, n = re.subn( + r"index_fill_\(\s*0,\s*idx_mapping,", + "index_fill_(0, idx_mapping.long(),", + src, +) +if n != 1: + raise SystemExit( + f"expected exactly one index_fill_(0, idx_mapping, ...) call in " + f"{path}, found {n} — image layout changed, refusing to patch" + ) +path.write_text(new) +print(f"Patched mamba_hybrid index_fill_ index dtype: {path}") +PY diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index ba6da59153..6570cc1fd9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -31,6 +31,11 @@ dynamo: # kimi_k3 tool-call/reasoning parser worker args. hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" +# Patches the image's mamba_hybrid postprocess_state: torch index_fill_ +# requires an int64 index but the runner passes the int32 idx_mapping, +# crashing the first decode step (IndexError: Expected dtype int64 for index). +setup_script: kimi-k3-container-deps.sh + slurm: time_limit: "8:00:00" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b94f744c83..6c39e4be50 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5127,5 +5127,6 @@ - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" + - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 1dd4e19d14..bff9b48af0 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -124,6 +124,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ recipes/vllm/kimi-k3/agentic || exit 1 + # In-container vLLM patch for the kimi-k3 image, referenced by the + # recipes' setup_script field (srt-slurm mounts configs/ at /configs). + cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/kimi-k3-container-deps.sh" \ + configs/kimi-k3-container-deps.sh || exit 1 fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" From 8fd319c06802dcf344080f517ee5fc457d71e570 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:07:41 -0400 Subject: [PATCH 10/21] =?UTF-8?q?feat:=20agentic=20experiment=20D=20?= =?UTF-8?q?=E2=80=94=20direct=20vllm=20serve=20via=20srt-slurm=20PR=20#278?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serve Kimi-K3 directly with vllm serve (srt-slurm PR #278 frontend.type: vllm, branch kylliang/direct-aggregate-vllm): no dynamo frontend/worker/ router, which removes the dynamo tokenizer/schema gaps entirely, and the OpenAI-frontend flags --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 become legitimate. PR #278 validates single-node only, so ship patches/srt-slurm-pr278-direct-vllm-multinode.patch extending it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology. Keeps the mamba_hybrid index-dtype container patch (engine bug is frontend-agnostic). 中文：智能体实验变体 D——通过 srt-slurm PR #278（frontend.type: vllm）直接以 vllm serve 提供服务：去除 dynamo 前端/worker/router，从根本上规避 dynamo 的 分词器与 schema 兼容问题，OpenAI 前端参数 --enable-auto-tool-choice --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 因此可用。PR #278 仅 支持单节点，故新增补丁将其扩展为 vLLM 原生多节点 serve（--master-addr/ --nnodes/--node-rank，非主节点 headless），以运行 2 节点 TP8xPP2 拓扑。保留 mamba_hybrid 索引类型容器补丁（引擎缺陷与前端无关）。 Co-Authored-By: Claude Fable 5 --- ...rt-slurm-pr278-direct-vllm-multinode.patch | 62 +++++++++++++++++++ .../agentic/agg-b200-tp8pp2-agentic.yaml | 32 +++++----- configs/nvidia-master.yaml | 4 +- perf-changelog.yaml | 4 +- runners/launch_b200-dgxc.sh | 20 +++--- 5 files changed, 91 insertions(+), 31 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch diff --git a/benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch b/benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch new file mode 100644 index 0000000000..9408bc7764 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch @@ -0,0 +1,62 @@ +diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py +index 74f673b..377606a 100644 +--- a/src/srtctl/backends/vllm.py ++++ b/src/srtctl/backends/vllm.py +@@ -716,25 +716,31 @@ class VLLMProtocol: + if frontend_type == "vllm": + if mode != "agg": + raise ValueError("frontend.type: vllm supports aggregate vLLM jobs only") +- if is_multi_node: +- raise ValueError("frontend.type: vllm currently supports single-node aggregate jobs only") + + config.pop("host", None) + config.pop("port", None) + config.pop("connector", None) + config.setdefault("served-model-name", served_model_name) + +- cmd.extend( +- [ +- "vllm", +- "serve", +- model_arg, +- "--host", +- "0.0.0.0", +- "--port", +- str(runtime.frontend_port), +- ] +- ) ++ node_rank = endpoint_nodes.index(process.node) ++ cmd.extend(["vllm", "serve", model_arg]) ++ if node_rank == 0: ++ cmd.extend(["--host", "0.0.0.0", "--port", str(runtime.frontend_port)]) ++ if is_multi_node: ++ # vLLM-native multi-node serve (torchrun-style): the leader owns ++ # the OpenAI server; other node ranks run headless engine workers. ++ cmd.extend( ++ [ ++ "--master-addr", ++ leader_ip, ++ "--nnodes", ++ str(len(endpoint_nodes)), ++ "--node-rank", ++ str(node_rank), ++ ] ++ ) ++ if node_rank > 0: ++ cmd.append("--headless") + if not self.set_cuda_visible_devices: + device_ids = ",".join(str(i) for i in sorted(process.gpu_indices)) + if device_ids: +diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py +index 1263ddc..0ef7ae4 100644 +--- a/src/srtctl/core/schema.py ++++ b/src/srtctl/core/schema.py +@@ -1587,8 +1587,6 @@ class SrtConfig: + raise ValidationError("frontend.type: vllm supports aggregate jobs only, not disaggregated layouts") + if self.resources.num_agg < 1: + raise ValidationError("frontend.type: vllm requires resources.agg_workers >= 1") +- if (self.resources.agg_nodes or 1) != 1: +- raise ValidationError("frontend.type: vllm currently supports single-node aggregate jobs only") + + def _validate_het_jobs(self): + """When ``resources.het_jobs`` is set to True, enforce supported shape. diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 6570cc1fd9..2e313acf70 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -19,17 +19,12 @@ identity: repo: "moonshotai/Kimi-K3" container: image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.4.0-kimi-k3-dev.1" +# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the +# InferenceX multinode patch): `vllm serve` owns the OpenAI port itself, so +# no Dynamo frontend/worker is involved and no dynamo install is needed. dynamo: - install: true - # Day-zero Kimi-K3 dynamo ("feat: Added support for Kimi-K3", tag - # v1.4.0-kimi-k3-dev.1 == this commit): adds the kimi_k3 tiktoken tokenizer - # to the rust frontend (dynamo <=1.2.1 only knows kimi/kimi_k2/kimi_k25/ - # deepseek_v3, so the model never registers and every request 404s) and the - # kimi_k3 tool-call/reasoning parser worker args. - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: false # Patches the image's mamba_hybrid postprocess_state: torch index_fill_ # requires an int64 index but the runner passes the int32 idx_mapping, @@ -55,7 +50,11 @@ infra: nats_max_payload_mb: 32 frontend: - type: dynamo + # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns + # the public port; rank-1 runs a headless engine worker (vLLM-native + # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, enabled by + # patches/srt-slurm-pr278-direct-vllm-multinode.patch). + type: vllm enable_multiple_frontends: false backend: @@ -86,14 +85,13 @@ backend: moe-backend: auto gpu-memory-utilization: 0.95 no-enable-flashinfer-autotune: true - # kimi_k3 parsers via dynamo's namespaced worker args (--dyn-*), which - # the day-zero K3 dynamo build pinned above provides. The vLLM - # OpenAI-frontend spellings (--enable-auto-tool-choice / - # --tool-call-parser) are rejected by the dynamo worker entrypoint as - # unrecognized arguments (different arg parser than `vllm serve`). - dyn-tool-call-parser: kimi_k3 + # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — + # legitimate here because this recipe serves directly with vllm serve + # (frontend.type: vllm), not through the dynamo worker entrypoint that + # rejects them. + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 # No explicit max-model-len: let vLLM derive the native 1M window from # the model config (agentic trajectories blow past any small cap, and # K3's KDA layers keep per-token KV small — only the 24 gated-MLA diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 89bf2fad2b..014afa7314 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8343,8 +8343,10 @@ kimik3-fp4-b200-dynamo-vllm-agentic: model-prefix: kimik3 runner: cluster:b200-dgxc precision: fp4 + # framework stays dynamo-vllm for launcher routing, but this variant serves + # DIRECTLY with vllm serve (srt-slurm PR #278 frontend.type: vllm + the + # InferenceX multinode patch) — no dynamo frontend/worker/router involved. framework: dynamo-vllm - router: { name: dynamo-router, version: "1.4.0-kimi-k3-dev.1" } multinode: true disagg: false scenarios: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6c39e4be50..03d735f567 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,9 +5122,9 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, and kimi_k3 parsers via dynamo's namespaced worker args --dyn-tool-call-parser kimi_k3 --reasoning-parser kimi_k3 --dyn-reasoning-parser kimi_k3 (the vLLM OpenAI-frontend spellings --enable-auto-tool-choice/--tool-call-parser are rejected by the dynamo worker entrypoint as unrecognized arguments)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3 (agentic experiment Variant D: the OpenAI-frontend parser flags are legitimate here because serving is DIRECT vllm serve, not the dynamo worker entrypoint that rejects them)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - - "Dynamo hash-pinned to ba83080ecd31c1ce918559e576d3c5bc9e092ff1 ('feat: Added support for Kimi-K3', tag v1.4.0-kimi-k3-dev.1, published 2026-07-27) via srt-slurm's hash-cached source install. Required because dynamo 1.2.1's rust frontend tokenizer rejects Kimi-K3's tiktoken model_type kimi_k3 (only kimi/kimi_k2/kimi_k25/deepseek_v3 supported), so the model never registered with the frontend and every chat completion 404'd, aborting the AgentX warmup (third sweep attempt); the TP8xPP2 engine itself loaded and served (health-ready in ~14 min via fastsafetensors)" + - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index bff9b48af0..fe8e518126 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -108,18 +108,16 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor - if [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic recipes use NVIDIA/srt-slurm v1.0.36, the upstream version - # validated in InferenceX PR #2302/#2341 for the vLLM agentic path - # (BenchmarkType.CUSTOM + benchmark.command/env, DynamoConfig.wheel, - # srun_options propagation, per-node DP, matching Dynamo health - # counts). Keep it pinned so sweeps are reproducible. Note the older - # cquil11/srt-slurm-nv cam/sa-submission-q2-2026 fork previously - # cloned here rejects newer recipe schema fields. - git clone --branch v1.0.36 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then + # Direct-vLLM agentic experiment (Variant D): srt-slurm PR #278 + # (kylliang/direct-aggregate-vllm) adds frontend.type: vllm — `vllm + # serve` owns the OpenAI port itself, no Dynamo layer. The InferenceX + # patch below extends it from single-node to vLLM-native multi-node + # serve (--master-addr/--nnodes/--node-rank + headless non-leader + # ranks) so the 2-node TP8xPP2 topology can run. + git clone --branch kylliang/direct-aggregate-vllm --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - # Overlay InferenceX-staged agentic recipes onto the clone (cp -rT so - # an upstream stub directory is merged rather than nested). + git apply "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/patches/srt-slurm-pr278-direct-vllm-multinode.patch" || exit 1 if [[ $MODEL_PREFIX == "kimik3" ]]; then mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ From c0ace4d01c6911ea3f341b5c8c9a7e39ea1b9b47 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:08:31 -0400 Subject: [PATCH 11/21] docs: point changelog and MODELS rows at experiment PR #2359 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将更新日志条目与 MODELS 表格行链接指向实验 PR #2359。 Co-Authored-By: Claude Fable 5 --- MODELS.md | 2 +- MODELS_zh.md | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODELS.md b/MODELS.md index 31cb9df252..b68b49759a 100644 --- a/MODELS.md +++ b/MODELS.md @@ -18,7 +18,7 @@ This document tracks every model benchmarked by InferenceX-e2e: when it was adde | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | TBD | Agentic coding | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | Agentic coding | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2359](https://github.com/SemiAnalysisAI/InferenceX/pull/2359)) | Agentic coding | | | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding | | | MiniMax-M3 | `minimaxm3` | 2026-06-12 ([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24 ([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | diff --git a/MODELS_zh.md b/MODELS_zh.md index e71067c13b..bcbbde0017 100644 --- a/MODELS_zh.md +++ b/MODELS_zh.md @@ -18,7 +18,7 @@ | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | 待定 | 智能体编码 | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2355](https://github.com/SemiAnalysisAI/InferenceX/pull/2355)) | 智能体编码 | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2359](https://github.com/SemiAnalysisAI/InferenceX/pull/2359)) | 智能体编码 | | | GLM-5.2 | `glm5.2` | 2026-07-18（[#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)） | 智能体编码 | | | MiniMax-M3 | `minimaxm3` | 2026-06-12（[#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24（[#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 03d735f567..c13907e0c6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5129,4 +5129,4 @@ - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2355 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2359 From 862024dc12f36dc71b31e966c38851e06594b249 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:21:28 -0400 Subject: [PATCH 12/21] fix: drop gpu-memory-utilization to 0.90 (flashinfer MoE workspace OOM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same engine-level OOM as the dynamo-frontend variants: the flashinfer trtllm MXFP4 MoE kernel allocates a ~1.6 GiB runtime workspace outside vLLM's memory pool on the first forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free. 中文：与 dynamo 前端变体相同的引擎级 OOM：flashinfer trtllm MXFP4 MoE 内核在 首个前向时于 vLLM 显存池外分配约 1.6 GiB 工作区，0.95 下仅剩约 1.35 GiB。 改为 0.90。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 7 ++++++- perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 2e313acf70..65b123d85f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -83,7 +83,12 @@ backend: trust-remote-code: true load-format: fastsafetensors moe-backend: auto - gpu-memory-utilization: 0.95 + # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a + # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first + # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first + # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 + # matches the GB200/GB300 agentic recipes. + gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — # legitimate here because this recipe serves directly with vllm serve diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c13907e0c6..36133bbe5e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5122,7 +5122,7 @@ description: - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.95, --no-enable-flashinfer-autotune, --trust-remote-code, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3 (agentic experiment Variant D: the OpenAI-frontend parser flags are legitimate here because serving is DIRECT vllm serve, not the dynamo worker entrypoint that rejects them)" + - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.90 (0.95 OOMs: the flashinfer trtllm MXFP4 MoE kernel allocates a ~1.6 GiB runtime workspace outside vLLM's pool on the first forward), --no-enable-flashinfer-autotune, --trust-remote-code, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3 (agentic experiment Variant D: the OpenAI-frontend parser flags are legitimate here because serving is DIRECT vllm serve, not the dynamo worker entrypoint that rejects them)" - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" From 4370988a8e5dbdf8e4ceba9323697598a4783399 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:03:53 -0400 Subject: [PATCH 13/21] fix: expandable_segments allocator, drop NCCL_CUMEM_ENABLE (prefill OOM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inherited from the closed dynamo-frontend variants (#2355/#2358): at gpu-mem-util 0.90 the first long-context MLA prefill OOM'd on a 2.92 GiB transient while 3.39 GiB sat reserved-but-unallocated (fragmentation). Set PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True and drop NCCL_CUMEM_ENABLE. 中文：继承自已关闭的 dynamo 前端变体（#2355/#2358）：0.90 显存利用率下首个 长上下文 MLA 预填充因 2.92 GiB 瞬时分配 OOM，而 3.39 GiB 处于已保留未分配 状态（碎片化）。设置 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True 并 移除 NCCL_CUMEM_ENABLE。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 9 ++++++++- perf-changelog.yaml | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 65b123d85f..b00c9dfeb1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -68,7 +68,14 @@ backend: # 300-second model-execution default. VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - NCCL_CUMEM_ENABLE: "1" + # expandable_segments: the dynamo-frontend variants OOM'd on a 2.92 GiB + # MLA long-context prefill transient (kv_b_proj in + # _compute_prefill_context) while 3.39 GiB sat reserved-but-unallocated — + # allocator fragmentation, exactly the case this mode fixes (and what the + # torch OOM message recommends; the DSv4 recipes set it too). + # NCCL_CUMEM_ENABLE dropped at the same time to trim NCCL's share of + # non-PyTorch device memory. + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 36133bbe5e..d21d3f153a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5126,6 +5126,7 @@ - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" + - "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True and NCCL_CUMEM_ENABLE dropped: the dynamo-frontend variants (#2355/#2358, closed in favor of this PR) OOM'd at gpu-mem-util 0.90 on a 2.92 GiB MLA long-context prefill transient (kv_b_proj in _compute_prefill_context) while 3.39 GiB sat reserved-but-unallocated — allocator fragmentation, fixed by expandable_segments (recommended by the torch OOM message; the DSv4 recipes set it); dropping NCCL_CUMEM_ENABLE trims NCCL's share of the ~8 GiB non-PyTorch device memory" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" From 0c5fe1140ac82b59d4b5d2338ba0fb677803ada8 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:11:45 -0400 Subject: [PATCH 14/21] Revert "fix: expandable_segments allocator, drop NCCL_CUMEM_ENABLE (prefill OOM)" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 4370988a8. The superseded direct-vllm run served the agentic benchmark for 24 minutes on the original env (NCCL_CUMEM_ENABLE=1, no expandable_segments) without any OOM — the allocator change was precautionary carryover from the closed dynamo-frontend variants and was never justified by evidence from this serving path. Restore the env that was demonstrably running. 中文：回滚 4370988a8。被中断的 direct-vllm 运行在原始环境 （NCCL_CUMEM_ENABLE=1、未设 expandable_segments）下已稳定运行智能体基准测试 24 分钟且无 OOM——该分配器改动只是从已关闭的 dynamo 前端变体沿袭的预防性 措施，并无本服务路径上的证据支持。恢复已被验证可运行的环境。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 9 +-------- perf-changelog.yaml | 1 - 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index b00c9dfeb1..65b123d85f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -68,14 +68,7 @@ backend: # 300-second model-execution default. VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # expandable_segments: the dynamo-frontend variants OOM'd on a 2.92 GiB - # MLA long-context prefill transient (kv_b_proj in - # _compute_prefill_context) while 3.39 GiB sat reserved-but-unallocated — - # allocator fragmentation, exactly the case this mode fixes (and what the - # torch OOM message recommends; the DSv4 recipes set it too). - # NCCL_CUMEM_ENABLE dropped at the same time to trim NCCL's share of - # non-PyTorch device memory. - PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d21d3f153a..36133bbe5e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5126,7 +5126,6 @@ - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - - "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True and NCCL_CUMEM_ENABLE dropped: the dynamo-frontend variants (#2355/#2358, closed in favor of this PR) OOM'd at gpu-mem-util 0.90 on a 2.92 GiB MLA long-context prefill transient (kv_b_proj in _compute_prefill_context) while 3.39 GiB sat reserved-but-unallocated — allocator fragmentation, fixed by expandable_segments (recommended by the torch OOM message; the DSv4 recipes set it); dropping NCCL_CUMEM_ENABLE trims NCCL's share of the ~8 GiB non-PyTorch device memory" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" From e675dd239d9f1897e23f602486c9efb02a2b329e Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:59:49 -0400 Subject: [PATCH 15/21] feat: add VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep prefix-cache blocks alive across agentic turn gaps, matching the GB200/GB300 AgentX recipes. 中文：新增 VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768，使前缀缓存块在智能体 回合间隔内保持留存，与 GB200/GB300 AgentX 配方一致。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 65b123d85f..7e9941dddb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -68,6 +68,10 @@ backend: # 300-second model-execution default. VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + # Keep prefix-cache blocks alive across agentic turn gaps (same value as + # the GB200/GB300 AgentX recipes) — trajectory reuse is the point of the + # agentic scenario. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" From 4b0c3a44ea8a9894ee1a5a6dc86cc61f54513461 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:07:43 -0400 Subject: [PATCH 16/21] feat: widen agentic conc list to 1/8/16/32 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将智能体并发列表从单点 8 扩展为 1/8/16/32。 Co-Authored-By: Claude Fable 5 --- configs/nvidia-master.yaml | 4 +--- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 014afa7314..0aab2ff0fb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8352,10 +8352,8 @@ kimik3-fp4-b200-dynamo-vllm-agentic: scenarios: agentic-coding: - search-space: - # Single-concurrency smoke test for the bring-up; widen the conc curve - # once the topology is proven green. - spec-decoding: none - conc-list: [8] + conc-list: [1, 8, 16, 32] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 36133bbe5e..5d2966a962 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5123,7 +5123,7 @@ - "Add Kimi-K3 MXFP4 B200 aggregated multinode Dynamo-vLLM agentic-coding bring-up (new model on B200; first kimik3 benchmark config)" - "Aggregated TP8 x PP2 across 2 B200 nodes (16 GPUs), plain TP (NOT TEP: ep 1, no enable-expert-parallel) — the native MXFP4 checkpoint (2.8T total params, ~1.4TB weights) does not fit one 8xB200 node, so TP8 shards attention/dense and PP2 splits the 93 layers. Aggregated mode (prefill num-worker 1 + decode num-worker 0, RECIPES.md section 5): one worker serves prefill and decode, no P/D KV transfer" - "Dedicated bring-up image vllm/vllm-openai:kimi-k3 with VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 (fuses the K3 LatentMoE tail path), --load-format fastsafetensors, --moe-backend auto, --gpu-memory-utilization 0.90 (0.95 OOMs: the flashinfer trtllm MXFP4 MoE kernel allocates a ~1.6 GiB runtime workspace outside vLLM's pool on the first forward), --no-enable-flashinfer-autotune, --trust-remote-code, --enable-auto-tool-choice, --tool-call-parser kimi_k3, --reasoning-parser kimi_k3 (agentic experiment Variant D: the OpenAI-frontend parser flags are legitimate here because serving is DIRECT vllm serve, not the dynamo worker entrypoint that rejects them)" - - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; single-concurrency smoke test at conc 8 (widen the curve once the topology is proven green)" + - "No explicit max-model-len (vLLM derives the native 1M window from the model config; K3's KDA layers keep per-token KV small — only the 24 gated-MLA layers hold cache), prefix caching on for trajectory reuse, max-num-batched-tokens 8192 so a single long prefill cannot OOM a pipeline stage; conc 1/8/16/32" - "DIRECT vLLM serving via srt-slurm PR #278 (kylliang/direct-aggregate-vllm, frontend.type: vllm): vllm serve owns the OpenAI port itself, removing the dynamo frontend/worker/router entirely (dynamo install: false) and with it the kimi_k3 tiktoken tokenizer gap that 404'd every request on dynamo <=1.2.1. PR #278 validates single-node only, so patches/srt-slurm-pr278-direct-vllm-multinode.patch extends it to vLLM-native multi-node serve (--master-addr/--nnodes/--node-rank, headless non-leader ranks) for the 2-node TP8xPP2 topology" - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" From 479b74bc237a0fd8ea74fe0ed9ef9cb6137df68a Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:41:24 -0400 Subject: [PATCH 17/21] fix: remove VLLM_PREFIX_CACHE_RETENTION_INTERVAL (K3 scheduler block) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine init hard-fails on Kimi-K3 with the GB200/GB300 AgentX value: "VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative and a multiple of scheduler_block_size (3145728)" — the KDA hybrid architecture gives K3 a 3.1M-token scheduler block. Default retention served fine in the earlier runs, so drop the override. 中文：移除 VLLM_PREFIX_CACHE_RETENTION_INTERVAL——Kimi-K3 的 KDA 混合架构使 scheduler_block_size 达 3145728，GB200/GB300 AgentX 的 32768 取值导致引擎 初始化直接失败（必须为其整数倍）。此前运行证明默认留存策略可正常服务， 故不再覆盖。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 7e9941dddb..8eed2ec13b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -68,10 +68,12 @@ backend: # 300-second model-execution default. VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # Keep prefix-cache blocks alive across agentic turn gaps (same value as - # the GB200/GB300 AgentX recipes) — trajectory reuse is the point of the - # agentic scenario. - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value + # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a + # scheduler_block_size of 3145728 and the interval must be a multiple of + # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative + # and a multiple of scheduler_block_size (3145728)"). Default retention + # served fine in earlier runs. NCCL_CUMEM_ENABLE: "1" TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" From 77988f71c093bb6dc85920a68b371f27cbf606e4 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:25:27 -0400 Subject: [PATCH 18/21] =?UTF-8?q?feat:=20agentic=20experiment=20E=20?= =?UTF-8?q?=E2=80=94=20SimpleCPUOffloadConnector=20CPU=20DRAM=20KV=20offlo?= =?UTF-8?q?ad?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Variant of the direct-vllm Kimi-K3 B200 agg TP8xPP2 agentic bring-up (#2359) with KV offload to CPU DRAM: SimpleCPUOffloadConnector (kv_role kv_both, lazy_offload false, enable_cross_layers_blocks true), cpu_bytes_to_use_per_rank 299,875,000,000 (the framework's 0.80-utilization budget for cluster:b200-dgxc: 2399 GB/node / 8 ranks), PYTHONHASHSEED=42 for cross-rank block-key hashing, explicit enable-prefix-caching, and dram/vllm-simple kv-offload labeling in the master entry. 中文：直接 vllm serve 的 Kimi-K3 B200 聚合式 TP8xPP2 智能体实验变体 E （基于 #2359）：通过 SimpleCPUOffloadConnector 将 KV 卸载至 CPU 内存 （kv_role kv_both、lazy_offload false、enable_cross_layers_blocks true）， cpu_bytes_to_use_per_rank 为 299,875,000,000（框架按 0.80 利用率对 cluster:b200-dgxc 的预算：每节点 2399 GB / 8 个 rank），设置 PYTHONHASHSEED=42 保证各 rank 前缀块键一致，显式启用前缀缓存，并在主配置 中以 dram/vllm-simple 标注 KV 卸载。 Co-Authored-By: Claude Fable 5 --- .../agentic/agg-b200-tp8pp2-agentic.yaml | 17 ++++++++++++++++- configs/nvidia-master.yaml | 8 +++++++- perf-changelog.yaml | 3 ++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 8eed2ec13b..991e9ef948 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -1,4 +1,4 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-agentic" +name: "kimik3-vllm-agg-b200-tp8pp2-simplecpu-agentic" # Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). # The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not @@ -31,6 +31,11 @@ dynamo: # crashing the first decode step (IndexError: Expected dtype int64 for index). setup_script: kimi-k3-container-deps.sh +environment: + # SimpleCPUOffloadConnector: identical prefixes must hash to identical + # block keys across ranks. + PYTHONHASHSEED: "42" + slurm: time_limit: "8:00:00" @@ -87,6 +92,16 @@ backend: tensor-parallel-size: 8 pipeline-parallel-size: 2 trust-remote-code: true + # SimpleCPUOffloadConnector (variant E): offload KV blocks to CPU DRAM. + # cpu_bytes_to_use_per_rank = TOTAL_CPU_DRAM_GB * 1e9 / GPU_COUNT with + # the framework's own budget math for cluster:b200-dgxc — DRAM + # min(3,095,781 MiB, 2,861,022 MiB cap) * 0.80 utilization = 2399 GB + # per node (matrix total-cpu-dram-gb), / 8 ranks = 299,875,000,000 + # bytes per rank. + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":299875000000,"lazy_offload":false,"enable_cross_layers_blocks":"true"}}' + # Explicit (default-on) — offloaded blocks are only useful if prefix + # caching reuses them across agentic turns. + enable-prefix-caching: true load-format: fastsafetensors moe-backend: auto # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0aab2ff0fb..6a1d9de30b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8351,8 +8351,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic: disagg: false scenarios: agentic-coding: - - search-space: + # Variant E: SimpleCPUOffloadConnector KV offload to CPU DRAM. + # dram-utilization 0.80 of the capped node DRAM = 2400 GB/node budget; + # the recipe hardcodes the resulting 300 GB/rank connector config. + - dram-utilization: 0.80 + search-space: - spec-decoding: none + kv-offloading: dram + kv-offload-backend: { name: vllm-simple, version: "kimi-k3" } conc-list: [1, 8, 16, 32] prefill: num-worker: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5d2966a962..127acb0e7e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5128,5 +5128,6 @@ - "Model pre-staged at /lustre/fsw/models/Kimi-K3 (moonshotai/Kimi-K3); launcher launch_b200-dgxc.sh gains the kimik3/fp4 model-path mapping, pins the agentic srt-slurm base to upstream NVIDIA/srt-slurm v1.0.36 (validated in #2302/#2341; replaces the cquil11/srt-slurm-nv fork branch whose older srtctl schema rejects newer recipe fields such as benchmark.aiperf_server_metrics), overlays the kimi-k3 agentic recipes onto the clone, and adds the agentic default_mounts (/aiperf_mmap_cache, /hf_hub_cache) already used by the GB200/GB300 agentic paths" - "aiperf conv-aware routing disabled (AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0, same opt-out as the GB300 aggregate AgentX recipes): aiperf's nvext.session_control is a removed POC field this dynamo build 400-rejects at warmup (fifth sweep attempt: tokenizer registered, engine served, all 5 warmup requests 400'd); with a single aggregate worker there is no P/D routing to bind" - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" + - "Agentic experiment Variant E (of the #2359 direct-vllm Variant D): SimpleCPUOffloadConnector KV offload to CPU DRAM via --kv-transfer-config (kv_role kv_both, lazy_offload false, enable_cross_layers_blocks true), cpu_bytes_to_use_per_rank 299,875,000,000 = the framework's 0.80-utilization DRAM budget for cluster:b200-dgxc (2399 GB/node / 8 ranks), PYTHONHASHSEED=42 so identical prefixes hash to identical block keys across ranks, explicit enable-prefix-caching, and kv-offloading dram / kv-offload-backend vllm-simple labeling in the master entry" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2359 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 833d4e8042bd818de8357ec5df22c2c6cff5d36f Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:26:16 -0400 Subject: [PATCH 19/21] docs: point changelog and MODELS rows at experiment PR #2370 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将更新日志条目与 MODELS 表格行链接指向实验 PR #2370。 Co-Authored-By: Claude Fable 5 --- MODELS.md | 2 +- MODELS_zh.md | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MODELS.md b/MODELS.md index b68b49759a..15de10cea6 100644 --- a/MODELS.md +++ b/MODELS.md @@ -18,7 +18,7 @@ This document tracks every model benchmarked by InferenceX-e2e: when it was adde | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | TBD | Agentic coding | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2359](https://github.com/SemiAnalysisAI/InferenceX/pull/2359)) | Agentic coding | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2370](https://github.com/SemiAnalysisAI/InferenceX/pull/2370)) | Agentic coding | | | GLM-5.2 | `glm5.2` | 2026-07-18 ([#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)) | Agentic coding | | | MiniMax-M3 | `minimaxm3` | 2026-06-12 ([#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24 ([#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)) | Single-turn 8k1k, Agentic coding | Single-turn 1k1k | diff --git a/MODELS_zh.md b/MODELS_zh.md index bcbbde0017..9381475e14 100644 --- a/MODELS_zh.md +++ b/MODELS_zh.md @@ -18,7 +18,7 @@ | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| | Qwen3.8 2.4T | `qwen3.8` | 待定 | 智能体编码 | | -| Kimi-K3 | `kimik3` | 2026-07-27 ([#2359](https://github.com/SemiAnalysisAI/InferenceX/pull/2359)) | 智能体编码 | | +| Kimi-K3 | `kimik3` | 2026-07-27 ([#2370](https://github.com/SemiAnalysisAI/InferenceX/pull/2370)) | 智能体编码 | | | GLM-5.2 | `glm5.2` | 2026-07-18（[#2268](https://github.com/SemiAnalysisAI/InferenceX/pull/2268)） | 智能体编码 | | | MiniMax-M3 | `minimaxm3` | 2026-06-12（[#1724](https://github.com/SemiAnalysisAI/InferenceX/pull/1724)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | | DeepSeek-V4-Pro | `dsv4` | 2026-04-24（[#1130](https://github.com/SemiAnalysisAI/InferenceX/pull/1130)） | 单轮 8k1k、智能体编码 | 单轮 1k1k | diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 127acb0e7e..fecd62735b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5130,4 +5130,4 @@ - "In-container vLLM patch via setup_script kimi-k3-container-deps.sh: the kimi-k3 image's first decode step crashes in the KDA hybrid-state postprocess (vllm/v1/worker/gpu/model_states/mamba_hybrid.py postprocess_state, IndexError: index_fill_(): Expected dtype int64 for index — torch requires an int64 index but the runner passes the int32 idx_mapping; sixth sweep attempt, first warmup request 500s then the model 503s). The patch coerces the index with .long(), is idempotent, and refuses to run if the image layout changed" - "Agentic experiment Variant E (of the #2359 direct-vllm Variant D): SimpleCPUOffloadConnector KV offload to CPU DRAM via --kv-transfer-config (kv_role kv_both, lazy_offload false, enable_cross_layers_blocks true), cpu_bytes_to_use_per_rank 299,875,000,000 = the framework's 0.80-utilization DRAM budget for cluster:b200-dgxc (2399 GB/node / 8 ranks), PYTHONHASHSEED=42 so identical prefixes hash to identical block keys across ranks, explicit enable-prefix-caching, and kv-offloading dram / kv-offload-backend vllm-simple labeling in the master entry" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2370 From 05eae6f979c4b54264dc0dcf583b13b420e420f6 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:45:24 -0400 Subject: [PATCH 20/21] fix: disable cross-layer blocks in SimpleCPUOffloadConnector (K3 hybrid) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With enable_cross_layers_blocks true, KV-cache initialization crashes on Kimi-K3's KDA hybrid: "shape '[64980, 64, 576]' is invalid for input of size 199618560" — a 12x element mismatch equal to the MLA layers per PP stage (24 gated-MLA layers / PP2), i.e. the cross-layer block folding mishandles the hybrid-model geometry. Run with cross-layer blocks off, matching the GB200 connector configs. 中文：enable_cross_layers_blocks 为 true 时，Kimi-K3 的 KDA 混合架构在 KV 缓存初始化阶段崩溃（元素数量差 12 倍，恰为每个 PP 阶段的 MLA 层数 24/2）——跨层块折叠未正确处理混合架构几何。改为关闭跨层块，与 GB200 连接器配置一致。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index 991e9ef948..e6a111d155 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -93,12 +93,17 @@ backend: pipeline-parallel-size: 2 trust-remote-code: true # SimpleCPUOffloadConnector (variant E): offload KV blocks to CPU DRAM. + # enable_cross_layers_blocks false: with true, KV-cache init crashes on + # K3's KDA hybrid — "shape '[64980, 64, 576]' is invalid for input of + # size 199618560", a 12x mismatch equal to the MLA layers per PP stage + # (24/2); the cross-layer block folding gets the hybrid geometry wrong. + # The GB200 recipes also run connectors with cross-layer blocks off. # cpu_bytes_to_use_per_rank = TOTAL_CPU_DRAM_GB * 1e9 / GPU_COUNT with # the framework's own budget math for cluster:b200-dgxc — DRAM # min(3,095,781 MiB, 2,861,022 MiB cap) * 0.80 utilization = 2399 GB # per node (matrix total-cpu-dram-gb), / 8 ranks = 299,875,000,000 # bytes per rank. - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":299875000000,"lazy_offload":false,"enable_cross_layers_blocks":"true"}}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":299875000000,"lazy_offload":false,"enable_cross_layers_blocks":false}}' # Explicit (default-on) — offloaded blocks are only useful if prefix # caching reuses them across agentic turns. enable-prefix-caching: true From b99c9ceb54cc0e35545235ad9feece64dcc7838e Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:24:49 -0400 Subject: [PATCH 21/21] fix: expandable_segments on the SimpleCPU-offload variant (prefill OOM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evidence from this variant's own conc16 run: a 2.36 GiB MLA long-context prefill transient OOM'd while 2.63 GiB sat reserved-but-unallocated — allocator fragmentation. The GPU-resident variant D runs green without this, but the offload connector's staging shifts the allocation layout enough to fragment. Set PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True; NCCL_CUMEM_ENABLE stays unchanged. 中文：本变体自身 conc16 运行的证据：MLA 长上下文预填充的 2.36 GiB 瞬时分配 OOM，而 2.63 GiB 处于已保留未分配状态——分配器碎片化。GPU 常驻 KV 的变体 D 无需此项即可通过，但卸载连接器的暂存缓冲改变了分配布局。设置 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True；NCCL_CUMEM_ENABLE 保持不变。 Co-Authored-By: Claude Fable 5 --- .../vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index e6a111d155..0ba9cfc1a1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -73,6 +73,13 @@ backend: # 300-second model-execution default. VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + # Evidence from THIS variant (conc16, SimpleCPU offload): a 2.36 GiB MLA + # prefill transient OOM'd while 2.63 GiB sat reserved-but-unallocated — + # allocator fragmentation, exactly what this mode fixes (and what the + # torch OOM message recommends). The GPU-resident variant D runs green + # without it; the offload connector's staging shifts the layout enough + # to fragment. NCCL_CUMEM_ENABLE stays unchanged. + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a # scheduler_block_size of 3145728 and the interval must be a multiple of