From 39de6943f46af2644cc502a94a59eaf977c9078f Mon Sep 17 00:00:00 2001 From: Julien Lin Date: Thu, 2 Jul 2026 01:57:11 -0700 Subject: [PATCH 1/3] added gb200 glm5 fp8 mtp recipes --- recipes/gb200-fp8/glm5-mtp.yaml | 223 ++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 recipes/gb200-fp8/glm5-mtp.yaml diff --git a/recipes/gb200-fp8/glm5-mtp.yaml b/recipes/gb200-fp8/glm5-mtp.yaml new file mode 100644 index 000000000..0f599c994 --- /dev/null +++ b/recipes/gb200-fp8/glm5-mtp.yaml @@ -0,0 +1,223 @@ +base: + name: gb200-fp8-glm5-mtp + model: + path: glm-5-fp8 + container: v0.5.13.post1 + precision: fp8 + identity: + model: + repo: zai-org/GLM-5-FP8 + revision: 4f96cc5eec29dcee5d6ded54f7ffe889438f9516 + frameworks: + dynamo: 1.1.0 + sglang: 0.5.13.post1 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + dynamo: + version: 1.1.0 + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + DYN_REQUEST_PLANE: nats + sglang_config: + prefill: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: false + max-running-requests: 136 + cuda-graph-max-bs: 136 + mem-fraction-static: 0.7 + context-length: 9280 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: false + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + max-running-requests: 28 + cuda-graph-max-bs: 28 + chunked-prefill-size: 64 + mem-fraction-static: 0.94 + context-length: 9280 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + health_check: + max_attempts: 360 + interval_seconds: 10 + benchmark: + type: sa-bench + req_rate: inf +# ################# 8k1k ################# +zip_override_mtp_8k1k_hightpt: + name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16] + backend: + decode_environment: + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + sglang_config: + decode: + cuda-graph-max-bs: [111, 234, 535] + data-parallel-size: 16 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: 16 + max-running-requests: [111, 234, 535] + mem-fraction-static: 0.85 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: deep_gemm + tensor-parallel-size: 16 + benchmark: + concurrencies: [[137], [620], [1305]] + isl: 8192 + osl: 1024 + resources: + decode_nodes: 4 + decode_workers: 1 + prefill_nodes: [2, 4, 8] + prefill_workers: [1, 2, 4] +zip_override_mtp_8k1k_lowlat: + name: [8k1k-1p4d_tp8, 8k1k-1p4d_tp8-conc16, 8k1k-1p8d_tp8, 8k1k-1p8d_tp8-conc1, 8k1k-1p8d_tp8-conc4] + backend: + sglang_config: + decode: + cuda-graph-max-bs: [28, 8, 12, 1, 2] + max-running-requests: [28, 8, 12, 1, 2] + benchmark: + concurrencies: [[136], [56], [120], [12], [31]] + isl: 8192 + osl: 1024 + resources: + decode_nodes: [8, 8, 16, 16, 16] + decode_workers: [4, 4, 8, 8, 8] + prefill_nodes: 2 + prefill_workers: 1 +# ################# 1k1k ################# +zip_override_mtp_1k1k_hightpt: + name: [1k1k-1p1d_dep16, 1k1k-1p1d_dep16-conc1024, 1k1k-1p1d_dep16-conc256, 1k1k-1p1d_dep16-conc512, 1k1k-1p1d_dep32] + backend: + decode_environment: + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: ['512', '512', '512', '512', '768'] + sglang_config: + decode: + context-length: 2112 + cuda-graph-max-bs: [1430, 969, 242, 482, 1251] + data-parallel-size: [16, 16, 16, 16, 32] + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: [16, 16, 16, 16, 32] + max-running-requests: [1430, 969, 242, 482, 1251] + mem-fraction-static: [0.85, 0.85, 0.85, 0.85, 0.7] + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: deep_gemm + tensor-parallel-size: [16, 16, 16, 16, 32] + prefill: + context-length: 2112 + cuda-graph-max-bs: 256 + max-running-requests: 256 + benchmark: + concurrencies: [[1614], [1213], [320], [625], [1446]] + isl: 1024 + osl: 1024 + resources: + decode_nodes: [4, 4, 4, 4, 8] + decode_workers: 1 + prefill_nodes: 2 + prefill_workers: 1 +zip_override_mtp_1k1k_lowlat: + name: [1k1k-1p4d_dtp8-conc1, 1k1k-1p4d_dtp8-conc16, 1k1k-1p4d_dtp8-conc4] + backend: + sglang_config: + decode: + context-length: 2112 + cuda-graph-max-bs: [8, 16, 8] + data-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + max-running-requests: [8, 16, 8] + prefill: + context-length: 2112 + cuda-graph-max-bs: 256 + max-running-requests: 256 + benchmark: + concurrencies: [[37], [88], [41]] + isl: 1024 + osl: 1024 + random_range_ratio: 1.0 + resources: + decode_nodes: 8 + decode_workers: 4 + prefill_nodes: 2 + prefill_workers: 1 From 4a37535c15f9f95ca0ba1ab39a5b14612a80079b Mon Sep 17 00:00:00 2001 From: Julien Lin Date: Thu, 2 Jul 2026 23:06:54 -0700 Subject: [PATCH 2/3] updated with tp8 --- recipes/gb200-fp8/glm5-mtp.yaml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/recipes/gb200-fp8/glm5-mtp.yaml b/recipes/gb200-fp8/glm5-mtp.yaml index 0f599c994..47148d2e2 100644 --- a/recipes/gb200-fp8/glm5-mtp.yaml +++ b/recipes/gb200-fp8/glm5-mtp.yaml @@ -197,27 +197,24 @@ zip_override_mtp_1k1k_hightpt: prefill_nodes: 2 prefill_workers: 1 zip_override_mtp_1k1k_lowlat: - name: [1k1k-1p4d_dtp8-conc1, 1k1k-1p4d_dtp8-conc16, 1k1k-1p4d_dtp8-conc4] + name: [1k1k-1p8d_tp8-conc1, 1k1k-1p8d_tp8-conc16, 1k1k-1p8d_tp8-conc4] backend: sglang_config: decode: context-length: 2112 - cuda-graph-max-bs: [8, 16, 8] - data-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: [8, 16, 8] + cuda-graph-max-bs: [1, 8, 2] + max-running-requests: [1, 8, 2] + mem-fraction-static: 0.8 prefill: context-length: 2112 cuda-graph-max-bs: 256 max-running-requests: 256 benchmark: - concurrencies: [[37], [88], [41]] + concurrencies: [[13], [113], [32]] isl: 1024 osl: 1024 - random_range_ratio: 1.0 resources: - decode_nodes: 8 - decode_workers: 4 + decode_nodes: 16 + decode_workers: 8 prefill_nodes: 2 prefill_workers: 1 From 1de8b99c9f3f224c38304998169ec8b5552b643f Mon Sep 17 00:00:00 2001 From: Julien Lin Date: Fri, 3 Jul 2026 02:39:31 -0700 Subject: [PATCH 3/3] updated dynamo version --- recipes/gb200-fp8/glm5-mtp.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/gb200-fp8/glm5-mtp.yaml b/recipes/gb200-fp8/glm5-mtp.yaml index 47148d2e2..f2b7b3f65 100644 --- a/recipes/gb200-fp8/glm5-mtp.yaml +++ b/recipes/gb200-fp8/glm5-mtp.yaml @@ -9,7 +9,7 @@ base: repo: zai-org/GLM-5-FP8 revision: 4f96cc5eec29dcee5d6ded54f7ffe889438f9516 frameworks: - dynamo: 1.1.0 + dynamo: 1.2.1 sglang: 0.5.13.post1 resources: gpu_type: gb200 @@ -17,7 +17,7 @@ base: frontend: type: dynamo dynamo: - version: 1.1.0 + version: 1.2.1 backend: prefill_environment: TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'