diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/16k_512/hightpt.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/16k_512/hightpt.yaml new file mode 100644 index 000000000..466d7f837 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/16k_512/hightpt.yaml @@ -0,0 +1,158 @@ +# B200-FP8 GLM5 16k512 disaggregated recipe - HIGH THROUGHPUT candidates +# Preserve the proven 8k1k topology families while scaling token and KV limits for 16k512. + +base: + name: "b200-fp8-glm5-dynamo-sglang" + + model: + path: "glm5-fp8" + container: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + container: + image: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + args: + enforce-disagg: true + dynamo: + install: false + version: null + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 160 + cuda-graph-max-bs: 160 + mem-fraction-static: 0.7 + context-length: 16896 + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 16896 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 16k512 throughput sweep: scaled from the 8k1k points #### + +zip_override_16k512_hightpt: + resources: + prefill_nodes: [2, 1, 1] + prefill_workers: [2, 1, 1] + decode_nodes: [1, 1, 2] + decode_workers: [1, 1, 2] + backend: + sglang_config: + decode: + data-parallel-size: 8 + enable-dp-lm-head: true + enable-dp-attention: true + load-balance-method: "total_tokens" + + max-running-requests: [288, 224, 208] + cuda-graph-max-bs: [288, 224, 208] + + benchmark: + isl: 16384 + osl: 512 + concurrencies: ["304", "240", "224"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/16k_512/lowlat.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/16k_512/lowlat.yaml new file mode 100644 index 000000000..e89a6486b --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/16k_512/lowlat.yaml @@ -0,0 +1,154 @@ +# B200-FP8 GLM5 16k512 disaggregated recipe - LOW LATENCY candidates + +base: + name: "b200-fp8-glm5-dynamo-sglang" + + model: + path: "glm5-fp8" + container: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + container: + image: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + args: + enforce-disagg: true + dynamo: + install: false + version: null + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 160 + cuda-graph-max-bs: 160 + mem-fraction-static: 0.7 + context-length: 16896 + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 16896 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 16k512 low-latency sweep #### + +zip_override_16k512_lowlat: + resources: + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: [2, 3, 4, 5, 7, 8] + decode_workers: [2, 3, 4, 5, 7, 8] + backend: + sglang_config: + decode: + data-parallel-size: 1 + + max-running-requests: [40, 24, 16, 12, 4, 1] + cuda-graph-max-bs: [40, 24, 16, 12, 4, 1] + + benchmark: + isl: 16384 + osl: 512 + concurrencies: ["128", "128", "96", "64", "32", "8"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/1k_1k/hightpt.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/1k_1k/hightpt.yaml new file mode 100644 index 000000000..72584281d --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/1k_1k/hightpt.yaml @@ -0,0 +1,161 @@ +# B200-FP8 GLM5 1k1k disaggregated recipe - 1k1k HIGH THROUGHPUT (4 points) + +base: + name: "b200-fp8-glm5-dynamo-sglang" + + model: + path: "glm5-fp8" + container: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + container: + image: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + + slurm: + # D-HH:MM:SS so YAML can't reparse it as a base-60 integer. = 1h30m per point. + time_limit: "0-01:30:00" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + args: + enforce-disagg: true + dynamo: + install: false + version: null + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 1k1k #### + +zip_override_1k1k_hightpt: + resources: + prefill_nodes: [1, 1, 1, 1] + prefill_workers: [1, 1, 1, 1] + decode_nodes: [1, 2, 3, 4] + decode_workers: [1, 2, 3, 4] + backend: + sglang_config: + decode: + data-parallel-size: 8 + enable-dp-lm-head: true + enable-dp-attention: true + load-balance-method: "total_tokens" + + max-running-requests: [2560, 1232, 784, 560] + cuda-graph-max-bs: [2560, 1232, 784, 560] + + benchmark: + isl: 1024 + osl: 1024 + concurrencies: ["2576", "1248", "800", "576"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/1k_1k/lowlat.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/1k_1k/lowlat.yaml new file mode 100644 index 000000000..f25bfe655 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/1k_1k/lowlat.yaml @@ -0,0 +1,158 @@ +# B200-FP8 GLM5 1k1k disaggregated recipe - 1k1k LOW LATENCY (2 points) + +base: + name: "b200-fp8-glm5-dynamo-sglang" + + model: + path: "glm5-fp8" + container: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + container: + image: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + + slurm: + # D-HH:MM:SS so YAML can't reparse it as a base-60 integer. = 1h30m per point. + time_limit: "0-01:30:00" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + args: + enforce-disagg: true + dynamo: + install: false + version: null + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 1k1k #### + +zip_override_1k1k_lowlat: + resources: + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: [8, 8] + decode_workers: [8, 8] + backend: + sglang_config: + decode: + data-parallel-size: 1 + + max-running-requests: [64, 1] + cuda-graph-max-bs: [64, 1] + + benchmark: + isl: 1024 + osl: 1024 + concurrencies: ["512x256x128x64x32", "16"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/8k_1k/hightpt.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/8k_1k/hightpt.yaml new file mode 100644 index 000000000..cac551bd1 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/8k_1k/hightpt.yaml @@ -0,0 +1,161 @@ +# B200-FP8 GLM5 8k1k disaggregated recipe - 8k1k HIGH THROUGHPUT (3 points) + +base: + name: "b200-fp8-glm5-dynamo-sglang" + + model: + path: "glm5-fp8" + container: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + container: + image: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + + slurm: + # D-HH:MM:SS so YAML can't reparse it as a base-60 integer. = 1h30m per point. + time_limit: "0-01:30:00" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + args: + enforce-disagg: true + dynamo: + install: false + version: null + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 8k1k #### + +zip_override_8k1k_hightpt: + resources: + prefill_nodes: [2, 1, 1] + prefill_workers: [2, 1, 1] + decode_nodes: [1, 1, 2] + decode_workers: [1, 1, 2] + backend: + sglang_config: + decode: + data-parallel-size: 8 + enable-dp-lm-head: true + enable-dp-attention: true + load-balance-method: "total_tokens" + + max-running-requests: [544, 224, 208] + cuda-graph-max-bs: [544, 224, 208] + + benchmark: + isl: 8192 + osl: 1024 + concurrencies: ["560", "240", "224"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/8k_1k/lowlat.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/8k_1k/lowlat.yaml new file mode 100644 index 000000000..a3bbced9e --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/Dynamo_Slurm/8k_1k/lowlat.yaml @@ -0,0 +1,158 @@ +# B200-FP8 GLM5 8k1k disaggregated recipe - 8k1k LOW LATENCY (6 points) + +base: + name: "b200-fp8-glm5-dynamo-sglang" + + model: + path: "glm5-fp8" + container: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + container: + image: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.4.0-inkling-dev.1" + + slurm: + # D-HH:MM:SS so YAML can't reparse it as a base-60 integer. = 1h30m per point. + time_limit: "0-01:30:00" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + args: + enforce-disagg: true + dynamo: + install: false + version: null + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 8k1k #### + +zip_override_8k1k_lowlat: + resources: + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: [2, 3, 4, 5, 7, 8] + decode_workers: [2, 3, 4, 5, 7, 8] + backend: + sglang_config: + decode: + data-parallel-size: 1 + + max-running-requests: [80, 48, 34, 22, 8, 1] + cuda-graph-max-bs: [80, 48, 34, 22, 8, 1] + + benchmark: + isl: 8192 + osl: 1024 + concurrencies: ["256", "256", "200", "128", "64", "12"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/16k_512/hightpt.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/16k_512/hightpt.yaml new file mode 100644 index 000000000..ca77c5f65 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/16k_512/hightpt.yaml @@ -0,0 +1,156 @@ +# B200-FP8 GLM5 16k512 disaggregated recipe - HIGH THROUGHPUT candidates +# Preserve the proven 8k1k topology families while scaling token and KV limits for 16k512. + +base: + name: "b200-fp8-glm5" + + model: + path: "glm5-fp8" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + frameworks: + dynamo: "1.1.0" + sglang: "0.5.11" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + dynamo: + version: "1.1.0" + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 160 + cuda-graph-max-bs: 160 + mem-fraction-static: 0.7 + context-length: 16896 + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 16896 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 16k512 throughput sweep: scaled from the 8k1k points #### + +zip_override_16k512_hightpt: + resources: + prefill_nodes: [2, 1, 1] + prefill_workers: [2, 1, 1] + decode_nodes: [1, 1, 2] + decode_workers: [1, 1, 2] + backend: + sglang_config: + decode: + data-parallel-size: 8 + enable-dp-lm-head: true + enable-dp-attention: true + load-balance-method: "total_tokens" + + max-running-requests: [288, 224, 208] + cuda-graph-max-bs: [288, 224, 208] + + benchmark: + isl: 16384 + osl: 512 + concurrencies: ["304", "240", "224"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/16k_512/lowlat.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/16k_512/lowlat.yaml new file mode 100644 index 000000000..f395b3113 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/16k_512/lowlat.yaml @@ -0,0 +1,152 @@ +# B200-FP8 GLM5 16k512 disaggregated recipe - LOW LATENCY candidates + +base: + name: "b200-fp8-glm5" + + model: + path: "glm5-fp8" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + frameworks: + dynamo: "1.1.0" + sglang: "0.5.11" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + dynamo: + version: "1.1.0" + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 160 + cuda-graph-max-bs: 160 + mem-fraction-static: 0.7 + context-length: 16896 + chunked-prefill-size: 131072 + max-prefill-tokens: 16384 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 16896 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 16k512 low-latency sweep #### + +zip_override_16k512_lowlat: + resources: + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: [2, 3, 4, 5, 7, 8] + decode_workers: [2, 3, 4, 5, 7, 8] + backend: + sglang_config: + decode: + data-parallel-size: 1 + + max-running-requests: [40, 24, 16, 12, 4, 1] + cuda-graph-max-bs: [40, 24, 16, 12, 4, 1] + + benchmark: + isl: 16384 + osl: 512 + concurrencies: ["128", "128", "96", "64", "32", "8"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/1k_1k/hightpt.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/1k_1k/hightpt.yaml new file mode 100644 index 000000000..a992d52c6 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/1k_1k/hightpt.yaml @@ -0,0 +1,155 @@ +# B200-FP8 GLM5 1k1k disaggregated recipe - 1k1k HIGH THROUGHPUT (4 points) + +base: + name: "b200-fp8-glm5" + + model: + path: "glm5-fp8" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + frameworks: + dynamo: "1.1.0" + sglang: "0.5.11" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + dynamo: + version: "1.1.0" + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 1k1k #### + +zip_override_1k1k_hightpt: + resources: + prefill_nodes: [1, 1, 1, 1] + prefill_workers: [1, 1, 1, 1] + decode_nodes: [1, 2, 3, 4] + decode_workers: [1, 2, 3, 4] + backend: + sglang_config: + decode: + data-parallel-size: 8 + enable-dp-lm-head: true + enable-dp-attention: true + load-balance-method: "total_tokens" + + max-running-requests: [2560, 1232, 784, 560] + cuda-graph-max-bs: [2560, 1232, 784, 560] + + benchmark: + isl: 1024 + osl: 1024 + concurrencies: ["2576", "1248", "800", "576"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/1k_1k/lowlat.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/1k_1k/lowlat.yaml new file mode 100644 index 000000000..e2ac834af --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/1k_1k/lowlat.yaml @@ -0,0 +1,152 @@ +# B200-FP8 GLM5 1k1k disaggregated recipe - 1k1k LOW LATENCY (2 points) + +base: + name: "b200-fp8-glm5" + + model: + path: "glm5-fp8" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + frameworks: + dynamo: "1.1.0" + sglang: "0.5.11" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + dynamo: + version: "1.1.0" + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 1k1k #### + +zip_override_1k1k_lowlat: + resources: + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: [8, 8] + decode_workers: [8, 8] + backend: + sglang_config: + decode: + data-parallel-size: 1 + + max-running-requests: [64, 1] + cuda-graph-max-bs: [64, 1] + + benchmark: + isl: 1024 + osl: 1024 + concurrencies: ["512x256x128x64x32", "16"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/8k_1k/hightpt.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/8k_1k/hightpt.yaml new file mode 100644 index 000000000..0eebace6d --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/8k_1k/hightpt.yaml @@ -0,0 +1,155 @@ +# B200-FP8 GLM5 8k1k disaggregated recipe - 8k1k HIGH THROUGHPUT (3 points) + +base: + name: "b200-fp8-glm5" + + model: + path: "glm5-fp8" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + frameworks: + dynamo: "1.1.0" + sglang: "0.5.11" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + dynamo: + version: "1.1.0" + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 8k1k #### + +zip_override_8k1k_hightpt: + resources: + prefill_nodes: [2, 1, 1] + prefill_workers: [2, 1, 1] + decode_nodes: [1, 1, 2] + decode_workers: [1, 1, 2] + backend: + sglang_config: + decode: + data-parallel-size: 8 + enable-dp-lm-head: true + enable-dp-attention: true + load-balance-method: "total_tokens" + + max-running-requests: [544, 224, 208] + cuda-graph-max-bs: [544, 224, 208] + + benchmark: + isl: 8192 + osl: 1024 + concurrencies: ["560", "240", "224"] diff --git a/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/8k_1k/lowlat.yaml b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/8k_1k/lowlat.yaml new file mode 100644 index 000000000..4d52da2ac --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/Disagg/Sglang/non_Dynamo_slurm/8k_1k/lowlat.yaml @@ -0,0 +1,152 @@ +# B200-FP8 GLM5 8k1k disaggregated recipe - 8k1k LOW LATENCY (6 points) + +base: + name: "b200-fp8-glm5" + + model: + path: "glm5-fp8" + container: "lmsysorg/sglang:v0.5.11-cu130" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + frameworks: + dynamo: "1.1.0" + sglang: "0.5.11" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + + frontend: + type: "dynamo" + dynamo: + version: "1.1.0" + + backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + NCCL_CUMEM_ENABLE: "1" + DYN_REQUEST_PLANE: "nats" + + sglang_config: + prefill: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "prefill" + disaggregation-transfer-backend: "nixl" + + # Size limits + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + + # Parallelism + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: "total_tokens" + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + decode: + # Model configuration + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + # Disaggregation mode + disaggregation-mode: "decode" + disaggregation-transfer-backend: "nixl" + + # Memory and token limits + mem-fraction-static: 0.8 + context-length: 9600 + + # Parallelism + tensor-parallel-size: 8 + expert-parallel-size: 1 + + # Backend + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + # Other flags + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 360 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + req_rate: "inf" + +#### 8k1k #### + +zip_override_8k1k_lowlat: + resources: + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: [2, 3, 4, 5, 7, 8] + decode_workers: [2, 3, 4, 5, 7, 8] + backend: + sglang_config: + decode: + data-parallel-size: 1 + + max-running-requests: [80, 48, 34, 22, 8, 1] + cuda-graph-max-bs: [80, 48, 34, 22, 8, 1] + + benchmark: + isl: 8192 + osl: 1024 + concurrencies: ["256", "256", "200", "128", "64", "12"] diff --git a/CrossCluster_Recipes/GLM5/B200/agg/Sglang/non_dyanmo_slurm/16k_512.yaml b/CrossCluster_Recipes/GLM5/B200/agg/Sglang/non_dyanmo_slurm/16k_512.yaml new file mode 100644 index 000000000..b156618dc --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/agg/Sglang/non_dyanmo_slurm/16k_512.yaml @@ -0,0 +1,89 @@ +# B200-FP8 GLM-5 aggregate serving recipe for 16K input / 512 output. +# One TP8 worker on one 8xB200 node. The sweep ends at concurrency 256: +# throughput is saturated there, while higher concurrency adds latency. + +base: + name: "b200-fp8-glm5-agg-16k512" + + model: + path: "glm5-fp8" + container: "lmsysorg/sglang:v0.5.15.post1-cu130" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + container: + image: "lmsysorg/sglang:v0.5.15.post1-cu130" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + + frontend: + type: "sglang" + args: + worker-startup-timeout-secs: 7200 + worker-startup-check-interval: 10 + + backend: + type: "sglang" + + aggregated_environment: + PYTHONNOUSERSITE: "1" + PYTHONUNBUFFERED: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "1" + NCCL_CUMEM_ENABLE: "1" + + sglang_config: + aggregated: + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + tool-call-parser: "glm47" + reasoning-parser: "glm45" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + attention-backend: "nsa" + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + mem-fraction-static: 0.85 + context-length: 16896 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 900 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + isl: 16384 + osl: 512 + random_range_ratio: 0.8 + req_rate: "inf" + +zip_override_agg_16k512: + backend: + sglang_config: + aggregated: + max-running-requests: [4, 8, 16, 32, 64, 128, 256] + cuda-graph-max-bs: [4, 8, 16, 32, 64, 128, 256] + + benchmark: + concurrencies: ["4", "8", "16", "32", "64", "128", "256"] diff --git a/CrossCluster_Recipes/GLM5/B200/agg/Sglang/non_dyanmo_slurm/1k_1k.yaml b/CrossCluster_Recipes/GLM5/B200/agg/Sglang/non_dyanmo_slurm/1k_1k.yaml new file mode 100644 index 000000000..48cf78cc6 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/agg/Sglang/non_dyanmo_slurm/1k_1k.yaml @@ -0,0 +1,89 @@ +# B200-FP8 GLM-5 aggregate serving recipe for 1K input / 1K output. +# One TP8 worker on one 8xB200 node. All points through concurrency 1024 +# completed successfully; throughput was still increasing at the final point. + +base: + name: "b200-fp8-glm5-agg-1k1k" + + model: + path: "glm5-fp8" + container: "lmsysorg/sglang:v0.5.15.post1-cu130" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + container: + image: "lmsysorg/sglang:v0.5.15.post1-cu130" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + + frontend: + type: "sglang" + args: + worker-startup-timeout-secs: 7200 + worker-startup-check-interval: 10 + + backend: + type: "sglang" + + aggregated_environment: + PYTHONNOUSERSITE: "1" + PYTHONUNBUFFERED: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "1" + NCCL_CUMEM_ENABLE: "1" + + sglang_config: + aggregated: + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + tool-call-parser: "glm47" + reasoning-parser: "glm45" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + attention-backend: "nsa" + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + mem-fraction-static: 0.85 + context-length: 2200 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 900 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + req_rate: "inf" + +zip_override_agg_1k1k: + backend: + sglang_config: + aggregated: + max-running-requests: [4, 8, 16, 32, 64, 128, 256, 512, 1024] + cuda-graph-max-bs: [4, 8, 16, 32, 64, 128, 256, 512, 1024] + + benchmark: + concurrencies: ["4", "8", "16", "32", "64", "128", "256", "512", "1024"] diff --git a/CrossCluster_Recipes/GLM5/B200/agg/Sglang/non_dyanmo_slurm/8k_1k.yaml b/CrossCluster_Recipes/GLM5/B200/agg/Sglang/non_dyanmo_slurm/8k_1k.yaml new file mode 100644 index 000000000..a99c876b2 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/B200/agg/Sglang/non_dyanmo_slurm/8k_1k.yaml @@ -0,0 +1,89 @@ +# B200-FP8 GLM-5 aggregate serving recipe for 8K input / 1K output. +# One TP8 worker on one 8xB200 node. The sweep ends at concurrency 256: +# throughput is saturated there, while higher concurrencies only add TTFT. + +base: + name: "b200-fp8-glm5-agg-8k1k" + + model: + path: "glm5-fp8" + container: "lmsysorg/sglang:v0.5.15.post1-cu130" + precision: "fp8" + + identity: + model: + repo: "zai-org/GLM-5-FP8" + revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" + container: + image: "lmsysorg/sglang:v0.5.15.post1-cu130" + + resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + + frontend: + type: "sglang" + args: + worker-startup-timeout-secs: 7200 + worker-startup-check-interval: 10 + + backend: + type: "sglang" + + aggregated_environment: + PYTHONNOUSERSITE: "1" + PYTHONUNBUFFERED: "1" + SGLANG_ENABLE_JIT_DEEPGEMM: "1" + NCCL_CUMEM_ENABLE: "1" + + sglang_config: + aggregated: + served-model-name: "GLM-5-FP8" + trust-remote-code: true + + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 1 + + tool-call-parser: "glm47" + reasoning-parser: "glm45" + quantization: "fp8" + kv-cache-dtype: "fp8_e4m3" + + attention-backend: "nsa" + nsa-decode-backend: "trtllm" + nsa-prefill-backend: "trtllm" + moe-runner-backend: "flashinfer_trtllm" + + mem-fraction-static: 0.85 + context-length: 9600 + chunked-prefill-size: 32768 + max-prefill-tokens: 32768 + + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + + health_check: + max_attempts: 900 + interval_seconds: 10 + + benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + random_range_ratio: 0.8 + req_rate: "inf" + +zip_override_agg_8k1k: + backend: + sglang_config: + aggregated: + max-running-requests: [4, 8, 16, 32, 64, 128, 256] + cuda-graph-max-bs: [4, 8, 16, 32, 64, 128, 256] + + benchmark: + concurrencies: ["4", "8", "16", "32", "64", "128", "256"] diff --git a/CrossCluster_Recipes/GLM5/gb300/1k_1k/hightpt/glm5-gb300-fp8-stp-hightpt-1p1d-bs2560-conc2576.yaml b/CrossCluster_Recipes/GLM5/gb300/1k_1k/hightpt/glm5-gb300-fp8-stp-hightpt-1p1d-bs2560-conc2576.yaml new file mode 100644 index 000000000..88adff9fb --- /dev/null +++ b/CrossCluster_Recipes/GLM5/gb300/1k_1k/hightpt/glm5-gb300-fp8-stp-hightpt-1p1d-bs2560-conc2576.yaml @@ -0,0 +1,109 @@ +# Generated from recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_hightpt[0] +# Topology: 1 prefill nodes + 1 decode nodes = 2 nodes / 8 GPUs +name: gb300-fp8-glm5-stp-1k1k-hightpt-1p1d-bs2560-conc2576 +model: + path: glm5-fp8 + container: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa + precision: fp8 +identity: + model: + repo: zai-org/GLM-5-FP8 + revision: 4f96cc5eec29dcee5d6ded54f7ffe889438f9516 + frameworks: + dynamo: 1.3.0 + sglang: 0.5.14 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 1 + decode_workers: 1 +frontend: + type: dynamo +dynamo: + version: 1.3.0 + install: false +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + sglang_config: + prefill: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.8 + context-length: 9600 + tensor-parallel-size: 8 + expert-parallel-size: 1 + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + data-parallel-size: 8 + enable-dp-lm-head: true + enable-dp-attention: true + load-balance-method: total_tokens + max-running-requests: 2560 + cuda-graph-max-bs: 2560 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '2576' diff --git a/CrossCluster_Recipes/GLM5/gb300/1k_1k/hightpt/glm5-gb300-fp8-stp-hightpt-1p2d-bs1232-conc1248.yaml b/CrossCluster_Recipes/GLM5/gb300/1k_1k/hightpt/glm5-gb300-fp8-stp-hightpt-1p2d-bs1232-conc1248.yaml new file mode 100644 index 000000000..45c7f02c8 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/gb300/1k_1k/hightpt/glm5-gb300-fp8-stp-hightpt-1p2d-bs1232-conc1248.yaml @@ -0,0 +1,109 @@ +# Generated from recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_hightpt[1] +# Topology: 1 prefill nodes + 2 decode nodes = 3 nodes / 12 GPUs +name: gb300-fp8-glm5-stp-1k1k-hightpt-1p2d-bs1232-conc1248 +model: + path: glm5-fp8 + container: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa + precision: fp8 +identity: + model: + repo: zai-org/GLM-5-FP8 + revision: 4f96cc5eec29dcee5d6ded54f7ffe889438f9516 + frameworks: + dynamo: 1.3.0 + sglang: 0.5.14 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 2 + decode_workers: 2 +frontend: + type: dynamo +dynamo: + version: 1.3.0 + install: false +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + sglang_config: + prefill: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.8 + context-length: 9600 + tensor-parallel-size: 8 + expert-parallel-size: 1 + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + data-parallel-size: 8 + enable-dp-lm-head: true + enable-dp-attention: true + load-balance-method: total_tokens + max-running-requests: 1232 + cuda-graph-max-bs: 1232 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: '1248' diff --git a/CrossCluster_Recipes/GLM5/gb300/1k_1k/lowlatency/glm5-gb300-fp8-stp-lowlatency-1p8d-bs64-conc32x64x128x256x512.yaml b/CrossCluster_Recipes/GLM5/gb300/1k_1k/lowlatency/glm5-gb300-fp8-stp-lowlatency-1p8d-bs64-conc32x64x128x256x512.yaml new file mode 100644 index 000000000..c713e40a4 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/gb300/1k_1k/lowlatency/glm5-gb300-fp8-stp-lowlatency-1p8d-bs64-conc32x64x128x256x512.yaml @@ -0,0 +1,106 @@ +# Generated from recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_lowlat[0] +# Topology: 1 prefill nodes + 8 decode nodes = 9 nodes / 36 GPUs +name: gb300-fp8-glm5-stp-1k1k-lowlatency-1p8d-bs64-conc32x64x128x256x512 +model: + path: glm5-fp8 + container: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa + precision: fp8 +identity: + model: + repo: zai-org/GLM-5-FP8 + revision: 4f96cc5eec29dcee5d6ded54f7ffe889438f9516 + frameworks: + dynamo: 1.3.0 + sglang: 0.5.14 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 8 + decode_workers: 8 +frontend: + type: dynamo +dynamo: + version: 1.3.0 + install: false +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + sglang_config: + prefill: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.8 + context-length: 9600 + tensor-parallel-size: 8 + expert-parallel-size: 1 + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + data-parallel-size: 1 + max-running-requests: 64 + cuda-graph-max-bs: 64 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + req_rate: inf + isl: 1024 + osl: 1024 + concurrencies: 32x64x128x256x512 diff --git a/CrossCluster_Recipes/GLM5/gb300/8k_1k/hightpt/glm5-gb300-fp8-stp-hightpt-2p6d-bs4096-conc512x1024x2048.yaml b/CrossCluster_Recipes/GLM5/gb300/8k_1k/hightpt/glm5-gb300-fp8-stp-hightpt-2p6d-bs4096-conc512x1024x2048.yaml new file mode 100644 index 000000000..df8f99bcb --- /dev/null +++ b/CrossCluster_Recipes/GLM5/gb300/8k_1k/hightpt/glm5-gb300-fp8-stp-hightpt-2p6d-bs4096-conc512x1024x2048.yaml @@ -0,0 +1,115 @@ +# Generated from recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[0] +# Topology: 2 prefill nodes + 6 decode nodes = 8 nodes / 32 GPUs +name: gb300-fp8-glm5-stp-8k1k-hightpt-2p6d-bs4096-conc512x1024x2048 +model: + path: glm5-fp8 + container: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa + precision: fp8 +identity: + model: + repo: zai-org/GLM-5-FP8 + revision: 4f96cc5eec29dcee5d6ded54f7ffe889438f9516 + frameworks: + dynamo: 1.3.0 + sglang: 0.5.14 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 1 + decode_nodes: 6 + decode_workers: 1 +frontend: + type: dynamo +dynamo: + version: 1.3.0 + install: false +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + sglang_config: + prefill: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.8 + context-length: 9600 + tensor-parallel-size: 24 + expert-parallel-size: 24 + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + data-parallel-size: 24 + enable-dp-lm-head: true + enable-dp-attention: true + moe-dense-tp-size: 1 + load-balance-method: total_tokens + ep-num-redundant-experts: 32 + ep-dispatch-algorithm: static + moe-a2a-backend: deepep + deepep-mode: low_latency + deepep-config: /configs/deepep_config.json + max-running-requests: 4096 + cuda-graph-max-bs: 4096 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: 512x1024x2048 diff --git a/CrossCluster_Recipes/GLM5/gb300/8k_1k/lowlatency/glm5-gb300-fp8-stp-lowlatency-1p3d-bs48-conc32x64x128x256.yaml b/CrossCluster_Recipes/GLM5/gb300/8k_1k/lowlatency/glm5-gb300-fp8-stp-lowlatency-1p3d-bs48-conc32x64x128x256.yaml new file mode 100644 index 000000000..c947dcc11 --- /dev/null +++ b/CrossCluster_Recipes/GLM5/gb300/8k_1k/lowlatency/glm5-gb300-fp8-stp-lowlatency-1p3d-bs48-conc32x64x128x256.yaml @@ -0,0 +1,106 @@ +# Generated from recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[1] +# Topology: 1 prefill nodes + 3 decode nodes = 4 nodes / 16 GPUs +name: gb300-fp8-glm5-stp-8k1k-lowlatency-1p3d-bs48-conc32x64x128x256 +model: + path: glm5-fp8 + container: nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa + precision: fp8 +identity: + model: + repo: zai-org/GLM-5-FP8 + revision: 4f96cc5eec29dcee5d6ded54f7ffe889438f9516 + frameworks: + dynamo: 1.3.0 + sglang: 0.5.14 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 3 + decode_workers: 3 +frontend: + type: dynamo +dynamo: + version: 1.3.0 + install: false +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + NCCL_CUMEM_ENABLE: '1' + DYN_REQUEST_PLANE: nats + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + sglang_config: + prefill: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + max-running-requests: 256 + cuda-graph-max-bs: 256 + mem-fraction-static: 0.7 + context-length: 9600 + chunked-prefill-size: 65536 + max-prefill-tokens: 8192 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + served-model-name: GLM-5-FP8 + trust-remote-code: true + quantization: fp8 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.8 + context-length: 9600 + tensor-parallel-size: 8 + expert-parallel-size: 1 + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + stream-interval: 30 + model-loader-extra-config: '{"enable_multithread_load": true}' + data-parallel-size: 1 + max-running-requests: 48 + cuda-graph-max-bs: 48 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: 32x64x128x256 diff --git a/recipes/gb300-fp8/glm5.yaml b/recipes/gb300-fp8/glm5.yaml index 8f9462f28..f446e4f3f 100644 --- a/recipes/gb300-fp8/glm5.yaml +++ b/recipes/gb300-fp8/glm5.yaml @@ -1,11 +1,11 @@ -# B200-FP8 GLM5 8k1k disaggregated recipe +# GB300-FP8 GLM5 disaggregated recipe base: name: "gb300-fp8-glm5" model: path: "glm5-fp8" - container: "sglang-v0.5.10-cu130.post1" + container: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa" precision: "fp8" identity: @@ -13,8 +13,8 @@ base: repo: "zai-org/GLM-5-FP8" revision: "4f96cc5eec29dcee5d6ded54f7ffe889438f9516" frameworks: - dynamo: "1.1.0.dev2" - sglang: "0.5.10.post1" + dynamo: "1.3.0" + sglang: "0.5.14" resources: gpu_type: "gb300" @@ -23,7 +23,8 @@ base: frontend: type: "dynamo" dynamo: - version: "1.1.0.dev2" + version: "1.3.0" + install: false backend: prefill_environment: