From 484c8a5f0ff848a8fa8619ee092090bec535dae0 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 14 Jul 2026 22:43:30 -0400 Subject: [PATCH 1/6] feat(qwen3.5-gb300-agentx): add Qwen3.5-NVFP4 GB300 SGLang AgentX benchmarks - Add qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg and -disagg configs - Add agg/disagg benchmark recipes and result files - Add GB300 AgentX launch script support - Add perf changelog entry for PR 2121 --- .../workflows/benchmark-multinode-tmpl.yml | 2 +- .github/workflows/run-sweep.yml | 8 +- ...g-gb300-tp4-c1-mtp-hicache-jid2191933.yaml | 114 +++++++++ ...-gb300-tp4-c96-mtp-hicache-jid2195211.yaml | 114 +++++++++ ...c128-mtp-hicache-convaware-jid2220685.yaml | 209 ++++++++++++++++ ...-c32-mtp-hicache-convaware-jid2212783.yaml | 207 ++++++++++++++++ ...-tp4-tp4-c64-mtp-convaware-jid2214439.yaml | 231 ++++++++++++++++++ ...-tp4-tp4-c96-mtp-convaware-jid2214440.yaml | 207 ++++++++++++++++ ...c192-mtp-hicache-convaware-jid2230562.yaml | 209 ++++++++++++++++ ...c256-mtp-hicache-convaware-jid2228613.yaml | 209 ++++++++++++++++ ...c384-mtp-hicache-convaware-jid2239740.yaml | 209 ++++++++++++++++ configs/nvidia-master.yaml | 179 ++++++++++++++ perf-changelog.yaml | 10 + runners/launch_gb300-nv.sh | 97 +++++--- utils/process_changelog.py | 2 +- 15 files changed, 1970 insertions(+), 37 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index b2219144f1..f98da238cf 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -240,7 +240,7 @@ jobs: ${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }} ${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }} ${{ inputs.kv-offload-backend != '' && inputs.kv-offload-backend != 'none' && inputs.kv-offload-backend != 'default' && inputs.kv-offload-backend || '' }} - c${{ inputs.conc != '' && inputs.conc || join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }} + c${{ join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }} steps: - name: Slurm cleanup (pre-run) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index b983982af8..69bbdbd0d6 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -556,7 +556,7 @@ jobs: precision: ${{ matrix.config.precision }} router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} - conc-list: '[${{ matrix.config.conc }}]' + conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} prefill-hardware: ${{ matrix.config.prefill.hardware }} @@ -577,7 +577,7 @@ jobs: decode-ep: ${{ matrix.config.decode.ep }} decode-dp-attn: ${{ matrix.config.decode.dp-attn }} decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} - conc: ${{ matrix.config.conc }} + conc: ${{ matrix.config.conc[0] }} kv-offloading: ${{ matrix.config.kv-offloading }} kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} @@ -705,7 +705,9 @@ jobs: needs.sweep-single-node-1k1k.result != 'skipped' || needs.sweep-single-node-8k1k.result != 'skipped' || needs.sweep-multi-node-1k1k.result != 'skipped' || - needs.sweep-multi-node-8k1k.result != 'skipped' + needs.sweep-multi-node-8k1k.result != 'skipped' || + needs.sweep-agentic.result != 'skipped' || + needs.sweep-multi-node-agentic.result != 'skipped' ) }} uses: ./.github/workflows/collect-results.yml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml new file mode 100644 index 0000000000..02c88cc299 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml @@ -0,0 +1,114 @@ +name: "agg-gb300-tp4-c1-mtp-hicache-jid2191933" + +# Agentic-coding SGLang aggregated recipe for Qwen3.5-397B-A17B-NVFP4 on GB300 +# (single aggregated worker, TP4, NEXTN MTP + hierarchical cache). +# +# Ported from the manually-run srtctl base:/zip_override_conc: starter to the +# flat single-variant agentic schema. Pure sglang (no dynamo): frontend.type +# sglang serves directly. Concurrency comes from the GHA matrix (exported as +# CONC into agentic_srt.sh), not the recipe. Serving tuning preserved verbatim +# from the source run; only CI scaffolding is added. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + nats_max_payload_mb: 8 + +frontend: + type: sglang + +backend: + type: sglang + aggregated_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 256 + max-running-requests: 1024 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + max-mamba-cache-size: 1024 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 1.01 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user into the container as a non-root uid; + # agentic_srt.sh's apt-get install git step needs EUID 0. Remap to uid 0 inside + # the container. srt-slurm renders empty-string values as flag-only srun args. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # Aggregated: one worker serves both prefill and decode, so GPU accounting is + # the single-worker form num_gpus = TP (not the disagg prefill+decode sum). + # Force the single-node post-proc path; TP must match sglang_config tp-size. + IS_MULTINODE: "false" + TP: "4" + # Match the source run's dataset (256k cc-traces-with-subagents variant). + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + # Container-side path of the aiperf mmap dataset cache; host-side mount wired + # via launch_gb300-nv.sh srtslurm.yaml default_mounts. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also via default_mounts). + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml new file mode 100644 index 0000000000..5891df5b32 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml @@ -0,0 +1,114 @@ +name: "agg-gb300-tp4-c96-mtp-hicache-jid2195211" + +# Agentic-coding SGLang aggregated recipe for Qwen3.5-397B-A17B-NVFP4 on GB300 +# (single aggregated worker, TP4, NEXTN MTP + hierarchical cache). +# +# Ported from the manually-run srtctl base:/zip_override_conc: starter to the +# flat single-variant agentic schema. Pure sglang (no dynamo): frontend.type +# sglang serves directly. Concurrency comes from the GHA matrix (exported as +# CONC into agentic_srt.sh), not the recipe. Serving tuning preserved verbatim +# from the source run; only CI scaffolding is added. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + nats_max_payload_mb: 8 + +frontend: + type: sglang + +backend: + type: sglang + aggregated_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + TORCH_CUDA_ARCH_LIST: '10.0' + PYTHONNOUSERSITE: '1' + NCCL_NVLS_ENABLE: '1' + SGL_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 256 + max-running-requests: 1024 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.8 + max-mamba-cache-size: 1024 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 1.05 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_through + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user into the container as a non-root uid; + # agentic_srt.sh's apt-get install git step needs EUID 0. Remap to uid 0 inside + # the container. srt-slurm renders empty-string values as flag-only srun args. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # Aggregated: one worker serves both prefill and decode, so GPU accounting is + # the single-worker form num_gpus = TP (not the disagg prefill+decode sum). + # Force the single-node post-proc path; TP must match sglang_config tp-size. + IS_MULTINODE: "false" + TP: "4" + # Match the source run's dataset (256k cc-traces-with-subagents variant). + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + # Container-side path of the aiperf mmap dataset cache; host-side mount wired + # via launch_gb300-nv.sh srtslurm.yaml default_mounts. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also via default_mounts). + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml new file mode 100644 index 0000000000..c2afab3c02 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml @@ -0,0 +1,209 @@ +name: "disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685" + +# Agentic-coding SGLang disaggregated 1P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + moe-a2a-backend: deepep + deepep-mode: low_latency + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: deep_gemm + speculative-moe-a2a-backend: deepep + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml new file mode 100644 index 0000000000..bcb7aa3b12 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml @@ -0,0 +1,207 @@ +name: "disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783" + +# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml new file mode 100644 index 0000000000..0c7a122994 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml @@ -0,0 +1,231 @@ +name: "disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439" + +# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300 (TP4 prefill / TP4 decode, NEXTN MTP + hierarchical cache). +# +# Ported from the manually-run srtctl base:/zip_override_conc: starter to the +# flat single-variant schema the agentic CI flow expects (modeled on the +# proven sglang/deepseek-v4/agentic recipes). Concurrency is NOT a recipe +# field here: the GHA matrix fans out one job per concurrency from the +# master-config conc-list, exporting CONC into agentic_srt.sh. Serving tuning +# (sglang_config, prefill/decode env, frontend, dynamo hash, topology) is +# preserved verbatim from the source run; only the CI scaffolding is added. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + # Dedicated etcd/nats node — under the dynamo router + multiple frontends + # the infra services need their own node (matches the proven agentic recipes). + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + # The sglang image is PEP 668 externally-managed; the runtime dynamo + # install (dynamo.hash source build) runs pip and fails with + # "externally-managed-environment" without this. + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: false + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + # gb300-nv: pyxis maps the calling user into the container as a non-root + # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to + # uid 0 inside the container. srt-slurm renders empty-string values as + # flag-only srun args. + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Match the source run's dataset (256k cc-traces-with-subagents variant). + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + # Per-session affinity so multi-turn KV cache is reused across turns + # (recipe name: "convaware"); build_replay_cmd adds + # --use-dynamo-conv-aware-routing for dynamo-* frameworks. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Container-side path of the aiperf mmap dataset cache; host-side mount + # wired via launch_gb300-nv.sh srtslurm.yaml default_mounts. + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + # Persistent HF hub cache (also via default_mounts) so the trace dataset + # isn't re-downloaded each run. + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml new file mode 100644 index 0000000000..53771b82d7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml @@ -0,0 +1,207 @@ +name: "disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440" + +# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: false + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml new file mode 100644 index 0000000000..1468a4b226 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml @@ -0,0 +1,209 @@ +name: "disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562" + +# Agentic-coding SGLang disaggregated 2P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 4 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + num_additional_frontends: 0 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + moe-a2a-backend: deepep + deepep-mode: low_latency + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: deep_gemm + speculative-moe-a2a-backend: deepep + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml new file mode 100644 index 0000000000..b6963a90b6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml @@ -0,0 +1,209 @@ +name: "disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613" + +# Agentic-coding SGLang disaggregated 3P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 2 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + num_additional_frontends: 0 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + moe-a2a-backend: deepep + deepep-mode: low_latency + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: deep_gemm + speculative-moe-a2a-backend: deepep + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml new file mode 100644 index 0000000000..1c2dc21cc8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml @@ -0,0 +1,209 @@ +name: "disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740" + +# Agentic-coding SGLang disaggregated 4P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 +# on GB300, ported to the flat single-variant agentic schema. Concurrency is +# NOT a recipe field: the GHA matrix fans out one job per concurrency and +# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, +# frontend, dynamo hash, topology) is preserved verbatim from the source run. + +model: + path: "qwen3.5-fp4" + container: "dynamo-sglang" + precision: "fp4" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 2 + prefill_workers: 4 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 + +frontend: + type: dynamo + enable_multiple_frontends: false + num_additional_frontends: 0 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: kv + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + PIP_BREAK_SYSTEM_PACKAGES: "1" + decode_environment: + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" + SGLANG_DISAGG_STAGING_BUFFER: '1' + NO_COLOR: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_ENABLE_SPEC_V2: '1' + PYTHONUNBUFFERED: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + PIP_BREAK_SYSTEM_PACKAGES: "1" + + sglang_config: + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 1.01 + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + moe-dense-tp-size: 1 + enable-dp-attention: true + enable-dp-lm-head: true + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + moe-a2a-backend: deepep + deepep-mode: low_latency + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: deep_gemm + speculative-moe-a2a-backend: deepep + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 64 + watchdog-timeout: 1000000 + decode-log-interval: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 73327351e1..c3e44880a3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11486,6 +11486,185 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 16 dp-attn: true + +qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [96] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + + +qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "3169427d2ed7ea6a0451e47ce443822d26893bae" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # 1P1D STP (TP4 prefill / TP4 decode). 2 GB300 nodes. + - spec-decoding: "mtp" + conc-list: [32] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [64] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [96] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + # 1P1D wide-EP (DEP4 prefill / DEP4 decode). 2 GB300 nodes. + - spec-decoding: "mtp" + conc-list: [128] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + # 2P1D wide-EP (2x DEP4 prefill / DEP4 decode). 3 GB300 nodes. + - spec-decoding: "mtp" + conc-list: [192] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + # 3P1D wide-EP (3x DEP4 prefill / DEP8 decode). 5 GB300 nodes. + - spec-decoding: "mtp" + conc-list: [256] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 4P1D wide-EP (4x DEP4 prefill / DEP8 decode). 6 GB300 nodes. + - spec-decoding: "mtp" + conc-list: [384] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + glm5-fp4-gb300-dynamo-sglang: image: lmsysorg/sglang:v0.5.11-cu130 model: nvidia/GLM-5.1-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d77c35f1fd..20999a7313 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4781,3 +4781,13 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Add Qwen3.5-397B-A17B-NVFP4 FP4 GB300 SGLang AgentX Pareto-frontier benchmarks via the srtctl/dynamo stack" + - "agg: single aggregated node (TP4, MTP/NEXTN, hierarchical KV cache), concurrencies [1, 96]" + - "disagg: 1P1D-4P1D (TP4/DEP4 prefill, TP4/DEP4/DEP8 decode, MTP/NEXTN, hicache, conv-aware routing) over 7 Pareto points at concurrency 32-384" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2; runner: gb300-nv" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2121 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 1381bd5db7..2b40810c36 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -17,6 +17,17 @@ export ENROOT_ROOTFS_WRITABLE=1 # write to it. export AIPERF_MMAP_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/ai-perf-cache" +# Persistent dynamo source-build cache. srtctl's hash-pinned dynamo install +# (_hash_cached_source_install) caches the built wheel + src tarball at +# /configs/dynamo-wheels/ with a .complete sentinel; on a warm cache the +# install is just `pip install` from the cache (no apt, no root). In CI /configs +# is the per-job srt-slurm checkout (cold every job → cold build needs apt + +# root, which the non-root server containers can't do), so persist and share +# the cache across jobs by bind-mounting this host dir at /configs/dynamo-wheels. +# Seed it once with a --container-remap-root build. 777 for multi-user runners. +export DYNAMO_WHEELS_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/dynamo-wheels" +mkdir -p "$DYNAMO_WHEELS_CACHE_HOST_PATH" + export HF_HUB_CACHE_HOST_PATH="/data/home/sa-shared/gharunners/hf-hub-cache" mkdir -p "$HF_HUB_CACHE_HOST_PATH" @@ -128,38 +139,57 @@ SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" if [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, - # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag - # (`srtctl apply --no-preflight`) — needed because: - # - # - We want MODEL_PATH=/scratch/models/DeepSeek-V4-Pro (node-local - # NVMe, fast) instead of the NFS path under /data/home/sa-shared. - # - /scratch only exists on GB300 compute nodes; it is NOT mounted - # on the GHA runner pod that invokes srtctl. - # - srtctl's pre-submit model check (_preflight_model in - # src/srtctl/core/validation.py) does a Path.is_dir() in-process - # on the invoking node — so it fails before sbatch is ever - # called with "Model alias 'X' resolved to '/scratch/...', - # but that path is unavailable". - # - --no-preflight skips just the optional Python-level FS check. - # vLLM still fails loudly at runtime if the path is genuinely - # missing on the compute node. - # - # All other upstream schema features we need are inherited from - # NVIDIA HEAD: - # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env - # (hook that hands off to benchmarks/multi_node/agentic_srt.sh) - # - DynamoConfig.wheel (so vllm recipes can pin the ai-dynamo wheel) - # - sbatch_directives / srun_options (top-level recipe fields) - git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - # 854b3fd = --no-preflight flag - # 6e34b8b = benchmark_stage propagates srun_options (needed for - # container-remap-root to reach the agentic_srt.sh srun) - git checkout 6e34b8b83229634d732e41a4e2d6595f46ef60b5 - mkdir -p recipes/vllm/deepseek-v4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic + if [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then + # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.22: the two features the + # cquil11 fork was pinned for are merged upstream (present in v1.0.22) — + # - `srtctl apply --no-preflight` (skip the in-process model FS check): + # model.path resolves to /scratch/models/Qwen3.5-397B-A17B-NVFP4 + # (compute-node-only NVMe), which the GHA runner pod can't stat, so + # the Path.is_dir() preflight would fail before sbatch is ever + # called. The engine still fails loudly at runtime if the path is + # genuinely missing on the compute node. + # - benchmark_stage propagates srun_options (container-remap-root must + # reach the agentic_srt.sh srun). + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.22 + mkdir -p recipes/sglang/qwen3.5 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ + recipes/sglang/qwen3.5 + else + # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, + # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag + # (`srtctl apply --no-preflight`) — needed because: + # + # - We want MODEL_PATH=/scratch/models/DeepSeek-V4-Pro (node-local + # NVMe, fast) instead of the NFS path under /data/home/sa-shared. + # - /scratch only exists on GB300 compute nodes; it is NOT mounted + # on the GHA runner pod that invokes srtctl. + # - srtctl's pre-submit model check (_preflight_model in + # src/srtctl/core/validation.py) does a Path.is_dir() in-process + # on the invoking node — so it fails before sbatch is ever + # called with "Model alias 'X' resolved to '/scratch/...', + # but that path is unavailable". + # - --no-preflight skips just the optional Python-level FS check. + # vLLM still fails loudly at runtime if the path is genuinely + # missing on the compute node. + # + # All other upstream schema features we need are inherited from + # NVIDIA HEAD: + # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env + # (hook that hands off to benchmarks/multi_node/agentic_srt.sh) + # - DynamoConfig.wheel (so vllm recipes can pin the ai-dynamo wheel) + # - sbatch_directives / srun_options (top-level recipe fields) + git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + # 854b3fd = --no-preflight flag + # 6e34b8b = benchmark_stage propagates srun_options (needed for + # container-remap-root to reach the agentic_srt.sh srun) + git checkout 6e34b8b83229634d732e41a4e2d6595f46ef60b5 + mkdir -p recipes/vllm/deepseek-v4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ + recipes/vllm/deepseek-v4/agentic + fi elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" @@ -281,6 +311,9 @@ srtctl_root: "${SRTCTL_ROOT}" # re-tokenized + re-written every job. default_mounts: "${AIPERF_MMAP_CACHE_HOST_PATH}": "/aiperf_mmap_cache" + # Warm dynamo source-build cache (nested over the auto /configs mount) so the + # hash-pinned install is a cache hit (pip-only, no apt/root) on every job. + "${DYNAMO_WHEELS_CACHE_HOST_PATH}": "/configs/dynamo-wheels" "${HF_HUB_CACHE_HOST_PATH}": "/hf_hub_cache" # Model path aliases diff --git a/utils/process_changelog.py b/utils/process_changelog.py index 311c8c9e22..c293dc2f80 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -270,7 +270,7 @@ def main(): # Validate final results structure validated = ChangelogMatrixEntry.model_validate(final_results) - print(validated.model_dump_json(by_alias=True)) + print(validated.model_dump_json(by_alias=True, exclude_none=True)) if __name__ == "__main__": From f0f925f5ccb6a2eb4915397e0eaad0362f188cc5 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Wed, 15 Jul 2026 15:27:51 -0700 Subject: [PATCH 2/6] qwen3.5 agentic: image 20260715-50d1edaa; prefill NEXTN MTP; fix agg GPU accounting - Bump the qwen3.5-fp4 GB300 agentic image (agg + disagg master-config keys and the changelog line) to lmsysorg/sglang:nightly-dev-cu13-20260715-50d1edaa. - Enable prefill-side NEXTN MTP (speculative-algorithm NEXTN, num-steps 3, eagle-topk 1, num-draft-tokens 4) in the 7 disagg recipes' prefill sglang_config. - Fix the agg GPU accounting: master-config decode num-worker 1 -> 0, and process_agentic_result.py zeroes the decode-side parallelism when num_decode_gpu <= 0, so TP/num_gpus reflect the single aggregated worker (4) instead of double-counting (8). Disagg configs are unaffected. --- ...ep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml | 4 ++++ ...d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml | 4 ++++ ...b300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml | 4 ++++ ...b300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml | 4 ++++ ...ep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml | 4 ++++ ...ep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml | 4 ++++ ...ep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml | 4 ++++ configs/nvidia-master.yaml | 8 ++++---- perf-changelog.yaml | 2 +- utils/agentic/aggregation/process_agentic_result.py | 10 ++++++++++ 10 files changed, 43 insertions(+), 5 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml index c2afab3c02..18d4b322b7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml @@ -145,6 +145,10 @@ backend: hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-ratio: 1.01 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 model-path: /model/ diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml index bcb7aa3b12..33319a1641 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml @@ -148,6 +148,10 @@ backend: hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-ratio: 1.01 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 model-path: /model/ diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml index 0c7a122994..9833142af5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml @@ -160,6 +160,10 @@ backend: hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-ratio: 1.01 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 model-path: /model/ diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml index 53771b82d7..eb2ffe45a9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml @@ -148,6 +148,10 @@ backend: hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-ratio: 1.01 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 model-path: /model/ diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml index 1468a4b226..cf5041c108 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml @@ -145,6 +145,10 @@ backend: hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-ratio: 1.01 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 model-path: /model/ diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml index b6963a90b6..9ecaaafe0a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml @@ -145,6 +145,10 @@ backend: hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-ratio: 1.01 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 model-path: /model/ diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml index 1c2dc21cc8..0ed641a2a0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml @@ -145,6 +145,10 @@ backend: hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-ratio: 1.01 + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 model-path: /model/ diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c0a767a912..a8c9293c31 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11831,7 +11831,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + image: lmsysorg/sglang:nightly-dev-cu13-20260715-50d1edaa model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: cluster:gb300-nv @@ -11855,7 +11855,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 4 ep: 1 dp-attn: false @@ -11871,14 +11871,14 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 4 ep: 1 dp-attn: false qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 + image: lmsysorg/sglang:nightly-dev-cu13-20260715-50d1edaa model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: cluster:gb300-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c4f07695c7..edf3216701 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4823,5 +4823,5 @@ - "Add Qwen3.5-397B-A17B-NVFP4 FP4 GB300 SGLang AgentX Pareto-frontier benchmarks via the srtctl/dynamo stack" - "agg: single aggregated node (TP4, MTP/NEXTN, hierarchical KV cache), concurrencies [1, 96]" - "disagg: 1P1D-4P1D (TP4/DEP4 prefill, TP4/DEP4/DEP8 decode, MTP/NEXTN, hicache, conv-aware routing) over 7 Pareto points at concurrency 32-384" - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2; runner: gb300-nv" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260715-50d1edaa; runner: gb300-nv" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2121 diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index 6b4acbc131..5d13fa0f1b 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -148,6 +148,16 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: num_prefill_gpu = prefill_num_workers * prefill_tp * prefill_pp * prefill_pcp_size num_decode_gpu = decode_num_workers * decode_tp * decode_pp * decode_pcp_size num_gpus = num_prefill_gpu + num_decode_gpu + # Aggregated configs set decode num-worker 0 (prefill+decode co-located on one + # worker), so there are no separate decode GPUs. Mirror process_result.py and drop + # the decode-side parallelism, so TP/EP and the per-GPU throughput denominator + # reflect the single aggregated worker instead of double-counting its GPUs. + if num_decode_gpu <= 0: + decode_tp = 0 + decode_ep = 0 + decode_pp = 1 + decode_dcp_size = 1 + decode_pcp_size = 1 tp = prefill_tp + decode_tp ep = max(prefill_ep, decode_ep) dp_attention = ( From 028f2ff0852bc722345e837b88fe42ea65e948d1 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Mon, 20 Jul 2026 11:37:49 -0700 Subject: [PATCH 3/6] qwen3.5 agentic: bump image to 20260716-b0b2dfbd Bump the qwen3.5-fp4 GB300 agentic image (agg + disagg master-config keys and the changelog line) to lmsysorg/sglang:nightly-dev-cu13-20260716-b0b2dfbd. --- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a8c9293c31..91f3f02148 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11831,7 +11831,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260715-50d1edaa + image: lmsysorg/sglang:nightly-dev-cu13-20260716-b0b2dfbd model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: cluster:gb300-nv @@ -11878,7 +11878,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260715-50d1edaa + image: lmsysorg/sglang:nightly-dev-cu13-20260716-b0b2dfbd model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: cluster:gb300-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index edf3216701..0a8724fae0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4823,5 +4823,5 @@ - "Add Qwen3.5-397B-A17B-NVFP4 FP4 GB300 SGLang AgentX Pareto-frontier benchmarks via the srtctl/dynamo stack" - "agg: single aggregated node (TP4, MTP/NEXTN, hierarchical KV cache), concurrencies [1, 96]" - "disagg: 1P1D-4P1D (TP4/DEP4 prefill, TP4/DEP4/DEP8 decode, MTP/NEXTN, hicache, conv-aware routing) over 7 Pareto points at concurrency 32-384" - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260715-50d1edaa; runner: gb300-nv" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260716-b0b2dfbd; runner: gb300-nv" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2121 From f1e6bcad34fdc8c20730194fc7d4f57dc793c200 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Mon, 20 Jul 2026 12:46:43 -0700 Subject: [PATCH 4/6] runners: add gb300-nv hardware metadata (available-cpu-dram-mib, gpus-per-node) The agentic DRAM-offload matrix logic requires available-cpu-dram-mib for any runner used by a kv-offloading: dram (hicache) config. gb300-nv had no hardware entry, so generate_sweep_configs test-config crashed for the gb300 agentic keys (qwen3.5 here, and dsv4 already on main). GB300 NVL72 uses the same 2x Grace / 4-GPU-per-node topology and LPDDR5X capacity as GB200, so mirror gb200-nv's available host DRAM (860_160 MiB) and gpus-per-node (4). --- configs/runners.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/runners.yaml b/configs/runners.yaml index 1386652f5a..c4bb40ba85 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -315,6 +315,9 @@ hardware: cluster:gb200-nv: available-cpu-dram-mib: 860_160 gpus-per-node: 4 + cluster:gb300-nv: + available-cpu-dram-mib: 860_160 + gpus-per-node: 4 cluster:mi300x-amds: available-cpu-dram-mib: 2_321_924 gpus-per-node: 8 From a46700eac5c8c66f8f81600bd67e8949ea15681f Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Fri, 24 Jul 2026 10:30:45 -0700 Subject: [PATCH 5/6] qwen3.5 agentic: bump image to 20260724-433429b1 --- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index d4e762af05..5391fa70b6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6328,7 +6328,7 @@ qwen3.5-fp4-gb300-dynamo-sglang: dp-attn: true qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260716-b0b2dfbd + image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: cluster:gb300-nv @@ -6375,7 +6375,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260716-b0b2dfbd + image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: cluster:gb300-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b43a744919..5b66e2e801 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4982,5 +4982,5 @@ - "Add Qwen3.5-397B-A17B-NVFP4 FP4 GB300 SGLang AgentX Pareto-frontier benchmarks via the srtctl/dynamo stack" - "agg: single aggregated node (TP4, MTP/NEXTN, hierarchical KV cache), concurrencies [1, 96]" - "disagg: 1P1D-4P1D (TP4/DEP4 prefill, TP4/DEP4/DEP8 decode, MTP/NEXTN, hicache, conv-aware routing) over 7 Pareto points at concurrency 32-384" - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260716-b0b2dfbd; runner: gb300-nv" + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1; runner: gb300-nv" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2121 From 32289c34d6091409fbf4b32085133f9bc5140a0d Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Fri, 24 Jul 2026 10:44:15 -0700 Subject: [PATCH 6/6] run-sweep: drop unneeded agentic collect-results gate Agentic sweep results are ingested by the dedicated trigger-agentic-ingest job (repository dispatch to InferenceX-app, main-push only), not through collect-results, whose gate intentionally excludes the agentic sweeps. This reverts the qwen branch's addition so run-sweep.yml matches main. --- .github/workflows/run-sweep.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 0cf4252385..8e8777cce9 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -867,9 +867,7 @@ jobs: needs.sweep-single-node-1k1k.result != 'skipped' || needs.sweep-single-node-8k1k.result != 'skipped' || needs.sweep-multi-node-1k1k.result != 'skipped' || - needs.sweep-multi-node-8k1k.result != 'skipped' || - needs.sweep-agentic.result != 'skipped' || - needs.sweep-multi-node-agentic.result != 'skipped' + needs.sweep-multi-node-8k1k.result != 'skipped' ) }} uses: ./.github/workflows/collect-results.yml