diff --git a/benchmarks/patches/vllm/minimax_m3_nvfp4_marlin.patch b/benchmarks/patches/vllm/minimax_m3_nvfp4_marlin.patch new file mode 100644 index 0000000000..a44bf655da --- /dev/null +++ b/benchmarks/patches/vllm/minimax_m3_nvfp4_marlin.patch @@ -0,0 +1,30 @@ +--- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py ++++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +@@ -176,2 +176,3 @@ + NvFp4MoeBackend.FLASHINFER_TRTLLM, ++ NvFp4MoeBackend.MARLIN, + } +--- a/vllm/model_executor/layers/fused_moe/experts/marlin_moe.py ++++ b/vllm/model_executor/layers/fused_moe/experts/marlin_moe.py +@@ -583,9 +583,12 @@ +- # Gated-activation params (used by SWIGLUOAI_UNINTERLEAVE on packed w13). +- # silu == swigluoai with alpha=1, beta=0; configs that don't set these +- # (plain silu) fall back to the silu identity. +- self.gemm1_alpha = ( +- quant_config.gemm1_alpha if quant_config.gemm1_alpha is not None else 1.0 +- ) +- self.gemm1_beta = ( +- quant_config.gemm1_beta if quant_config.gemm1_beta is not None else 0.0 +- ) ++ if self.gemm1_clamp_limit is None: ++ self.gemm1_clamp_limit = moe_config.swiglu_limit ++ ++ gemm1_alpha = quant_config.gemm1_alpha ++ if gemm1_alpha is None: ++ gemm1_alpha = moe_config.swiglu_alpha ++ self.gemm1_alpha = 1.0 if gemm1_alpha is None else gemm1_alpha ++ ++ gemm1_beta = quant_config.gemm1_beta ++ if gemm1_beta is None: ++ gemm1_beta = moe_config.swiglu_beta ++ self.gemm1_beta = 0.0 if gemm1_beta is None else gemm1_beta diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_rtx6000pro.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_rtx6000pro.sh new file mode 100755 index 0000000000..8f2aa5dc6b --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_rtx6000pro.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# MiniMax-M3 NVFP4 RTX PRO 6000 Blackwell single-node vLLM recipe. +# This is the PCIe/SM120 counterpart to minimaxm3_fp4_b200.sh. It keeps +# the ModelOpt NVFP4, FP8 KV-cache, and MSA block-size settings while using +# NCCL collectives instead of the B200-tuned FlashInfer/TRT-LLM all-reduce. +# +# The pinned vLLM image does not contain the MiniMax-M3 Marlin fixes tracked +# by vLLM PRs #45836 and #48929. Apply the narrow compatibility patch covered +# by docs/waiver/2306.md before importing vLLM. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + EP_SIZE \ + DP_ATTENTION \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +nvidia-smi + +SERVER_LOG=/workspace/server.log +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" + +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_FLOAT32_MATMUL_PRECISION=high + +if [ "${DP_ATTENTION}" = "true" ]; then + PARALLEL_ARGS=( + --tensor-parallel-size 1 + --data-parallel-size "$TP" + --enable-expert-parallel + ) +elif [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS=( + --tensor-parallel-size "$TP" + --enable-expert-parallel + ) +else + PARALLEL_ARGS=(--tensor-parallel-size "$TP") +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + setup_eval_context + MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi +start_gpu_monitor + +VLLM_SITE_PACKAGES="$( + python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])' +)" +VLLM_MARLIN_PATCH=/workspace/benchmarks/patches/vllm/minimax_m3_nvfp4_marlin.patch +if ! patch --batch --forward --fuzz=0 -p1 -d "$VLLM_SITE_PACKAGES" \ + < "$VLLM_MARLIN_PATCH"; then + echo "Failed to apply the pinned MiniMax-M3 Marlin compatibility patch" >&2 + exit 1 +fi + +set -x +vllm serve "$MODEL" --port "$PORT" \ + "${PARALLEL_ARGS[@]}" \ + --disable-custom-all-reduce \ + --gpu-memory-utilization "$GPU_MEM_UTIL" \ + --max-model-len "$MAX_MODEL_LEN" \ + --kv-cache-dtype fp8 \ + --block-size 128 \ + --language-model-only \ + --attention-backend TRITON_ATTN \ + --moe-backend marlin \ + --max-cudagraph-capture-size 2048 \ + --max-num-seqs "$CONC" \ + --max-num-batched-tokens "$((ISL * 2))" \ + --stream-interval 20 \ + --no-enable-prefix-caching \ + --trust-remote-code > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code + +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +stop_gpu_monitor +set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_rtx6000pro_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_rtx6000pro_mtp.sh new file mode 100755 index 0000000000..618578a58b --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_rtx6000pro_mtp.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash + +# MiniMax-M3 NVFP4 RTX PRO 6000 Blackwell single-node vLLM recipe with +# EAGLE3 speculative decoding. The SM120 target and draft both use Triton +# attention; the target's NVFP4 experts use Marlin. +# +# The pinned vLLM image does not contain the MiniMax-M3 Marlin fixes tracked +# by vLLM PRs #45836 and #48929. Apply the narrow compatibility patch covered +# by docs/waiver/2306.md before importing vLLM. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + EP_SIZE \ + DP_ATTENTION \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +SERVED_MODEL_NAME="$MODEL" +TARGET_MODEL_PATH="${MODEL_PATH:-$MODEL}" +if [[ "$TARGET_MODEL_PATH" != /* ]]; then + hf download "$TARGET_MODEL_PATH" +fi + +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" +hf download "$DRAFT_MODEL" + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +nvidia-smi + +SERVER_LOG=/workspace/server.log +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" +NUM_SPEC_TOKENS="${NUM_SPEC_TOKENS:-3}" + +if [[ ! "$NUM_SPEC_TOKENS" =~ ^[1-9][0-9]*$ ]]; then + echo "NUM_SPEC_TOKENS must be a positive integer, got: $NUM_SPEC_TOKENS" >&2 + exit 1 +fi + +# vLLM's speculative decode capture size is measured in tokens. Each active +# sequence can contribute one accepted token plus NUM_SPEC_TOKENS draft tokens. +CUDAGRAPH_CAPTURE_SIZE="$((CONC * (NUM_SPEC_TOKENS + 1)))" + +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_FLOAT32_MATMUL_PRECISION=high + +if [ "${DP_ATTENTION}" = "true" ]; then + PARALLEL_ARGS=( + --tensor-parallel-size 1 + --data-parallel-size "$TP" + --enable-expert-parallel + ) + DRAFT_TP=1 +elif [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS=( + --tensor-parallel-size "$TP" + --enable-expert-parallel + ) + DRAFT_TP="$TP" +else + PARALLEL_ARGS=(--tensor-parallel-size "$TP") + DRAFT_TP="$TP" +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + setup_eval_context + MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi +start_gpu_monitor + +VLLM_SITE_PACKAGES="$( + python3 -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])' +)" +VLLM_MARLIN_PATCH=/workspace/benchmarks/patches/vllm/minimax_m3_nvfp4_marlin.patch +if ! patch --batch --forward --fuzz=0 -p1 -d "$VLLM_SITE_PACKAGES" \ + < "$VLLM_MARLIN_PATCH"; then + echo "Failed to apply the pinned MiniMax-M3 Marlin compatibility patch" >&2 + exit 1 +fi + +SPECULATIVE_CONFIG="$( + printf '{"method":"eagle3","model":"%s","num_speculative_tokens":%d,"draft_tensor_parallel_size":%d,"attention_backend":"TRITON_ATTN"}' \ + "$DRAFT_MODEL" "$NUM_SPEC_TOKENS" "$DRAFT_TP" +)" + +set -x +vllm serve "$TARGET_MODEL_PATH" \ + --served-model-name "$SERVED_MODEL_NAME" \ + --port "$PORT" \ + "${PARALLEL_ARGS[@]}" \ + --disable-custom-all-reduce \ + --gpu-memory-utilization "$GPU_MEM_UTIL" \ + --max-model-len "$MAX_MODEL_LEN" \ + --kv-cache-dtype fp8 \ + --block-size 128 \ + --language-model-only \ + --attention-backend TRITON_ATTN \ + --moe-backend marlin \ + --max-cudagraph-capture-size "$CUDAGRAPH_CAPTURE_SIZE" \ + --max-num-seqs "$CONC" \ + --max-num-batched-tokens "$((ISL * 2))" \ + --speculative-config "$SPECULATIVE_CONFIG" \ + --stream-interval 20 \ + --no-enable-prefix-caching \ + --trust-remote-code > "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$SERVED_MODEL_NAME" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --trust-remote-code \ + --use-chat-template + +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +stop_gpu_monitor +set +x diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a388cad95d..cd0455d58f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7347,6 +7347,44 @@ minimaxm3-fp8-b300-vllm: - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128 } - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 } +# MiniMax-M3 NVFP4 single-node vLLM sweep using 4 of 8 RTX PRO 6000 GPUs. +minimaxm3-fp4-rtx6000pro-vllm: + image: vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41@sha256:6af4be7ae69a5f424de85b3d514ac79778bdcf4a9d05f93ec908a4095e0f1253 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: rtx6000pro-lat + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-list: [1, 4, 16, 64] } + # Marlin EP4 uses the standard NCCL expert exchange; DeepEP/NIXL + # batched activation layouts are not compatible. + - { tp: 4, ep: 4, conc-list: [1, 4, 16, 64] } + +# EAGLE3 speculative-decoding twin of the RTX PRO 6000 MiniMax-M3 NVFP4 +# sweep. The external Inferact/MiniMax-M3-EAGLE3 draft uses three speculative +# tokens and Triton attention on SM120; prompts use the model's chat template. +minimaxm3-fp4-rtx6000pro-vllm-mtp: + image: vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41@sha256:6af4be7ae69a5f424de85b3d514ac79778bdcf4a9d05f93ec908a4095e0f1253 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: rtx6000pro-lat + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, conc-list: [1, 4, 16, 64], spec-decoding: mtp } + - { tp: 4, ep: 4, conc-list: [1, 4, 16, 64], spec-decoding: mtp } + # MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B300 single-node vLLM — FP4 variant # of minimaxm3-fp8-b300-vllm. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm # PR #46380) is baked into the perf container image, so no runtime patch is diff --git a/configs/runners.yaml b/configs/runners.yaml index 851b821ba2..4c6b11069d 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -164,6 +164,10 @@ labels: - gb300-nv_0 - gb300-nv_1 - gb300-nv_2 + rtx6000pro: + - rtx6000pro-lat_00 + rtx6000pro-lat: + - rtx6000pro-lat_00 cluster:h100-cw: - h100-cw_00 - h100-cw_01 @@ -251,6 +255,8 @@ labels: - gb300-nv_0 - gb300-nv_1 - gb300-nv_2 + cluster:rtx6000pro-lat: + - rtx6000pro-lat_00 cluster:mi300x-amds: - mi300x-amds_00 - mi300x-amds_01 @@ -315,6 +321,9 @@ hardware: cluster:gb200-nv: available-cpu-dram-mib: 860_160 gpus-per-node: 4 + cluster:rtx6000pro-lat: + available-cpu-dram-mib: 1_500_000 + gpus-per-node: 8 cluster:mi300x-amds: available-cpu-dram-mib: 2_321_924 gpus-per-node: 8 diff --git a/docs/waiver/2306.md b/docs/waiver/2306.md new file mode 100644 index 0000000000..4703704d42 --- /dev/null +++ b/docs/waiver/2306.md @@ -0,0 +1,47 @@ +# Runtime patch waiver for PR 2306 + +
+ +**English** | [中文](./2306_zh.md) + +
+ +## Scope + +The MiniMax-M3 NVFP4 RTX PRO 6000 STP and EAGLE3 MTP recipes apply +`benchmarks/patches/vllm/minimax_m3_nvfp4_marlin.patch` to the vLLM Python +package in their pinned container before starting the server. The patch: + +- admits Marlin to the clamped NVFP4 MoE backend allowlist; and +- makes Marlin fall back to the model's SwiGLU clamp, alpha, and beta when + those values are absent from the quantization config. + +It does not modify a CUDA kernel, checkpoint weight, or benchmark result. + +## Why the unmodified image cannot run this benchmark + +MiniMax-M3 sets a SwiGLU clamp. The pinned vLLM image consequently filters its +NVFP4 MoE candidates to FlashInfer TRT-LLM, whose device guard accepts SM100 +but rejects the RTX PRO 6000's SM120 compute capability. Explicitly selecting +Marlin in the unmodified image is also rejected by that allowlist. If only the +allowlist is widened, Marlin substitutes plain-SiLU defaults for the model's +alpha and beta and produces incorrect output. + +The recipe therefore selects `--moe-backend marlin` and carries the minimum +Python adapter fix required to preserve the model's activation parameters. + +## Upstream tracking + +The Marlin clamp allowlist is upstream in +https://github.com/vllm-project/vllm/pull/45836. The model-config parameter +fallback is the Marlin portion of +https://github.com/vllm-project/vllm/pull/48929. The latter pull request +includes focused tests and reports coherent deterministic MiniMax-M3 NVFP4 +output after the fix, where the unpatched Marlin path produced garbled output. + +## Removal plan + +Replace the pinned image with the first suitable upstream vLLM image that +contains the Marlin fix, then remove the runtime patch application and patch +file. Revalidate STP and EAGLE3 MTP TP4 and TP4/EP4 8k/1k generation on SM120 +before removing this waiver. diff --git a/docs/waiver/2306_zh.md b/docs/waiver/2306_zh.md new file mode 100644 index 0000000000..c5a115e92b --- /dev/null +++ b/docs/waiver/2306_zh.md @@ -0,0 +1,41 @@ +# PR 2306 运行时补丁豁免 + +
+ +[English](./2306.md) | **中文** + +
+ +## 范围 + +MiniMax-M3 NVFP4 RTX PRO 6000 的 STP 和 EAGLE3 MTP 配方会在启动服务前,将 +`benchmarks/patches/vllm/minimax_m3_nvfp4_marlin.patch` 应用到锁定镜像中的 +vLLM Python 包。该补丁: + +- 将 Marlin 加入支持 clamp 的 NVFP4 混合专家(MoE)后端允许列表;以及 +- 当量化配置未提供 SwiGLU clamp、alpha 和 beta 时,让 Marlin 回退到模型配置中的对应值。 + +该补丁不修改 CUDA 内核、模型权重或基准测试结果。 + +## 为何无法直接使用未修改的镜像 + +MiniMax-M3 设置了 SwiGLU clamp,因此锁定版本的 vLLM 会把 NVFP4 MoE 候选后端 +限制为 FlashInfer TRT-LLM。该后端的设备检查只接受 SM100,不接受 RTX PRO 6000 +的 SM120 计算能力。未修改的镜像还会通过允许列表拒绝显式选择 Marlin。如果仅 +扩大允许列表,Marlin 会使用普通 SiLU 的 alpha、beta 默认值,并产生错误输出。 + +因此,该配方显式选择 `--moe-backend marlin`,并携带保留模型激活参数所需的最小 +Python 适配层修复。 + +## 上游跟踪 + +Marlin 的 clamp 允许列表修复已由 +https://github.com/vllm-project/vllm/pull/45836 合入上游。模型配置参数回退逻辑来自 +https://github.com/vllm-project/vllm/pull/48929 中的 Marlin 部分。后者包含针对性 +测试,并报告修复后 MiniMax-M3 NVFP4 的确定性输出从乱码恢复为连贯文本。 + +## 移除计划 + +当包含 Marlin 修复的合适 vLLM 上游镜像发布后,更新锁定镜像,并移除运行时补丁 +应用逻辑及补丁文件。移除此豁免前,需要在 SM120 上重新验证 STP 和 EAGLE3 MTP +的 TP4 和 TP4/EP4 8k/1k 生成。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 975c917743..76e172289b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5066,3 +5066,19 @@ description: - "Bump SGLang container image from lmsysorg/sglang:v0.5.12-cu130 to lmsysorg/sglang:v0.5.15.post1-cu130 (https://github.com/sgl-project/sglang/releases/tag/v0.5.15.post1)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2313 + +- config-keys: + - minimaxm3-fp4-rtx6000pro-vllm + - minimaxm3-fp4-rtx6000pro-vllm-mtp + scenario-type: + - fixed-seq-len + description: + - "Add MiniMax-M3 NVFP4 single-node vLLM benchmarking using four GPUs per job on the 8x RTX PRO 6000 Blackwell Latitude runner" + - "Sweep 8k/1k at concurrency 1, 4, 16, and 64 for both GPU-resident TP4 and TEP4 (TP4 attention with EP4 MoE, without DP attention), using FP8 KV cache, MSA block size 128, and NCCL collectives for the PCIe-only topology" + - "Pin the CUDA 13.0.1 MiniMax-M3 performance image by digest and force the non-FlashInfer Marlin MoE backend on SM120" + - "Apply the narrow vLLM adapter fixes tracked by upstream PRs #45836 and #48929 so Marlin preserves MiniMax-M3's SwiGLU clamp, alpha, and beta; documented by docs/waiver/2306.md" + - "Use Triton attention because MiniMax-M3 requires block size 128, which the image's default SM120 FlashInfer attention backend does not accept" + - "Add an EAGLE3 MTP twin using Inferact/MiniMax-M3-EAGLE3 with three speculative tokens, Triton attention for the SM120 draft, chat-formatted prompts, and the same TP4/TEP4 concurrency sweep" + - "Right-size max-num-seqs to benchmark concurrency for both recipes; size the MTP CUDA-graph token cap to concurrency times four so the draft does not reserve the previous 2048-token graph envelope" + - "Disable NCCL IB/RoCE discovery on this single-node runner to avoid an NCCL 2.28.9 bnxt_re initialization segfault" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2306 diff --git a/runners/launch_rtx6000pro-lat.sh b/runners/launch_rtx6000pro-lat.sh new file mode 100755 index 0000000000..93007ee127 --- /dev/null +++ b/runners/launch_rtx6000pro-lat.sh @@ -0,0 +1,137 @@ +#!/usr/bin/bash +set -euo pipefail + +# This runner executes directly on the single RTX PRO 6000 GPU node. Docker +# therefore owns a separate image cache from the node's RKE2/containerd cache. +HF_HUB_CACHE_MOUNT="${HF_HUB_CACHE_MOUNT:-/var/lib/inferencex/hf-hub-cache}" +export HF_HUB_CACHE="${HF_HUB_CACHE:-/mnt/hf_hub_cache/}" +PORT="${PORT:-8888}" + +# NCCL 2.28.9 segfaults while probing this node's bnxt_re RDMA devices. +# These jobs are single-node, so disable only the IB/RoCE transport while +# preserving CUDA P2P and shared-memory transports plus a caller override. +export NCCL_IB_DISABLE="${NCCL_IB_DISABLE:-1}" + +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE must be set}" +: "${IMAGE:?IMAGE must be set}" +: "${EXP_NAME:?EXP_NAME must be set}" +: "${PRECISION:?PRECISION must be set}" + +mkdir -p "$HF_HUB_CACHE_MOUNT" + +export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" +if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then + echo "GPU_COUNT must be a positive integer, got: $GPU_COUNT" >&2 + exit 1 +fi + +export CUDA_VISIBLE_DEVICES +CUDA_VISIBLE_DEVICES="$(seq -s, 0 "$((GPU_COUNT - 1))")" + +# Some Slurm/enroot configs spell registry paths as nvcr.io#namespace/image. +# Docker requires the normal slash form. +DOCKER_IMAGE="${IMAGE//#//}" + +SPEC_SUFFIX="" +if [[ "${SPEC_DECODING:-}" == "mtp" ]]; then + SPEC_SUFFIX="_mtp" +fi + +export SCENARIO_SUBDIR="${SCENARIO_SUBDIR:-fixed_seq_len/}" +SCENARIO_SUBDIR="${SCENARIO_SUBDIR#/}" +SCENARIO_SUBDIR="${SCENARIO_SUBDIR%/}/" +BENCH_SCRIPT="benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_rtx6000pro${SPEC_SUFFIX}.sh" + +if [[ ! -f "$GITHUB_WORKSPACE/$BENCH_SCRIPT" ]]; then + echo "Benchmark script not found: $GITHUB_WORKSPACE/$BENCH_SCRIPT" >&2 + exit 1 +fi + +server_name="bmk-server-${RUNNER_NAME:-rtx6000pro-lat}" +server_name="${server_name//[^a-zA-Z0-9_.-]/-}" + +cleanup() { + docker rm -f "$server_name" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +# Clear a container left behind by a cancelled or interrupted workflow. +cleanup + +docker run \ + --rm \ + --pull=missing \ + --name="$server_name" \ + --runtime=nvidia \ + --gpus="$GPU_COUNT" \ + --network=host \ + --ipc=host \ + --privileged \ + --shm-size=32g \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + --security-opt seccomp=unconfined \ + --cap-add=SYS_PTRACE \ + --volume "$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE" \ + --volume "$GITHUB_WORKSPACE:/workspace/" \ + --workdir=/workspace/ \ + --env HF_TOKEN \ + --env HF_HUB_CACHE \ + --env MODEL \ + --env MODEL_PREFIX \ + --env MODEL_PATH \ + --env TP \ + --env PP_SIZE \ + --env DCP_SIZE \ + --env PCP_SIZE \ + --env EP_SIZE \ + --env DP_SIZE \ + --env DP_ATTENTION \ + --env GPU_COUNT \ + --env CONC \ + --env MAX_MODEL_LEN \ + --env ISL \ + --env OSL \ + --env FRAMEWORK \ + --env PRECISION \ + --env DISAGG \ + --env SPEC_DECODING \ + --env NUM_SPEC_TOKENS \ + --env RUN_EVAL \ + --env EVAL_ONLY \ + --env EVAL_LIMIT \ + --env EVAL_MAX_MODEL_LEN \ + --env RUNNER_TYPE \ + --env RUNNER_NAME \ + --env RESULT_FILENAME \ + --env RESULT_DIR \ + --env RANDOM_RANGE_RATIO \ + --env GPU_MEM_UTIL \ + --env AIPERF_FAILED_REQUEST_THRESHOLD \ + --env KV_OFFLOADING \ + --env KV_OFFLOAD_BACKEND \ + --env KV_OFFLOAD_BACKEND_METADATA \ + --env ROUTER_METADATA \ + --env KV_P2P_TRANSFER \ + --env TOTAL_CPU_DRAM_GB \ + --env DURATION \ + --env SCENARIO_TYPE \ + --env SCENARIO_SUBDIR \ + --env IS_AGENTIC \ + --env SWEBENCH_GEN_MODE \ + --env SWEBENCH_USE_MODAL \ + --env MODAL_TOKEN_ID \ + --env MODAL_TOKEN_SECRET \ + --env PROFILE \ + --env SGLANG_TORCH_PROFILER_DIR \ + --env VLLM_TORCH_PROFILER_DIR \ + --env VLLM_RPC_TIMEOUT \ + --env PYTHONDONTWRITEBYTECODE \ + --env PYTHONPYCACHEPREFIX=/tmp/pycache/ \ + --env PORT="$PORT" \ + --env CUDA_DEVICE_ORDER=PCI_BUS_ID \ + --env CUDA_VISIBLE_DEVICES \ + --env NCCL_IB_DISABLE \ + --entrypoint=/bin/bash \ + "$DOCKER_IMAGE" \ + "$BENCH_SCRIPT"