diff --git a/.github/workflows/sglang_benchmark_workflow.yaml b/.github/workflows/sglang_benchmark_workflow.yaml index 2baf431a12e0..3e03c22d5bc4 100644 --- a/.github/workflows/sglang_benchmark_workflow.yaml +++ b/.github/workflows/sglang_benchmark_workflow.yaml @@ -5,8 +5,8 @@ on: branches: - qwen3_5_v0.5.15_dflash schedule: - # Daily at 22:00 Beijing time (14:00 UTC) - - cron: "0 14 * * *" + # Daily at 04:00 AM Beijing time (20:00 UTC) + - cron: "0 20 * * *" workflow_dispatch: inputs: run_mode: @@ -24,6 +24,7 @@ on: type: choice options: - all + - load_test - pure_text - concurrency - request_rate @@ -46,6 +47,8 @@ env: CONTAINER_MODELS_ROOT: /models SGLANG_BENCHMARK_EXAMPLE_ROOT: /models/benchamark_example SGLANG_BENCHMARK_RESULTS_DIR: benchmark_test_results + SGLANG_BENCHMARK_DATASETS_ROOT: /models/benchmark_datasets + SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH: /models/benchmark_datasets/ShareGPT_V3_unfiltered_cleaned_split.json TVM_FFI_DISABLE_TORCH_C_DLPACK: "1" SGLANG_GITHUB_REPO_URL: ${{ github.event.pull_request.head.repo.clone_url || 'https://github.com/apinge/sglang.git' }} SGLANG_GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -80,8 +83,8 @@ jobs: if event_name == "schedule" or ( event_name == "workflow_dispatch" and run_mode == "benchmark" ): - if benchmark_target == "all": - targets = ["pure_text", "concurrency", "request_rate"] + if benchmark_target in {"all", "load_test"}: + targets = ["load_test"] elif benchmark_target in {"pure_text", "concurrency", "request_rate"}: targets = [benchmark_target] else: @@ -271,19 +274,51 @@ jobs: ls -lah "${{ env.HOST_MODELS_ROOT }}" du -sh "${MODEL_PATH_HOST}" + - name: Validate benchmark datasets + if: ${{ matrix.job_mode == 'accuracy' || matrix.job_mode == 'benchmark' }} + run: | + set -euo pipefail + + dataset_path="${{ env.HOST_MODELS_ROOT }}/benchmark_datasets/ShareGPT_V3_unfiltered_cleaned_split.json" + if [ ! -f "${dataset_path}" ]; then + echo "Expected ShareGPT dataset not found on host: ${dataset_path}" + echo "Container path will be: ${{ env.SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH }}" + echo "Prepare it once on the runner with:" + echo " bash scripts/ci/prepare_benchmark_datasets.sh ${{ env.HOST_MODELS_ROOT }}/benchmark_datasets" + exit 1 + fi + + python3 - <<'PY' "${dataset_path}" + import json + import sys + from pathlib import Path + + path = Path(sys.argv[1]) + json.loads(path.read_text(encoding="utf-8")) + print(f"Validated ShareGPT dataset: {path} ({path.stat().st_size} bytes)") + PY + - name: Validate benchmark scripts - if: ${{ matrix.job_mode == 'benchmark' && matrix.benchmark_target != 'pure_text' }} + if: ${{ matrix.job_mode == 'benchmark' && (matrix.benchmark_target == 'load_test' || matrix.benchmark_target == 'concurrency' || matrix.benchmark_target == 'request_rate') }} run: | set -euo pipefail benchmark_example_host="${{ env.HOST_MODELS_ROOT }}/benchamark_example" - script_path="${benchmark_example_host}/${BENCHMARK_TARGET}/run.sh" - if [ ! -f "${script_path}" ]; then - echo "Expected benchmark script not found on host: ${script_path}" - echo "Container path will be: ${{ env.SGLANG_BENCHMARK_EXAMPLE_ROOT }}/${BENCHMARK_TARGET}/run.sh" - exit 1 + if [ "${{ matrix.benchmark_target }}" = "load_test" ]; then + benchmark_targets=(concurrency request_rate) + else + benchmark_targets=("${{ matrix.benchmark_target }}") fi + for target in "${benchmark_targets[@]}"; do + script_path="${benchmark_example_host}/${target}/run.sh" + if [ ! -f "${script_path}" ]; then + echo "Expected benchmark script not found on host: ${script_path}" + echo "Container path will be: ${{ env.SGLANG_BENCHMARK_EXAMPLE_ROOT }}/${target}/run.sh" + exit 1 + fi + done + ls -lah "${benchmark_example_host}" - name: Generate Dockerfile @@ -349,6 +384,8 @@ jobs: -e SGLANG_BENCHMARK_PORT=${{ env.SGLANG_BENCHMARK_PORT }} \ -e SGLANG_BENCHMARK_EXAMPLE_ROOT=${{ env.SGLANG_BENCHMARK_EXAMPLE_ROOT }} \ -e SGLANG_BENCHMARK_RESULTS_DIR=${{ env.SGLANG_BENCHMARK_RESULTS_DIR }} \ + -e SGLANG_BENCHMARK_DATASETS_ROOT=${{ env.SGLANG_BENCHMARK_DATASETS_ROOT }} \ + -e SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH=${{ env.SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH }} \ -e TVM_FFI_DISABLE_TORCH_C_DLPACK=${{ env.TVM_FFI_DISABLE_TORCH_C_DLPACK }} \ -v ${{ env.HOST_MODELS_ROOT }}:${{ env.CONTAINER_MODELS_ROOT }}:ro \ --security-opt seccomp=unconfined \ @@ -423,9 +460,9 @@ jobs: set -ex docker exec "${CONTAINER_NAME}" bash -lc "bash scripts/ci/sglang_benchmark_workflow.sh pure_text \"${MODEL_NAME}\" \"${MODEL_PATH}\" \"${MODEL_TP}\" 1" - - name: Run selected load tests + - name: Run load tests if: ${{ matrix.job_mode == 'benchmark' }} - timeout-minutes: 300 + timeout-minutes: 120 run: | set -ex docker exec "${CONTAINER_NAME}" bash -lc "bash scripts/ci/sglang_benchmark_workflow.sh ${BENCHMARK_TARGET} \"${MODEL_NAME}\" \"${MODEL_PATH}\" \"${MODEL_TP}\" 1 45" @@ -473,6 +510,8 @@ jobs: target = os.environ.get("BENCHMARK_TARGET", "pure_text") if target == "accuracy": target = "pure_text" + elif target == "load_test": + target = "pure_text, concurrency, request_rate" f.write(f"- Target: `{target}`\n") for log_file in log_files: f.write(f"- Log: `{log_file}`\n") @@ -487,4 +526,16 @@ jobs: - name: Clean up if: always() run: | + set -euo pipefail + + # Only remove resources created by this workflow job. Do not prune or + # delete other images/containers on the shared CI runner. + echo "Stopping and removing CI container: ${CONTAINER_NAME}" docker rm -f "${CONTAINER_NAME}" || true + + echo "Removing CI-built image for this job: ${LOCAL_IMAGE_NAME}" + docker rmi -f "${LOCAL_IMAGE_NAME}" || true + + rm -f Dockerfile.mod || true + + echo "CI cleanup complete (container + image for this job only)." diff --git a/scripts/ci/sglang_benchmark_workflow.sh b/scripts/ci/sglang_benchmark_workflow.sh index 9c3b86d6d40d..e750f1492491 100755 --- a/scripts/ci/sglang_benchmark_workflow.sh +++ b/scripts/ci/sglang_benchmark_workflow.sh @@ -22,6 +22,9 @@ PURE_TEXT_INPUT_TOKENS=${SGLANG_BENCHMARK_PURE_TEXT_INPUT_TOKENS:-8000} PURE_TEXT_OUTPUT_TOKENS=${SGLANG_BENCHMARK_PURE_TEXT_OUTPUT_TOKENS:-500} PURE_TEXT_NUM_PROMPTS=${SGLANG_BENCHMARK_PURE_TEXT_NUM_PROMPTS:-32} PURE_TEXT_MAX_CONCURRENCY=${SGLANG_BENCHMARK_PURE_TEXT_MAX_CONCURRENCY:-1} +BENCHMARK_DATASETS_ROOT=${SGLANG_BENCHMARK_DATASETS_ROOT:-/models/benchmark_datasets} +SHAREGPT_DATASET_FILENAME=${SGLANG_BENCHMARK_SHAREGPT_FILENAME:-ShareGPT_V3_unfiltered_cleaned_split.json} +SHAREGPT_DATASET_PATH=${SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH:-${BENCHMARK_DATASETS_ROOT}/${SHAREGPT_DATASET_FILENAME}} if [[ -z "${SGLANG_BENCHMARK_PORT:-}" && -f "${SERVER_PORT_FILE}" ]]; then PORT=$(<"${SERVER_PORT_FILE}") @@ -432,6 +435,8 @@ run_external_benchmark() { export BASE_URL="http://127.0.0.1:${PORT}" export SGLANG_BENCHMARK_MODEL_PATH="${MODEL_PATH}" export SGLANG_BENCHMARK_PORT="${PORT}" + export SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH="${SHAREGPT_DATASET_PATH}" + export DATASET_PATH="${SHAREGPT_DATASET_PATH}" bash -euo pipefail "./run.sh" ) | tee "${log_path}" @@ -454,6 +459,31 @@ ensure_server_ready() { return 1 } +validate_sharegpt_dataset() { + if python3 - <<'PY' "${SHAREGPT_DATASET_PATH}" +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +if not path.is_file(): + raise SystemExit(f"ShareGPT dataset not found: {path}") +try: + json.loads(path.read_text(encoding="utf-8")) +except json.JSONDecodeError as exc: + raise SystemExit(f"ShareGPT dataset is not valid JSON: {path} ({exc})") +print(f"Using ShareGPT dataset: {path}") +PY + then + return 0 + fi + + echo "Expected local ShareGPT dataset at: ${SHAREGPT_DATASET_PATH}" + echo "Prepare it once on the runner host with:" + echo " bash scripts/ci/prepare_benchmark_datasets.sh /raid/models/benchmark_datasets" + return 1 +} + run_pure_text_benchmark() { local log_path="${BENCHMARK_RESULTS_DIR}/pure_text_benchmark_${MODEL_NAME}_TP${TP}_EP${EP}.log" @@ -468,11 +498,14 @@ run_pure_text_benchmark() { echo "max concurrency: ${PURE_TEXT_MAX_CONCURRENCY}" echo "num prompts: ${PURE_TEXT_NUM_PROMPTS}" echo "dataset-name: random" + echo "dataset-path: ${SHAREGPT_DATASET_PATH}" + validate_sharegpt_dataset python3 -m sglang.bench_serving \ --backend sglang \ --model "${MODEL_PATH}" \ --dataset-name random \ + --dataset-path "${SHAREGPT_DATASET_PATH}" \ --host localhost \ --port "${PORT}" \ --num-prompts "${PURE_TEXT_NUM_PROMPTS}" \ @@ -617,6 +650,12 @@ PY elif [[ "${TYPE}" == "concurrency" || "${TYPE}" == "request_rate" ]]; then run_external_benchmark "${TYPE}" +elif [[ "${TYPE}" == "load_test" ]]; then + ensure_server_ready + run_pure_text_benchmark + run_external_benchmark "concurrency" + run_external_benchmark "request_rate" + elif [[ "${TYPE}" == "pure_text" ]]; then run_pure_text_benchmark @@ -640,7 +679,7 @@ elif [[ "${TYPE}" == "performance" ]]; then else echo "Unknown TYPE: ${TYPE}" - echo "Usage: $0 {launch|evaluation|pure_text|concurrency|request_rate|performance} [model_name] [model_path] [TP] [EP] [timeout]" + echo "Usage: $0 {launch|evaluation|load_test|pure_text|concurrency|request_rate|performance} [model_name] [model_path] [TP] [EP] [timeout]" exit 1 fi