Skip to content

[CI] Optimize CI workflow #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
apinge merged 2 commits into from
Jul 31, 2026
Merged

[CI] Optimize CI workflow #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0cc7a1a
Optimize CI workflow
sammysun0711 Jul 30, 2026
95863db
Use pre-download shargpt dataset for benchmark test
sammysun0711 Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 63 additions & 12 deletions .github/workflows/sglang_benchmark_workflow.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ on:
branches:
- qwen3_5_v0.5.15_dflash
schedule:
# Daily at 22:00 Beijing time (14:00 UTC)
- cron: "0 14 * * *"
# Daily at 04:00 AM Beijing time (20:00 UTC)
- cron: "0 20 * * *"
workflow_dispatch:
inputs:
run_mode:
Expand All @@ -24,6 +24,7 @@ on:
type: choice
options:
- all
- load_test
- pure_text
- concurrency
- request_rate
Expand All @@ -46,6 +47,8 @@ env:
CONTAINER_MODELS_ROOT: /models
SGLANG_BENCHMARK_EXAMPLE_ROOT: /models/benchamark_example
SGLANG_BENCHMARK_RESULTS_DIR: benchmark_test_results
SGLANG_BENCHMARK_DATASETS_ROOT: /models/benchmark_datasets
SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH: /models/benchmark_datasets/ShareGPT_V3_unfiltered_cleaned_split.json
TVM_FFI_DISABLE_TORCH_C_DLPACK: "1"
SGLANG_GITHUB_REPO_URL: ${{ github.event.pull_request.head.repo.clone_url || 'https://github.com/apinge/sglang.git' }}
SGLANG_GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
Expand Down Expand Up @@ -80,8 +83,8 @@ jobs:
if event_name == "schedule" or (
event_name == "workflow_dispatch" and run_mode == "benchmark"
):
if benchmark_target == "all":
targets = ["pure_text", "concurrency", "request_rate"]
if benchmark_target in {"all", "load_test"}:
targets = ["load_test"]
elif benchmark_target in {"pure_text", "concurrency", "request_rate"}:
targets = [benchmark_target]
else:
Expand Down Expand Up @@ -271,19 +274,51 @@ jobs:
ls -lah "${{ env.HOST_MODELS_ROOT }}"
du -sh "${MODEL_PATH_HOST}"

- name: Validate benchmark datasets
if: ${{ matrix.job_mode == 'accuracy' || matrix.job_mode == 'benchmark' }}
run: |
set -euo pipefail

dataset_path="${{ env.HOST_MODELS_ROOT }}/benchmark_datasets/ShareGPT_V3_unfiltered_cleaned_split.json"
if [ ! -f "${dataset_path}" ]; then
echo "Expected ShareGPT dataset not found on host: ${dataset_path}"
echo "Container path will be: ${{ env.SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH }}"
echo "Prepare it once on the runner with:"
echo " bash scripts/ci/prepare_benchmark_datasets.sh ${{ env.HOST_MODELS_ROOT }}/benchmark_datasets"
exit 1
fi

python3 - <<'PY' "${dataset_path}"
import json
import sys
from pathlib import Path

path = Path(sys.argv[1])
json.loads(path.read_text(encoding="utf-8"))
print(f"Validated ShareGPT dataset: {path} ({path.stat().st_size} bytes)")
PY
Comment on lines +291 to +299

- name: Validate benchmark scripts
if: ${{ matrix.job_mode == 'benchmark' && matrix.benchmark_target != 'pure_text' }}
if: ${{ matrix.job_mode == 'benchmark' && (matrix.benchmark_target == 'load_test' || matrix.benchmark_target == 'concurrency' || matrix.benchmark_target == 'request_rate') }}
run: |
set -euo pipefail

benchmark_example_host="${{ env.HOST_MODELS_ROOT }}/benchamark_example"
script_path="${benchmark_example_host}/${BENCHMARK_TARGET}/run.sh"
if [ ! -f "${script_path}" ]; then
echo "Expected benchmark script not found on host: ${script_path}"
echo "Container path will be: ${{ env.SGLANG_BENCHMARK_EXAMPLE_ROOT }}/${BENCHMARK_TARGET}/run.sh"
exit 1
if [ "${{ matrix.benchmark_target }}" = "load_test" ]; then
benchmark_targets=(concurrency request_rate)
else
benchmark_targets=("${{ matrix.benchmark_target }}")
fi

for target in "${benchmark_targets[@]}"; do
script_path="${benchmark_example_host}/${target}/run.sh"
if [ ! -f "${script_path}" ]; then
echo "Expected benchmark script not found on host: ${script_path}"
echo "Container path will be: ${{ env.SGLANG_BENCHMARK_EXAMPLE_ROOT }}/${target}/run.sh"
exit 1
fi
done

ls -lah "${benchmark_example_host}"

- name: Generate Dockerfile
Expand Down Expand Up @@ -349,6 +384,8 @@ jobs:
-e SGLANG_BENCHMARK_PORT=${{ env.SGLANG_BENCHMARK_PORT }} \
-e SGLANG_BENCHMARK_EXAMPLE_ROOT=${{ env.SGLANG_BENCHMARK_EXAMPLE_ROOT }} \
-e SGLANG_BENCHMARK_RESULTS_DIR=${{ env.SGLANG_BENCHMARK_RESULTS_DIR }} \
-e SGLANG_BENCHMARK_DATASETS_ROOT=${{ env.SGLANG_BENCHMARK_DATASETS_ROOT }} \
-e SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH=${{ env.SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH }} \
-e TVM_FFI_DISABLE_TORCH_C_DLPACK=${{ env.TVM_FFI_DISABLE_TORCH_C_DLPACK }} \
-v ${{ env.HOST_MODELS_ROOT }}:${{ env.CONTAINER_MODELS_ROOT }}:ro \
--security-opt seccomp=unconfined \
Expand Down Expand Up @@ -423,9 +460,9 @@ jobs:
set -ex
docker exec "${CONTAINER_NAME}" bash -lc "bash scripts/ci/sglang_benchmark_workflow.sh pure_text \"${MODEL_NAME}\" \"${MODEL_PATH}\" \"${MODEL_TP}\" 1"

- name: Run selected load tests
- name: Run load tests
if: ${{ matrix.job_mode == 'benchmark' }}
timeout-minutes: 300
timeout-minutes: 120
run: |
set -ex
docker exec "${CONTAINER_NAME}" bash -lc "bash scripts/ci/sglang_benchmark_workflow.sh ${BENCHMARK_TARGET} \"${MODEL_NAME}\" \"${MODEL_PATH}\" \"${MODEL_TP}\" 1 45"
Expand Down Expand Up @@ -473,6 +510,8 @@ jobs:
target = os.environ.get("BENCHMARK_TARGET", "pure_text")
if target == "accuracy":
target = "pure_text"
elif target == "load_test":
target = "pure_text, concurrency, request_rate"
f.write(f"- Target: `{target}`\n")
for log_file in log_files:
f.write(f"- Log: `{log_file}`\n")
Expand All @@ -487,4 +526,16 @@ jobs:
- name: Clean up
if: always()
run: |
set -euo pipefail

# Only remove resources created by this workflow job. Do not prune or
# delete other images/containers on the shared CI runner.
echo "Stopping and removing CI container: ${CONTAINER_NAME}"
docker rm -f "${CONTAINER_NAME}" || true

echo "Removing CI-built image for this job: ${LOCAL_IMAGE_NAME}"
docker rmi -f "${LOCAL_IMAGE_NAME}" || true

rm -f Dockerfile.mod || true

echo "CI cleanup complete (container + image for this job only)."
41 changes: 40 additions & 1 deletion scripts/ci/sglang_benchmark_workflow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ PURE_TEXT_INPUT_TOKENS=${SGLANG_BENCHMARK_PURE_TEXT_INPUT_TOKENS:-8000}
PURE_TEXT_OUTPUT_TOKENS=${SGLANG_BENCHMARK_PURE_TEXT_OUTPUT_TOKENS:-500}
PURE_TEXT_NUM_PROMPTS=${SGLANG_BENCHMARK_PURE_TEXT_NUM_PROMPTS:-32}
PURE_TEXT_MAX_CONCURRENCY=${SGLANG_BENCHMARK_PURE_TEXT_MAX_CONCURRENCY:-1}
BENCHMARK_DATASETS_ROOT=${SGLANG_BENCHMARK_DATASETS_ROOT:-/models/benchmark_datasets}
SHAREGPT_DATASET_FILENAME=${SGLANG_BENCHMARK_SHAREGPT_FILENAME:-ShareGPT_V3_unfiltered_cleaned_split.json}
SHAREGPT_DATASET_PATH=${SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH:-${BENCHMARK_DATASETS_ROOT}/${SHAREGPT_DATASET_FILENAME}}

if [[ -z "${SGLANG_BENCHMARK_PORT:-}" && -f "${SERVER_PORT_FILE}" ]]; then
PORT=$(<"${SERVER_PORT_FILE}")
Expand Down Expand Up @@ -432,6 +435,8 @@ run_external_benchmark() {
export BASE_URL="http://127.0.0.1:${PORT}"
export SGLANG_BENCHMARK_MODEL_PATH="${MODEL_PATH}"
export SGLANG_BENCHMARK_PORT="${PORT}"
export SGLANG_BENCHMARK_SHAREGPT_DATASET_PATH="${SHAREGPT_DATASET_PATH}"
export DATASET_PATH="${SHAREGPT_DATASET_PATH}"
bash -euo pipefail "./run.sh"
) | tee "${log_path}"

Expand All @@ -454,6 +459,31 @@ ensure_server_ready() {
return 1
}

validate_sharegpt_dataset() {
if python3 - <<'PY' "${SHAREGPT_DATASET_PATH}"
import json
import sys
from pathlib import Path

path = Path(sys.argv[1])
if not path.is_file():
raise SystemExit(f"ShareGPT dataset not found: {path}")
try:
json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise SystemExit(f"ShareGPT dataset is not valid JSON: {path} ({exc})")
print(f"Using ShareGPT dataset: {path}")
Comment on lines +464 to +475
PY
then
return 0
fi

echo "Expected local ShareGPT dataset at: ${SHAREGPT_DATASET_PATH}"
echo "Prepare it once on the runner host with:"
echo " bash scripts/ci/prepare_benchmark_datasets.sh /raid/models/benchmark_datasets"
return 1
}

run_pure_text_benchmark() {
local log_path="${BENCHMARK_RESULTS_DIR}/pure_text_benchmark_${MODEL_NAME}_TP${TP}_EP${EP}.log"

Expand All @@ -468,11 +498,14 @@ run_pure_text_benchmark() {
echo "max concurrency: ${PURE_TEXT_MAX_CONCURRENCY}"
echo "num prompts: ${PURE_TEXT_NUM_PROMPTS}"
echo "dataset-name: random"
echo "dataset-path: ${SHAREGPT_DATASET_PATH}"
validate_sharegpt_dataset

python3 -m sglang.bench_serving \
--backend sglang \
--model "${MODEL_PATH}" \
--dataset-name random \
--dataset-path "${SHAREGPT_DATASET_PATH}" \
--host localhost \
--port "${PORT}" \
--num-prompts "${PURE_TEXT_NUM_PROMPTS}" \
Expand Down Expand Up @@ -617,6 +650,12 @@ PY
elif [[ "${TYPE}" == "concurrency" || "${TYPE}" == "request_rate" ]]; then
run_external_benchmark "${TYPE}"

elif [[ "${TYPE}" == "load_test" ]]; then
ensure_server_ready
run_pure_text_benchmark
run_external_benchmark "concurrency"
run_external_benchmark "request_rate"

elif [[ "${TYPE}" == "pure_text" ]]; then
run_pure_text_benchmark

Expand All @@ -640,7 +679,7 @@ elif [[ "${TYPE}" == "performance" ]]; then

else
echo "Unknown TYPE: ${TYPE}"
echo "Usage: $0 {launch|evaluation|pure_text|concurrency|request_rate|performance} [model_name] [model_path] [TP] [EP] [timeout]"
echo "Usage: $0 {launch|evaluation|load_test|pure_text|concurrency|request_rate|performance} [model_name] [model_path] [TP] [EP] [timeout]"
exit 1
fi

Expand Down
Loading