e2e Test - MI355X production Qwen3.5 srt-slurm 1P1D MoRI validation #5000
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End Tests | |
| run-name: e2e Test - ${{ inputs.test-name || inputs.generate-cli-command || github.event.inputs.generate-cli-command }} | |
| permissions: | |
| contents: read | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: false | |
| type: string | |
| default: "" | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| require-power: | |
| description: "Fail fixed-sequence jobs when GPU power is invalid" | |
| required: false | |
| type: boolean | |
| default: false | |
| power-producer-sha: | |
| description: "Expected srt-slurm producer git commit for power validation" | |
| required: false | |
| type: string | |
| default: "" | |
| agentx-fast: | |
| description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile." | |
| required: false | |
| type: boolean | |
| default: false | |
| eval-limit: | |
| description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" | |
| required: false | |
| type: string | |
| default: "" | |
| swebench-gen-mode: | |
| description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-base-ref: | |
| description: "Trusted-dispatch mode: base SHA for process_changelog.py" | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-head-ref: | |
| description: "Trusted-dispatch mode: exact approved PR head SHA" | |
| required: false | |
| type: string | |
| default: "" | |
| trim-conc: | |
| description: "Trim each configuration to its minimum concurrency" | |
| required: false | |
| type: boolean | |
| default: false | |
| all-evals: | |
| description: "Expand evaluation selection to every fixed-sequence configuration" | |
| required: false | |
| type: boolean | |
| default: false | |
| evals-only: | |
| description: "Suppress throughput and run only evaluations" | |
| required: false | |
| type: boolean | |
| default: false | |
| fail-fast: | |
| description: "Cancel the rest of each matrix after its first failure" | |
| required: false | |
| type: boolean | |
| default: false | |
| pr-labels-json: | |
| description: "Labels from the source PR for priority scoring" | |
| required: false | |
| type: string | |
| default: "[]" | |
| workflow_call: | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: false | |
| type: string | |
| default: "" | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| require-power: | |
| description: "Fail fixed-sequence jobs when GPU power is invalid" | |
| required: false | |
| type: boolean | |
| default: false | |
| power-producer-sha: | |
| description: "Expected srt-slurm producer git commit for power validation" | |
| required: false | |
| type: string | |
| default: "" | |
| agentx-fast: | |
| description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile." | |
| required: false | |
| type: boolean | |
| default: false | |
| eval-limit: | |
| description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" | |
| required: false | |
| type: string | |
| default: "" | |
| swebench-gen-mode: | |
| description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-base-ref: | |
| description: "Trusted-dispatch mode: base SHA for process_changelog.py" | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-head-ref: | |
| description: "Trusted-dispatch mode: exact approved PR head SHA" | |
| required: false | |
| type: string | |
| default: "" | |
| trim-conc: | |
| description: "Trim each configuration to its minimum concurrency" | |
| required: false | |
| type: boolean | |
| default: false | |
| all-evals: | |
| description: "Expand evaluation selection to every fixed-sequence configuration" | |
| required: false | |
| type: boolean | |
| default: false | |
| evals-only: | |
| description: "Suppress throughput and run only evaluations" | |
| required: false | |
| type: boolean | |
| default: false | |
| fail-fast: | |
| description: "Cancel the rest of each matrix after its first failure" | |
| required: false | |
| type: boolean | |
| default: false | |
| pr-labels-json: | |
| description: "Labels from the source PR for priority scoring" | |
| required: false | |
| type: string | |
| default: "[]" | |
| jobs: | |
| get-jobs: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| single-node-config: ${{ steps.get-jobs.outputs.single-node-config }} | |
| multi-node-config: ${{ steps.get-jobs.outputs.multi-node-config }} | |
| eval-config: ${{ steps.get-jobs.outputs.eval-config }} | |
| multi-node-eval-config: ${{ steps.get-jobs.outputs.multi-node-eval-config }} | |
| agentic-config: ${{ steps.get-jobs.outputs.agentic-config }} | |
| agentic-eval-config: ${{ steps.get-jobs.outputs.agentic-eval-config }} | |
| multi-node-agentic-config: ${{ steps.get-jobs.outputs.multi-node-agentic-config }} | |
| steps: | |
| - name: Checkout code (ref) | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ inputs.ref }} | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Checkout code (default) | |
| if: ${{ !inputs.ref || inputs.ref == '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ github.sha }} | |
| - name: Checkout priority scheduler tooling | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ github.workflow_sha }} | |
| path: .ci-priority | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - id: get-jobs | |
| env: | |
| PR_LABELS: ${{ inputs.pr-labels-json || toJson(github.event.pull_request.labels.*.name) }} | |
| CHANGELOG_BASE_REF: ${{ inputs.changelog-base-ref }} | |
| CHANGELOG_HEAD_REF: ${{ inputs.changelog-head-ref }} | |
| TRIM_CONC: ${{ inputs.trim-conc }} | |
| ALL_EVALS: ${{ inputs.all-evals }} | |
| EVALS_ONLY: ${{ inputs.evals-only }} | |
| run: | | |
| PRIORITY_ROOT="${GITHUB_WORKSPACE}" | |
| if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then | |
| PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority" | |
| fi | |
| if [ -n "$CHANGELOG_BASE_REF" ] || [ -n "$CHANGELOG_HEAD_REF" ]; then | |
| if [ -z "$CHANGELOG_BASE_REF" ] || [ -z "$CHANGELOG_HEAD_REF" ]; then | |
| echo "Both changelog-base-ref and changelog-head-ref are required" >&2 | |
| exit 1 | |
| fi | |
| CMD=( | |
| uv run --no-project --with pydantic --with pyyaml --python 3.12 | |
| "${PRIORITY_ROOT}/utils/process_changelog.py" | |
| --changelog-file "${GITHUB_WORKSPACE}/perf-changelog.yaml" | |
| --base-ref "$CHANGELOG_BASE_REF" | |
| --head-ref "$CHANGELOG_HEAD_REF" | |
| ) | |
| if [ "$TRIM_CONC" = "true" ]; then | |
| CMD+=(--trim-conc) | |
| fi | |
| if [ "$ALL_EVALS" = "true" ]; then | |
| CMD+=(--all-evals) | |
| fi | |
| if [ "$EVALS_ONLY" = "true" ]; then | |
| CMD+=(--evals-only) | |
| fi | |
| RAW_CONFIG_JSON=$("${CMD[@]}") | |
| CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON") | |
| else | |
| GENERATE_COMMAND="${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}" | |
| if [ -z "$GENERATE_COMMAND" ]; then | |
| echo "generate-cli-command is required outside trusted changelog dispatch mode" >&2 | |
| exit 1 | |
| fi | |
| CONFIG_JSON=$(uv run --no-project --with pydantic --with pyyaml --python 3.12 \ | |
| "${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py" \ | |
| $GENERATE_COMMAND) | |
| fi | |
| score_matrix() { | |
| local family="$1" | |
| uv run --no-project --with pyyaml --python 3.12 "${PRIORITY_ROOT}/utils/ci_priority.py" \ | |
| --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ | |
| --event-name "${{ github.event_name }}" \ | |
| --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}:${family}" \ | |
| --labels-json "$PR_LABELS" | |
| } | |
| AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('run-eval', False)]))" | score_matrix agentic) | |
| AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))" | score_matrix agentic-eval) | |
| MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))" | score_matrix multi-agentic) | |
| SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) | |
| MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) | |
| EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) | |
| MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-eval) | |
| { | |
| echo "agentic-config=$AGENTIC" | |
| echo "agentic-eval-config=$AGENTIC_EVAL" | |
| echo "multi-node-agentic-config=$MULTI_AGENTIC" | |
| echo "single-node-config=$SINGLE" | |
| echo "multi-node-config=$MULTI" | |
| echo "eval-config=$EVALS" | |
| echo "multi-node-eval-config=$MULTI_EVAL" | |
| } >> "$GITHUB_OUTPUT" | |
| test-sweep-multi-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-config) }} | |
| secrets: inherit | |
| with: | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| exp-name: ${{ matrix.config.exp-name }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-pp: ${{ matrix.config.prefill.pp }} | |
| prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} | |
| prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-pp: ${{ matrix.config.decode.pp }} | |
| decode-dcp-size: ${{ matrix.config.decode.dcp-size }} | |
| decode-pcp-size: ${{ matrix.config.decode.pcp-size }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| run-eval: false | |
| require-power: ${{ inputs.require-power }} | |
| power-producer-sha: ${{ inputs.power-producer-sha }} | |
| ref: ${{ inputs.ref }} | |
| test-sweep-multi-node-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node eval / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-eval-config) }} | |
| secrets: inherit | |
| with: | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| exp-name: ${{ matrix.config.exp-name }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-pp: ${{ matrix.config.prefill.pp }} | |
| prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} | |
| prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-pp: ${{ matrix.config.decode.pp }} | |
| decode-dcp-size: ${{ matrix.config.decode.dcp-size }} | |
| decode-pcp-size: ${{ matrix.config.decode.pcp-size }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| run-eval: true | |
| eval-only: true | |
| eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} | |
| ref: ${{ inputs.ref }} | |
| test-sweep-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.agentic-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: agentic / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.agentic-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| tp: ${{ matrix.config.tp }} | |
| pp: ${{ matrix.config.pp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} | |
| kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} | |
| total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} | |
| duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} | |
| agentx-fast: ${{ inputs.agentx-fast }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ 'false' }} | |
| run-eval: false | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-agentic-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.agentic-eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: agentic eval / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.agentic-eval-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| tp: ${{ matrix.config.tp }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config.kv-offload-backend }} | |
| total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} | |
| duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} | |
| agentx-fast: ${{ inputs.agentx-fast }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| spec-decoding: 'none' | |
| disagg: ${{ 'false' }} | |
| run-eval: true | |
| eval-only: true | |
| eval-limit: ${{ inputs.eval-limit }} | |
| swebench-gen-mode: ${{ inputs.swebench-gen-mode }} | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-multi-node-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-agentic-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node agentic / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-pp: ${{ matrix.config.prefill.pp }} | |
| prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} | |
| prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-pp: ${{ matrix.config.decode.pp }} | |
| decode-dcp-size: ${{ matrix.config.decode.dcp-size }} | |
| decode-pcp-size: ${{ matrix.config.decode.pcp-size }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| conc: ${{ matrix.config.conc[0] }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} | |
| kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} | |
| duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} | |
| agentx-fast: ${{ inputs.agentx-fast }} | |
| run-eval: false | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-single-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.single-node-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: single-node / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.single-node-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| tp: ${{ matrix.config.tp }} | |
| pp: ${{ matrix.config.pp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| run-eval: false | |
| require-power: ${{ inputs.require-power }} | |
| ref: ${{ inputs.ref }} | |
| test-sweep-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: eval / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.eval-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| tp: ${{ matrix.config.tp }} | |
| pp: ${{ matrix.config.pp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| run-eval: true | |
| eval-only: true | |
| eval-limit: ${{ inputs.eval-limit }} | |
| ref: ${{ inputs.ref }} | |
| collect-results: | |
| needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic] | |
| if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped') }} | |
| uses: ./.github/workflows/collect-results.yml | |
| secrets: inherit | |
| with: | |
| result-prefix: "bmk" | |
| collect-evals: | |
| needs: [test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals] | |
| if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped') }} | |
| uses: ./.github/workflows/collect-evals.yml | |
| secrets: inherit | |
| calc-success-rate: | |
| needs: [collect-results, collect-evals] | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| env: | |
| RESULTS_DIR: "results/" | |
| STATS_FILENAME: "run_stats" | |
| GITHUB_TOKEN: ${{ secrets.REPO_PAT }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| token: ${{ secrets.REPO_PAT }} | |
| fetch-depth: 0 | |
| - name: Download results artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: ${{ env.RESULTS_DIR }} | |
| pattern: results_* | |
| - name: Install python dependencies | |
| run: pip install PyGithub | |
| - name: Calculate success rate | |
| run: python3 utils/calc_success_rate.py "$STATS_FILENAME" | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: "run-stats" | |
| path: ${{ env.STATS_FILENAME }}.json |