e2e Test - k3 a8w4 env fix -- 124154a8 control -- perf c8 k=3 3600s #4850
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End Tests | |
| run-name: e2e Test - ${{ inputs.test-name || inputs.generate-cli-command || github.event.inputs.generate-cli-command }} | |
| permissions: | |
| contents: read | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: false | |
| type: string | |
| default: "" | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| agentx-fast: | |
| description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile." | |
| required: false | |
| type: boolean | |
| default: false | |
| eval-limit: | |
| description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" | |
| required: false | |
| type: string | |
| default: "" | |
| swebench-gen-mode: | |
| description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-base-ref: | |
| description: "Trusted-dispatch mode: base SHA for process_changelog.py" | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-head-ref: | |
| description: "Trusted-dispatch mode: exact approved PR head SHA" | |
| required: false | |
| type: string | |
| default: "" | |
| trim-conc: | |
| description: "Trim each configuration to its minimum concurrency" | |
| required: false | |
| type: boolean | |
| default: false | |
| all-evals: | |
| description: "Expand evaluation selection to every fixed-sequence configuration" | |
| required: false | |
| type: boolean | |
| default: false | |
| evals-only: | |
| description: "Suppress throughput and run only evaluations" | |
| required: false | |
| type: boolean | |
| default: false | |
| fail-fast: | |
| description: "Cancel the rest of each matrix after its first failure" | |
| required: false | |
| type: boolean | |
| default: false | |
| pr-labels-json: | |
| description: "Labels from the source PR for priority scoring" | |
| required: false | |
| type: string | |
| default: "[]" | |
| workflow_call: | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: false | |
| type: string | |
| default: "" | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| agentx-fast: | |
| description: "AgentX fast feedback: one warmup request per lane and a 20-minute profile." | |
| required: false | |
| type: boolean | |
| default: false | |
| eval-limit: | |
| description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" | |
| required: false | |
| type: string | |
| default: "" | |
| swebench-gen-mode: | |
| description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-base-ref: | |
| description: "Trusted-dispatch mode: base SHA for process_changelog.py" | |
| required: false | |
| type: string | |
| default: "" | |
| changelog-head-ref: | |
| description: "Trusted-dispatch mode: exact approved PR head SHA" | |
| required: false | |
| type: string | |
| default: "" | |
| trim-conc: | |
| description: "Trim each configuration to its minimum concurrency" | |
| required: false | |
| type: boolean | |
| default: false | |
| all-evals: | |
| description: "Expand evaluation selection to every fixed-sequence configuration" | |
| required: false | |
| type: boolean | |
| default: false | |
| evals-only: | |
| description: "Suppress throughput and run only evaluations" | |
| required: false | |
| type: boolean | |
| default: false | |
| fail-fast: | |
| description: "Cancel the rest of each matrix after its first failure" | |
| required: false | |
| type: boolean | |
| default: false | |
| pr-labels-json: | |
| description: "Labels from the source PR for priority scoring" | |
| required: false | |
| type: string | |
| default: "[]" | |
| jobs: | |
| get-jobs: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| single-node-config: ${{ steps.get-jobs.outputs.single-node-config }} | |
| multi-node-config: ${{ steps.get-jobs.outputs.multi-node-config }} | |
| eval-config: ${{ steps.get-jobs.outputs.eval-config }} | |
| multi-node-eval-config: ${{ steps.get-jobs.outputs.multi-node-eval-config }} | |
| agentic-config: ${{ steps.get-jobs.outputs.agentic-config }} | |
| agentic-eval-config: ${{ steps.get-jobs.outputs.agentic-eval-config }} | |
| multi-node-agentic-config: ${{ steps.get-jobs.outputs.multi-node-agentic-config }} | |
| steps: | |
| - name: Checkout code (ref) | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ inputs.ref }} | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Checkout code (default) | |
| if: ${{ !inputs.ref || inputs.ref == '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ github.sha }} | |
| - name: Checkout priority scheduler tooling | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ github.workflow_sha }} | |
| path: .ci-priority | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - id: get-jobs | |
| env: | |
| PR_LABELS: ${{ inputs.pr-labels-json || toJson(github.event.pull_request.labels.*.name) }} | |
| CHANGELOG_BASE_REF: ${{ inputs.changelog-base-ref }} | |
| CHANGELOG_HEAD_REF: ${{ inputs.changelog-head-ref }} | |
| TRIM_CONC: ${{ inputs.trim-conc }} | |
| ALL_EVALS: ${{ inputs.all-evals }} | |
| EVALS_ONLY: ${{ inputs.evals-only }} | |
| run: | | |
| PRIORITY_ROOT="${GITHUB_WORKSPACE}" | |
| if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then | |
| PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority" | |
| fi | |
| if [ -n "$CHANGELOG_BASE_REF" ] || [ -n "$CHANGELOG_HEAD_REF" ]; then | |
| if [ -z "$CHANGELOG_BASE_REF" ] || [ -z "$CHANGELOG_HEAD_REF" ]; then | |
| echo "Both changelog-base-ref and changelog-head-ref are required" >&2 | |
| exit 1 | |
| fi | |
| CMD=( | |
| uv run --no-project --with pydantic --with pyyaml --python 3.12 | |
| "${PRIORITY_ROOT}/utils/process_changelog.py" | |
| --changelog-file "${GITHUB_WORKSPACE}/perf-changelog.yaml" | |
| --base-ref "$CHANGELOG_BASE_REF" | |
| --head-ref "$CHANGELOG_HEAD_REF" | |
| ) | |
| if [ "$TRIM_CONC" = "true" ]; then | |
| CMD+=(--trim-conc) | |
| fi | |
| if [ "$ALL_EVALS" = "true" ]; then | |
| CMD+=(--all-evals) | |
| fi | |
| if [ "$EVALS_ONLY" = "true" ]; then | |
| CMD+=(--evals-only) | |
| fi | |
| RAW_CONFIG_JSON=$("${CMD[@]}") | |
| CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON") | |
| else | |
| GENERATE_COMMAND="${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}" | |
| if [ -z "$GENERATE_COMMAND" ]; then | |
| echo "generate-cli-command is required outside trusted changelog dispatch mode" >&2 | |
| exit 1 | |
| fi | |
| CONFIG_JSON=$(uv run --no-project --with pydantic --with pyyaml --python 3.12 \ | |
| "${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py" \ | |
| $GENERATE_COMMAND) | |
| fi | |
| score_matrix() { | |
| local family="$1" | |
| uv run --no-project --with pyyaml --python 3.12 "${PRIORITY_ROOT}/utils/ci_priority.py" \ | |
| --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ | |
| --event-name "${{ github.event_name }}" \ | |
| --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}:${family}" \ | |
| --labels-json "$PR_LABELS" | |
| } | |
| AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('run-eval', False)]))" | score_matrix agentic) | |
| AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))" | score_matrix agentic-eval) | |
| MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))" | score_matrix multi-agentic) | |
| SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) | |
| MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) | |
| EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) | |
| MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-eval) | |
| { | |
| echo "agentic-config=$AGENTIC" | |
| echo "agentic-eval-config=$AGENTIC_EVAL" | |
| echo "multi-node-agentic-config=$MULTI_AGENTIC" | |
| echo "single-node-config=$SINGLE" | |
| echo "multi-node-config=$MULTI" | |
| echo "eval-config=$EVALS" | |
| echo "multi-node-eval-config=$MULTI_EVAL" | |
| } >> "$GITHUB_OUTPUT" | |
| test-sweep-multi-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-config) }} | |
| secrets: inherit | |
| with: | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| exp-name: ${{ matrix.config.exp-name }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-pp: ${{ matrix.config.prefill.pp }} | |
| prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} | |
| prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-pp: ${{ matrix.config.decode.pp }} | |
| decode-dcp-size: ${{ matrix.config.decode.dcp-size }} | |
| decode-pcp-size: ${{ matrix.config.decode.pcp-size }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| run-eval: false | |
| ref: ${{ inputs.ref }} | |
| test-sweep-multi-node-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node eval / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-eval-config) }} | |
| secrets: inherit | |
| with: | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| exp-name: ${{ matrix.config.exp-name }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-pp: ${{ matrix.config.prefill.pp }} | |
| prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} | |
| prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-pp: ${{ matrix.config.decode.pp }} | |
| decode-dcp-size: ${{ matrix.config.decode.dcp-size }} | |
| decode-pcp-size: ${{ matrix.config.decode.pcp-size }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| run-eval: true | |
| eval-only: true | |
| eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} | |
| ref: ${{ inputs.ref }} | |
| test-sweep-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.agentic-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: agentic / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.agentic-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| tp: ${{ matrix.config.tp }} | |
| pp: ${{ matrix.config.pp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} | |
| kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} | |
| total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} | |
| duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} | |
| agentx-fast: ${{ inputs.agentx-fast }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ 'false' }} | |
| run-eval: false | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-agentic-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.agentic-eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: agentic eval / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.agentic-eval-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| tp: ${{ matrix.config.tp }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config.kv-offload-backend }} | |
| total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} | |
| duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} | |
| agentx-fast: ${{ inputs.agentx-fast }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| # Was hardcoded 'none'. The launcher derives the benchmark script | |
| # suffix from this, so a DSpark eval resolved the NON-DSpark script | |
| # name -- and on this branch only kimik3_fp4_mi355x_mtp.sh exists, | |
| # so run 30989466944 died with exit 127, "No such file or | |
| # directory". Where a base script does exist this is worse than a | |
| # crash: the eval would quietly measure a NO-SPECULATION run and | |
| # report it as the DSpark quality gate. The sibling `agentic /` job | |
| # has passed the matrix value all along. | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ 'false' }} | |
| run-eval: true | |
| eval-only: true | |
| eval-limit: ${{ inputs.eval-limit }} | |
| swebench-gen-mode: ${{ inputs.swebench-gen-mode }} | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-multi-node-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-agentic-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node agentic / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-pp: ${{ matrix.config.prefill.pp }} | |
| prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} | |
| prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-pp: ${{ matrix.config.decode.pp }} | |
| decode-dcp-size: ${{ matrix.config.decode.dcp-size }} | |
| decode-pcp-size: ${{ matrix.config.decode.pcp-size }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| conc: ${{ matrix.config.conc[0] }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} | |
| kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} | |
| duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} | |
| agentx-fast: ${{ inputs.agentx-fast }} | |
| run-eval: false | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-single-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.single-node-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: single-node / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.single-node-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| tp: ${{ matrix.config.tp }} | |
| pp: ${{ matrix.config.pp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| run-eval: false | |
| ref: ${{ inputs.ref }} | |
| test-sweep-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: eval / | |
| strategy: | |
| fail-fast: ${{ inputs.fail-fast }} | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.eval-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| tp: ${{ matrix.config.tp }} | |
| pp: ${{ matrix.config.pp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| run-eval: true | |
| eval-only: true | |
| eval-limit: ${{ inputs.eval-limit }} | |
| ref: ${{ inputs.ref }} | |
| collect-results: | |
| needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic] | |
| if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped') }} | |
| uses: ./.github/workflows/collect-results.yml | |
| secrets: inherit | |
| with: | |
| result-prefix: "bmk" | |
| collect-evals: | |
| needs: [test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals] | |
| if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped') }} | |
| uses: ./.github/workflows/collect-evals.yml | |
| secrets: inherit | |
| calc-success-rate: | |
| needs: [collect-results, collect-evals] | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| env: | |
| RESULTS_DIR: "results/" | |
| STATS_FILENAME: "run_stats" | |
| GITHUB_TOKEN: ${{ secrets.REPO_PAT }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| token: ${{ secrets.REPO_PAT }} | |
| fetch-depth: 0 | |
| - name: Download results artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: ${{ env.RESULTS_DIR }} | |
| pattern: results_* | |
| - name: Install python dependencies | |
| run: pip install PyGithub | |
| - name: Calculate success rate | |
| run: python3 utils/calc_success_rate.py "$STATS_FILENAME" | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: "run-stats" | |
| path: ${{ env.STATS_FILENAME }}.json |