e2e Test - GLM-5.2 RTX normal + MTP real verify smoke r2 #4377
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End Tests | |
| run-name: e2e Test - ${{ inputs.test-name || inputs.generate-cli-command || github.event.inputs.generate-cli-command }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: true | |
| type: string | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| eval-limit: | |
| description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" | |
| required: false | |
| type: string | |
| default: "" | |
| swebench-gen-mode: | |
| description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." | |
| required: false | |
| type: string | |
| default: "" | |
| workflow_call: | |
| inputs: | |
| generate-cli-command: | |
| description: "Command passed to generate matrix script" | |
| required: true | |
| type: string | |
| test-name: | |
| description: "Name for this test run" | |
| required: false | |
| type: string | |
| ref: | |
| description: "Ref (branch/sha) to checkout for generating configs" | |
| required: false | |
| type: string | |
| duration-override: | |
| description: "Override matrix.config.duration (seconds). Empty = use matrix value." | |
| required: false | |
| type: string | |
| default: "" | |
| eval-limit: | |
| description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" | |
| required: false | |
| type: string | |
| default: "" | |
| swebench-gen-mode: | |
| description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." | |
| required: false | |
| type: string | |
| default: "" | |
| jobs: | |
| get-jobs: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| single-node-config: ${{ steps.get-jobs.outputs.single-node-config }} | |
| multi-node-config: ${{ steps.get-jobs.outputs.multi-node-config }} | |
| eval-config: ${{ steps.get-jobs.outputs.eval-config }} | |
| multi-node-eval-config: ${{ steps.get-jobs.outputs.multi-node-eval-config }} | |
| agentic-config: ${{ steps.get-jobs.outputs.agentic-config }} | |
| agentic-eval-config: ${{ steps.get-jobs.outputs.agentic-eval-config }} | |
| multi-node-agentic-config: ${{ steps.get-jobs.outputs.multi-node-agentic-config }} | |
| steps: | |
| - name: Checkout code (ref) | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ inputs.ref }} | |
| - name: Checkout code (default) | |
| if: ${{ !inputs.ref || inputs.ref == '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ github.sha }} | |
| - name: Checkout priority scheduler tooling | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: ${{ github.workflow_sha }} | |
| path: .ci-priority | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 | |
| - id: get-jobs | |
| env: | |
| PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} | |
| run: | | |
| CONFIG_JSON=$(uv run --no-project --with pydantic --with pyyaml --python 3.12 \ | |
| ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py \ | |
| ${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}) | |
| PRIORITY_ROOT="${GITHUB_WORKSPACE}" | |
| if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then | |
| PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority" | |
| fi | |
| score_matrix() { | |
| local family="$1" | |
| uv run --no-project --with pyyaml --python 3.12 "${PRIORITY_ROOT}/utils/ci_priority.py" \ | |
| --policy "${PRIORITY_ROOT}/configs/ci-priority.yaml" \ | |
| --event-name "${{ github.event_name }}" \ | |
| --queue-namespace "${{ github.run_id }}:${{ github.run_attempt }}:${family}" \ | |
| --labels-json "$PR_LABELS" | |
| } | |
| AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and not x.get('run-eval', False)]))" | score_matrix agentic) | |
| AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' not in x and x.get('run-eval', False)]))" | score_matrix agentic-eval) | |
| MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x]))" | score_matrix multi-agentic) | |
| SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) | |
| MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) | |
| EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) | |
| MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-eval) | |
| echo "agentic-config=$AGENTIC" >> $GITHUB_OUTPUT | |
| echo "agentic-eval-config=$AGENTIC_EVAL" >> $GITHUB_OUTPUT | |
| echo "multi-node-agentic-config=$MULTI_AGENTIC" >> $GITHUB_OUTPUT | |
| echo "single-node-config=$SINGLE" >> $GITHUB_OUTPUT | |
| echo "multi-node-config=$MULTI" >> $GITHUB_OUTPUT | |
| echo "eval-config=$EVALS" >> $GITHUB_OUTPUT | |
| echo "multi-node-eval-config=$MULTI_EVAL" >> $GITHUB_OUTPUT | |
| test-sweep-multi-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-config) }} | |
| secrets: inherit | |
| with: | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| exp-name: ${{ matrix.config.exp-name }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-pp: ${{ matrix.config.prefill.pp }} | |
| prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} | |
| prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-pp: ${{ matrix.config.decode.pp }} | |
| decode-dcp-size: ${{ matrix.config.decode.dcp-size }} | |
| decode-pcp-size: ${{ matrix.config.decode.pcp-size }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| run-eval: false | |
| ref: ${{ inputs.ref }} | |
| test-sweep-multi-node-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node eval / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-eval-config) }} | |
| secrets: inherit | |
| with: | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| exp-name: ${{ matrix.config.exp-name }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-pp: ${{ matrix.config.prefill.pp }} | |
| prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} | |
| prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-pp: ${{ matrix.config.decode.pp }} | |
| decode-dcp-size: ${{ matrix.config.decode.dcp-size }} | |
| decode-pcp-size: ${{ matrix.config.decode.pcp-size }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| run-eval: true | |
| eval-only: true | |
| eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} | |
| ref: ${{ inputs.ref }} | |
| test-sweep-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.agentic-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: agentic / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.agentic-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| tp: ${{ matrix.config.tp }} | |
| pp: ${{ matrix.config.pp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} | |
| kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} | |
| total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} | |
| duration: ${{ inputs.duration-override != '' && inputs.duration-override || matrix.config.duration }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ 'false' }} | |
| run-eval: false | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-agentic-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.agentic-eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: agentic eval / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.agentic-eval-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| tp: ${{ matrix.config.tp }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config.kv-offload-backend }} | |
| total-cpu-dram-gb: ${{ matrix.config.total-cpu-dram-gb }} | |
| duration: ${{ inputs.duration-override != '' && inputs.duration-override || matrix.config.duration }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| spec-decoding: 'none' | |
| disagg: ${{ 'false' }} | |
| run-eval: true | |
| eval-only: true | |
| eval-limit: ${{ inputs.eval-limit }} | |
| swebench-gen-mode: ${{ inputs.swebench-gen-mode }} | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-multi-node-agentic: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.multi-node-agentic-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-multinode-tmpl.yml | |
| name: multi-node agentic / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: '0' | |
| osl: '0' | |
| max-model-len: '0' | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| conc-list: ${{ toJson(matrix.config.conc) }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| prefill-hardware: ${{ matrix.config.prefill.hardware }} | |
| decode-hardware: ${{ matrix.config.decode.hardware }} | |
| prefill-num-worker: ${{ matrix.config.prefill.num-worker }} | |
| prefill-tp: ${{ matrix.config.prefill.tp }} | |
| prefill-pp: ${{ matrix.config.prefill.pp }} | |
| prefill-dcp-size: ${{ matrix.config.prefill.dcp-size }} | |
| prefill-pcp-size: ${{ matrix.config.prefill.pcp-size }} | |
| prefill-ep: ${{ matrix.config.prefill.ep }} | |
| prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }} | |
| prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }} | |
| decode-num-worker: ${{ matrix.config.decode.num-worker }} | |
| decode-tp: ${{ matrix.config.decode.tp }} | |
| decode-pp: ${{ matrix.config.decode.pp }} | |
| decode-dcp-size: ${{ matrix.config.decode.dcp-size }} | |
| decode-pcp-size: ${{ matrix.config.decode.pcp-size }} | |
| decode-ep: ${{ matrix.config.decode.ep }} | |
| decode-dp-attn: ${{ matrix.config.decode.dp-attn }} | |
| decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }} | |
| conc: ${{ matrix.config.conc[0] }} | |
| kv-offloading: ${{ matrix.config.kv-offloading }} | |
| kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }} | |
| kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }} | |
| duration: ${{ inputs.duration-override != '' && inputs.duration-override || matrix.config.duration }} | |
| run-eval: false | |
| scenario-type: agentic-coding | |
| ref: ${{ inputs.ref }} | |
| test-sweep-single-node: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.single-node-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: single-node / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.single-node-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| tp: ${{ matrix.config.tp }} | |
| pp: ${{ matrix.config.pp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| run-eval: false | |
| ref: ${{ inputs.ref }} | |
| test-sweep-evals: | |
| needs: get-jobs | |
| if: ${{ needs.get-jobs.outputs.eval-config != '[]' }} | |
| uses: ./.github/workflows/benchmark-tmpl.yml | |
| name: eval / | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{ fromJson(needs.get-jobs.outputs.eval-config) }} | |
| secrets: inherit | |
| with: | |
| exp-name: ${{ matrix.config.exp-name }} | |
| isl: ${{ matrix.config.isl }} | |
| osl: ${{ matrix.config.osl }} | |
| max-model-len: ${{ matrix.config.max-model-len }} | |
| runner: ${{ matrix.config.runner }} | |
| priority: ${{ matrix.config.priority }} | |
| queue-token: ${{ matrix.config['queue-token'] }} | |
| image: ${{ matrix.config.image }} | |
| model: ${{ matrix.config.model }} | |
| model-prefix: ${{ matrix.config.model-prefix }} | |
| framework: ${{ matrix.config.framework }} | |
| precision: ${{ matrix.config.precision }} | |
| router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} | |
| kv-p2p-transfer: ${{ matrix.config['kv-p2p-transfer'] || '' }} | |
| tp: ${{ matrix.config.tp }} | |
| pp: ${{ matrix.config.pp }} | |
| dcp-size: ${{ matrix.config.dcp-size }} | |
| pcp-size: ${{ matrix.config.pcp-size }} | |
| ep: ${{ matrix.config.ep }} | |
| dp-attn: ${{ matrix.config.dp-attn }} | |
| conc: ${{ matrix.config.conc }} | |
| spec-decoding: ${{ matrix.config.spec-decoding }} | |
| disagg: ${{ matrix.config.disagg }} | |
| run-eval: true | |
| eval-only: true | |
| eval-limit: ${{ inputs.eval-limit }} | |
| ref: ${{ inputs.ref }} | |
| collect-results: | |
| needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic] | |
| if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped') }} | |
| uses: ./.github/workflows/collect-results.yml | |
| secrets: inherit | |
| with: | |
| result-prefix: "bmk" | |
| collect-evals: | |
| needs: [test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals] | |
| if: ${{ always() && (needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped') }} | |
| uses: ./.github/workflows/collect-evals.yml | |
| secrets: inherit | |
| calc-success-rate: | |
| needs: [collect-results, collect-evals] | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| env: | |
| RESULTS_DIR: "results/" | |
| STATS_FILENAME: "run_stats" | |
| GITHUB_TOKEN: ${{ secrets.REPO_PAT }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| token: ${{ secrets.REPO_PAT }} | |
| fetch-depth: 0 | |
| - name: Download results artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: ${{ env.RESULTS_DIR }} | |
| pattern: results_* | |
| - name: Install python dependencies | |
| run: pip install PyGithub | |
| - name: Calculate success rate | |
| run: python3 utils/calc_success_rate.py $STATS_FILENAME | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: "run-stats" | |
| path: ${{ env.STATS_FILENAME }}.json |