diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5ab5160d76..1150dc7df5 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -166,6 +166,27 @@ and eval jobs retain their canonical settings. Adding or removing the modifier restarts the active sweep. Fast-mode runs are not eligible for artifact reuse after merge. +## Trusted External-Fork Sweep Dispatch (PoC) + +Public-fork `pull_request` workflows receive no repository secrets. For an +external PR, the ordinary `run-sweep.yml` run therefore validates the +changelog but does not fan out onto GPU runners. A maintainer with `write`, +`maintain`, or `admin` permission can add any modifier labels first, then apply +one primary sweep label to approve the PR's exact current head SHA. +`trusted-external-sweep.yml` then dispatches `e2e-tests.yml` from `main`, pins +both the approved head and GitHub's merge SHA, and runs the generated matrix +with the trusted workflow's secrets. + +The approval is revision-specific. A later push is not trusted automatically; +remove and re-add the primary sweep label to approve the new SHA. The trusted +dispatcher never checks out or executes PR code itself. + +This proof of concept produces benchmark and evaluation artifacts through the +End-to-End Tests workflow. Those runs are not yet eligible for +`/reuse-sweep-run`, which currently accepts only `run-sweep.yml` runs. The PoC +also fans out the selected matrix immediately; it does not reproduce +`run-sweep.yml`'s canary-first sequencing. + ## Reusing an Approved PR Full Sweep `[skip-sweep]` skips PR benchmark setup only; changelog and reuse checks still diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 34fba860f4..06604b2f77 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -1,13 +1,17 @@ name: End-to-End Tests run-name: e2e Test - ${{ inputs.test-name || inputs.generate-cli-command || github.event.inputs.generate-cli-command }} +permissions: + contents: read + on: workflow_dispatch: inputs: generate-cli-command: description: "Command passed to generate matrix script" - required: true + required: false type: string + default: "" test-name: description: "Name for this test run" required: false @@ -36,12 +40,48 @@ on: required: false type: string default: "" + changelog-base-ref: + description: "Trusted-dispatch mode: base SHA for process_changelog.py" + required: false + type: string + default: "" + changelog-head-ref: + description: "Trusted-dispatch mode: exact approved PR head SHA" + required: false + type: string + default: "" + trim-conc: + description: "Trim each configuration to its minimum concurrency" + required: false + type: boolean + default: false + all-evals: + description: "Expand evaluation selection to every fixed-sequence configuration" + required: false + type: boolean + default: false + evals-only: + description: "Suppress throughput and run only evaluations" + required: false + type: boolean + default: false + fail-fast: + description: "Cancel the rest of each matrix after its first failure" + required: false + type: boolean + default: false + pr-labels-json: + description: "Labels from the source PR for priority scoring" + required: false + type: string + default: "[]" workflow_call: inputs: generate-cli-command: description: "Command passed to generate matrix script" - required: true + required: false type: string + default: "" test-name: description: "Name for this test run" required: false @@ -70,6 +110,41 @@ on: required: false type: string default: "" + changelog-base-ref: + description: "Trusted-dispatch mode: base SHA for process_changelog.py" + required: false + type: string + default: "" + changelog-head-ref: + description: "Trusted-dispatch mode: exact approved PR head SHA" + required: false + type: string + default: "" + trim-conc: + description: "Trim each configuration to its minimum concurrency" + required: false + type: boolean + default: false + all-evals: + description: "Expand evaluation selection to every fixed-sequence configuration" + required: false + type: boolean + default: false + evals-only: + description: "Suppress throughput and run only evaluations" + required: false + type: boolean + default: false + fail-fast: + description: "Cancel the rest of each matrix after its first failure" + required: false + type: boolean + default: false + pr-labels-json: + description: "Labels from the source PR for priority scoring" + required: false + type: string + default: "[]" jobs: get-jobs: @@ -88,6 +163,8 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ inputs.ref }} + fetch-depth: 0 + persist-credentials: false - name: Checkout code (default) if: ${{ !inputs.ref || inputs.ref == '' }} @@ -105,15 +182,50 @@ jobs: - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - id: get-jobs env: - PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} + PR_LABELS: ${{ inputs.pr-labels-json || toJson(github.event.pull_request.labels.*.name) }} + CHANGELOG_BASE_REF: ${{ inputs.changelog-base-ref }} + CHANGELOG_HEAD_REF: ${{ inputs.changelog-head-ref }} + TRIM_CONC: ${{ inputs.trim-conc }} + ALL_EVALS: ${{ inputs.all-evals }} + EVALS_ONLY: ${{ inputs.evals-only }} run: | - CONFIG_JSON=$(uv run --no-project --with pydantic --with pyyaml --python 3.12 \ - ${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py \ - ${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}) PRIORITY_ROOT="${GITHUB_WORKSPACE}" if [ -d "${GITHUB_WORKSPACE}/.ci-priority" ]; then PRIORITY_ROOT="${GITHUB_WORKSPACE}/.ci-priority" fi + if [ -n "$CHANGELOG_BASE_REF" ] || [ -n "$CHANGELOG_HEAD_REF" ]; then + if [ -z "$CHANGELOG_BASE_REF" ] || [ -z "$CHANGELOG_HEAD_REF" ]; then + echo "Both changelog-base-ref and changelog-head-ref are required" >&2 + exit 1 + fi + CMD=( + uv run --no-project --with pydantic --with pyyaml --python 3.12 + "${PRIORITY_ROOT}/utils/process_changelog.py" + --changelog-file "${GITHUB_WORKSPACE}/perf-changelog.yaml" + --base-ref "$CHANGELOG_BASE_REF" + --head-ref "$CHANGELOG_HEAD_REF" + ) + if [ "$TRIM_CONC" = "true" ]; then + CMD+=(--trim-conc) + fi + if [ "$ALL_EVALS" = "true" ]; then + CMD+=(--all-evals) + fi + if [ "$EVALS_ONLY" = "true" ]; then + CMD+=(--evals-only) + fi + RAW_CONFIG_JSON=$("${CMD[@]}") + CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON") + else + GENERATE_COMMAND="${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}" + if [ -z "$GENERATE_COMMAND" ]; then + echo "generate-cli-command is required outside trusted changelog dispatch mode" >&2 + exit 1 + fi + CONFIG_JSON=$(uv run --no-project --with pydantic --with pyyaml --python 3.12 \ + "${GITHUB_WORKSPACE}/utils/matrix_logic/generate_sweep_configs.py" \ + $GENERATE_COMMAND) + fi score_matrix() { local family="$1" uv run --no-project --with pyyaml --python 3.12 "${PRIORITY_ROOT}/utils/ci_priority.py" \ @@ -129,13 +241,15 @@ jobs: MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-eval) - echo "agentic-config=$AGENTIC" >> $GITHUB_OUTPUT - echo "agentic-eval-config=$AGENTIC_EVAL" >> $GITHUB_OUTPUT - echo "multi-node-agentic-config=$MULTI_AGENTIC" >> $GITHUB_OUTPUT - echo "single-node-config=$SINGLE" >> $GITHUB_OUTPUT - echo "multi-node-config=$MULTI" >> $GITHUB_OUTPUT - echo "eval-config=$EVALS" >> $GITHUB_OUTPUT - echo "multi-node-eval-config=$MULTI_EVAL" >> $GITHUB_OUTPUT + { + echo "agentic-config=$AGENTIC" + echo "agentic-eval-config=$AGENTIC_EVAL" + echo "multi-node-agentic-config=$MULTI_AGENTIC" + echo "single-node-config=$SINGLE" + echo "multi-node-config=$MULTI" + echo "eval-config=$EVALS" + echo "multi-node-eval-config=$MULTI_EVAL" + } >> "$GITHUB_OUTPUT" test-sweep-multi-node: needs: get-jobs @@ -143,7 +257,7 @@ jobs: uses: ./.github/workflows/benchmark-multinode-tmpl.yml name: multi-node / strategy: - fail-fast: false + fail-fast: ${{ inputs.fail-fast }} matrix: config: ${{ fromJson(needs.get-jobs.outputs.multi-node-config) }} secrets: inherit @@ -194,7 +308,7 @@ jobs: uses: ./.github/workflows/benchmark-multinode-tmpl.yml name: multi-node eval / strategy: - fail-fast: false + fail-fast: ${{ inputs.fail-fast }} matrix: config: ${{ fromJson(needs.get-jobs.outputs.multi-node-eval-config) }} secrets: inherit @@ -247,7 +361,7 @@ jobs: uses: ./.github/workflows/benchmark-tmpl.yml name: agentic / strategy: - fail-fast: false + fail-fast: ${{ inputs.fail-fast }} matrix: config: ${{ fromJson(needs.get-jobs.outputs.agentic-config) }} secrets: inherit @@ -291,7 +405,7 @@ jobs: uses: ./.github/workflows/benchmark-tmpl.yml name: agentic eval / strategy: - fail-fast: false + fail-fast: ${{ inputs.fail-fast }} matrix: config: ${{ fromJson(needs.get-jobs.outputs.agentic-eval-config) }} secrets: inherit @@ -332,7 +446,7 @@ jobs: uses: ./.github/workflows/benchmark-multinode-tmpl.yml name: multi-node agentic / strategy: - fail-fast: false + fail-fast: ${{ inputs.fail-fast }} matrix: config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-config) }} secrets: inherit @@ -388,7 +502,7 @@ jobs: uses: ./.github/workflows/benchmark-tmpl.yml name: single-node / strategy: - fail-fast: false + fail-fast: ${{ inputs.fail-fast }} matrix: config: ${{ fromJson(needs.get-jobs.outputs.single-node-config) }} secrets: inherit @@ -425,7 +539,7 @@ jobs: uses: ./.github/workflows/benchmark-tmpl.yml name: eval / strategy: - fail-fast: false + fail-fast: ${{ inputs.fail-fast }} matrix: config: ${{ fromJson(needs.get-jobs.outputs.eval-config) }} secrets: inherit @@ -498,7 +612,7 @@ jobs: run: pip install PyGithub - name: Calculate success rate - run: python3 utils/calc_success_rate.py $STATS_FILENAME + run: python3 utils/calc_success_rate.py "$STATS_FILENAME" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 9d52fda0b9..298cefaf49 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -190,6 +190,7 @@ jobs: ( github.event_name == 'pull_request' && !github.event.pull_request.draft && + github.event.pull_request.head.repo.full_name == github.repository && needs.check-changelog.outputs.skip-pr-sweep != 'true' && ( contains(github.event.pull_request.labels.*.name, 'sweep-enabled') || diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index e9140b5d3d..c1f11f34f8 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -15,6 +15,7 @@ on: - ".github/workflows/run-sweep.yml" - ".github/workflows/speedbench-al.yml" - ".github/workflows/test-changelog-gate.yml" + - ".github/workflows/trusted-external-sweep.yml" - "benchmarks/benchmark_lib.sh" - "configs/ci-priority.yaml" - "benchmarks/multi_node/amd_utils/job.slurm" diff --git a/.github/workflows/trusted-external-sweep.yml b/.github/workflows/trusted-external-sweep.yml new file mode 100644 index 0000000000..2a7a486ae2 --- /dev/null +++ b/.github/workflows/trusted-external-sweep.yml @@ -0,0 +1,143 @@ +name: Trusted External Sweep Dispatch +run-name: Trusted external sweep - PR #${{ github.event.pull_request.number }} @ ${{ github.event.pull_request.head.sha }} + +# This workflow is intentionally control-plane only. It never checks out or +# executes pull-request code. A write-authorized maintainer applies one primary +# sweep label to approve the exact external head SHA, then this trusted event +# dispatches the existing e2e workflow from main with repository secrets. +on: + pull_request_target: + branches: + - main + types: + - labeled + +permissions: + actions: write + contents: read + issues: write + pull-requests: read + +jobs: + dispatch: + if: >- + github.event.pull_request.head.repo.full_name != github.repository && + contains( + fromJson('["sweep-enabled","full-sweep-enabled","non-canary-full-sweep-enabled","full-sweep-fail-fast","full-sweep-fail-fast-no-canary"]'), + github.event.label.name + ) + runs-on: ubuntu-latest + steps: + - name: Authorize exact external revision and dispatch + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ github.token }} + script: | + const trustedPermissions = new Set(['admin', 'maintain', 'write']); + const primaryLabels = new Set([ + 'sweep-enabled', + 'full-sweep-enabled', + 'non-canary-full-sweep-enabled', + 'full-sweep-fail-fast', + 'full-sweep-fail-fast-no-canary', + ]); + + const permission = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: context.actor, + }); + if (!trustedPermissions.has(permission.data.permission)) { + core.setFailed( + `@${context.actor} has ${permission.data.permission} permission; ` + + 'write, maintain, or admin permission is required to authorize external code.', + ); + return; + } + + const eventPull = context.payload.pull_request; + const {data: pull} = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: eventPull.number, + }); + if (pull.state !== 'open' || pull.draft) { + core.setFailed('The pull request must be open and ready for review.'); + return; + } + if (pull.head.repo?.full_name === context.payload.repository.full_name) { + core.setFailed('Trusted external dispatch is only for fork pull requests.'); + return; + } + if (pull.head.sha !== eventPull.head.sha) { + core.setFailed( + `The PR advanced from ${eventPull.head.sha} to ${pull.head.sha}; ` + + 'remove and re-add the sweep label to approve the new revision.', + ); + return; + } + + const labels = pull.labels.map((label) => label.name); + const primary = labels.filter((label) => primaryLabels.has(label)); + if (primary.length !== 1 || primary[0] !== context.payload.label.name) { + core.setFailed(`Expected exactly the newly applied primary sweep label; found ${primary}.`); + return; + } + if (!pull.merge_commit_sha) { + core.setFailed('GitHub has not produced a merge commit; resolve conflicts and re-add the label.'); + return; + } + + const marker = `External PR #${pull.number} @ ${pull.head.sha.slice(0, 12)}`; + const dispatchStarted = Date.now(); + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'e2e-tests.yml', + ref: context.payload.repository.default_branch, + inputs: { + 'test-name': marker, + 'ref': pull.merge_commit_sha, + 'changelog-base-ref': pull.base.sha, + 'changelog-head-ref': pull.head.sha, + 'trim-conc': String(primary[0] === 'sweep-enabled'), + 'all-evals': String(labels.includes('all-evals')), + 'evals-only': String(labels.includes('evals-only')), + 'fail-fast': String(primary[0].includes('fail-fast')), + 'agentx-fast': String(labels.includes('agentx-fast')), + 'pr-labels-json': JSON.stringify(labels), + }, + }); + + let dispatchedRun; + for (let attempt = 0; attempt < 15 && !dispatchedRun; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 2000)); + const runs = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'e2e-tests.yml', + event: 'workflow_dispatch', + per_page: 20, + }); + dispatchedRun = runs.data.workflow_runs.find((run) => + run.display_title === `e2e Test - ${marker}` && + Date.parse(run.created_at) >= dispatchStarted - 5000 + ); + } + + const runLink = dispatchedRun + ? `[trusted sweep run](${dispatchedRun.html_url})` + : '[End-to-End Tests workflow](https://github.com/SemiAnalysisAI/InferenceX/actions/workflows/e2e-tests.yml)'; + const body = [ + `Dispatched ${runLink} for approved external revision \`${pull.head.sha}\`.`, + 'New commits are not trusted automatically; remove and re-add the primary sweep label to approve a new SHA.', + '', + `已为获批的外部提交 \`${pull.head.sha}\` 调度${runLink}。`, + '后续新提交不会自动获得信任；如需批准新的 SHA，请移除并重新添加主扫描标签。', + ].join('\n'); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pull.number, + body, + }); diff --git a/utils/changelog_gate_tests/test_run_sweep_gating.py b/utils/changelog_gate_tests/test_run_sweep_gating.py index 0489cb4649..ed30ec5ad9 100644 --- a/utils/changelog_gate_tests/test_run_sweep_gating.py +++ b/utils/changelog_gate_tests/test_run_sweep_gating.py @@ -201,8 +201,12 @@ def _eval(expr: str, ctx: dict) -> bool: def _ctx(sc: dict) -> dict: return { "github.event_name": sc["event"], + "github.repository": "SemiAnalysisAI/InferenceX", "github.event.action": sc.get("action"), "github.event.pull_request.draft": sc.get("draft", False), + "github.event.pull_request.head.repo.full_name": sc.get( + "head_repo", "SemiAnalysisAI/InferenceX" + ), "github.event.pull_request.labels.*.name": sc.get("labels", []), "github.event.label.name": sc.get("label_name"), "vars.PRIORITY_SCHEDULER_ENABLED": sc.get("scheduler_enabled", "true"), @@ -286,6 +290,10 @@ def run_dag(sc: dict) -> tuple[str, str, str]: ("PR-sync-no-sweep-label", {**_PR, "action": "synchronize", "labels": []}, ("success", "skipped", "SKIP")), + ("PR-sync-external-fork-defers-to-trusted-dispatch", + {**_PR, "action": "synchronize", "labels": ["full-sweep-enabled"], + "head_repo": "external/InferenceX"}, + ("success", "success", "SKIP")), ("PR-labeled-with-sweep-label", {**_PR, "action": "labeled", "label_name": "full-sweep-enabled", "labels": ["full-sweep-enabled"]}, ("success", "skipped", "RUN")), @@ -464,6 +472,9 @@ def reference_gate(sc: dict) -> tuple[str, str, str]: labels = set(sc.get("labels", [])) draft = sc.get("draft", False) is_pr = sc["event"] == "pull_request" + is_internal_pr = sc.get("head_repo", "SemiAnalysisAI/InferenceX") == ( + "SemiAnalysisAI/InferenceX" + ) action = sc.get("action") check_runs = ( @@ -499,6 +510,7 @@ def reference_gate(sc: dict) -> tuple[str, str, str]: ) event_ok = ( (not draft) + and is_internal_pr and bool(labels & SWEEP_LABELS) and action_ok and "[skip-sweep]" not in sc.get("msg", "")