Stage results for PR #653
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Stage PR Results | |
| run-name: Stage results for PR #${{ github.event.issue.number }} | |
| on: | |
| issue_comment: | |
| types: [created] | |
| permissions: {} | |
| concurrency: | |
| group: stage-results-pr-${{ github.event.issue.number }} | |
| cancel-in-progress: false | |
| jobs: | |
| dispatch: | |
| name: Authorize and dispatch staging | |
| if: >- | |
| github.event.issue.pull_request && | |
| startsWith(github.event.comment.body, '/stage-results') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| contents: read | |
| issues: read | |
| pull-requests: write # Post staging status as github-actions rather than a PAT user | |
| steps: | |
| - name: Authorize request and resolve source run | |
| id: request | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| with: | |
| github-token: ${{ github.token }} | |
| script: | | |
| const body = context.payload.comment.body.trim(); | |
| const command = /^\/stage-results(?:\s+(?<runId>\d+))?$/u.exec(body); | |
| if (!command) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: 'Usage: `/stage-results` or `/stage-results <run-id>`.', | |
| }); | |
| core.setFailed('Unsupported /stage-results syntax'); | |
| return; | |
| } | |
| const actor = context.payload.comment.user.login; | |
| const permissionResponse = await github.rest.repos.getCollaboratorPermissionLevel({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| username: actor, | |
| }); | |
| const permission = permissionResponse.data.permission; | |
| if (!['admin', 'maintain', 'write'].includes(permission)) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: `@${actor} \`/stage-results\` requires write access to this repository.`, | |
| }); | |
| core.setFailed(`Actor ${actor} has ${permission} permission`); | |
| return; | |
| } | |
| const pull = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.issue.number, | |
| }); | |
| const fullSweepLabels = [ | |
| 'full-sweep-enabled', | |
| 'non-canary-full-sweep-enabled', | |
| 'full-sweep-fail-fast', | |
| 'full-sweep-fail-fast-no-canary', | |
| ]; | |
| const pullLabels = new Set( | |
| pull.data.labels | |
| .map((label) => (typeof label === 'string' ? label : label.name)) | |
| .filter(Boolean), | |
| ); | |
| const hasFullSweepLabel = fullSweepLabels.some((label) => pullLabels.has(label)); | |
| if (!hasFullSweepLabel) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: `@${actor} \`/stage-results\` requires a completed run from a PR using one of: ${fullSweepLabels.map((label) => `\`${label}\``).join(', ')}.`, | |
| }); | |
| core.setFailed('PR does not have a full-sweep label'); | |
| return; | |
| } | |
| const timelineEvents = await github.paginate( | |
| github.rest.issues.listEventsForTimeline, | |
| { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| per_page: 100, | |
| }, | |
| ); | |
| const fullSweepLabelEvents = timelineEvents | |
| .filter( | |
| (event) => | |
| ['labeled', 'unlabeled'].includes(event.event) && | |
| fullSweepLabels.includes(event.label?.name), | |
| ) | |
| .toSorted( | |
| (left, right) => | |
| Date.parse(left.created_at ?? '') - Date.parse(right.created_at ?? ''), | |
| ); | |
| const fullSweepLabelsAt = (createdAt) => { | |
| const candidateCreatedAt = Date.parse(createdAt); | |
| if (!Number.isFinite(candidateCreatedAt)) return []; | |
| const activeLabels = new Set(); | |
| for (const event of fullSweepLabelEvents) { | |
| const eventCreatedAt = Date.parse(event.created_at ?? ''); | |
| if (!Number.isFinite(eventCreatedAt)) continue; | |
| if (eventCreatedAt > candidateCreatedAt) break; | |
| if (event.event === 'labeled') activeLabels.add(event.label.name); | |
| else activeLabels.delete(event.label.name); | |
| } | |
| return [...activeLabels]; | |
| }; | |
| const pullCommits = await github.paginate(github.rest.pulls.listCommits, { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.issue.number, | |
| per_page: 100, | |
| }); | |
| const pullCommitShas = new Set(pullCommits.map((commit) => commit.sha)); | |
| const usableArtifactNames = async (runId) => { | |
| const artifacts = await github.paginate( | |
| github.rest.actions.listWorkflowRunArtifacts, | |
| { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: runId, | |
| per_page: 100, | |
| }, | |
| ); | |
| return artifacts | |
| .filter((artifact) => !artifact.expired) | |
| .map((artifact) => artifact.name); | |
| }; | |
| const hasBenchmarkData = (names) => names.some( | |
| (name) => | |
| name === 'results_bmk' || | |
| name === 'eval_results_all' || | |
| name.startsWith('bmk_agentic_'), | |
| ); | |
| const validateRun = async (candidate, allowHistoricalAssociation = false) => { | |
| if ( | |
| candidate.path !== '.github/workflows/run-sweep.yml' || | |
| candidate.event !== 'pull_request' | |
| ) { | |
| return { valid: false, reason: 'not a pull-request run of run-sweep.yml' }; | |
| } | |
| const isAssociatedWithPull = candidate.pull_requests?.some( | |
| (candidatePull) => candidatePull.number === context.issue.number, | |
| ); | |
| if ( | |
| !pullCommitShas.has(candidate.head_sha) && | |
| !(allowHistoricalAssociation && isAssociatedWithPull) | |
| ) { | |
| return { valid: false, reason: 'its head commit is not in the PR commit list' }; | |
| } | |
| if (fullSweepLabelsAt(candidate.created_at).length === 0) { | |
| return { | |
| valid: false, | |
| reason: 'it was not created while a full-sweep label was applied', | |
| }; | |
| } | |
| const stageableConclusions = new Set(['success', 'cancelled', 'failure']); | |
| if ( | |
| candidate.status !== 'completed' || | |
| !stageableConclusions.has(candidate.conclusion) | |
| ) { | |
| return { | |
| valid: false, | |
| reason: `it is ${candidate.status}/${candidate.conclusion}`, | |
| }; | |
| } | |
| // Cancelled or failed sweeps may still contain useful partial results. The | |
| // artifact checks below keep empty or metadata-only runs ineligible. | |
| const artifactNames = await usableArtifactNames(candidate.id); | |
| if (!artifactNames.includes('changelog-metadata')) { | |
| return { valid: false, reason: 'it has no unexpired changelog-metadata artifact' }; | |
| } | |
| if (!hasBenchmarkData(artifactNames)) { | |
| return { valid: false, reason: 'it has no unexpired benchmark result artifacts' }; | |
| } | |
| return { valid: true }; | |
| }; | |
| let run; | |
| const requestedRunId = command.groups?.runId; | |
| if (requestedRunId) { | |
| const response = await github.rest.actions.getWorkflowRun({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: Number(requestedRunId), | |
| }); | |
| run = response.data; | |
| const validation = await validateRun(run, true); | |
| if (!validation.valid) { | |
| core.setFailed(`Run ${requestedRunId} cannot be staged because ${validation.reason}`); | |
| return; | |
| } | |
| } else { | |
| const candidates = await github.paginate( | |
| github.rest.actions.listWorkflowRuns, | |
| { | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| workflow_id: 'run-sweep.yml', | |
| event: 'pull_request', | |
| branch: pull.data.head.ref, | |
| status: 'completed', | |
| per_page: 100, | |
| }, | |
| ); | |
| for (const candidate of candidates) { | |
| const validation = await validateRun(candidate); | |
| if (validation.valid) { | |
| run = candidate; | |
| break; | |
| } | |
| } | |
| if (!run) { | |
| core.setFailed( | |
| `No stageable completed run-sweep.yml run with unexpired staging artifacts was found for PR #${context.issue.number}`, | |
| ); | |
| return; | |
| } | |
| } | |
| core.setOutput('run-id', String(run.id)); | |
| core.setOutput('run-attempt', String(run.run_attempt ?? 1)); | |
| core.setOutput('run-date', run.created_at.slice(0, 10)); | |
| core.setOutput('requested-by', actor); | |
| core.setOutput('run-url', run.html_url); | |
| - name: Acknowledge staging request | |
| id: acknowledge | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| RUN_ID: ${{ steps.request.outputs.run-id }} | |
| RUN_URL: ${{ steps.request.outputs.run-url }} | |
| REQUESTED_BY: ${{ steps.request.outputs.requested-by }} | |
| with: | |
| github-token: ${{ github.token }} | |
| script: | | |
| const body = [ | |
| `@${process.env.REQUESTED_BY} staging [run ${process.env.RUN_ID}](${process.env.RUN_URL}). Existing staged runs will be preserved; a completion link will be posted here.`, | |
| '', | |
| `@${process.env.REQUESTED_BY} 正在将[运行 ${process.env.RUN_ID}](${process.env.RUN_URL})发布到预发布环境。已有的预发布运行会继续保留；完成后将在此处提供链接。`, | |
| ].join('\n'); | |
| const comment = await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body, | |
| }); | |
| core.setOutput('comment-id', String(comment.data.id)); | |
| - name: Dispatch InferenceX-app staging workflow | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| RUN_ID: ${{ steps.request.outputs.run-id }} | |
| RUN_ATTEMPT: ${{ steps.request.outputs.run-attempt }} | |
| RUN_DATE: ${{ steps.request.outputs.run-date }} | |
| REQUESTED_BY: ${{ steps.request.outputs.requested-by }} | |
| COMMENT_ID: ${{ steps.acknowledge.outputs.comment-id }} | |
| with: | |
| github-token: ${{ secrets.INFX_FRONTEND_PAT }} | |
| script: | | |
| await github.rest.repos.createDispatchEvent({ | |
| owner: 'SemiAnalysisAI', | |
| repo: 'InferenceX-app', | |
| event_type: 'stage-results', | |
| client_payload: { | |
| 'source-repository': `${context.repo.owner}/${context.repo.repo}`, | |
| 'pr-number': String(context.issue.number), | |
| 'run-id': process.env.RUN_ID, | |
| 'run-attempt': process.env.RUN_ATTEMPT, | |
| 'run-date': process.env.RUN_DATE, | |
| 'requested-by': process.env.REQUESTED_BY, | |
| 'comment-id': process.env.COMMENT_ID, | |
| }, | |
| }); |