diff --git a/.github/workflows/pr-experiment-commenter.yml b/.github/workflows/pr-experiment-commenter.yml new file mode 100644 index 0000000000..0a130b5cc4 --- /dev/null +++ b/.github/workflows/pr-experiment-commenter.yml @@ -0,0 +1,27 @@ +name: PR Experiment Commenter + +on: + issue_comment: + types: [created] + +jobs: + comment-experiment-links: + if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/gcbrun exp ') + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - name: Check out the repository to the runner + uses: actions/checkout@v4 + - name: Parse gcbrun command and post experiment links + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const scriptPath = path.join(process.cwd(), 'ci', 'pr_exp_comment.js'); + const code = fs.readFileSync(scriptPath, 'utf8'); + eval(code); + await runPrExperimentCommenter({ github, context }); diff --git a/ci/pr_exp_comment.js b/ci/pr_exp_comment.js new file mode 100644 index 0000000000..eda38f77db --- /dev/null +++ b/ci/pr_exp_comment.js @@ -0,0 +1,140 @@ +const DEFAULT_CLUSTER = 'llm-experiment'; +const LARGE_CLUSTER = 'llm-experiment-large'; +const DEFAULT_LOCATION = 'us-central1-c'; +const LARGE_LOCATION = 'us-central1'; +const BENCHMARK_SET = 'comparison'; + +const TRIGGER_COMMAND = '/gcbrun'; +const TRIAL_BUILD_COMMAND_STR = `${TRIGGER_COMMAND} exp `; +const SKIP_COMMAND_STR = `${TRIGGER_COMMAND} skip`; + +const PR_LINK_PREFIX = 'https://github.com/google/oss-fuzz-gen/pull'; +const JOB_LINK_PREFIX = 'https://console.cloud.google.com/kubernetes/job///default'; +const REPORT_LINK_PREFIX = 'https://llm-exp.oss-fuzz.com/Result-reports/ofg-pr'; +const BUCKET_LINK_PREFIX = 'https://console.cloud.google.com/storage/browser/oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr'; +const BUCKET_GS_LINK_PREFIX = 'gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/ofg-pr'; + +/** + * Parses command arguments from /gcbrun exp comment. + * @param {string} comment - The comment body + * @returns {Object|null} Parsed args {nameSuffix, benchmarkSet} or null if invalid + */ +function parseGcbrunArgs(comment) { + if (!comment.startsWith(TRIAL_BUILD_COMMAND_STR)) { + return null; + } + + const args = comment.slice(TRIAL_BUILD_COMMAND_STR.length).trim().split(/\s+/); + console.log('Parsed args:', args); + + let nameSuffix = ''; + let benchmarkSet = BENCHMARK_SET; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '-n' && i + 1 < args.length) { + nameSuffix = args[i + 1]; + } else if (args[i] === '-b' && i + 1 < args.length) { + benchmarkSet = args[i + 1]; + } + } + + return { nameSuffix, benchmarkSet }; +} + +/** + * Prepares and generates the key experiment information. + * @param {number} prId - Pull request ID + * @param {string} nameSuffix - Name suffix for the experiment + * @param {string} benchmarkSet - Benchmark set name + * @returns {Object} Generated experiment info with links and job name + */ +function prepareExperimentInfo(prId, nameSuffix, benchmarkSet) { + const experimentName = `${prId}-${nameSuffix}`; + + const gkeJobName = `ofg-pr-${experimentName}`; + + const now = new Date(); + const date = `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, '0')}-${String(now.getDate()).padStart(2, '0')}`; + + // Base path for reports and buckets + const basePath = `${date}-${prId}-${nameSuffix}-${benchmarkSet}`; + + // GKE job link + let gkeJobLink = `${JOB_LINK_PREFIX}/ofg-pr-${experimentName}`; + gkeJobLink = gkeJobLink.replace('', DEFAULT_LOCATION); + gkeJobLink = gkeJobLink.replace('', DEFAULT_CLUSTER); + + // PR link + const prLink = `${PR_LINK_PREFIX}/${prId}`; + + // Report link + const reportLink = `${REPORT_LINK_PREFIX}/${basePath}/index.html`; + + // Bucket links + const bucketLink = `${BUCKET_LINK_PREFIX}/${basePath}`; + const bucketGsLink = `${BUCKET_GS_LINK_PREFIX}/${basePath}`; + + return { + gkeJobName, + gkeJobLink, + prLink, + reportLink, + bucketLink, + bucketGsLink + }; +} + +/** + * Parses /gcbrun exp commands from PR comments and posts experiment links. + * @param {Object} params - Parameters object + * @param {Object} params.github - GitHub API client from actions/github-script + * @param {Object} params.context - GitHub Actions context with event payload + */ +async function runPrExperimentCommenter({ github, context }) { + const comment = context.payload.comment.body; + const prNumber = context.payload.issue.number; + + const parsedArgs = parseGcbrunArgs(comment); + if (!parsedArgs) { + console.log('Not a valid /gcbrun exp command'); + return; + } + + const { nameSuffix, benchmarkSet } = parsedArgs; + if (!nameSuffix) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: 'Error: Missing required `-n` (name suffix) parameter in /gcbrun command.' + }); + return; + } + + const experimentInfo = prepareExperimentInfo(prNumber, nameSuffix, benchmarkSet); + + console.log( + `Requesting a GKE experiment named ${experimentInfo.gkeJobName}:\n` + + `PR: ${experimentInfo.prLink}\n` + + `JOB: ${experimentInfo.gkeJobLink}\n` + + `REPORT: ${experimentInfo.reportLink}\n` + + `BUCKET: ${experimentInfo.bucketLink}\n` + + `BUCKET GS: ${experimentInfo.bucketGsLink}` + ); + + const body = `Requested GKE Job: ${experimentInfo.gkeJobName} + + JOB: ${experimentInfo.gkeJobLink} + REPORT: ${experimentInfo.reportLink} + BUCKET: ${experimentInfo.bucketLink} + BUCKET (GS): ${experimentInfo.bucketGsLink}`; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: body + }); + + console.log(`Posted experiment links for PR #${prNumber}, job: ${experimentInfo.gkeJobName}`); +} diff --git a/ci/request_pr_exp.py b/ci/request_pr_exp.py index a24b27d42f..3d71c31d7f 100644 --- a/ci/request_pr_exp.py +++ b/ci/request_pr_exp.py @@ -69,17 +69,41 @@ DEFAULT_VERTEX_AI_LOCATION = 'us-central1' VERTEX_AI_LOCATIONS = { 'vertex_ai_gemini-pro': - 'asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,asia-southeast1,australia-southeast1,europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west2,europe-west3,europe-west4,europe-west6,europe-west8,europe-west9,southamerica-east1,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,' + 'asia-southeast1,australia-southeast1,europe-central2,europe-north1,' + 'europe-southwest1,europe-west1,europe-west2,europe-west3,' + 'europe-west4,europe-west6,europe-west8,europe-west9,' + 'southamerica-east1,us-central1,us-east1,us-east4,us-east5,' + 'us-south1,us-west1,us-west4'), 'vertex_ai_gemini-ultra': - 'asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,asia-southeast1,australia-southeast1,europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west2,europe-west3,europe-west4,europe-west6,europe-west8,europe-west9,southamerica-east1,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,' + 'asia-southeast1,australia-southeast1,europe-central2,europe-north1,' + 'europe-southwest1,europe-west1,europe-west2,europe-west3,' + 'europe-west4,europe-west6,europe-west8,europe-west9,' + 'southamerica-east1,us-central1,us-east1,us-east4,us-east5,' + 'us-south1,us-west1,us-west4'), 'vertex_ai_gemini-1-5': - 'asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,asia-southeast1,australia-southeast1,europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west2,europe-west3,europe-west4,europe-west6,europe-west8,europe-west9,southamerica-east1,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,' + 'asia-southeast1,australia-southeast1,europe-central2,europe-north1,' + 'europe-southwest1,europe-west1,europe-west2,europe-west3,' + 'europe-west4,europe-west6,europe-west8,europe-west9,' + 'southamerica-east1,us-central1,us-east1,us-east4,us-east5,' + 'us-south1,us-west1,us-west4'), 'vertex_ai_gemini-1-5-chat': - 'asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,asia-southeast1,australia-southeast1,europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west2,europe-west3,europe-west4,europe-west6,europe-west8,europe-west9,southamerica-east1,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('asia-east1,asia-east2,asia-northeast1,asia-northeast3,asia-south1,' + 'asia-southeast1,australia-southeast1,europe-central2,europe-north1,' + 'europe-southwest1,europe-west1,europe-west2,europe-west3,' + 'europe-west4,europe-west6,europe-west8,europe-west9,' + 'southamerica-east1,us-central1,us-east1,us-east4,us-east5,' + 'us-south1,us-west1,us-west4'), 'vertex_ai_gemini-2-flash': - 'europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west4,europe-west8,europe-west9,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4', + ('europe-central2,europe-north1,europe-southwest1,europe-west1,' + 'europe-west4,europe-west8,europe-west9,us-central1,us-east1,' + 'us-east4,us-east5,us-south1,us-west1,us-west4'), 'vertex_ai_gemini-2-flash-chat': - 'europe-central2,europe-north1,europe-southwest1,europe-west1,europe-west4,europe-west8,europe-west9,us-central1,us-east1,us-east4,us-east5,us-south1,us-west1,us-west4' + ('europe-central2,europe-north1,europe-southwest1,europe-west1,' + 'europe-west4,europe-west8,europe-west9,us-central1,us-east1,' + 'us-east4,us-east5,us-south1,us-west1,us-west4') }