diff --git a/.github/workflows/nightly_benchmarks.yml b/.github/workflows/nightly_benchmarks.yml new file mode 100644 index 00000000..22752873 --- /dev/null +++ b/.github/workflows/nightly_benchmarks.yml @@ -0,0 +1,32 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Nightly Benchmarks + +on: + schedule: + - cron: "0 0 * * *" # Runs at 00:00 UTC every day + +permissions: + contents: read + pull-requests: write + +jobs: + run_benchmarks: + name: Run Single Host Benchmarks + uses: google-ml-infra/bap/.github/workflows/run-benchmarks.yaml@fd37d1f65a06f0124d9d2022d97efa8dc8812557 # main + with: + registry_file: "tpu_sync/benchmarks/benchmark_registry.pbtxt" + bap_ref: "fd37d1f65a06f0124d9d2022d97efa8dc8812557" + runner: "linux-x86-n2-8" diff --git a/.github/workflows/postsubmit_benchmarks.yml b/.github/workflows/postsubmit_benchmarks.yml new file mode 100644 index 00000000..5632eace --- /dev/null +++ b/.github/workflows/postsubmit_benchmarks.yml @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Postsubmit Benchmarks + +on: + # Required for internal services to invoke workflow runs. + # Required inputs: benchmark_filter, environment_filter, custom_metadata. + workflow_dispatch: + inputs: + benchmark_filter: + description: "Regex to filter by benchmark name (e.g., 'h2d_d2h_gating')." + required: false + type: string + default: "" + environment_filter: + description: "Regex to filter by environment configuration ID (e.g., 'tpu-v5e-single-node')." + required: false + type: string + default: "" + custom_metadata: + description: "JSON string of metadata to append to the benchmark result. Overwrites static metadata from the registry but is overwritten by dynamic metadata from the workload." + required: false + type: string + default: "{}" + + push: + branches: + - main + +permissions: + contents: read + pull-requests: write + +concurrency: + # Group by workflow name and branch to ensure runs execute sequentially. + # This prevents parallel postsubmit runs from clashing and ensures accurate + # reporting history. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + run_benchmarks: + name: Run Single Host Benchmarks + uses: google-ml-infra/bap/.github/workflows/run-benchmarks.yaml@fd37d1f65a06f0124d9d2022d97efa8dc8812557 # main + with: + registry_file: "tpu_sync/benchmarks/benchmark_registry.pbtxt" + bap_ref: "fd37d1f65a06f0124d9d2022d97efa8dc8812557" + runner: "linux-x86-n2-8" + publish_metrics: true + pub_sub_gcp_project_id: "ml-oss-benchmarking-production" + pub_sub_gcp_topic_id: "public-results-prod" + benchmark_filter: ${{ inputs.benchmark_filter || '' }} + environment_filter: ${{ inputs.environment_filter || '' }} + custom_metadata: ${{ inputs.custom_metadata || '{}' }} diff --git a/.github/workflows/presubmit_benchmarks.yml b/.github/workflows/presubmit_benchmarks.yml new file mode 100644 index 00000000..737e6a5f --- /dev/null +++ b/.github/workflows/presubmit_benchmarks.yml @@ -0,0 +1,31 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Presubmit Benchmarks + +on: + pull_request: + +permissions: + contents: read + pull-requests: write + +jobs: + run_benchmarks: + name: Run Single Host Benchmarks + uses: google-ml-infra/bap/.github/workflows/run-benchmarks.yaml@fd37d1f65a06f0124d9d2022d97efa8dc8812557 # main + with: + registry_file: "tpu_sync/benchmarks/benchmark_registry.pbtxt" + bap_ref: "fd37d1f65a06f0124d9d2022d97efa8dc8812557" + runner: "linux-x86-n2-8" diff --git a/.github/workflows/run_benchmarks.yml b/.github/workflows/run_benchmarks.yml index d8ba44d9..5f69b2ab 100644 --- a/.github/workflows/run_benchmarks.yml +++ b/.github/workflows/run_benchmarks.yml @@ -15,15 +15,39 @@ name: Run Benchmarks on: + # Allow manual runs from the Actions tab for testing workflow_dispatch: - - push: - branches: - - main - - pull_request: - schedule: - - cron: "0 0 * * *" # Runs at 00:00 UTC every day + inputs: + ab_mode: + description: "Run A/B comparison (baseline vs experiment)." + required: false + type: boolean + default: false + baseline_ref: + description: "Git ref for the baseline in A/B mode. Defaults to PR base or main." + required: false + type: string + default: "" + experiment_ref: + description: "Git ref for the experiment in A/B mode. Defaults to current commit SHA." + required: false + type: string + default: "" + benchmark_filter: + description: "Regex to filter by benchmark name (e.g., 'h2d_d2h_gating')." + required: false + type: string + default: "" + environment_filter: + description: "Regex to filter by environment configuration ID (e.g., 'tpu-v5e-single-node')." + required: false + type: string + default: "" + custom_metadata: + description: "JSON string of metadata to append to the benchmark result. Overwrites static metadata from the registry but is overwritten by dynamic metadata from the workload." + required: false + type: string + default: "{}" permissions: contents: read @@ -32,11 +56,14 @@ permissions: jobs: run_benchmarks: name: Run Single Host Benchmarks - uses: google-ml-infra/bap/.github/workflows/run-benchmarks.yaml@affd28b492f67c242afa8b119e590b25816b7b88 + uses: google-ml-infra/bap/.github/workflows/run-benchmarks.yaml@fd37d1f65a06f0124d9d2022d97efa8dc8812557 # main with: registry_file: "tpu_sync/benchmarks/benchmark_registry.pbtxt" - ml_actions_ref: "affd28b492f67c242afa8b119e590b25816b7b88" + bap_ref: "fd37d1f65a06f0124d9d2022d97efa8dc8812557" runner: "linux-x86-n2-8" - publish_metrics: false - pub_sub_gcp_project_id: "ml-oss-benchmarking-production" - pub_sub_gcp_topic_id: "public-results-prod" + ab_mode: ${{ inputs.ab_mode }} + baseline_ref: ${{ inputs.baseline_ref }} + experiment_ref: ${{ inputs.experiment_ref }} + benchmark_filter: ${{ inputs.benchmark_filter }} + environment_filter: ${{ inputs.environment_filter }} + custom_metadata: ${{ inputs.custom_metadata }} diff --git a/.github/workflows/update_d2h_h2d_record.yml b/.github/workflows/update_d2h_h2d_record.yml index cd01c94f..4081bc0b 100644 --- a/.github/workflows/update_d2h_h2d_record.yml +++ b/.github/workflows/update_d2h_h2d_record.yml @@ -1,16 +1,16 @@ name: Record H2D/D2H Baselines + on: workflow_dispatch: -permissions: { contents: read, pull-requests: write } +permissions: + contents: read + pull-requests: write jobs: record: - uses: google-ml-infra/bap/.github/workflows/run-benchmarks.yaml@affd28b492f67c242afa8b119e590b25816b7b88 + uses: google-ml-infra/bap/.github/workflows/run-benchmarks.yaml@fd37d1f65a06f0124d9d2022d97efa8dc8812557 # main with: registry_file: "tpu_sync/benchmarks/benchmark_registry_record.pbtxt" - ml_actions_ref: "affd28b492f67c242afa8b119e590b25816b7b88" + ml_actions_ref: "fd37d1f65a06f0124d9d2022d97efa8dc8812557" runner: "linux-x86-n2-32" - publish_metrics: false - pub_sub_gcp_project_id: "ml-oss-benchmarking-production" - pub_sub_gcp_topic_id: "public-results-prod"