From 0636abdc229218b098b251760063b6aee53f1f16 Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Wed, 18 Sep 2024 18:58:34 -0700 Subject: [PATCH] Switch build_test_all_bazel to new dockerfile and runners. (#18533) Progress on https://github.com/iree-org/iree/issues/15332 and https://github.com/iree-org/iree/issues/18238. Fixes https://github.com/iree-org/iree/issues/16915. This switches the `build_test_all_bazel` CI job from the `gcr.io/iree-oss/base-bleeding-edge` Dockerfile using GCP for remote cache storage to the `ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64` Dockerfile with no remote cache. With no cache, this job takes between 18 and 25 minutes. Early testing also showed times as long as 60 minutes, if the Docker command and runner are both not optimally configured for Bazel (e.g. not using a RAM disk). The job is also moved from running on every commit to running on a nightly schedule while we evaluate how frequently it breaks and how long it takes to run. If we set up a new remote cache (https://bazel.build/remote/caching), we can move it back to running more regularly. --- .bazelversion | 2 +- .github/workflows/ci.yml | 31 ----------- .github/workflows/ci_linux_x64_bazel.yml | 52 +++++++++++++++++++ WORKSPACE | 14 ++++- build_tools/bazel/build_test_all.sh | 4 +- build_tools/bazel/install_bazelisk.sh | 22 ++++++++ build_tools/bazel/workspace.bzl | 1 + .../compiler/Codegen/Common/GPU/BUILD.bazel | 1 + .../Codegen/Common/GPU/CMakeLists.txt | 1 + .../docs/developers/general/contributing.md | 12 ++--- .../docs/developers/general/github-actions.md | 4 +- 11 files changed, 102 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/ci_linux_x64_bazel.yml create mode 100755 build_tools/bazel/install_bazelisk.sh diff --git a/.bazelversion b/.bazelversion index f22d756da39d..643916c03f1f 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.5.0 +7.3.1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a900e8982a7..b94a4fc58698 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,34 +52,6 @@ jobs: setup: uses: ./.github/workflows/setup.yml - ################################### Basic #################################### - # Jobs that build all of IREE "normally" - ############################################################################## - - # TODO(#18238): migrate to new runner cluster - # build_test_all_bazel: - # needs: setup - # if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_all_bazel') - # runs-on: - # - self-hosted # must come first - # - runner-group=${{ needs.setup.outputs.runner-group }} - # - environment=${{ needs.setup.outputs.runner-env }} - # - cpu - # - os-family=Linux - # steps: - # - name: "Checking out repository" - # uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - # with: - # submodules: true - # - name: "Building and testing with Bazel" - # env: - # IREE_WRITE_REMOTE_BAZEL_CACHE: ${{ needs.setup.outputs.write-caches }} - # run: | - # ./build_tools/github_actions/docker_run.sh \ - # --env "IREE_WRITE_REMOTE_BAZEL_CACHE=${IREE_WRITE_REMOTE_BAZEL_CACHE}" \ - # gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \ - # ./build_tools/bazel/build_test_all.sh - ############################### Configurations ############################### # Jobs that build IREE in some non-default configuration ############################################################################## @@ -327,9 +299,6 @@ jobs: needs: - setup - # Toolchains - # - build_test_all_bazel # Currently disabled. - # Accelerators # - test_nvidia_a100 diff --git a/.github/workflows/ci_linux_x64_bazel.yml b/.github/workflows/ci_linux_x64_bazel.yml new file mode 100644 index 000000000000..3e5fd711af86 --- /dev/null +++ b/.github/workflows/ci_linux_x64_bazel.yml @@ -0,0 +1,52 @@ +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: CI - Linux x64 bazel + +on: + pull_request: + paths: + - ".github/workflows/ci_linux_x64_bazel.yml" + schedule: + # Weekday mornings at 09:15 UTC = 01:15 PST (UTC - 8). + - cron: "15 9 * * 1-5" + workflow_dispatch: + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). The workflow name is prepended to avoid conflicts between + # different workflows. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + linux_x64_bazel: + runs-on: azure-linux-scale + container: + image: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74 + # Mount a RAM disk and point the Bazel sandbox at it using our custom environment variable. + options: --mount type=tmpfs,destination=/dev/shm + env: + SANDBOX_BASE: /dev/shm + defaults: + run: + shell: bash + steps: + - name: "Checking out repository" + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + submodules: true + - name: Install Python requirements + run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt + - name: Build and test with Bazel + env: + IREE_CUDA_DEPS_DIR: /usr/local/iree_cuda_deps + run: | + ./build_tools/bazel/install_bazelisk.sh 1.21.0 + cp ./build_tools/docker/context/fetch_cuda_deps.sh /usr/local/bin + /usr/local/bin/fetch_cuda_deps.sh ${IREE_CUDA_DEPS_DIR} + ./build_tools/bazel/build_test_all.sh diff --git a/WORKSPACE b/WORKSPACE index 3c38e4026aa5..2cd66907626d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -39,7 +39,19 @@ new_local_repository( load("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") -llvm_configure(name = "llvm-project") +llvm_configure( + name = "llvm-project", + # Keep this in sync with the targets in iree_llvm.cmake. + targets = [ + "AArch64", + "ARM", + "RISCV", + "X86", + "NVPTX", + "AMDGPU", + "WebAssembly", + ], +) ############################################################################### diff --git a/build_tools/bazel/build_test_all.sh b/build_tools/bazel/build_test_all.sh index 92c70564d281..881e7c72f3f2 100755 --- a/build_tools/bazel/build_test_all.sh +++ b/build_tools/bazel/build_test_all.sh @@ -29,7 +29,7 @@ set -xeuo pipefail -IREE_READ_REMOTE_BAZEL_CACHE="${IREE_READ_REMOTE_BAZEL_CACHE:-1}" +IREE_READ_REMOTE_BAZEL_CACHE="${IREE_READ_REMOTE_BAZEL_CACHE:-0}" IREE_WRITE_REMOTE_BAZEL_CACHE="${IREE_WRITE_REMOTE_BAZEL_CACHE:-0}" BAZEL_BIN="${BAZEL_BIN:-$(which bazel)}" SANDBOX_BASE="${SANDBOX_BASE:-}" @@ -147,6 +147,7 @@ fi if (( IREE_READ_REMOTE_BAZEL_CACHE == 1 )); then BAZEL_TEST_CMD+=(--config=remote_cache_bazel_ci) + BAZEL_TEST_CMD+=(--config=rs) fi if (( IREE_WRITE_REMOTE_BAZEL_CACHE != 1 )); then @@ -160,7 +161,6 @@ BAZEL_TEST_CMD+=( --test_tag_filters="${TEST_TAG_FILTERS?}" --keep_going --test_output=errors - --config=rs --config=generic_clang ) diff --git a/build_tools/bazel/install_bazelisk.sh b/build_tools/bazel/install_bazelisk.sh new file mode 100755 index 000000000000..68c849657d7c --- /dev/null +++ b/build_tools/bazel/install_bazelisk.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -euox pipefail + +BAZELISK_VERSION="$1" + +ARCH="$(dpkg --print-architecture)" + +curl --silent --fail --show-error --location \ + "https://github.com/bazelbuild/bazelisk/releases/download/v${BAZELISK_VERSION}/bazelisk-linux-${ARCH}" \ + --output bazelisk + +cp ./bazelisk /usr/local/bin/bazel +chmod +x /usr/local/bin/bazel +cp ./bazelisk /usr/local/bin/bazelisk +chmod +x /usr/local/bin/bazelisk +rm ./bazelisk diff --git a/build_tools/bazel/workspace.bzl b/build_tools/bazel/workspace.bzl index e03dd5ce44d6..654649508d3c 100644 --- a/build_tools/bazel/workspace.bzl +++ b/build_tools/bazel/workspace.bzl @@ -17,6 +17,7 @@ CUDA_TOOLKIT_ROOT_ENV_KEY = "IREE_CUDA_TOOLKIT_ROOT" # because CUDA toolkit detection differs depending on whether it is # stripped down or not). # TODO: Simplify this on the CMake/docker side and update here to match. +# TODO(#15332): Dockerfiles no longer include these deps. Simplify. CUDA_DEPS_DIR_FOR_CI_ENV_KEY = "IREE_CUDA_DEPS_DIR" def cuda_auto_configure_impl(repository_ctx): diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel index 8bdd9982aff2..55e1b247dd39 100644 --- a/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel +++ b/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel @@ -85,6 +85,7 @@ iree_compiler_cc_library( ], hdrs = [ "GPUPatterns.h", + "GPUTileSwizzleUtils.h", "GPUVectorDistribution.h", "Passes.h", ], diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt index 82387adad22c..4ded89f40fa0 100644 --- a/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt +++ b/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt @@ -44,6 +44,7 @@ iree_cc_library( CommonGPUPasses HDRS "GPUPatterns.h" + "GPUTileSwizzleUtils.h" "GPUVectorDistribution.h" "Passes.h" SRCS diff --git a/docs/website/docs/developers/general/contributing.md b/docs/website/docs/developers/general/contributing.md index 2d3ad6c2a982..e5b55800f623 100644 --- a/docs/website/docs/developers/general/contributing.md +++ b/docs/website/docs/developers/general/contributing.md @@ -371,22 +371,22 @@ runner-env: [testing|prod] Copy/paste any of these at the bottom of a PR description to change what the CI runs. -* Also run GPU tests (opt-in due to low availability): +* Skip all CI builds and tests, e.g. for comment-only changes: ``` text - ci-extra: test_nvidia_t4,test_amd_mi250,test_amd_mi300,test_amd_w7900 + skip-ci: Comment-only change. ``` -* Skip all CI builds and tests, e.g. for comment-only changes: +* Only run runtime builds: ``` text - skip-ci: Comment-only change. + ci-exactly: runtime ``` -* Only run Bazel builds, e.g. for changes only affecting Bazel rules: +* Only run ONNX tests: ``` text - ci-exactly: build_test_all_bazel + ci-exactly: build_packages,test_onnx ``` For example, this PR opted in to running the `build_test_all_windows` job: diff --git a/docs/website/docs/developers/general/github-actions.md b/docs/website/docs/developers/general/github-actions.md index 94d08ae1649b..b91007c6eff3 100644 --- a/docs/website/docs/developers/general/github-actions.md +++ b/docs/website/docs/developers/general/github-actions.md @@ -133,8 +133,10 @@ Workflow file | Build status | Event triggers [`ci_linux_x64_clang_tsan.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_clang_tsan.yml) | [![CI - Linux x64 clang TSan](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_tsan.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_tsan.yml?query=branch%3Amain+event%3Aschedule) | `schedule` [`ci_linux_x64_clang_debug.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_clang_debug.yml) | [![CI - Linux x64 clang debug](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_debug.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_debug.yml?query=branch%3Amain+event%3Aschedule) | `schedule` [`ci_linux_x64_gcc.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_gcc.yml) | [![CI - Linux x64 gcc](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_gcc.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_gcc.yml?query=branch%3Amain+event%3Aschedule) | `schedule` +[`ci_linux_x64_clang_byollvm.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_clang_byollvm.yml) | [![CI - Linux x64 clang_byollvm](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_byollvm.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_byollvm.yml?query=branch%3Amain+event%3Aschedule) | `schedule` +[`ci_linux_x64_bazel.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_bazel.yml) | [![CI - Linux x64 bazel](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_bazel.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_bazel.yml?query=branch%3Amain+event%3Aschedule) | `schedule` - + ### Other workflows