Switch build_test_all_bazel to new dockerfile and runners. (iree-org#…

…18533) Progress on iree-org#15332 and iree-org#18238. Fixes iree-org#16915. This switches the `build_test_all_bazel` CI job from the `gcr.io/iree-oss/base-bleeding-edge` Dockerfile using GCP for remote cache storage to the `ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64` Dockerfile with no remote cache. With no cache, this job takes between 18 and 25 minutes. Early testing also showed times as long as 60 minutes, if the Docker command and runner are both not optimally configured for Bazel (e.g. not using a RAM disk). The job is also moved from running on every commit to running on a nightly schedule while we evaluate how frequently it breaks and how long it takes to run. If we set up a new remote cache (https://bazel.build/remote/caching), we can move it back to running more regularly.
nirvedhmeshram · Sep 19, 2024 · 0636abd · 0636abd
1 parent 3a62d5c
commit 0636abd
Show file tree

Hide file tree

Showing 11 changed files with 102 additions and 42 deletions.
diff --git a/.bazelversion b/.bazelversion
@@ -1 +1 @@
-6.5.0
+7.3.1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -52,34 +52,6 @@ jobs:
   setup:
     uses: ./.github/workflows/setup.yml
 
-  ################################### Basic ####################################
-  # Jobs that build all of IREE "normally"
-  ##############################################################################
-
-  # TODO(#18238): migrate to new runner cluster
-  # build_test_all_bazel:
-  #   needs: setup
-  #   if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_all_bazel')
-  #   runs-on:
-  #     - self-hosted # must come first
-  #     - runner-group=${{ needs.setup.outputs.runner-group }}
-  #     - environment=${{ needs.setup.outputs.runner-env }}
-  #     - cpu
-  #     - os-family=Linux
-  #   steps:
-  #     - name: "Checking out repository"
-  #       uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-  #       with:
-  #         submodules: true
-  #     - name: "Building and testing with Bazel"
-  #       env:
-  #         IREE_WRITE_REMOTE_BAZEL_CACHE: ${{ needs.setup.outputs.write-caches }}
-  #       run: |
-  #         ./build_tools/github_actions/docker_run.sh \
-  #           --env "IREE_WRITE_REMOTE_BAZEL_CACHE=${IREE_WRITE_REMOTE_BAZEL_CACHE}" \
-  #           gcr.io/iree-oss/base-bleeding-edge@sha256:cf2e78194e64fd0166f4141317366261d7a62432b72e9a324cb8c2ff4e1a515a \
-  #           ./build_tools/bazel/build_test_all.sh
-
   ############################### Configurations ###############################
   # Jobs that build IREE in some non-default configuration
   ##############################################################################
@@ -327,9 +299,6 @@ jobs:
     needs:
       - setup
 
-      # Toolchains
-      # - build_test_all_bazel  # Currently disabled.
-
       # Accelerators
       # - test_nvidia_a100
 

diff --git a/.github/workflows/ci_linux_x64_bazel.yml b/.github/workflows/ci_linux_x64_bazel.yml
@@ -0,0 +1,52 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+name: CI - Linux x64 bazel
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/ci_linux_x64_bazel.yml"
+  schedule:
+    # Weekday mornings at 09:15 UTC = 01:15 PST (UTC - 8).
+    - cron: "15 9 * * 1-5"
+  workflow_dispatch:
+
+concurrency:
+  # A PR number if a pull request and otherwise the commit hash. This cancels
+  # queued and in-progress runs for the same PR (presubmit) or commit
+  # (postsubmit). The workflow name is prepended to avoid conflicts between
+  # different workflows.
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  linux_x64_bazel:
+    runs-on: azure-linux-scale
+    container:
+      image: ghcr.io/iree-org/cpubuilder_ubuntu_jammy_x86_64@sha256:2b2ad51d7de988be13086bc618d89d2ba47fbf09eb5b38c60dce82b595fb1c74
+      # Mount a RAM disk and point the Bazel sandbox at it using our custom environment variable.
+      options: --mount type=tmpfs,destination=/dev/shm
+      env:
+        SANDBOX_BASE: /dev/shm
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: "Checking out repository"
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          submodules: true
+      - name: Install Python requirements
+        run: python3 -m pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt
+      - name: Build and test with Bazel
+        env:
+          IREE_CUDA_DEPS_DIR: /usr/local/iree_cuda_deps
+        run: |
+          ./build_tools/bazel/install_bazelisk.sh 1.21.0
+          cp ./build_tools/docker/context/fetch_cuda_deps.sh /usr/local/bin
+          /usr/local/bin/fetch_cuda_deps.sh ${IREE_CUDA_DEPS_DIR}
+          ./build_tools/bazel/build_test_all.sh
diff --git a/WORKSPACE b/WORKSPACE
@@ -39,7 +39,19 @@ new_local_repository(
 
 load("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure")
 
-llvm_configure(name = "llvm-project")
+llvm_configure(
+    name = "llvm-project",
+    # Keep this in sync with the targets in iree_llvm.cmake.
+    targets = [
+        "AArch64",
+        "ARM",
+        "RISCV",
+        "X86",
+        "NVPTX",
+        "AMDGPU",
+        "WebAssembly",
+    ],
+)
 
 ###############################################################################
 

diff --git a/build_tools/bazel/build_test_all.sh b/build_tools/bazel/build_test_all.sh
@@ -29,7 +29,7 @@
 
 set -xeuo pipefail
 
-IREE_READ_REMOTE_BAZEL_CACHE="${IREE_READ_REMOTE_BAZEL_CACHE:-1}"
+IREE_READ_REMOTE_BAZEL_CACHE="${IREE_READ_REMOTE_BAZEL_CACHE:-0}"
 IREE_WRITE_REMOTE_BAZEL_CACHE="${IREE_WRITE_REMOTE_BAZEL_CACHE:-0}"
 BAZEL_BIN="${BAZEL_BIN:-$(which bazel)}"
 SANDBOX_BASE="${SANDBOX_BASE:-}"
@@ -147,6 +147,7 @@ fi
 
 if (( IREE_READ_REMOTE_BAZEL_CACHE == 1 )); then
   BAZEL_TEST_CMD+=(--config=remote_cache_bazel_ci)
+  BAZEL_TEST_CMD+=(--config=rs)
 fi
 
 if (( IREE_WRITE_REMOTE_BAZEL_CACHE != 1 )); then
@@ -160,7 +161,6 @@ BAZEL_TEST_CMD+=(
   --test_tag_filters="${TEST_TAG_FILTERS?}"
   --keep_going
   --test_output=errors
-  --config=rs
   --config=generic_clang
 )
 

diff --git a/build_tools/bazel/install_bazelisk.sh b/build_tools/bazel/install_bazelisk.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+set -euox pipefail
+
+BAZELISK_VERSION="$1"
+
+ARCH="$(dpkg --print-architecture)"
+
+curl --silent --fail --show-error --location \
+    "https://github.com/bazelbuild/bazelisk/releases/download/v${BAZELISK_VERSION}/bazelisk-linux-${ARCH}" \
+    --output bazelisk
+
+cp ./bazelisk /usr/local/bin/bazel
+chmod +x /usr/local/bin/bazel
+cp ./bazelisk /usr/local/bin/bazelisk
+chmod +x /usr/local/bin/bazelisk
+rm ./bazelisk
diff --git a/build_tools/bazel/workspace.bzl b/build_tools/bazel/workspace.bzl
@@ -17,6 +17,7 @@ CUDA_TOOLKIT_ROOT_ENV_KEY = "IREE_CUDA_TOOLKIT_ROOT"
 # because CUDA toolkit detection differs depending on whether it is
 # stripped down or not).
 # TODO: Simplify this on the CMake/docker side and update here to match.
+# TODO(#15332): Dockerfiles no longer include these deps. Simplify.
 CUDA_DEPS_DIR_FOR_CI_ENV_KEY = "IREE_CUDA_DEPS_DIR"
 
 def cuda_auto_configure_impl(repository_ctx):

diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel
@@ -85,6 +85,7 @@ iree_compiler_cc_library(
     ],
     hdrs = [
         "GPUPatterns.h",
+        "GPUTileSwizzleUtils.h",
         "GPUVectorDistribution.h",
         "Passes.h",
     ],

diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt
@@ -44,6 +44,7 @@ iree_cc_library(
     CommonGPUPasses
   HDRS
     "GPUPatterns.h"
+    "GPUTileSwizzleUtils.h"
     "GPUVectorDistribution.h"
     "Passes.h"
   SRCS

diff --git a/docs/website/docs/developers/general/contributing.md b/docs/website/docs/developers/general/contributing.md
@@ -371,22 +371,22 @@ runner-env: [testing|prod]
 Copy/paste any of these at the bottom of a PR description to change what the CI
 runs.
 
-* Also run GPU tests (opt-in due to low availability):
+* Skip all CI builds and tests, e.g. for comment-only changes:
 
     ``` text
-    ci-extra: test_nvidia_t4,test_amd_mi250,test_amd_mi300,test_amd_w7900
+    skip-ci: Comment-only change.
     ```
 
-* Skip all CI builds and tests, e.g. for comment-only changes:
+* Only run runtime builds:
 
     ``` text
-    skip-ci: Comment-only change.
+    ci-exactly: runtime
     ```
 
-* Only run Bazel builds, e.g. for changes only affecting Bazel rules:
+* Only run ONNX tests:
 
     ``` text
-    ci-exactly: build_test_all_bazel
+    ci-exactly: build_packages,test_onnx
     ```
 
 For example, this PR opted in to running the `build_test_all_windows` job:

diff --git a/docs/website/docs/developers/general/github-actions.md b/docs/website/docs/developers/general/github-actions.md
@@ -133,8 +133,10 @@ Workflow file | Build status | Event triggers
 [`ci_linux_x64_clang_tsan.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_clang_tsan.yml) | [![CI - Linux x64 clang TSan](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_tsan.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_tsan.yml?query=branch%3Amain+event%3Aschedule) | `schedule`
 [`ci_linux_x64_clang_debug.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_clang_debug.yml) | [![CI - Linux x64 clang debug](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_debug.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_debug.yml?query=branch%3Amain+event%3Aschedule) | `schedule`
 [`ci_linux_x64_gcc.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_gcc.yml) | [![CI - Linux x64 gcc](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_gcc.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_gcc.yml?query=branch%3Amain+event%3Aschedule) | `schedule`
+[`ci_linux_x64_clang_byollvm.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_clang_byollvm.yml) | [![CI - Linux x64 clang_byollvm](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_byollvm.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_clang_byollvm.yml?query=branch%3Amain+event%3Aschedule) | `schedule`
+[`ci_linux_x64_bazel.yml`](https://github.com/iree-org/iree/blob/main/.github/workflows/ci_linux_x64_bazel.yml) | [![CI - Linux x64 bazel](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_bazel.yml/badge.svg?query=branch%3Amain+event%3Aschedule)](https://github.com/iree-org/iree/actions/workflows/ci_linux_x64_bazel.yml?query=branch%3Amain+event%3Aschedule) | `schedule`
 
-<!-- TODO(scotttodd): the remains of ci.yml (bazel, runtime builds, etc.) -->
+<!-- TODO(scotttodd): the remains of ci.yml (runtime builds, etc.) -->
 <!-- TODO(scotttodd): cross-compilation jobs -->
 
 ### Other workflows