From 3068e6f2f61e796a5ae1e026bb8cd38939f1eef3 Mon Sep 17 00:00:00 2001 From: Stella Laurenzo Date: Wed, 22 Jan 2025 18:45:12 -0800 Subject: [PATCH] Rework cmake build to a better super-project structure. (#17) * Removes some obsolete CI packaging steps that need to be reworked in a followon. * Reworks patches directory to be based on `git format-patch` and `git am` for a more sustainable dev flow. * Takes a fine tooth comb to version propagation in the sub-projects and burns out a lot of implicit dependencies on the host environment. * Layers the sub-project builds for isolation and separation of concerns so that there is less chance of implicitly depending on a byproduct out of order. * Implements automatic toolchain management so that sub-projects can be configured to build with an in-project built LLVM and/or HIP compiler. * Includes upstream patches for working around various CMake issues. * Removes most stringly propagated CMake variables in favor for explicit declaration in sub-projects. * Adds the beginning of a build manual. * Adds background build job pools that sub-projects can opt into if they have low concurrency/high latency. --- .github/workflows/build_linux_packages.yml | 40 +- .github/workflows/ci.yml | 6 +- .github/workflows/nightly_staging_release.yml | 3 - .pre-commit-config.yaml | 3 + CMakeLists.txt | 399 +---------- README.md | 12 +- base/CMakeLists.txt | 4 + base/README.md | 3 + base/rocm-cmake/CMakeLists.txt | 10 + base/rocm-core/CMakeLists.txt | 15 + base/rocm_smi_lib/CMakeLists.txt | 16 + base/rocprofiler-register/CMakeLists.txt | 18 + build_tools/apply_patches.sh | 31 - build_tools/fetch_sources.py | 36 +- build_tools/merge_dist_dir.cmake | 64 ++ build_tools/save_patches.sh | 51 ++ cmake/therock_job_pools.cmake | 17 + cmake/therock_subproject.cmake | 646 ++++++++++++++++++ cmake/therock_subproject_dep_provider.cmake | 58 ++ comm_libs/CMakeLists.txt | 5 + comm_libs/rccl/CMakeLists.txt | 33 + .../rccl/post_hook.cmake | 4 +- compiler/CMakeLists.txt | 2 + compiler/amd-llvm/CMakeLists.txt | 28 + compiler/amd-llvm/post_hook.cmake | 12 + compiler/amd-llvm/pre_hook.cmake | 16 + compiler/hipify/CMakeLists.txt | 18 + compiler/hipify/post_hook.cmake | 9 + components/amd-llvm/CMakeLists.txt | 41 -- core/CMakeLists.txt | 3 + core/ROCR-Runtime/CMakeLists.txt | 23 + .../ROCR-Runtime/post_hook.cmake | 1 - core/clr/CMakeLists.txt | 27 + core/clr/post_hook.cmake | 16 + core/rocminfo/CMakeLists.txt | 11 + core/rocminfo/post_hook.cmake | 9 + docs/development/build_system.md | 218 ++++++ ...g-buildable-with-an-LLVM-configured-.patch | 89 +++ ..._PERMISSIONS-when-installing-scripts.patch | 25 + ...TFORM-auto-detect-if-already-defined.patch | 155 +++++ ...1-Make-rccl-version-detection-robust.patch | 94 +++ 41 files changed, 1781 insertions(+), 490 deletions(-) create mode 100644 base/CMakeLists.txt create mode 100644 base/README.md create mode 100644 base/rocm-cmake/CMakeLists.txt create mode 100644 base/rocm-core/CMakeLists.txt create mode 100644 base/rocm_smi_lib/CMakeLists.txt create mode 100644 base/rocprofiler-register/CMakeLists.txt delete mode 100755 build_tools/apply_patches.sh create mode 100644 build_tools/merge_dist_dir.cmake create mode 100755 build_tools/save_patches.sh create mode 100644 cmake/therock_job_pools.cmake create mode 100644 cmake/therock_subproject.cmake create mode 100644 cmake/therock_subproject_dep_provider.cmake create mode 100644 comm_libs/CMakeLists.txt create mode 100644 comm_libs/rccl/CMakeLists.txt rename cmake/extensions/clr_post.cmake => comm_libs/rccl/post_hook.cmake (78%) create mode 100644 compiler/CMakeLists.txt create mode 100644 compiler/amd-llvm/CMakeLists.txt create mode 100644 compiler/amd-llvm/post_hook.cmake create mode 100644 compiler/amd-llvm/pre_hook.cmake create mode 100644 compiler/hipify/CMakeLists.txt create mode 100644 compiler/hipify/post_hook.cmake delete mode 100644 components/amd-llvm/CMakeLists.txt create mode 100644 core/CMakeLists.txt create mode 100644 core/ROCR-Runtime/CMakeLists.txt rename cmake/extensions/rocr_post.cmake => core/ROCR-Runtime/post_hook.cmake (92%) create mode 100644 core/clr/CMakeLists.txt create mode 100644 core/clr/post_hook.cmake create mode 100644 core/rocminfo/CMakeLists.txt create mode 100644 core/rocminfo/post_hook.cmake create mode 100644 docs/development/build_system.md create mode 100644 patches/rocm-6.3.1/HIPIFY/0001-Make-hipify-clang-buildable-with-an-LLVM-configured-.patch create mode 100644 patches/rocm-6.3.1/HIPIFY/0002-Pass-USE_SOURCE_PERMISSIONS-when-installing-scripts.patch create mode 100644 patches/rocm-6.3.1/clr/0001-Disable-HIP_PLATFORM-auto-detect-if-already-defined.patch create mode 100644 patches/rocm-6.3.1/rccl/0001-Make-rccl-version-detection-robust.patch diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index b53608d..4d0e0fd 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -53,6 +53,8 @@ jobs: - name: Fetch sources run: | + git config --global user.email "nobody@amd.com" + git config --global user.name "Nobody" ./build_tools/fetch_sources.py --depth 1 # The full checkout is very large: ~16GB, 9 of which is GIT stuff. @@ -75,9 +77,9 @@ jobs: -DTHEROCK_PACKAGE_VERSION="${package_version}" cmake --build build - - name: Build Tarballs - run: | - (cd build && cpack -G TXZ) + # - name: Build Tarballs + # run: | + # (cd build && cpack -G TXZ) - name: Report if: ${{ !cancelled() }} @@ -85,23 +87,23 @@ jobs: ls -lh build ccache -s - - name: Upload runtime artifacts - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: TheRock-runtime-linux-x86_64-tarball - path: | - build/TheRock-amdgpu-runtime-*.tar.xz - if-no-files-found: warn + # - name: Upload runtime artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ !cancelled() }} + # with: + # name: TheRock-runtime-linux-x86_64-tarball + # path: | + # build/TheRock-amdgpu-runtime-*.tar.xz + # if-no-files-found: warn - - name: Upload compiler artifacts - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: TheRock-compiler-linux-x86_64-tarball - path: | - build/TheRock-amdgpu-compiler*.tar.xz - if-no-files-found: warn + # - name: Upload compiler artifacts + # uses: actions/upload-artifact@v4 + # if: ${{ !cancelled() }} + # with: + # name: TheRock-compiler-linux-x86_64-tarball + # path: | + # build/TheRock-amdgpu-compiler*.tar.xz + # if-no-files-found: warn - name: Save cache uses: actions/cache/save@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4e91b1..7df43f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,6 @@ jobs: name: Build Linux Packages uses: ./.github/workflows/build_linux_packages.yml - build_python_packages: - name: Build Python Packages - uses: ./.github/workflows/build_python_packages.yml + # build_python_packages: + # name: Build Python Packages + # uses: ./.github/workflows/build_python_packages.yml diff --git a/.github/workflows/nightly_staging_release.yml b/.github/workflows/nightly_staging_release.yml index 2b583ca..dfcb91a 100644 --- a/.github/workflows/nightly_staging_release.yml +++ b/.github/workflows/nightly_staging_release.yml @@ -1,9 +1,6 @@ name: Nightly Staging Release on: - schedule: - - cron: "0 10 * * *" - workflow_dispatch: inputs: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9affa9b..c88605f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,8 +5,11 @@ repos: rev: v3.2.0 hooks: - id: trailing-whitespace + exclude: ^patches/ - id: end-of-file-fixer + exclude: ^patches/ - id: check-yaml + exclude: ^patches/ - id: check-added-large-files - repo: https://github.com/psf/black rev: 22.10.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index a5c6a0e..317caa2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,7 @@ +if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") + message(FATAL_ERROR "In-source builds are not allowed. Please create a separate build directory.") +endif() + cmake_minimum_required(VERSION 3.18) # Set the default build type to Release if not specified @@ -9,15 +13,23 @@ project(THEROCK) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include(ExternalProject) +include(therock_subproject) +include(therock_job_pools) ################################################################################ # Options ################################################################################ +option(THEROCK_INTERACTIVE "Enable interactive build mode which will cause all sub-project invocations to stream to the console (reducing parallelism)" OFF) +set(THEROCK_BACKGROUND_BUILD_JOBS "0" CACHE STRING "Number of jobs to reserve for projects marked for background building (empty=auto or a number)") + set(THEROCK_PACKAGE_VERSION "git" CACHE STRING "Sets the package version string") set(ROCM_GIT_DIR "${THEROCK_SOURCE_DIR}/sources" CACHE PATH "Directory of rocm-org repo checkout") message(STATUS "ROCM_GIT_DIR is set to: ${ROCM_GIT_DIR}") +# Library specific enable flags. +option(THEROCK_ENABLE_RCCL "Enable the comm_libs/rccl sub-project" ON) + # Initialize the install directory. if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "${THEROCK_SOURCE_DIR}/install" CACHE PATH "" FORCE) @@ -27,9 +39,10 @@ endif() set(ROCM_MAJOR_VERSION 6) set(ROCM_MINOR_VERSION 3) set(ROCM_PATCH_VERSION 1) -set(ROCM_VERSION - "${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}.${ROCM_PATCH_VERSION}") -set(AMDGPU_TARGETS "gfx90a gfx940 gfx942 gfx1100" CACHE STRING "AMDGPU Targets") +# set(AMDGPU_TARGETS "gfx90a gfx940 gfx942 gfx1100" CACHE STRING "AMDGPU Targets") +# TODO: Shard jobs by gfx family and have better ergonomics for multiple family +# specific packages. +set(AMDGPU_TARGETS "gfx1100" CACHE STRING "AMDGPU Targets") ################################################################################ # Global setup @@ -46,38 +59,11 @@ set(STAGING_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/staging_install") # External project setup ################################################################################ -option(ALWAYS_BUILD_SUBPROJECTS - "Don't let the brittle CMake external project machinery decide if a sub-project needs to rebuild. Always run the underlying build." - OFF) -set(FIND_PACKAGE_OPTIONS) +# On some distributions, this will install to lib64. We would like +# consistency in built packages, so hard-code it. +set(CMAKE_INSTALL_LIBDIR "lib") -macro(add_package_path PackageName path) - list(APPEND FIND_PACKAGE_OPTIONS "-D${PackageName}_DIR=${path}") -endmacro() -set(DEFAULT_CMAKE_ARGS - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_PLATFORM_NO_VERSIONED_SONAME=${CMAKE_PLATFORM_NO_VERSIONED_SONAME} - -DPython3_EXECUTABLE=${Python3_EXECUTABLE} - -DPython3_FIND_VIRTUALENV=${Python3_FIND_VIRTUALENV} - -DTHEROCK_SOURCE_DIR=${THEROCK_SOURCE_DIR} - -DROCM_GIT_DIR=${ROCM_GIT_DIR} - -DROCM_MAJOR_VERSION=${ROCM_MAJOR_VERSION} - -DROCM_MINOR_VERSION=${ROCM_MINOR_VERSION} - -DROCM_PATCH_VERSION=${ROCM_PATCH_VERSION} - -DROCM_VERSION=${ROCM_VERSION} - "-DROCM_PATH=${STAGING_INSTALL_DIR}" - "-DCPACK_PACKAGING_INSTALL_PREFIX=${STAGING_INSTALL_DIR}" - "-DAMDGPU_TARGETS=${AMDGPU_TARGETS}" - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER} - -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER} - # On some distributions, this will install to lib64. We would like - # consistency in built packages, so hard-code it. - -DCMAKE_INSTALL_LIBDIR=lib - -DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${CMAKE_CURRENT_SOURCE_DIR}/cmake/therock_external_project_include.cmake -) if(CMAKE_C_VISIBILITY_PRESET) list(APPEND DEFAULT_CMAKE_ARGS ${CMAKE_C_VISIBILITY_PRESET}) endif() @@ -85,271 +71,17 @@ if(CMAKE_CXX_VISIBILITY_PRESET) list(APPEND DEFAULT_CMAKE_ARGS ${CMAKE_CXX_VISIBILITY_PRESET}) endif() -################################################################################ -# rocm-cmake -################################################################################ - -ExternalProject_Add( - rocm-cmake - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocm-cmake - SOURCE_DIR "${ROCM_GIT_DIR}/rocm-cmake" - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) - -add_package_path(ROCM "${STAGING_INSTALL_DIR}/share/rocm/cmake") - -################################################################################ -# rocprofiler-register -# This is a stub that helps runtime libraries and profiles rendezvous -################################################################################ - -ExternalProject_Add( - rocprofiler-register - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocprofiler-register - SOURCE_DIR "${ROCM_GIT_DIR}/rocprofiler-register" - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) -add_package_path(rocprofiler-register "${STAGING_INSTALL_DIR}/lib/cmake/rocprofiler-register") - -################################################################################ -# LLVM -################################################################################ - -ExternalProject_Add( - amd-llvm - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/amd-llvm - SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/components/amd-llvm" - DEPENDS - rocm-cmake - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - # We install llvm in its own sub-directory. - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR}/lib/llvm - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) - -add_package_path(amd_comgr "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/amd_comgr") -add_package_path(Clang "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/clang") -add_package_path(LLD "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/lld") -add_package_path(LLVM "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/llvm") -add_package_path(AMDDeviceLibs "${STAGING_INSTALL_DIR}/lib/llvm/lib/cmake/AMDDeviceLibs") - -################################################################################ -# ROCR-Runtime -################################################################################ - -ExternalProject_Add( - ROCR-Runtime - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ROCR-Runtime - SOURCE_DIR "${ROCM_GIT_DIR}/ROCR-Runtime" - DEPENDS - amd-llvm # Build dep only - rocprofiler-register - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} - -DBUILD_SHARED_LIBS=ON - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) - -add_package_path(hsakmt "${STAGING_INSTALL_DIR}/lib/cmake/hsakmt") -add_package_path(hsa-runtime64 "${STAGING_INSTALL_DIR}/lib/cmake/hsa-runtime64") - -################################################################################ -# rocm-core -################################################################################ - -ExternalProject_Add( - rocm-core - DEPENDS - amd-llvm - rocm-cmake - ROCR-Runtime - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocm-core - SOURCE_DIR "${ROCM_GIT_DIR}/rocm-core" - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} - "-DTHEROCK_INSTALL_RPATH=lib$lib/llvm/lib" - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) - -################################################################################ -# rocminfo -################################################################################ - -ExternalProject_Add( - rocminfo - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocminfo - SOURCE_DIR "${ROCM_GIT_DIR}/rocminfo" - DEPENDS - rocm-core - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) - -################################################################################ -# HIPCC -################################################################################ - -ExternalProject_Add( - hipcc - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/hipcc - # Can't build directly as part of LLVM, because we want hipcc staged under - # bin, not llvm/bin. - SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/hipcc" - DEPENDS - amd-llvm # runtime - rocminfo # runtime - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - # We install llvm in its own sub-directory. - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) - -################################################################################ -# CLR -################################################################################ - -ExternalProject_Add( - clr - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/clr - SOURCE_DIR "${ROCM_GIT_DIR}/clr" - DEPENDS - amd-llvm - hipcc - rocm-core - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} - -DHIP_PLATFORM=amd - "-DOFFLOAD_ARCH_STR=${AMDGPU_TARGETS}" - "-DHIP_CLANG_PATH=${STAGING_INSTALL_DIR}/llvm/bin" - # Some junk needs this to be defined but is special cased so if empty, - # bad things don't happen. - "-DHIPCC_BIN_DIR=${STAGING_INSTALL_DIR}/bin" - "-DHIP_COMMON_DIR=${ROCM_GIT_DIR}/HIP" - # What is this? - -DROCM_PATCH_VERSION=99999 - -D__HIP_ENABLE_PCH=OFF - -DCLR_BUILD_HIP=ON - # Force CMake to reconfigure if key values change. - "-D_STAMP=${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}.${ROCM_PATCH_VERSION}" - - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) - -add_package_path(HIP "${STAGING_INSTALL_DIR}/lib/cmake/hip") -add_package_path(hip-lang "${STAGING_INSTALL_DIR}/lib/cmake/hip-lang") -add_package_path(hiprtc "${STAGING_INSTALL_DIR}/lib/cmake/hiprtc") - -################################################################################ -# ROCm SMI -################################################################################ - -ExternalProject_Add( - rocm_smi_lib - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rocm_smi_lib - SOURCE_DIR "${ROCM_GIT_DIR}/rocm_smi_lib" - DEPENDS - rocm-core - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) - -add_package_path(rocm-smi "${STAGING_INSTALL_DIR}/lib/cmake/rocm-smi") - -################################################################################ -# HIPIFY -################################################################################ - -ExternalProject_Add( - hipify - PREFIX ${CMAKE_CURRENT_BINARY_DIR}/hipify - SOURCE_DIR "${ROCM_GIT_DIR}/HIPIFY" - DEPENDS - amd-llvm - clr - CMAKE_ARGS - ${DEFAULT_CMAKE_ARGS} - ${FIND_PACKAGE_OPTIONS} - -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} - USES_TERMINAL_CONFIGURE TRUE - USES_TERMINAL_BUILD TRUE - BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -) - -################################################################################ -# RCCL -################################################################################ +# Add subdirectories in dependency DAG order (which happens to be semi-alpha: +# don't be fooled). +add_subdirectory(base) +add_subdirectory(compiler) +add_subdirectory(core) +add_subdirectory(comm_libs) -# TODO: Re-enable rccl after adapting to 6.3.x build option differences. -# Now seems to depend on rocprofiler. -# ExternalProject_Add( -# rccl -# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/rccl -# SOURCE_DIR "${ROCM_GIT_DIR}/rccl" -# DEPENDS -# clr -# hipcc -# hipify -# rocminfo -# CMAKE_ARGS -# ${DEFAULT_CMAKE_ARGS} -# ${FIND_PACKAGE_OPTIONS} -# # TODO: Audit how the 6.3.x build finds its C++ compiler. -# #-DCMAKE_CXX_COMPILER=${STAGING_INSTALL_DIR}/bin/hipcc -# -DCMAKE_CXX_COMPILER=${STAGING_INSTALL_DIR}/lib/llvm/bin/clang++ -# -DCMAKE_INSTALL_PREFIX=${STAGING_INSTALL_DIR} -# -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -# USES_TERMINAL_CONFIGURE TRUE -# USES_TERMINAL_BUILD TRUE -# BUILD_ALWAYS ${ALWAYS_BUILD_SUBPROJECTS} -# ) -################################################################################ -# Testing -################################################################################ +# ################################################################################ +# # Testing +# ################################################################################ add_executable( dlopen-hip @@ -357,78 +89,3 @@ add_executable( ) target_link_libraries(dlopen-hip dl) -################################################################################ -# Packaging -# Since our notion of packaging does not follow the actual sub-projects, -# we repackage based on what they have installed. -################################################################################ - -install( - CODE "set(STAGING_INSTALL_DIR ${STAGING_INSTALL_DIR})" - ALL_COMPONENTS -) - -install( - SCRIPT "cmake/custom_install_amdgpu_runtime.cmake" - COMPONENT amdgpu-runtime -) - -install( - SCRIPT "cmake/custom_install_amdgpu_runtime_dev.cmake" - COMPONENT amdgpu-runtime-dev -) - -install( - SCRIPT "cmake/custom_install_amdgpu_compiler.cmake" - COMPONENT amdgpu-compiler -) - -string(TOLOWER "${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}" _package_sysarch) -message(STATUS "CPack: SYSARCH=${_package_sysarch}, PACKAGE_VERSION=${THEROCK_PACKAGE_VERSION}") - -set(CPACK_PACKAGE_NAME "TheRock") -set(CPACK_PACKAGE_VENDOR "Advanced Micro Devices") -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Standalone amdgpu runtime packages") -set(CPACK_PACKAGE_VERSION "${THEROCK_PACKAGE_VERSION}") -set(CPACK_PACKAGE_VERSION_MAJOR "${ROCM_MAJOR_VERSION}") -set(CPACK_PACKAGE_VERSION_MINOR "${ROCM_MINOR_VERSION}") -set(CPACK_PACKAGE_VERSION_PATCH "${ROCM_PATCH_VERSION}") -set(CPACK_PACKAGE_INSTALL_DIRECTORY "TheRock-amdgpu") -set(CPACK_PACKAGE_FILE_NAME "") -set(CPACK_COMPONENTS_ALL - amdgpu-compiler - amdgpu-runtime - amdgpu-runtime-dev -) - -# CPack Archive Generation Options. -set(CPACK_ARCHIVE_COMPONENT_INSTALL ON) -set(CPACK_ARCHIVE_THREADS 0) -set( - CPACK_ARCHIVE_AMDGPU-RUNTIME_FILE_NAME - "TheRock-amdgpu-runtime-${_package_sysarch}-${THEROCK_PACKAGE_VERSION}") -set( - CPACK_ARCHIVE_AMDGPU-RUNTIME-DEV_FILE_NAME - "TheRock-amdgpu-runtime-dev-${_package_sysarch}-${THEROCK_PACKAGE_VERSION}") -set( - CPACK_ARCHIVE_AMDGPU-COMPILER_FILE_NAME - "TheRock-amdgpu-compiler-${_package_sysarch}-${THEROCK_PACKAGE_VERSION}") - -include(CPack) - -cpack_add_component( - amdgpu-runtime - DISPLAY_NAME "AMD GPU Runtime" - ARCHIVE_FILE "${CPACK_ARCHIVE_AMDGPU-RUNTIME_FILE_NAME}" -) -cpack_add_component( - amdgpu-runtime-dev - DISPLAY_NAME "AMD GPU Development Components" - DEPENDS amdgpu-runtime - ARCHIVE_FILE "${CPACK_ARCHIVE_AMDGPU-RUNTIME-DEV_FILE_NAME}" -) -cpack_add_component( - amdgpu-compiler - DISPLAY_NAME "AMD GPU Compiler" - ARCHIVE_FILE "${CPACK_ARCHIVE_AMDGPU-COMPILER_FILE_NAME}" -) diff --git a/README.md b/README.md index bb8a940..b172649 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # TheRock + The HIP Environment and ROCm Kit - A lightweight open source build system for HIP and ROCm [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) @@ -6,6 +7,7 @@ The HIP Environment and ROCm Kit - A lightweight open source build system for HI # Install Deps ## Common + ``` pip install CppHeaderParser ``` @@ -46,22 +48,16 @@ This will also apply the patches to the downloaded source files. ## Manually Checkout the latest development branch with + ``` mkdir -p ~/github/rocm cd ~/github/rocm repo init -u https://github.com/ROCm/ROCm.git -m tools/rocm-build/rocm-6.3.1.xml -b roc-6.3.x repo sync -j16 ``` -Checkout out latest LLVM sources - -``` -cd ~/github/rocm/llvm-project -git fetch --all -llvm-project$ git checkout rocm-org/amd-staging -``` -Latest `HIP` and `clr` should be on the `develop` branch Checkout TheRock build tools + ``` cd ~/github/ git clone https://github.com/nod-ai/TheRock diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt new file mode 100644 index 0000000..66e327d --- /dev/null +++ b/base/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(rocm-cmake) +add_subdirectory(rocm-core) +add_subdirectory(rocm_smi_lib) +add_subdirectory(rocprofiler-register) diff --git a/base/README.md b/base/README.md new file mode 100644 index 0000000..082ce04 --- /dev/null +++ b/base/README.md @@ -0,0 +1,3 @@ +# `base` Directory + +Projects in `base` are either third party or leaf libraries upon which everything else depends. diff --git a/base/rocm-cmake/CMakeLists.txt b/base/rocm-cmake/CMakeLists.txt new file mode 100644 index 0000000..98de608 --- /dev/null +++ b/base/rocm-cmake/CMakeLists.txt @@ -0,0 +1,10 @@ +################################################################################ +# rocm-cmake +################################################################################ + +therock_cmake_subproject_declare(rocm-cmake + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/rocm-cmake" +) +therock_cmake_subproject_provide_package(rocm-cmake + ROCM share/rocmcmakebuildtools/cmake) +therock_cmake_subproject_activate(rocm-cmake) diff --git a/base/rocm-core/CMakeLists.txt b/base/rocm-core/CMakeLists.txt new file mode 100644 index 0000000..f94a4b8 --- /dev/null +++ b/base/rocm-core/CMakeLists.txt @@ -0,0 +1,15 @@ +################################################################################ +# rocm-core +################################################################################ + +therock_cmake_subproject_declare(rocm-core + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/rocm-core" + CMAKE_ARGS + "-DBUILD_SHARED_LIBS=ON" + "-DROCM_VERSION=${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}.${ROCM_PATCH_VERSION}" +) +therock_cmake_subproject_glob_c_sources(rocm-core + SUBDIRS . +) +therock_cmake_subproject_provide_package(rocm-core rocm-core lib/cmake/rocm-core) +therock_cmake_subproject_activate(rocm-core) diff --git a/base/rocm_smi_lib/CMakeLists.txt b/base/rocm_smi_lib/CMakeLists.txt new file mode 100644 index 0000000..2b8a62d --- /dev/null +++ b/base/rocm_smi_lib/CMakeLists.txt @@ -0,0 +1,16 @@ +################################################################################ +# rocm_smi_lib +################################################################################ + +therock_cmake_subproject_declare(rocm_smi_lib + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/rocm_smi_lib" + INTERFACE_LINK_DIRS + "lib" +) +therock_cmake_subproject_glob_c_sources(rocm_smi_lib + SUBDIRS + include + src +) +therock_cmake_subproject_provide_package(rocm_smi_lib rocm_smi lib/cmake/rocm_smi) +therock_cmake_subproject_activate(rocm_smi_lib) diff --git a/base/rocprofiler-register/CMakeLists.txt b/base/rocprofiler-register/CMakeLists.txt new file mode 100644 index 0000000..2c8e0cf --- /dev/null +++ b/base/rocprofiler-register/CMakeLists.txt @@ -0,0 +1,18 @@ +################################################################################ +# rocprofiler-register +# This is a stub that helps runtime libraries and profiles rendezvous +################################################################################ + +therock_cmake_subproject_declare(rocprofiler-register + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/rocprofiler-register" + INTERFACE_LINK_DIRS + "lib" +) +therock_cmake_subproject_glob_c_sources(rocprofiler-register + SUBDIRS + source/include + source/lib +) +therock_cmake_subproject_provide_package(rocprofiler-register + rocprofiler-register lib/cmake/rocprofiler-register) +therock_cmake_subproject_activate(rocprofiler-register) diff --git a/build_tools/apply_patches.sh b/build_tools/apply_patches.sh deleted file mode 100755 index 4c6c16e..0000000 --- a/build_tools/apply_patches.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Applies patches that we are maintaining on the ROCM sources. -# These are developed against HEAD. -# Must be run from the repo checkout dir. -set -euxo pipefail - -repo_dir="$(pwd)" -this_dir="$(cd $(dirname $0) && pwd)" -root_dir="$(cd $this_dir/.. && pwd)" - -function stash_changes() { - local repo_name="$1" - cd $repo_dir/$repo_name - git add -A - git stash -} - -function apply_patch() { - local repo_name="$1" - local patch_file="$2" - cd $repo_dir/$repo_name - echo "Applying $patch_file to $repo_name" - patch -p1 < $root_dir/patches/$patch_file -} - -echo "Running from $PWD" - -# Example: -# stash_changes clr -# apply_patch clr clr-disable-hipconfig-check.patch -# apply_patch clr clr-respect-no-versioned-soname.patch diff --git a/build_tools/fetch_sources.py b/build_tools/fetch_sources.py index 9080add..4d43b39 100755 --- a/build_tools/fetch_sources.py +++ b/build_tools/fetch_sources.py @@ -9,13 +9,15 @@ import subprocess import sys -DEFAULT_SOURCES_DIR = Path(__file__).resolve().parent.parent / "sources" +THEROCK_DIR = Path(__file__).resolve().parent.parent +DEFAULT_SOURCES_DIR = THEROCK_DIR / "sources" +PATCHES_DIR = THEROCK_DIR / "patches" def exec(args: list[str | Path], cwd: Path): args = [str(arg) for arg in args] print(f"++ Exec [{cwd}]$ {shlex.join(args)}") - subprocess.check_call(args, cwd=str(cwd)) + subprocess.check_call(args, cwd=str(cwd), stdin=subprocess.DEVNULL) def run(args): @@ -42,16 +44,25 @@ def run(args): exec(["repo", "sync", "-j16"] + args.projects, cwd=repo_dir) populate_ancillary_sources(args) - - # Patches. - if not args.no_patch: - apply_patches(args) + apply_patches(args) def apply_patches(args): - # TODO: Can just merge this script in here if it survives. - script = Path(__file__).resolve().parent / "apply_patches.sh" - exec([script], cwd=args.dir) + if not args.patch_tag: + print("Not patching (no --patch-tag specified)") + patch_version_dir: Path = PATCHES_DIR / args.patch_tag + if not patch_version_dir.exists(): + print(f"ERROR: Patch directory {patch_version_dir} does not exist") + for patch_project_dir in patch_version_dir.iterdir(): + print(f"* Processing project patch directory {patch_project_dir}:") + project_dir: Path = args.dir / patch_project_dir.name + if not project_dir.exists(): + print(f"WARNING: Source directory {project_dir} does not exist. Skipping.") + continue + patch_files = list(patch_project_dir.glob("*.patch")) + patch_files.sort() + print(f"Applying {len(patch_files)} patches") + exec(["git", "am"] + patch_files, cwd=project_dir) def populate_ancillary_sources(args): @@ -93,7 +104,12 @@ def main(argv): help="Branch to sync with repo tool", default="roc-6.3.x", ) - parser.add_argument("--no-patch", action="store_true", help="Disable patching") + parser.add_argument( + "--patch-tag", + type=str, + default="rocm-6.3.1", + help="Patch tag to apply to sources after sync", + ) parser.add_argument( "--depth", type=int, help="Git depth to pass to repo", default=None ) diff --git a/build_tools/merge_dist_dir.cmake b/build_tools/merge_dist_dir.cmake new file mode 100644 index 0000000..6d30874 --- /dev/null +++ b/build_tools/merge_dist_dir.cmake @@ -0,0 +1,64 @@ +# Standalone CMake script to merge a list of directories into one output +# directory, respecting symbolic links and warning on namespace collisions. +# By default, this hard links files vs copying. The contents of the out +# directory will be recursively deleted. +# +# Usage: +# cmake -P merge_dist_dir.cmake {out_directory} {src_directory...} +# Do not follow symlinks by default. +cmake_policy(SET CMP0009 NEW) + +function(parse_args) + # Parse cmake args. Script invocation is like: ... -P this.cmake + set(_all_args) + set(_index 0) + while(_index LESS ${CMAKE_ARGC}) + list(APPEND _all_args "${CMAKE_ARGV${_index}}") + math(EXPR _index "${_index} + 1") + endwhile() + list(FIND _all_args "-P" _index) + if(_index LESS 0) + message(FATAL_ERROR "Could not find -P argument") + endif() + math(EXPR _index "${_index} + 2") + list(SUBLIST _all_args ${_index} -1 _all_args) + + list(POP_FRONT _all_args _out_dir) + list(REVERSE _all_args) + set(link_out_dir "${_out_dir}" PARENT_SCOPE) + set(link_from_dirs "${_all_args}" PARENT_SCOPE) +endfunction() + +parse_args() + +# Remove the directory children (we do this vs removing the directory itself +# so as to cause less churn to tools that may have it as a cwd). +file(MAKE_DIRECTORY "${link_out_dir}") +file(GLOB existing_children "${link_out_dir}/*") +foreach(existing_child ${existing_children}) + file(REMOVE_RECURSE "${existing_child}") +endforeach() + +foreach(from_dir ${link_from_dirs}) + file(GLOB_RECURSE rel_paths LIST_DIRECTORIES true RELATIVE "${from_dir}" + "${from_dir}/*") + foreach(rel_path ${rel_paths}) + set(src_path "${from_dir}/${rel_path}") + set(dst_path "${link_out_dir}/${rel_path}") + if(IS_DIRECTORY "${src_path}") + make_directory("${dst_path}") + else() + if(EXISTS "${dst_path}") + message(WARNING "Distribution directory path already exists: ${rel_path} (in ${dst_path}). " + "This typically means that more than one sub-project provided the same file.") + endif() + if(IS_SYMLINK "${src_path}") + file(READ_SYMLINK "${src_path}" symlink) + file(CREATE_LINK "${symlink}" "${dst_path}" SYMBOLIC) + else() + # Hard link regular files. + file(CREATE_LINK "${src_path}" "${dst_path}" COPY_ON_ERROR) + endif() + endif() + endforeach() +endforeach() diff --git a/build_tools/save_patches.sh b/build_tools/save_patches.sh new file mode 100755 index 0000000..b560dc9 --- /dev/null +++ b/build_tools/save_patches.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Saves patches that are maintained locally to make TheRock work with a given +# branch. +# Usage: save_patches.sh base_tag project_name +# The base_tag must be checked out on the current branch. All commits above it +# will be dumped as patches. +set -euo pipefail + +repo_dir="$(pwd)" +this_dir="$(cd $(dirname $0) && pwd)" +root_dir="$(cd $this_dir/.. && pwd)" + +# Command line args. +set +eu +version_tag="$1" +shift +project_name="$1" +shift +set -eu + +if [ -z "$version_tag" ] || [ -z "$project_name" ]; then + echo "Syntax: save_patches.sh " + exit 1 +fi + +# Directories. +patch_dir="$root_dir/patches/$version_tag/$project_name" +mkdir -p $patch_dir +source_dir="$root_dir/sources/$project_name" +if ! [ -d "$source_dir" ]; then + echo "Source directory not found: $source_dir" + exit 1 +fi + +# Get the most recent upstream base commit. +base_commit="$(cd $source_dir && git rev-list -1 "tags/$version_tag")" +if [ -z "$base_commit" ]; then + echo "Could not find base commit for tags/$version_tag in $source_dir" + exit 1 +fi +echo "Base commit: $base_commit" + +# Remove existing patch files. +for existing in $patch_dir/*.patch; do + if [ -f "$existing" ]; then + rm "$existing" + fi +done + +# And format. +(cd $source_dir && git format-patch -o $patch_dir "$base_commit") diff --git a/cmake/therock_job_pools.cmake b/cmake/therock_job_pools.cmake new file mode 100644 index 0000000..3ebd95d --- /dev/null +++ b/cmake/therock_job_pools.cmake @@ -0,0 +1,17 @@ +include(ProcessorCount) + +function(therock_setup_job_pools) + set(_background_jobs "${THEROCK_BACKGROUND_BUILD_JOBS}") + if(NOT _background_jobs OR _background_jobs LESS_EQUAL 0) + ProcessorCount(CORE_COUNT) + math(EXPR _background_jobs "${CORE_COUNT} / 10") + if(_background_jobs LESS 2) + set(_background_jobs 2) + endif() + message(STATUS "Configuring background job pool for ${_background_jobs} concurrent jobs") + endif() + + set_property(GLOBAL APPEND PROPERTY JOB_POOLS therock_background=${_background_jobs}) +endfunction() + +therock_setup_job_pools() diff --git a/cmake/therock_subproject.cmake b/cmake/therock_subproject.cmake new file mode 100644 index 0000000..bae2019 --- /dev/null +++ b/cmake/therock_subproject.cmake @@ -0,0 +1,646 @@ +# therock_subproject.cmake +# Facilities for defining build subprojects. This has some similarity to the +# built-in ExternalProject and FetchContent facilities, but it is intended to +# be performant and ergonomic for a super project of our scale where the sources +# of the subprojects are expected to be modified as part of the super-project +# development flow. + +# Global properties. +# THEROCK_DEFAULT_CMAKE_VARS: +# List of CMake variables that will be injected by default into the +# project_init.cmake file of each subproject. +set_property(GLOBAL PROPERTY THEROCK_DEFAULT_CMAKE_VARS + CMAKE_BUILD_TYPE + CMAKE_PROGRAM_PATH + CMAKE_PLATFORM_NO_VERSIONED_SONAME + Python3_EXECUTABLE + Python3_FIND_VIRTUALENV + THEROCK_SOURCE_DIR + ROCM_GIT_DIR + AMDGPU_TARGETS +) + +if(CMAKE_C_VISIBILITY_PRESET) + list(APPEND THEROCK_DEFAULT_CMAKE_VARS ${CMAKE_C_VISIBILITY_PRESET}) +endif() +if(CMAKE_CXX_VISIBILITY_PRESET) + list(APPEND THEROCK_DEFAULT_CMAKE_VARS ${CMAKE_CXX_VISIBILITY_PRESET}) +endif() + +# therock_cmake_subproject_declare +# This declares a cmake based subproject by setting a number of key properties +# and setting up boiler-plate targets. +# +# Arguments: +# NAME: Globally unique subproject name. This will become the stem of various +# targets and therefore must be unique (even for nested projects) and a valid +# target identifier. +# ACTIVATE: Option to signify that this call should end by calling +# therock_cmake_subproject_activate. Do not specify this option if wishing to +# further configure the sub-project. +# SOURCE_DIR: Absolute path to the external source directory. +# DIR_PREFIX: By default, directories named "build", "stage", "stamp" are +# created. But if there are multiple sub-projects in a parent dir, then they +# all must have a distinct prefix (not recommended). +# INSTALL_DESTINATION: Sub-directory within the stage/dist directory where this +# sub-project installs. Defaults to empty, meaning that it installs at the top +# of the namespace. +# CMAKE_ARGS: Additional CMake configure arguments. +# BUILD_DEPS: Projects which must build and provide their packages prior to this +# one. +# RUNTIME_DEPS: Projects which must build prior to this one and whose install +# files must be distributed with this project's artifacts in order to function. +# INTERFACE_LINK_DIRS: Relative paths within the install tree which dependent +# sub-projects must add to their runtime link library path. +# INTERFACE_PROGRAM_DIRS: Relative paths within the install tree which +# dependent sub-projects must add to their program search path. +# IGNORE_PACKAGES: List of find_package package names to ignore, even if they +# are advertised by the super-project. These packages will always fall-through +# to the system resolver. +# COMPILER_TOOLCHAIN: Uses a built compiler toolchain instead of the +# super-project specified C/C++ compiler. This will add an implicit dep on +# the named compiler sub-project and reconfigure CMAKE_C(XX)_COMPILER options. +# Only a fixed set of supported toolchains are supported (currently "amd-llvm"). +# BACKGROUND_BUILD: Option to indicate that the subproject does low concurrency, +# high latency build steps. It will be run in the backgroun in a job pool that +# allows some overlapping of work (controlled by THEROCK_BACKGROUND_BUILD_JOBS). +function(therock_cmake_subproject_declare target_name) + cmake_parse_arguments( + PARSE_ARGV 1 ARG + "ACTIVATE;EXCLUDE_FROM_ALL;BACKGROUND_BUILD" + "EXTERNAL_SOURCE_DIR;DIR_PREFIX;INSTALL_DESTINATION;COMPILER_TOOLCHAIN;INTERFACE_PROGRAM_DIRS" + "BUILD_DEPS;RUNTIME_DEPS;CMAKE_ARGS;INTERFACE_LINK_DIRS;IGNORE_PACKAGES" + ) + if(TARGET "${target_name}") + message(FATAL_ERROR "Cannot declare subproject '${target_name}': a target with that name already exists") + endif() + + message(STATUS "Including subproject ${target_name} (from ${ARG_EXTERNAL_SOURCE_DIR})") + add_custom_target("${target_name}" COMMENT "Top level target to build the ${target_name} sub-project") + + # Build directory. + set(_binary_dir "${CMAKE_CURRENT_BINARY_DIR}/${ARG_DIR_PREFIX}build") + make_directory("${_binary_dir}") + + # Stage directory. + set(_stage_dir "${CMAKE_CURRENT_BINARY_DIR}/${ARG_DIR_PREFIX}stage") + make_directory("${_stage_dir}") + + # Dist directory. + set(_dist_dir "${CMAKE_CURRENT_BINARY_DIR}/${ARG_DIR_PREFIX}dist") + make_directory("${_dist_dir}") + + # Stamp directory. + set(_stamp_dir "${CMAKE_CURRENT_BINARY_DIR}/${ARG_DIR_PREFIX}stamp") + make_directory("${_stamp_dir}") + + # Collect LINK_DIRS and PROGRAM_DIRS from explicit args and RUNTIME_DEPS. + _therock_cmake_subproject_collect_runtime_deps( + _private_link_dirs _private_program_dirs _transitive_runtime_deps + _transitive_configure_depend_files + ${ARG_RUNTIME_DEPS}) + set(_declared_link_dirs "${ARG_INTERFACE_LINK_DIRS}") + _therock_cmake_subproject_absolutize(_declared_link_dirs "${_stage_dir}") + # The link dirs that we advertise combine interface link dirs of runtime deps + # and any that we declared. + list(APPEND _interface_link_dirs ${_private_link_dirs} ${_declared_link_dirs}) + # Collect program dirs from explicit args and RUNTIME_DEPS. + set(_declared_program_dirs "${ARG_INTERFACE_PROGRAM_DIRS}") + _therock_cmake_subproject_absolutize(_declared_program_dirs "${_dist_dir}") + # The program dirs that we advertise combine interface program dirs of + # runtime deps and any that we declared. + list(APPEND _interface_program_dirs ${_private_program_dirs} ${_declared_program_dirs}) + + # Dedup transitives. + list(REMOVE_DUPLICATES _private_link_dirs) + list(REMOVE_DUPLICATES _interface_link_dirs) + list(REMOVE_DUPLICATES _transitive_runtime_deps) + list(REMOVE_DUPLICATES _private_program_dirs) + list(REMOVE_DUPLICATES _interface_program_dirs) + list(REMOVE_DUPLICATES _transitive_configure_depend_files) + + # Build pool determination. + set(_build_pool) + if(ARG_BACKGROUND_BUILD) + set(_build_pool "therock_background") + endif() + + set_target_properties("${target_name}" PROPERTIES + THEROCK_SUBPROJECT cmake + THEROCK_BUILD_POOL "${_build_pool}" + THEROCK_EXCLUDE_FROM_ALL "${ARG_EXCLUDE_FROM_ALL}" + THEROCK_EXTERNAL_SOURCE_DIR "${ARG_EXTERNAL_SOURCE_DIR}" + THEROCK_BINARY_DIR "${_binary_dir}" + THEROCK_DIST_DIR "${_dist_dir}" + THEROCK_STAGE_DIR "${_stage_dir}" + THEROCK_INSTALL_DESTINATION "${ARG_INSTALL_DESTINATION}" + THEROCK_STAMP_DIR "${_stamp_dir}" + THEROCK_CMAKE_SOURCE_DIR "${ARG_EXTERNAL_SOURCE_DIR}" + THEROCK_CMAKE_PROJECT_INIT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${ARG_BUILD_DIR}_init.cmake" + THEROCK_CMAKE_PROJECT_TOOLCHAIN_FILE "${CMAKE_CURRENT_BINARY_DIR}/${ARG_BUILD_DIR}_toolchain.cmake" + THEROCK_CMAKE_ARGS "${ARG_CMAKE_ARGS}" + # Non-transitive build deps. + THEROCK_BUILD_DEPS "${ARG_BUILD_DEPS}" + # Transitive runtime deps. + THEROCK_RUNTIME_DEPS "${_transitive_runtime_deps}" + # That this project compiles with. + THEROCK_PRIVATE_LINK_DIRS "${_private_link_dirs}" + # Link dirs that are advertised to dependents + THEROCK_INTERFACE_LINK_DIRS "${_interface_link_dirs}" + # Program dirs that this sub-project must configure with. + THEROCK_PRIVATE_PROGRAM_DIRS "${_private_program_dirs}" + # Program dirs that are advertised to dependents. + THEROCK_INTERFACE_PROGRAM_DIRS "${_interface_program_dirs}" + THEROCK_IGNORE_PACKAGES "${ARG_IGNORE_PACKAGES}" + THEROCK_COMPILER_TOOLCHAIN "${ARG_COMPILER_TOOLCHAIN}" + # Any extra depend files that must be added to the configure phase of dependents. + THEROCK_INTERFACE_CONFIGURE_DEPEND_FILES "${_transitive_configure_depend_files}" + ) + + if(ARG_ACTIVATE) + therock_cmake_subproject_activate("${target_name}") + endif() +endfunction() + +# therock_cmake_subproject_provide_package +# Declares that a subproject provides a given package which should be findable +# with `find_package(package_name)` at the given path relative to its install +# directory. +function(therock_cmake_subproject_provide_package target_name package_name relative_path) + string(APPEND CMAKE_MESSAGE_INDENT " ") + get_target_property(_existing_packages "${target_name}" THEROCK_PROVIDE_PACKAGES) + if(${package_name} IN_LIST _existing_packages) + message(FATAL_ERROR "Package defined multiple times on sub-project ${target_name}: ${package_name}") + endif() + set_property(TARGET "${target_name}" APPEND PROPERTY THEROCK_PROVIDE_PACKAGES "${package_name}") + set(_relpath_name THEROCK_PACKAGE_RELPATH_${package_name}) + set_property(TARGET "${target_name}" PROPERTY ${_relpath_name} "${relative_path}") + message(STATUS "PROVIDE ${package_name} = ${relative_path} (from ${target_name})") +endfunction() + +# therock_cmake_subproject_activate +# If using multi-step setup (i.e. without 'ACTIVATE' on the declare), then this +# must be called once all configuration is complete. +function(therock_cmake_subproject_activate target_name) + _therock_assert_is_cmake_subproject("${target_name}") + + # Get properties. + get_target_property(_binary_dir "${target_name}" THEROCK_BINARY_DIR) + get_target_property(_build_deps "${target_name}" THEROCK_BUILD_DEPS) + get_target_property(_build_pool "${target_name}" THEROCK_BUILD_POOL) + get_target_property(_compiler_toolchain "${target_name}" THEROCK_COMPILER_TOOLCHAIN) + get_target_property(_transitive_configure_depend_files "${target_name}" THEROCK_INTERFACE_CONFIGURE_DEPEND_FILES) + get_target_property(_dist_dir "${target_name}" THEROCK_DIST_DIR) + get_target_property(_runtime_deps "${target_name}" THEROCK_RUNTIME_DEPS) + get_target_property(_cmake_args "${target_name}" THEROCK_CMAKE_ARGS) + get_target_property(_cmake_project_init_file "${target_name}" THEROCK_CMAKE_PROJECT_INIT_FILE) + get_target_property(_cmake_project_toolchain_file "${target_name}" THEROCK_CMAKE_PROJECT_TOOLCHAIN_FILE) + get_target_property(_cmake_source_dir "${target_name}" THEROCK_CMAKE_SOURCE_DIR) + get_target_property(_exclude_from_all "${target_name}" THEROCK_EXCLUDE_FROM_ALL) + get_target_property(_external_source_dir "${target_name}" THEROCK_EXTERNAL_SOURCE_DIR) + get_target_property(_ignore_packages "${target_name}" THEROCK_IGNORE_PACKAGES) + get_target_property(_install_destination "${target_name}" THEROCK_INSTALL_DESTINATION) + get_target_property(_private_link_dirs "${target_name}" THEROCK_PRIVATE_LINK_DIRS) + get_target_property(_private_program_dirs "${target_name}" THEROCK_PRIVATE_PROGRAM_DIRS) + get_target_property(_stage_dir "${target_name}" THEROCK_STAGE_DIR) + get_target_property(_sources "${target_name}" SOURCES) + get_target_property(_stamp_dir "${target_name}" THEROCK_STAMP_DIR) + + # Handle optional properties. + if(NOT _sources) + set(_sources) + endif() + + # Defaults. + set(_configure_comment_suffix) + set(_build_comment_suffix) + + # Detect pre/post hooks. + set(_pre_hook_path "${CMAKE_CURRENT_SOURCE_DIR}/pre_hook.cmake") + if(NOT EXISTS "${_pre_hook_path}") + set(_pre_hook_path) + endif() + set(_post_hook_path "${CMAKE_CURRENT_SOURCE_DIR}/post_hook.cmake") + if(NOT EXISTS "${_post_hook_path}") + set(_post_hook_path) + endif() + + # Report transitive runtime deps. + if(_runtime_deps) + list(JOIN _runtime_deps " " _runtime_deps_pretty) + message(STATUS " RUNTIME_DEPS: ${_runtime_deps_pretty}") + endif() + + get_property(_mirror_cmake_vars GLOBAL PROPERTY THEROCK_DEFAULT_CMAKE_VARS) + + # Handle compiler toolchain. + set(_compiler_toolchain_addl_depends) + set(_compiler_toolchain_init_contents) + set(_build_env_pairs) + _therock_cmake_subproject_setup_toolchain("${_compiler_toolchain}" "${_cmake_project_toolchain_file}") + + # Customize any other super-project CMake variables that are captured by + # _init.cmake. + if(_private_program_dirs) + set(CMAKE_PROGRAM_PATH ${_private_program_dirs} ${CMAKE_PROGRAM_PATH}) + foreach(_message_contents ${_private_program_dirs}) + message(STATUS " PROGRAM_DIR: ${_message_contents}") + endforeach() + endif() + + # Generate the project_init.cmake + set(_dep_provider_file) + if(_build_deps OR _runtime_deps) + set(_dep_provider_file "${THEROCK_SOURCE_DIR}/cmake/therock_subproject_dep_provider.cmake") + endif() + set(_injected_file "${THEROCK_SOURCE_DIR}/cmake/therock_external_project_include.cmake") + set(_init_contents) + foreach(_var_name ${_mirror_cmake_vars}) + string(APPEND _init_contents "set(${_var_name} \"@${_var_name}@\" CACHE STRING \"\" FORCE)\n") + endforeach() + _therock_cmake_subproject_setup_deps(_deps_contents ${_build_deps} ${_runtime_deps}) + string(APPEND _init_contents "${_deps_contents}") + string(APPEND _init_contents "set(THEROCK_IGNORE_PACKAGES \"@_ignore_packages@\")\n") + foreach(_private_link_dir ${_private_link_dirs}) + message(STATUS " LINK_DIR: ${_private_link_dir}") + string(APPEND _init_contents "string(APPEND CMAKE_EXE_LINKER_FLAGS \" -Wl,-rpath-link,${_private_link_dir}\")\n") + string(APPEND _init_contents "string(APPEND CMAKE_SHARED_LINKER_FLAGS \" -Wl,-rpath-link,${_private_link_dir}\")\n") + endforeach() + string(APPEND _init_contents "${_compiler_toolchain_init_contents}") + if(_dep_provider_file) + string(APPEND _init_contents "include(${_dep_provider_file})\n") + endif() + if(_pre_hook_path) + string(APPEND _init_contents "include(@_pre_hook_path@)\n") + endif() + if(_post_hook_path) + string(APPEND _init_contents "cmake_language(DEFER CALL include \"@_post_hook_path@\")\n") + endif() + string(APPEND _init_contents "include(${_injected_file})\n") + file(CONFIGURE OUTPUT "${_cmake_project_init_file}" CONTENT "${_init_contents}" @ONLY ESCAPE_QUOTES) + + # Transform build and run deps from target form (i.e. 'ROCR-Runtime' to a dependency + # on the stage.stamp file). These are a dependency for configure. + _therock_cmake_subproject_deps_to_stamp(_configure_dep_stamps stage.stamp ${_build_deps} ${_runtime_deps}) + + # Target flags. + set(_all_option) + if(NOT _exclude_from_all) + set(_all_option "ALL") + endif() + + # configure target + message(STATUS " CONFIGURE_DEPENDS: ${_transitive_configure_depend_files} ") + set(_configure_stamp_file "${_stamp_dir}/configure.stamp") + set(_configure_comment_suffix " (in background)") + set(_terminal_option) + set(_build_terminal_option "USES_TERMINAL") + if(THEROCK_INTERACTIVE) + set(_terminal_option "USES_TERMINAL") + set(_configure_comment_suffix) + elseif(_build_pool) + message(STATUS " JOB_POOL: ${_build_pool}") + set(_build_terminal_option JOB_POOL "${_build_pool}") + set(_build_comment_suffix " (in background)") + endif() + set(_stage_destination_dir "${_stage_dir}") + if(_install_destination) + cmake_path(APPEND _stage_destination_dir "${_install_destination}") + endif() + add_custom_command( + OUTPUT "${_configure_stamp_file}" + COMMAND "${CMAKE_COMMAND}" + "-G${CMAKE_GENERATOR}" + "-B${_binary_dir}" + "-S${_cmake_source_dir}" + "-DCPACK_PACKAGING_INSTALL_PREFIX=${STAGING_INSTALL_DIR}" + "-DCMAKE_INSTALL_PREFIX=${_stage_destination_dir}" + "-DCMAKE_TOOLCHAIN_FILE=${_cmake_project_toolchain_file}" + "-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${_cmake_project_init_file}" + ${_cmake_args} + COMMAND "${CMAKE_COMMAND}" -E touch "${_configure_stamp_file}" + WORKING_DIRECTORY "${_binary_dir}" + COMMENT "Configure sub-project ${target_name}${_configure_comment_suffix}" + ${_terminal_option} + BYPRODUCTS + "${_binary_dir}/CMakeCache.txt" + "${_binary_dir}/cmake_install.cmake" + DEPENDS + "${_cmake_source_dir}/CMakeLists.txt" + "${_cmake_project_toolchain_file}" + "${_cmake_project_init_file}" + "${_injected_file}" + ${_dep_provider_file} + ${_configure_dep_stamps} + ${_pre_hook_path} + ${_post_hook_path} + ${_compiler_toolchain_addl_depends} + ${_transitive_configure_depend_files} + + # TODO: Have a mechanism for adding more depends for better rebuild ergonomics + ) + add_custom_target( + "${target_name}+configure" + ${_all_option} + DEPENDS "${_configure_stamp_file}" + ) + add_dependencies("${target_name}" "${target_name}+configure") + + # build target. + set(_build_stamp_file "${_stamp_dir}/build.stamp") + add_custom_command( + OUTPUT "${_build_stamp_file}" + COMMAND "${CMAKE_COMMAND}" -E env ${_build_env_pairs} -- "${CMAKE_COMMAND}" "--build" "${_binary_dir}" + COMMAND "${CMAKE_COMMAND}" -E touch "${_build_stamp_file}" + WORKING_DIRECTORY "${_binary_dir}" + COMMENT "Building sub-project ${target_name}${_build_comment_suffix}" + ${_build_terminal_option} + DEPENDS + "${_configure_stamp_file}" + ${_sources} + ) + add_custom_target( + "${target_name}+build" + ${_all_option} + DEPENDS + "${_build_stamp_file}" + ) + add_dependencies("${target_name}" "${target_name}+build") + + # stage install target. + set(_stage_stamp_file "${_stamp_dir}/stage.stamp") + add_custom_command( + OUTPUT "${_stage_stamp_file}" + COMMAND "${CMAKE_COMMAND}" --install "${_binary_dir}" + COMMAND "${CMAKE_COMMAND}" -E touch "${_stage_stamp_file}" + WORKING_DIRECTORY "${_binary_dir}" + COMMENT "Stage installing sub-project ${target_name}" + ${_terminal_option} + DEPENDS + "${_build_stamp_file}" + ) + add_custom_target( + "${target_name}+stage" + ${_all_option} + DEPENDS + "${_stage_stamp_file}" + ) + add_dependencies("${target_name}" "${target_name}+stage") + + # dist install target. + set(_dist_stamp_file "${_stamp_dir}/dist.stamp") + set(_merge_dist_script "${THEROCK_SOURCE_DIR}/build_tools/merge_dist_dir.cmake") + _therock_cmake_subproject_get_stage_dirs( + _dist_source_dirs "${target_name}" ${_runtime_deps}) + add_custom_command( + OUTPUT "${_dist_stamp_file}" + COMMAND "${CMAKE_COMMAND}" -P "${_merge_dist_script}" "${_dist_dir}" ${_dist_source_dirs} + COMMAND "${CMAKE_COMMAND}" -E touch "${_dist_stamp_file}" + COMMENT "Merging sub-project dist directory for ${target_name}" + ${_terminal_option} + DEPENDS + "${_stage_stamp_file}" + "${_merge_dist_script}" + ) + add_custom_target( + "${target_name}+dist" + ${_all_option} + DEPENDS + "${_dist_stamp_file}" + ) + add_dependencies("${target_name}" "${target_name}+dist") + + # expunge target + add_custom_target( + "${target_name}+expunge" + COMMAND + ${CMAKE_COMMAND} -E rm -rf "${_binary_dir}" "${_stage_dir}" "${_stamp_dir}" "${_dist_dir}" + ) +endfunction() + +# therock_cmake_subproject_glob_c_sources +# Adds C/C++ sources from given project subdirectories to the list of sources for +# a sub-project. This allows the super-project build system to know when to +# re-trigger the build step of the sub-project. There are many issues with globs +# in CMake, but as an ergonomic win, this is deemed an acceptable compromise +# to a large degree of explicitness. +function(therock_cmake_subproject_glob_c_sources target_name) + cmake_parse_arguments( + PARSE_ARGV 1 ARG + "" + "" + "SUBDIRS" + ) + get_target_property(_project_source_dir "${target_name}" THEROCK_EXTERNAL_SOURCE_DIR) + set(_globs) + foreach(_subdir ${ARG_SUBDIRS}) + set(_s "${_project_source_dir}/${_subdir}") + list(APPEND _globs + "${_s}/*.h" + "${_s}/*.hpp" + "${_s}/*.inc" + "${_s}/*.cc" + "${_s}/*.cpp" + "${_s}/*.c" + ) + endforeach() + file(GLOB_RECURSE _files LIST_DIRECTORIES FALSE + CONFIGURE_DEPENDS + ${_globs} + ) + target_sources("${target_name}" PRIVATE ${_files}) +endfunction() + +function(_therock_assert_is_cmake_subproject target_name) + # Make sure it is a sub-project. + get_target_property(_is_subproject "${target_name}" THEROCK_SUBPROJECT) + if(NOT _is_subproject STREQUAL "cmake") + message(FATAL_ERROR "Target ${target_name} is not a sub-project") + endif() +endfunction() + +# Builds a CMake language fragment to set up a dependency provider such that +# it handles super-project provided dependencies locally. +function(_therock_cmake_subproject_setup_deps out_contents) + string(APPEND CMAKE_MESSAGE_INDENT " ") + set(_contents "set(THEROCK_PROVIDED_PACKAGES)\n") + set(_already_provided) + foreach(dep_target ${ARGN}) + _therock_assert_is_cmake_subproject("${dep_target}") + + get_target_property(_provides "${dep_target}" THEROCK_PROVIDE_PACKAGES) + if(_provides) + foreach(_package_name ${_provides}) + if(_package_name IN_LIST _already_provided) + continue() + endif() + list(APPEND _already_provided "${_package_name}") + get_target_property(_stage_dir "${dep_target}" THEROCK_STAGE_DIR) + set(_relpath_name THEROCK_PACKAGE_RELPATH_${_package_name}) + get_target_property(_relpath "${dep_target}" ${_relpath_name}) + if(NOT _stage_dir OR NOT _relpath) + message(FATAL_ERROR "Missing package info props for ${_package_name} on ${dep_target}: '${_stage_dir}' ${_relpath_name}='${_relpath}'") + endif() + set(_find_package_path "${_stage_dir}") + cmake_path(APPEND _find_package_path "${_relpath}") + message(STATUS "INJECT ${_package_name} = ${_find_package_path} (from ${dep_target})") + string(APPEND _contents "set(THEROCK_PACKAGE_DIR_${_package_name} \"${_find_package_path}\")\n") + string(APPEND _contents "list(APPEND THEROCK_PROVIDED_PACKAGES ${_package_name})\n") + endforeach() + endif() + endforeach() + set("${out_contents}" "${_contents}" PARENT_SCOPE) +endfunction() + +# Gets the staging install directories for a list of subproject deps. +function(_therock_cmake_subproject_get_stage_dirs out_dirs) + set(_dirs) + foreach(target_name ${ARGN}) + get_target_property(_stage_dir "${target_name}" THEROCK_STAGE_DIR) + if(NOT _stage_dir) + message(FATAL_ERROR "Sub-project target ${target_name} does not have a stage install dir") + endif() + list(APPEND _dirs "${_stage_dir}") + endforeach() + set(${out_dirs} "${_dirs}" PARENT_SCOPE) +endfunction() + +# Transforms a list of sub-project targets to corresponding stamp files of +# `stamp_name`. These are the actual build system deps that are encoded in the +# commands (whereas the target names are just for humans). +function(_therock_cmake_subproject_deps_to_stamp out_stamp_files stamp_name) + set(_stamp_files) + foreach(target_name ${ARGN}) + _therock_assert_is_cmake_subproject("${target_name}") + get_target_property(_stamp_dir "${target_name}" THEROCK_STAMP_DIR) + if(NOT _stamp_dir) + message(FATAL_ERROR "Sub-project is missing a stamp dir: ${target_name}") + endif() + + list(APPEND _stamp_files "${_stamp_dir}/${stamp_name}") + endforeach() + set(${out_stamp_files} "${_stamp_files}" PARENT_SCOPE) +endfunction() + +# For a list of targets, gets absolute paths for all interface link directories +# and transitive runtime deps. Both lists may contain duplicates if the DAG +# includes the same dep multiple times. +function(_therock_cmake_subproject_collect_runtime_deps + out_link_dirs out_program_dirs out_transitive_deps + out_transitive_configure_depend_files) + set(_link_dirs) + set(_program_dirs) + set(_transitive_deps) + set(_transitive_configure_depend_files) + foreach(target_name ${ARGN}) + _therock_assert_is_cmake_subproject("${target_name}") + get_target_property(_declared_configure_depend_files "${target_name}" THEROCK_INTERFACE_CONFIGURE_DEPEND_FILES) + list(APPEND _transitive_configure_depend_files ${_declared_configure_depend_files}) + get_target_property(_stamp_dir "${target_name}" THEROCK_STAMP_DIR) + + # Link dirs. + get_target_property(_link_dir "${target_name}" THEROCK_INTERFACE_LINK_DIRS) + list(APPEND _link_dirs ${_link_dir}) + + # Transitive runtime target deps. + get_target_property(_deps "${target_name}" THEROCK_RUNTIME_DEPS) + list(APPEND _transitive_deps ${_deps} ${target_name}) + + # If we have program dirs, then this target's 'dist' phase has to become + # a transitive dep for all future configures (by default the build graph + # only depends on the 'stage' phase). + get_target_property(_program_dir "${target_name}" THEROCK_INTERFACE_PROGRAM_DIRS) + if(_program_dir) + list(APPEND _program_dirs ${_program_dir}) + list(APPEND _transitive_configure_depend_files "${_stamp_dir}/dist.stamp") + endif() + endforeach() + set("${out_link_dirs}" "${_link_dirs}" PARENT_SCOPE) + set("${out_program_dirs}" "${_program_dirs}" PARENT_SCOPE) + set("${out_transitive_deps}" "${_transitive_deps}" PARENT_SCOPE) + set("${out_transitive_configure_depend_files}" "${_transitive_configure_depend_files}" PARENT_SCOPE) +endfunction() + +# Transforms a list to be absolute paths if not already. +function(_therock_cmake_subproject_absolutize list_var relative_to) + set(_dirs "${${list_var}}") + set(_abs_dirs) + foreach(_dir ${_dirs}) + cmake_path(ABSOLUTE_PATH _dir BASE_DIRECTORY "${relative_to}" NORMALIZE) + list(APPEND _abs_dirs "${_dir}") + endforeach() + set("${list_var}" "${_abs_dirs}" PARENT_SCOPE) +endfunction() + +# Writes a toolchain file and sets variables in the parent scope of the +# therock_cmake_subproject_activate prior to initializing sub-project arguments +# to configure the toolchain based on the user-provided COMPILER_TOOLCHAIN. +# +# Toolchain menemonics: +# * amd-llvm: Locally build compiler/amd-llvm toolchain as a standalone +# tool. While this can compiler HIP code, it does not natively have access +# to a ROCM installation for headers, etc. +# * amd-hip: Extends the amd-llvm toolchain to also depend on HIP, making +# it ready to use to compile HIP code. +function(_therock_cmake_subproject_setup_toolchain compiler_toolchain toolchain_file) + string(APPEND CMAKE_MESSAGE_INDENT " ") + set(_toolchain_contents) + + # General settings applicable to all toolchains. + string(APPEND _toolchain_contents "set(CMAKE_INSTALL_LIBDIR @CMAKE_INSTALL_LIBDIR@)\n") + string(APPEND _toolchain_contents "set(CMAKE_PLATFORM_NO_VERSIONED_SONAME @CMAKE_PLATFORM_NO_VERSIONED_SONAME@)\n") + + # Propagate super-project flags to the sub-project by default. + string(APPEND _toolchain_contents "set(CMAKE_C_COMPILER @CMAKE_C_COMPILER@)\n") + string(APPEND _toolchain_contents "set(CMAKE_CXX_COMPILER @CMAKE_CXX_COMPILER@)\n") + string(APPEND _toolchain_contents "set(CMAKE_LINKER @CMAKE_LINKER@)\n") + string(APPEND _toolchain_contents "set(CMAKE_C_COMPILER_LAUNCHER @CMAKE_C_COMPILER_LAUNCHER@)\n") + string(APPEND _toolchain_contents "set(CMAKE_CXX_COMPILER_LAUNCHER @CMAKE_CXX_COMPILER_LAUNCHER@)\n") + string(APPEND _toolchain_contents "set(CMAKE_C_FLAGS_INIT @CMAKE_C_FLAGS@)\n") + string(APPEND _toolchain_contents "set(CMAKE_CXX_FLAGS_INIT @CMAKE_CXX_FLAGS@)\n") + string(APPEND _toolchain_contents "set(CMAKE_EXE_LINKER_FLAGS_INIT @CMAKE_EXE_LINKER_FLAGS@)\n") + string(APPEND _toolchain_contents "set(CMAKE_SHARED_LINKER_FLAGS_INIT @CMAKE_SHARED_LINKER_FLAGS@)\n") + + if(NOT compiler_toolchain) + # Make any additional customizations if no toolchain specified. + elseif(compiler_toolchain STREQUAL "amd-llvm" OR compiler_toolchain STREQUAL "amd-hip") + # amd-llvm toolchain: The private built LLVM in isolation. + _therock_assert_is_cmake_subproject("amd-llvm") + get_target_property(_amd_llvm_dist_dir amd-llvm THEROCK_DIST_DIR) + get_target_property(_amd_llvm_stamp_dir amd-llvm THEROCK_STAMP_DIR) + # Add a dependency on the toolchain's dist + set(AMD_LLVM_C_COMPILER "${_amd_llvm_dist_dir}/lib/llvm/bin/clang") + set(AMD_LLVM_CXX_COMPILER "${_amd_llvm_dist_dir}/lib/llvm/bin/clang++") + set(AMD_LLVM_LINKER "${_amd_llvm_dist_dir}/lib/llvm/bin/lld") + list(APPEND _compiler_toolchain_addl_depends "${_amd_llvm_stamp_dir}/dist.stamp") + string(APPEND _toolchain_contents "set(CMAKE_C_COMPILER @AMD_LLVM_C_COMPILER@)\n") + string(APPEND _toolchain_contents "set(CMAKE_CXX_COMPILER @AMD_LLVM_CXX_COMPILER@)\n") + string(APPEND _toolchain_contents "set(CMAKE_LINKER @AMD_LLVM_LINKER@)\n") + + message(STATUS "Compiler toolchain ${compiler_toolchain}:") + string(APPEND CMAKE_MESSAGE_INDENT " ") + message(STATUS "CMAKE_C_COMPILER = ${AMD_LLVM_C_COMPILER}") + message(STATUS "CMAKE_CXX_COMPILER = ${AMD_LLVM_CXX_COMPILER}") + message(STATUS "CMAKE_LINKER = ${AMD_LLVM_LINKER}") + else() + message(FATAL_ERROR "Unsupported COMPILER_TOOLCHAIN = ${compiler_toolchain} (supported: 'amd-llvm' or none)") + endif() + + # Configure additional HIP dependencies. + if (compiler_toolchain STREQUAL "amd-hip") + _therock_assert_is_cmake_subproject("hip-clr") + get_target_property(_hip_dist_dir hip-clr THEROCK_DIST_DIR) + get_target_property(_hip_stamp_dir hip-clr THEROCK_STAMP_DIR) + # Add a dependency on HIP's stamp. + set(_amd_llvm_device_lib_path "${_amd_llvm_dist_dir}/lib/llvm/amdgcn/bitcode") + list(APPEND _compiler_toolchain_addl_depends "${_hip_stamp_dir}/dist.stamp") + string(APPEND _toolchain_contents "string(APPEND CMAKE_CXX_FLAGS_INIT \" --hip-path=@_hip_dist_dir@\")\n") + string(APPEND _toolchain_contents "string(APPEND CMAKE_CXX_FLAGS_INIT \" --hip-device-lib-path=@_amd_llvm_device_lib_path@\")\n") + message(STATUS "HIP_DIR = ${_hip_dist_dir}") + endif() + + set(_compiler_toolchain_addl_depends "${_compiler_toolchain_addl_depends}" PARENT_SCOPE) + set(_compiler_toolchain_init_contents "${_compiler_toolchain_init_contents}" PARENT_SCOPE) + set(_build_env_pairs "${_build_env_pairs}" PARENT_SCOPE) + file(CONFIGURE OUTPUT "${toolchain_file}" CONTENT "${_toolchain_contents}" @ONLY ESCAPE_QUOTES) +endfunction() diff --git a/cmake/therock_subproject_dep_provider.cmake b/cmake/therock_subproject_dep_provider.cmake new file mode 100644 index 0000000..b94057f --- /dev/null +++ b/cmake/therock_subproject_dep_provider.cmake @@ -0,0 +1,58 @@ +# find_package dependency provider +# This is injected into sub-projects that contain dependencies. It runs in a +# context with the following variables defined at the top level: +# THEROCK_PROVIDED_PACKAGES: Package names that are to be provided from the +# super-project +# THEROCK_PACKAGE_DIR_${package_name}: Directory in the super-project to +# resolve the dependency. +# THEROCK_IGNORE_PACKAGES: Packages to ignore, even if they are in +# THEROCK_PROVIDED_PACKAGES, falling back to the system resolver. +# See: _therock_cmake_subproject_setup_deps which assembles these variables +macro(therock_dependency_provider method package_name) + if("${package_name}" IN_LIST THEROCK_PROVIDED_PACKAGES AND NOT + "${package_name}" IN_LIST THEROCK_IGNORE_PACKAGES) + # It is quite hard to completely neuter find_package so that for an + # arbitrary signature it will only attempt to find from one specified path. + # This is important because it "latches" and if any find_package manages + # to escape to the system, it will likely find a library from outside the + # super-project, which can cause all kinds of hard to diagnose issues. + # For background, read carefully: + # https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-search-procedure + # We opt-to rewrite the signature, removing any options that connote an + # implicit search behavior and then rewrite the signature to be explicit. + # We further do this in a function to avoid macro namespace pollution, since + # the find_package itself must be evaluated in the caller-scope. + therock_reparse_super_project_find_package( + "${THEROCK_PACKAGE_DIR_${package_name}}" "${package_name}" ${ARGN}) + find_package(${_therock_rewritten_superproject_find_package_sig}) + else() + message(STATUS "Resolving system find_package(${package_name}) (not found in super-project ${THEROCK_PROVIDED_PACKAGES})") + find_package(${package_name} ${ARGN} BYPASS_PROVIDER) + endif() +endmacro() +if(THEROCK_PROVIDED_PACKAGES) + message(STATUS "Resolving packages from super-project: ${THEROCK_PROVIDED_PACKAGES}") + cmake_language( + SET_DEPENDENCY_PROVIDER therock_dependency_provider + SUPPORTED_METHODS FIND_PACKAGE + ) +endif() + +function(therock_reparse_super_project_find_package superproject_path package_name) + # We parse the arguments we want dropped from the find_package and then use + # what was unparsed. + cmake_parse_arguments(PARSE_ARGV 1 UNUSED + "BYPASS_PROVIDER;CONFIG;NO_DEFAULT_PATH;NO_CMAKE_PATH;NO_CMAKE_ENVIRONMENT_PATH;NO_SYSTEM_ENVIRONMENT_PATH;NO_CMAKE_PACKAGE_REGISTRY" + "" + "HINTS;PATHS" + ) + if(NOT superproject_path) + message(FATAL_ERROR "Super-project package path not found for ${package_name}") + endif() + + set(_rewritten ${UNUSED_UNPARSED_ARGUMENTS}) + list(APPEND _rewritten CONFIG BYPASS_PROVIDER NO_DEFAULT_PATH PATHS ${superproject_path}) + list(JOIN _rewritten " " _rewritten_pretty) + message(STATUS "Resolving super-project find_package(${_rewritten_pretty})") + set(_therock_rewritten_superproject_find_package_sig ${_rewritten} PARENT_SCOPE) +endfunction() diff --git a/comm_libs/CMakeLists.txt b/comm_libs/CMakeLists.txt new file mode 100644 index 0000000..1c2a093 --- /dev/null +++ b/comm_libs/CMakeLists.txt @@ -0,0 +1,5 @@ +if(THEROCK_ENABLE_RCCL) + add_subdirectory(rccl) +else() + message(STATUS "Not building RCCL (THEROCK_ENABLE_RCCL=OFF)") +endif() diff --git a/comm_libs/rccl/CMakeLists.txt b/comm_libs/rccl/CMakeLists.txt new file mode 100644 index 0000000..eb7e285 --- /dev/null +++ b/comm_libs/rccl/CMakeLists.txt @@ -0,0 +1,33 @@ +################################################################################# +# rccl +################################################################################# + +therock_cmake_subproject_declare(rccl + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/rccl" + # High latency LTO link of a single library. + BACKGROUND_BUILD + CMAKE_ARGS + -DHIP_PLATFORM=amd + -DROCM_PATH= + -DROCM_DIR= + "-DEXPLICIT_ROCM_VERSION=${ROCM_MAJOR_VERSION}.${ROCM_MINOR_VERSION}.${ROCM_PATCH_VERSION}" + COMPILER_TOOLCHAIN + amd-hip + IGNORE_PACKAGES + # The current version of rccl needs to download a 2y old version of rocm-cmake + # to work and it will only do so if the system resolver reports it not found + # without any other error (which due to the signature, our resolver will + # hard fail). Once fixed, `rocm-core` should be added to the BUILD_DEPS + # and this removed: https://github.com/nod-ai/TheRock/issues/18 + ROCM + RUNTIME_DEPS + hip-clr + hipify + rocm_smi_lib + rocprofiler-register +) +therock_cmake_subproject_glob_c_sources(rccl + SUBDIRS + src +) +therock_cmake_subproject_activate(rccl) diff --git a/cmake/extensions/clr_post.cmake b/comm_libs/rccl/post_hook.cmake similarity index 78% rename from cmake/extensions/clr_post.cmake rename to comm_libs/rccl/post_hook.cmake index 8923aa0..f1eb3fe 100644 --- a/cmake/extensions/clr_post.cmake +++ b/comm_libs/rccl/post_hook.cmake @@ -3,9 +3,7 @@ include(therock_rpath) therock_set_install_rpath( TARGETS - amdhip64 - hiprtc + rccl PATHS . - llvm/lib ) diff --git a/compiler/CMakeLists.txt b/compiler/CMakeLists.txt new file mode 100644 index 0000000..e046074 --- /dev/null +++ b/compiler/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(amd-llvm) +add_subdirectory(hipify) diff --git a/compiler/amd-llvm/CMakeLists.txt b/compiler/amd-llvm/CMakeLists.txt new file mode 100644 index 0000000..44fb756 --- /dev/null +++ b/compiler/amd-llvm/CMakeLists.txt @@ -0,0 +1,28 @@ +################################################################################ +# amd-llvm +################################################################################ + +therock_cmake_subproject_declare(amd-llvm + # Note that LLVM top level CMakeLists.txt is in the llvm subdir of the + # monorepo. + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/llvm" + INTERFACE_PROGRAM_DIRS + lib/llvm/bin + BUILD_DEPS + rocm-cmake + # The entire LLVM install tree is placed inside of the overall ROCM lib/llvm + # directory. + INSTALL_DESTINATION "lib/llvm" + INTERFACE_LINK_DIRS + "lib/llvm/lib" +) +# Note that we do not trigger on LLVM source changes because the monorepo is +# too large to glob like that. Consider having a project dev mode option for +# enabling better ergonomics here. + +therock_cmake_subproject_provide_package(amd-llvm amd_comgr lib/llvm/lib/cmake/amd_comgr) +therock_cmake_subproject_provide_package(amd-llvm Clang lib/llvm/lib/cmake/clang) +therock_cmake_subproject_provide_package(amd-llvm LLD lib/llvm/lib/cmake/lld) +therock_cmake_subproject_provide_package(amd-llvm LLVM lib/llvm/lib/cmake/llvm) +therock_cmake_subproject_provide_package(amd-llvm AMDDeviceLibs lib/llvm/lib/cmake/AMDDeviceLibs) +therock_cmake_subproject_activate(amd-llvm) diff --git a/compiler/amd-llvm/post_hook.cmake b/compiler/amd-llvm/post_hook.cmake new file mode 100644 index 0000000..32709e6 --- /dev/null +++ b/compiler/amd-llvm/post_hook.cmake @@ -0,0 +1,12 @@ +list(APPEND CMAKE_MODULE_PATH "${THEROCK_SOURCE_DIR}/cmake") +include(therock_rpath) + +# This should be fixed upstream to use LLVM's native RPATH setting logic (since +# the LLVM layout is already correct and we shouldn't have to muck with it). +# https://github.com/nod-ai/TheRock/issues/19 +therock_set_install_rpath( + TARGETS + amd_comgr + PATHS + . +) diff --git a/compiler/amd-llvm/pre_hook.cmake b/compiler/amd-llvm/pre_hook.cmake new file mode 100644 index 0000000..6c43d1a --- /dev/null +++ b/compiler/amd-llvm/pre_hook.cmake @@ -0,0 +1,16 @@ +# Build LLVM and the comgr dependency. +# Note that in LLVM "BUILD_SHARED_LIBS" enables an unsupported development mode. +# The flag you want for a shared library build is LLVM_BUILD_LLVM_DYLIB. +set(BUILD_SHARED_LIBS OFF) +set(LLVM_BUILD_LLVM_DYLIB ON) +set(LLVM_LINK_LLVM_DYLIB ON) + +# Set the LLVM_ENABLE_PROJECTS variable before including LLVM's CMakeLists.txt +set(BUILD_TESTING OFF CACHE BOOL "DISABLE BUILDING TESTS IN SUBPROJECTS" FORCE) +set(LLVM_ENABLE_PROJECTS "compiler-rt;lld;clang" CACHE STRING "Enable LLVM projects" FORCE) +set(LLVM_TARGETS_TO_BUILD "AMDGPU;X86" CACHE STRING "Enable LLVM Targets" FORCE) +set(LLVM_EXTERNAL_DEVICE_LIBS_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/device-libs" CACHE STRING "Device libs path" FORCE) +set(LLVM_EXTERNAL_PROJECTS "amddevice-libs;amdcomgr;hipcc" CACHE STRING "Enable extra projects" FORCE) +set(LLVM_EXTERNAL_AMDDEVICE_LIBS_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/device-libs") +set(LLVM_EXTERNAL_AMDCOMGR_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/comgr") +set(LLVM_EXTERNAL_HIPCC_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/hipcc") diff --git a/compiler/hipify/CMakeLists.txt b/compiler/hipify/CMakeLists.txt new file mode 100644 index 0000000..65a0730 --- /dev/null +++ b/compiler/hipify/CMakeLists.txt @@ -0,0 +1,18 @@ +################################################################################# +# HIPIFY +################################################################################# + +therock_cmake_subproject_declare(hipify + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/HIPIFY" + INTERFACE_PROGRAM_DIRS + bin + CMAKE_ARGS + -DHIPIFY_INSTALL_CLANG_HEADERS=OFF + RUNTIME_DEPS + amd-llvm +) +therock_cmake_subproject_glob_c_sources(hipify + SUBDIRS + src +) +therock_cmake_subproject_activate(hipify) diff --git a/compiler/hipify/post_hook.cmake b/compiler/hipify/post_hook.cmake new file mode 100644 index 0000000..bc732df --- /dev/null +++ b/compiler/hipify/post_hook.cmake @@ -0,0 +1,9 @@ +list(APPEND CMAKE_MODULE_PATH "${THEROCK_SOURCE_DIR}/cmake") +include(therock_rpath) + +therock_set_install_rpath( + TARGETS + hipify-clang + PATHS + ../lib/llvm/lib +) diff --git a/components/amd-llvm/CMakeLists.txt b/components/amd-llvm/CMakeLists.txt deleted file mode 100644 index dacad04..0000000 --- a/components/amd-llvm/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -cmake_minimum_required(VERSION 3.18) -project(THEROCK-AMD-LLVM) - -# Build LLVM and the comgr dependency. -# Note that in LLVM "BUILD_SHARED_LIBS" enables an unsupported development mode. -# The flag you want for a shared library build is LLVM_BUILD_LLVM_DYLIB. -set(BUILD_SHARED_LIBS OFF) -set(LLVM_BUILD_LLVM_DYLIB ON) -# TODO: Depending on what is getting released, it may or may not be paying -# for itself to dynamically link. -set(LLVM_LINK_LLVM_DYLIB OFF) -set(LLVM_DIR "${ROCM_GIT_DIR}/llvm-project") -message(STATUS "LLVM_DIR is set to: ${LLVM_DIR}") - -# Set the LLVM_ENABLE_PROJECTS variable before including LLVM's CMakeLists.txt -set(BUILD_TESTING OFF CACHE BOOL "DISABLE BUILDING TESTS IN SUBPROJECTS" FORCE) -set(LLVM_ENABLE_PROJECTS "compiler-rt;lld;clang" CACHE STRING "Enable LLVM projects" FORCE) -set(LLVM_TARGETS_TO_BUILD "AMDGPU;X86" CACHE STRING "Enable LLVM Targets" FORCE) -set(LLVM_EXTERNAL_DEVICE_LIBS_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/device-libs" CACHE STRING "Device libs path" FORCE) -set(LLVM_EXTERNAL_PROJECTS "amddevice-libs;amdcomgr" CACHE STRING "Enable extra projects" FORCE) -set(LLVM_EXTERNAL_AMDCOMGR_SOURCE_DIR "${LLVM_DIR}/amd/comgr") -set(LLVM_EXTERNAL_AMDDEVICE_LIBS_SOURCE_DIR "${ROCM_GIT_DIR}/llvm-project/amd/device-libs") -#set(ROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_NEW "llvm/amdgcn-new") -# hipcc expects bit codes under amdgcn. -#set(ROCM_DEVICE_LIBS_BITCODE_INSTALL_LOC_OLD "amdgcn") - -# Now include LLVM's source directory -add_subdirectory(${LLVM_DIR}/llvm ${CMAKE_BINARY_DIR}/llvm) - -# add_custom_target( -# amd-bitcodes-symlink ALL -# WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} -# COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/.. -# COMMAND ${CMAKE_COMMAND} -E create_symlink amdgcn ${CMAKE_INSTALL_PREFIX}/../amdgcn -# ) -install(CODE "execute_process( \ - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/.. \ - COMMAND ${CMAKE_COMMAND} -E create_symlink llvm/amdgcn ${CMAKE_INSTALL_PREFIX}/../amdgcn \ - COMMAND_ERROR_IS_FATAL ANY \ - )" -) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt new file mode 100644 index 0000000..5898d94 --- /dev/null +++ b/core/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(ROCR-Runtime) +add_subdirectory(clr) +add_subdirectory(rocminfo) diff --git a/core/ROCR-Runtime/CMakeLists.txt b/core/ROCR-Runtime/CMakeLists.txt new file mode 100644 index 0000000..02dcb19 --- /dev/null +++ b/core/ROCR-Runtime/CMakeLists.txt @@ -0,0 +1,23 @@ +################################################################################ +# ROCR-Runtime +################################################################################ + +therock_cmake_subproject_declare(ROCR-Runtime + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/ROCR-Runtime" + CMAKE_ARGS + "-DBUILD_SHARED_LIBS=ON" + BUILD_DEPS + amd-llvm + RUNTIME_DEPS + rocprofiler-register + INTERFACE_LINK_DIRS + "lib" +) +therock_cmake_subproject_glob_c_sources(ROCR-Runtime + SUBDIRS + libhsakmt + runtime +) +therock_cmake_subproject_provide_package(ROCR-Runtime hsakmt lib/cmake/hsakmt) +therock_cmake_subproject_provide_package(ROCR-Runtime hsa-runtime64 lib/cmake/hsa-runtime64) +therock_cmake_subproject_activate(ROCR-Runtime) diff --git a/cmake/extensions/rocr_post.cmake b/core/ROCR-Runtime/post_hook.cmake similarity index 92% rename from cmake/extensions/rocr_post.cmake rename to core/ROCR-Runtime/post_hook.cmake index 5eca160..ee3055b 100644 --- a/cmake/extensions/rocr_post.cmake +++ b/core/ROCR-Runtime/post_hook.cmake @@ -6,5 +6,4 @@ therock_set_install_rpath( hsa-runtime64 PATHS . - llvm/lib ) diff --git a/core/clr/CMakeLists.txt b/core/clr/CMakeLists.txt new file mode 100644 index 0000000..025d47e --- /dev/null +++ b/core/clr/CMakeLists.txt @@ -0,0 +1,27 @@ +################################################################################ +# clr +################################################################################ + +therock_cmake_subproject_declare(hip-clr + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/clr" + CMAKE_ARGS + -DHIP_PLATFORM=amd + "-DHIP_COMMON_DIR=${ROCM_GIT_DIR}/HIP" + -DCLR_BUILD_HIP=ON + # Legacy: Disable various auto-detection logic that breaks out of jail + # and can use local machine tools. + -DHIPCC_BIN_DIR= + BUILD_DEPS + rocm-cmake + RUNTIME_DEPS + amd-llvm + rocm-core + rocprofiler-register + ROCR-Runtime + INTERFACE_LINK_DIRS + "lib" +) +therock_cmake_subproject_glob_c_sources(hip-clr SUBDIRS .) +therock_cmake_subproject_provide_package(hip-clr hip lib/cmake/hip) +therock_cmake_subproject_provide_package(hip-clr HIP lib/cmake/hip) +therock_cmake_subproject_activate(hip-clr) diff --git a/core/clr/post_hook.cmake b/core/clr/post_hook.cmake new file mode 100644 index 0000000..b725d70 --- /dev/null +++ b/core/clr/post_hook.cmake @@ -0,0 +1,16 @@ +list(APPEND CMAKE_MODULE_PATH "${THEROCK_SOURCE_DIR}/cmake") +include(therock_rpath) + +therock_set_install_rpath( + TARGETS + amdhip64 + hiprtc + PATHS + . + llvm/lib +) + +# See: https://github.com/nod-ai/TheRock/issues/21 +if(HIPCC_BIN_DIR) + message(FATAL_ERROR "The legacy HIPCC_BIN_DIR was somehow set, indicating a bug in the clr CMake files: ${HIPCC_BIN_DIR}") +endif() diff --git a/core/rocminfo/CMakeLists.txt b/core/rocminfo/CMakeLists.txt new file mode 100644 index 0000000..d877c1a --- /dev/null +++ b/core/rocminfo/CMakeLists.txt @@ -0,0 +1,11 @@ +################################################################################ +# rocminfo +################################################################################ + +therock_cmake_subproject_declare(rocminfo + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/rocminfo" + RUNTIME_DEPS + ROCR-Runtime +) +therock_cmake_subproject_glob_c_sources(rocminfo SUBDIRS .) +therock_cmake_subproject_activate(rocminfo) diff --git a/core/rocminfo/post_hook.cmake b/core/rocminfo/post_hook.cmake new file mode 100644 index 0000000..d34ecf4 --- /dev/null +++ b/core/rocminfo/post_hook.cmake @@ -0,0 +1,9 @@ +list(APPEND CMAKE_MODULE_PATH "${THEROCK_SOURCE_DIR}/cmake") +include(therock_rpath) + +therock_set_install_rpath( + TARGETS + rocminfo + PATHS + ../lib +) diff --git a/docs/development/build_system.md b/docs/development/build_system.md new file mode 100644 index 0000000..06a887f --- /dev/null +++ b/docs/development/build_system.md @@ -0,0 +1,218 @@ +# TheROCK Build System Manual + +The ROCM distribution consists of many component projects that form a DAG +consisting of build and runtime dependencies between projects. Each individual +sub-project is CMake based and dependencies between them are generally +resolved via `find_package`. This means that it is possible to build each +project in isolation. + +However, working with the individual pieces is not well suited for many tasks, +most notably CI and full-stack development workflows. TheROCK provides a CMake +base super-project and monorepo like organization of the source and build to +make building and testing more of a one stop shop. + +## Terminology + +Super-Project +: This refers to `TheROCK` project itself as the container of sub-projects. + +Sub-Project +: Each individual piece or standalone dependency of the ROCM system is +referred to as a sub-project. + +Build Phases +: Each sub-project is built in several phases: `configure`, `build`, `stage`, +and `dist`. Inter-project dependencies are taken at a build phase +granularity, allowing a degree of parallelism in the case of a more +limited dependency. For each project in the tree, a specific phase can +be built interactively by appending `+phase` to the sub-project's target +name. + +Build Dependency +: If a sub-project dependency is a build dependency, it is not required to be +co-resident in a unified install tree in order to function. + +Runtime Dependency +: If a sub-project is a runtime dependency that all/part of it must be +co-resident in the unified install tree of the depending project in order +to function. + +Utility Targets +: Sub-projects may expose additional utility targets that can be accessed as +`+utility`. Currently, this just includes `+expunge`, which removes all +configured/built files related to the sub-project. + +Stamps +: All Build Phases depend on a set of stamp files and produce a stamp file +named `phase.stamp`. This produces a worst-case ordering DAG between +phases. If a stamp file is removed (manually or via `clean`), the phase +and all of its dependencies will re-run, regardless of whether any sources +changed. + +## Directory Layout + +While the namespace of sub-projects is flat (every sub-project is a CMake target +name and therefore must be globally unique), the repository itself is organized +into a hierarchy with individual sub-projects at the leaves. Since the `ninja` +build system has good scoping of parts of the build by sub-directory (via the +implicit `all` pseudo-target), this provides some developer ergonomics, allowing +easy partial builds/cleans or deletions of parts of the build tree while +working. + +The following structure is presently used: + +- `base/` : Utility sub-projects that are dependency free or minimally + co-dependent, providing core dependencies for the rest of the system. For + standalone builds, bundled external libraries will be rooted here. +- `compiler/` : Compiler sub-projects, most notably the AMD-LLVM build, `hipcc`, + etc. +- `core/` : Core runtime sub-projects, including the low-level ROCR-Runtime and + higher level HIP runtimes. +- TODO: others + +Note that there is nothing in the build system which ensures naming consistency, +however, we try to name leaf directories after their global sub-project +target name (i.e. `ROCR-Runtime` or `amd-llvm`) and if possible, the `project()` +name of the sub-project itself, if that is something we control. Consistency +just makes everyone's lives easier, even at the expense of sometimes not +having a naming/capitalization convention that is uniform. + +## Build Directory Layout + +Each sub-project, by default, uses a standard directory layout for its build: + +- `build/` : The `CMAKE_BINARY_DIR` for the project, containing the + `CMakeCache.txt`, etc. Once an initial super-project build is performed, + developers can interact directly with the CMake build here if they prefer to + work on a single component. The `configure` phase populates this directory + by running the initial `cmake` configure. The `build` phase compiles the + `all` target. +- `stage/` : Local staging install directory for the sub-project. The + sub-projects install components (or just default install) will populate + this directory during the `stage` build phase. Note that for sub-projects + with runtime deps, this will be a "torn" directory in that it may contain + shared libraries whose dependencies cannot be found because their RPATH is + configured for the standard directory layout, which presumes that the project + is bundled with its runtime deps. +- `dist/` : This is populated by the `dist` build phase by hard linking (or + copying if hard-links are not possible) the cone of all runtime dep project's + `stage/` and this project's `stage/` directory contents. In this way, each + individual project's `dist/` directory should consist of a self-contained + slice that is relocatable and usable (for testing, etc). Top-level + distributions are created in this same way by having runtime deps on all + relevant sub-projects. Keeping sub-projects isolated in this way aids in + testing, especially to ensure that dependencies are declared and layered + properly. +- `_init.cmake` : Generated as part of the super-project to set all necessary + CMake settings to build the sub-project. This is injected at configure time + via the `CMAKE_PROJECT_TOP_LEVEL_INCLUDES` facility and serves some other + ancillary functions as well: + - Loads sub-project specific `pre` and `post` CMake file for further + sub-project customization needed as part of the integrated whole. + - Installs a [CMake dependency provider](https://cmake.org/cmake/help/latest/command/cmake_language.html#dependency-providers) + which rewrites `find_package` calls for any packages provided as part + of super-project deps appropriately. +- `stamp/` : Directory of `{phase}.stamp` files that are used to control + build sequencing. + +## CMake Configuration Options + +TODO + +## Developer Cookbook + +TheROCK aims to not just be a CI tool but to be a daily driver for developer +and end-users who wish to consume a source build of ROCM. This section contains +some advice that may help such users be more productive. + +### Building part of the tree + +Ninja's built-in directory scoping allows easy partial-builds: + +Example: + +``` +ninja compiler/amd-llvm/all +``` + +Dependencies are tracked so if building a leaf project, it will build projects +it depends on. Note that there is an implicit `all` target at each directory +level, so the path can be arbitrarily fine-grained. + +Similarly, cleaning the super-project and configure state can be done with: + +``` +ninja -t clean compiler/amd-llvm/all +``` + +Note that this will just clean the stamp files and byproducts that the +super-project knows about, causing a subsequent build to perform all build +phases regardless of whether any contents changed. Notably at present, this +does not clean the built files (this may change in the future), so a subsequent +build will still use CMake's own cache to avoid a full rebuild. + +Sometimes a sub-project just needs to be removed entirely and rebuilt from +scratch. For this, you can do it the manual way via +`rm -Rf compiler/amd-llvm; cmake .` (note that if using a big hammer like this, +you need to regenerate the super-project build system because you just deleted +part of it). Or you can invoke the expunge sub-target: + +``` +ninja amd-llvm+expunge +``` + +## Adding Sub-Projects + +The entire sub-project facility is defined in `cmake/therock_subproject.cmake` +and it may be useful to refer to that if doing anything advanced. This section +attempts to document the basics. + +Consider an example that is typical. This is taken from the tree but annotated +with comments, describing what is going on. + +```cmake +# Create a CMake target named `ROCR-Runtime` and set it up as a sub-project. +# This will also cause phase specific convenience targets like +# `ROCR-Runtime+build` to be created. Additional calls are needed to further +# set up the sub-project, and the sequence must terminate with a call to +# `therock_cmake_subproject_activate()` +# Specific settings used here: +# * EXTERNAL_SOURCE_DIR: Tells the system that the sources are located +# somewhere else (in this case in the standard location we use for `repo`). +# * CMAKE_ARGS: Additional arguments to pass to CMake. This is in addition +# to a number of default arguments. +# * BUILD_DEPS: Sub-projects that must be built and staged before this +# project's configure phase can run. +# * RUNTIME_DEPS: Sub-projects that should be considered a BUILD_DEP and +# also are required to be in a unified distribution tree at runtime. +therock_cmake_subproject_declare(ROCR-Runtime + EXTERNAL_SOURCE_DIR "${ROCM_GIT_DIR}/ROCR-Runtime" + CMAKE_ARGS + "-DBUILD_SHARED_LIBS=ON" + BUILD_DEPS + amd-llvm + RUNTIME_DEPS + rocprofiler-register +) + +# By default, the super-project treats the sources for the sub-project like a +# black box. This means that if you change the sources and rebuild, nothing +# will happen (unless if you clean or invoke the sub-project build directly). +# This directive tells the super-project that it should rebuild if C source +# files in any of the given sub-directories are modified. +therock_cmake_subproject_glob_c_sources(ROCR-Runtime + SUBDIRS + libhsakmt + runtime +) + +# Tells the build system that this sub-project is expected to produce two +# `find_package` packages for consumers. The path is relative to the unified +# install directory layout. +therock_cmake_subproject_provide_package(ROCR-Runtime hsakmt lib/cmake/hsakmt) +therock_cmake_subproject_provide_package(ROCR-Runtime hsa-runtime64 lib/cmake/hsa-runtime64) + +# Activates the sub-project once all customization is done. This is analogous +# to `FetchContent_MakeAvailable()` for that facility. +therock_cmake_subproject_activate(ROCR-Runtime) +``` diff --git a/patches/rocm-6.3.1/HIPIFY/0001-Make-hipify-clang-buildable-with-an-LLVM-configured-.patch b/patches/rocm-6.3.1/HIPIFY/0001-Make-hipify-clang-buildable-with-an-LLVM-configured-.patch new file mode 100644 index 0000000..c393a72 --- /dev/null +++ b/patches/rocm-6.3.1/HIPIFY/0001-Make-hipify-clang-buildable-with-an-LLVM-configured-.patch @@ -0,0 +1,89 @@ +From bb11a94929ddc927ed2c76ca0d2aff47e0dadb73 Mon Sep 17 00:00:00 2001 +From: Stella Laurenzo +Date: Tue, 21 Jan 2025 17:37:09 -0800 +Subject: [PATCH 1/2] Make hipify-clang buildable with an LLVM configured to + link libLLVM.so. + +Without these changes, if building against and LLVM configured with `-DLLVM_LINK_LLVM_DYLIB=ON`, the resulting binary will have ODR violations resulting in the dread duplicate CL option registered errors at runtime. + +In order to work in this case, executables must be uniformly linked in the same way that LLVM binaries themselves are: by setting `LLVM_LINK_COMPONENTS` vs depending directly on the (static) individual libraries. This should work in all linking modes. + +Note that the problem originate because an LLVM configured in this way will configure any clang libraries to carry a transitive dep on libLLVM. Therefore, it is illegal to depend on clang (static) libraries and LLVM static libraries in combination. + +There is a step further that could be taken to also support building against a shared libClang, but this is merely an optimization vs an error: since libClang is a leaf library, it is ok to depend on its static components. Perhaps total package size could be reduced by also supporting libClang, but this is left for the future. +--- + CMakeLists.txt | 51 ++++++++++++++++++++++++++++++-------------------- + 1 file changed, 31 insertions(+), 20 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 1b51a16e..d2055a01 100755 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -64,7 +64,35 @@ if (NOT HIPIFY_CLANG_TESTS_ONLY) + file(GLOB_RECURSE HIPIFY_SOURCES src/*.cpp) + file(GLOB_RECURSE HIPIFY_HEADERS src/*.h) + +- add_llvm_executable(hipify-clang ${HIPIFY_SOURCES} ${HIPIFY_HEADERS}) ++ # In order to be compatible with an LLVM that uses LLVM_LINK_LLVM_DYLIB=ON, ++ # must link core LLVM libraries via LLVM_LINK_COMPONENTS vs directly adding ++ # them. ++ set(HIPIFY_ADDL_LINK_LIBS) ++ set(LLVM_LINK_COMPONENTS ++ ProfileData ++ Support ++ MCParser ++ MC ++ BitReader ++ Option ++ Core ++ ) ++ if(LLVM_PACKAGE_VERSION VERSION_GREATER "9.0.1") ++ list(APPEND LLVM_LINK_COMPONENTS FrontendOpenMP) ++ endif() ++ ++ if(LLVM_PACKAGE_VERSION VERSION_EQUAL "15.0.0" OR LLVM_PACKAGE_VERSION VERSION_GREATER "15.0.0") ++ list(APPEND LLVM_LINK_COMPONENTS WindowsDriver) ++ list(APPEND HIPIFY_ADDL_LINK_LIBS clangSupport) ++ endif() ++ ++ if(LLVM_PACKAGE_VERSION VERSION_GREATER "6.0.1") ++ list(APPEND HIPIFY_ADDL_LINK_LIBS clangToolingInclusions) ++ endif() ++ ++ add_llvm_executable(hipify-clang ++ ${HIPIFY_SOURCES} ${HIPIFY_HEADERS} ++ ) + target_link_directories(hipify-clang PRIVATE ${LLVM_LIBRARY_DIRS}) + + if(HIPIFY_INCLUDE_IN_HIP_SDK) +@@ -109,25 +137,8 @@ if (NOT HIPIFY_CLANG_TESTS_ONLY) + clangToolingCore + clangRewrite + clangBasic +- LLVMProfileData +- LLVMSupport +- LLVMMCParser +- LLVMMC +- LLVMBitReader +- LLVMOption +- LLVMCore) +- +- if(LLVM_PACKAGE_VERSION VERSION_GREATER "6.0.1") +- target_link_libraries(hipify-clang PRIVATE clangToolingInclusions) +- endif() +- +- if(LLVM_PACKAGE_VERSION VERSION_GREATER "9.0.1") +- target_link_libraries(hipify-clang PRIVATE LLVMFrontendOpenMP) +- endif() +- +- if(LLVM_PACKAGE_VERSION VERSION_EQUAL "15.0.0" OR LLVM_PACKAGE_VERSION VERSION_GREATER "15.0.0") +- target_link_libraries(hipify-clang PRIVATE LLVMWindowsDriver clangSupport) +- endif() ++ ${HIPIFY_ADDL_LINK_LIBS} ++ ) + + if(LLVM_PACKAGE_VERSION VERSION_EQUAL "16.0.0" OR LLVM_PACKAGE_VERSION VERSION_GREATER "16.0.0") + if(MSVC) +-- +2.43.0 + diff --git a/patches/rocm-6.3.1/HIPIFY/0002-Pass-USE_SOURCE_PERMISSIONS-when-installing-scripts.patch b/patches/rocm-6.3.1/HIPIFY/0002-Pass-USE_SOURCE_PERMISSIONS-when-installing-scripts.patch new file mode 100644 index 0000000..b3d4cd1 --- /dev/null +++ b/patches/rocm-6.3.1/HIPIFY/0002-Pass-USE_SOURCE_PERMISSIONS-when-installing-scripts.patch @@ -0,0 +1,25 @@ +From c60987bfeb913591805627291ad68fad7a50d958 Mon Sep 17 00:00:00 2001 +From: Stella Laurenzo +Date: Tue, 21 Jan 2025 21:18:45 -0800 +Subject: [PATCH 2/2] Pass USE_SOURCE_PERMISSIONS when installing scripts. + +Without this, hipify-perl and friends are installed with non-executable permissions, requiring after the fact fixup. +--- + CMakeLists.txt | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index d2055a01..be5b2a95 100755 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -192,6 +192,7 @@ if (NOT HIPIFY_CLANG_TESTS_ONLY) + install( + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bin + DESTINATION . ++ USE_SOURCE_PERMISSIONS + PATTERN "hipify-perl" + PATTERN "*.sh" + PATTERN "findcode.sh" EXCLUDE +-- +2.43.0 + diff --git a/patches/rocm-6.3.1/clr/0001-Disable-HIP_PLATFORM-auto-detect-if-already-defined.patch b/patches/rocm-6.3.1/clr/0001-Disable-HIP_PLATFORM-auto-detect-if-already-defined.patch new file mode 100644 index 0000000..31ed30e --- /dev/null +++ b/patches/rocm-6.3.1/clr/0001-Disable-HIP_PLATFORM-auto-detect-if-already-defined.patch @@ -0,0 +1,155 @@ +From 54553db0d22c76c25863b5af9467d03afb94a9d3 Mon Sep 17 00:00:00 2001 +From: Stella Laurenzo +Date: Wed, 22 Jan 2025 14:01:50 -0800 +Subject: [PATCH] Disable HIP_PLATFORM auto-detect if already defined. + +This code appears to be old copy-pasta and is effectively just a no-op if -DHIP_PLATFORM=amd is passed explicitly. It has a number of nit-picky checks which will fail in this case for no real reason. + +This is actually worse than that because it leaks global variables that establish a different default value/scoping vs the definitions in the inner hipamd/CMakeLists.txt. I opted to rewrite this section to scope the detection logic and keep it from leaking with respect to the inner build file. The detection logic is still not great but at least will not actively cause damage or make control flow hard to reason about. It seems like this is for legacy/compatibility so I opted to simply sequester it vs applying more diligence to it. + +Also fixes a bug in hip-config.cmake where it enforces that hipcc exists even if not taking the install branch in the code above. + +See: https://github.com/nod-ai/TheRock/issues/21 +--- + CMakeLists.txt | 43 ++++++++++++++++++++++++-------------- + hipamd/CMakeLists.txt | 5 ++++- + hipamd/hip-config.cmake.in | 21 +++++++++++-------- + 3 files changed, 43 insertions(+), 26 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index cc9cf1654..ad2201073 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -40,40 +40,51 @@ endif() + ############# + # Build steps + ############# +-if(CLR_BUILD_HIP) +- if (UNIX) +- set(HIPCC_EXECUTABLE "hipcc") +- set(HIPCONFIG_EXECUTABLE "hipconfig") +- else() +- set(HIPCC_EXECUTABLE "hipcc.exe") +- set(HIPCONFIG_EXECUTABLE "hipconfig.exe") +- endif() ++ ++# Attempt to auto-detect HIP_PLATFORM by interrogating hipconfig. This is kept ++# for compatibility: users are advised to pass HIP_PLATFORM explicitly. ++# Sets the HIP_PLATFORM variable in the parent scope. ++function(_hip_clr_auto_detect_hip_platform) ++ if (UNIX) ++ set(HIPCC_EXECUTABLE "hipcc") ++ set(HIPCONFIG_EXECUTABLE "hipconfig") ++ else() ++ set(HIPCC_EXECUTABLE "hipcc.exe") ++ set(HIPCONFIG_EXECUTABLE "hipconfig.exe") ++ endif() + + # Set default HIPCC_BIN_DIR to /opt/rocm/bin + if(NOT DEFINED HIPCC_BIN_DIR AND UNIX) +- set(HIPCC_BIN_DIR "/opt/rocm/bin" CACHE STRING "Default hipcc directory on linux.") ++ set(HIPCC_BIN_DIR "/opt/rocm/bin") + endif() +- message(STATUS "HIPCC Binary Directory: ${HIPCC_BIN_DIR}") ++ message(STATUS "Auto-detect HIP_PLATFORM from HIPCC Binary Directory: ${HIPCC_BIN_DIR}") + + if(NOT EXISTS ${HIPCC_BIN_DIR}/${HIPCONFIG_EXECUTABLE}) + message(FATAL_ERROR "Please pass hipcc/build or hipcc/bin using -DHIPCC_BIN_DIR.") + endif() + +- message(STATUS "HIP Common Directory: ${HIP_COMMON_DIR}") +- if(NOT DEFINED HIP_COMMON_DIR) +- message(FATAL_ERROR "Please pass HIP using -DHIP_COMMON_DIR. HIP_COMMON_DIR is incorrect") +- endif() + # Determine HIP_PLATFORM + if(NOT DEFINED HIP_PLATFORM) + if(NOT DEFINED ENV{HIP_PLATFORM}) + execute_process(COMMAND ${HIPCC_BIN_DIR}/${HIPCONFIG_EXECUTABLE} --platform +- OUTPUT_VARIABLE HIP_PLATFORM ++ OUTPUT_VARIABLE _detected_hip_platform + OUTPUT_STRIP_TRAILING_WHITESPACE) ++ set(HIP_PLATFORM "${_detected_hip_platform}" PARENT_SCOPE) + else() +- set(HIP_PLATFORM $ENV{HIP_PLATFORM} CACHE STRING "HIP Platform") ++ set(HIP_PLATFORM "$ENV{HIP_PLATFORM}" PARENT_SCOPE) + endif() + endif() ++endfunction() ++if(CLR_BUILD_HIP) ++ message(STATUS "HIP Common Directory: ${HIP_COMMON_DIR}") ++ if(NOT DEFINED HIP_COMMON_DIR) ++ message(FATAL_ERROR "Please pass HIP using -DHIP_COMMON_DIR. HIP_COMMON_DIR is incorrect") ++ endif() ++ if(NOT DEFINED HIP_PLATFORM) ++ _hip_clr_auto_detect_hip_platform() ++ endif() + endif() ++ + if(CLR_BUILD_HIP) + option(BUILD_SHARED_LIBS "Build the shared library" ON) + if (NOT BUILD_SHARED_LIBS) +diff --git a/hipamd/CMakeLists.txt b/hipamd/CMakeLists.txt +index 9538dbbad..7babbfc36 100755 +--- a/hipamd/CMakeLists.txt ++++ b/hipamd/CMakeLists.txt +@@ -53,7 +53,8 @@ if(MSVC) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG:FULL") + endif() + +-set(HIPCC_BIN_DIR "" CACHE STRING "HIPCC and HIPCONFIG binary directories") ++# Set default HIPCC_BIN_DIR to /opt/rocm/bin ++set(HIPCC_BIN_DIR "/opt/rocm/bin" CACHE STRING "HIPCC and HIPCONFIG binary directories") + + if(__HIP_ENABLE_PCH) + set(_pchStatus 1) +@@ -393,9 +394,11 @@ endif() + install(FILES ${PROJECT_BINARY_DIR}/include/hip/hip_version.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hip) + ++set(HIP_INSTALLS_HIPCC OFF) + if (NOT ${HIPCC_BIN_DIR} STREQUAL "") + file(TO_CMAKE_PATH "${HIPCC_BIN_DIR}" HIPCC_BIN_DIR) + if(EXISTS ${HIPCC_BIN_DIR}) ++ set(HIP_INSTALLS_HIPCC ON) + install(PROGRAMS ${HIPCC_BIN_DIR}/${HIPCC_EXECUTABLE} DESTINATION bin) + install(PROGRAMS ${HIPCC_BIN_DIR}/${HIPCONFIG_EXECUTABLE} DESTINATION bin) + install(PROGRAMS ${HIPCC_BIN_DIR}/hipcc.pl DESTINATION bin) +diff --git a/hipamd/hip-config.cmake.in b/hipamd/hip-config.cmake.in +index 1fcdcd970..d033c09ac 100755 +--- a/hipamd/hip-config.cmake.in ++++ b/hipamd/hip-config.cmake.in +@@ -101,12 +101,14 @@ set_and_check( hip_INCLUDE_DIR "@PACKAGE_INCLUDE_INSTALL_DIR@" ) + set_and_check( hip_INCLUDE_DIRS "${hip_INCLUDE_DIR}" ) + set_and_check( hip_LIB_INSTALL_DIR "@PACKAGE_LIB_INSTALL_DIR@" ) + set_and_check( hip_BIN_INSTALL_DIR "@PACKAGE_BIN_INSTALL_DIR@" ) +-if (WIN32) +- set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc.exe") +- set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig.exe") +-else() +- set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc") +- set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig") ++if("@HIP_INSTALLS_HIPCC@") ++ if (WIN32) ++ set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc.exe") ++ set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig.exe") ++ else() ++ set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc") ++ set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig") ++ endif() + endif() + + if(NOT DEFINED HIP_PLATFORM) +@@ -140,6 +142,7 @@ set(HIP_LIB_INSTALL_DIR ${hip_LIB_INSTALL_DIR}) + set(HIP_BIN_INSTALL_DIR ${hip_BIN_INSTALL_DIR}) + set(HIP_LIBRARIES ${hip_LIBRARIES}) + set(HIP_LIBRARY ${hip_LIBRARY}) +-set(HIP_HIPCC_EXECUTABLE ${hip_HIPCC_EXECUTABLE}) +-set(HIP_HIPCONFIG_EXECUTABLE ${hip_HIPCONFIG_EXECUTABLE}) +- ++if("@HIP_INSTALLS_HIPCC@") ++ set(HIP_HIPCC_EXECUTABLE ${hip_HIPCC_EXECUTABLE}) ++ set(HIP_HIPCONFIG_EXECUTABLE ${hip_HIPCONFIG_EXECUTABLE}) ++endif() +-- +2.43.0 + diff --git a/patches/rocm-6.3.1/rccl/0001-Make-rccl-version-detection-robust.patch b/patches/rocm-6.3.1/rccl/0001-Make-rccl-version-detection-robust.patch new file mode 100644 index 0000000..1c6e20b --- /dev/null +++ b/patches/rocm-6.3.1/rccl/0001-Make-rccl-version-detection-robust.patch @@ -0,0 +1,94 @@ +From 81394fa82ea2422cef4648ce24bbfdd30ca1d42d Mon Sep 17 00:00:00 2001 +From: Stella Laurenzo +Date: Wed, 22 Jan 2025 16:20:07 -0800 +Subject: [PATCH] Make rccl version detection robust. + +* Accept an EXPLICIT_ROCM_VERSION and use that vs inspecting the environment if provided. +* Use CMake's built in file reading support vs execute_process (without error checking) to avoid silent but deadly later failures. +* Properly quote some comparisons to avoid syntax errors if they happen to have an empty string. +* Guard against ROCM_PATH being an empty string, avoiding stray path extensions to root directories, etc. +--- + CMakeLists.txt | 32 +++++++++++++++++++++----------- + 1 file changed, 21 insertions(+), 11 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 966c5cc..857467f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -129,10 +129,12 @@ endif() + set(CMAKE_INSTALL_PREFIX "${ROCM_PATH}" CACHE PATH "") + set(CMAKE_CXX_STANDARD 14) # We use C++14 features, this will add compile option: -std=c++14 + set(CMAKE_CXX_EXTENSIONS OFF) # Without this line, it will add -std=gnu++14 instead, which has some issues. +-list(APPEND CMAKE_PREFIX_PATH # Add ROCM_PATH to CMake search paths (for finding HIP / HSA +- ${ROCM_PATH} +- ${ROCM_PATH}/hip +- ${ROCM_PATH}/llvm) ++if(ROCM_PATH) ++ list(APPEND CMAKE_PREFIX_PATH # Add ROCM_PATH to CMake search paths (for finding HIP / HSA ++ ${ROCM_PATH} ++ ${ROCM_PATH}/hip ++ ${ROCM_PATH}/llvm) ++endif() + + # Check for required dependencies + #================================================================================================== +@@ -165,10 +167,15 @@ execute_process( + message(STATUS "${COMPILER_EXE_NAME} HIP version: ${hip_version_string}") + + ## Check for ROCm version +-execute_process( +- COMMAND bash "-c" "cat ${ROCM_PATH}/.info/version" +- OUTPUT_VARIABLE rocm_version_string +-) ++set(EXPLICIT_ROCM_VERSION "" CACHE STRING "Explicit ROCM version to compile to (auto detect if empty)") ++if(EXPLICIT_ROCM_VERSION) ++ set(rocm_version_string "${EXPLICIT_ROCM_VERSION}") ++elseif(ROCM_PATH) ++ message(STATUS "Reading ROCM version from ${ROCM_PATH}/.info/version") ++ file(READ "${ROCM_PATH}/.info/version" rocm_version_string) ++else() ++ message(FATAL_ERROR "Could not determine ROCM version (set EXPLICIT_ROCM_VERSION or set ROCM_PATH to a valid installation)") ++endif() + string(REGEX MATCH "([0-9]+)\\.([0-9]+)\\.([0-9]+)" rocm_version_matches ${rocm_version_string}) + if (rocm_version_matches) + set(ROCM_MAJOR_VERSION ${CMAKE_MATCH_1}) +@@ -197,7 +204,7 @@ unset(CMAKE_REQUIRED_LIBRARIES) + + ### Check for indirect function call support + if(ENABLE_IFC) +- if(${hip_version_string} VERSION_GREATER_EQUAL "5.5.30201") ++ if("${hip_version_string}" VERSION_GREATER_EQUAL "5.5.30201") + set(IFC_ENABLED ON) + message(STATUS "Indirect function call enabled") + else() +@@ -209,7 +216,7 @@ else() + endif() + + ## Check for LL128 support +-if(${hip_version_string} VERSION_GREATER_EQUAL "6.1.33591") ++if("${hip_version_string}" VERSION_GREATER_EQUAL "6.1.33591") + set(LL128_ENABLED ON) + message(STATUS "RCCL LL128 protocol enabled") + else() +@@ -556,6 +563,9 @@ endif() + # Hipify source files (copy of source generated into hipify directory) + #================================================================================================== + find_program(hipify-perl_executable hipify-perl) ++if(NOT hipify-perl_executable) ++ message(FATAL_ERROR "hipify-perl not found") ++endif() + set(HIPIFY_DIR "${CMAKE_CURRENT_BINARY_DIR}/hipify") + + ## Loop over each source file to hipify +@@ -685,7 +695,7 @@ if(${HIP_CONTIGUOUS_MEMORY}) + else() + message(STATUS "HIP_CONTIGUOUS_MEMORY disabled") + endif() +-if(${hip_version_string} VERSION_GREATER_EQUAL "5.7.31920") ++if("${hip_version_string}" VERSION_GREATER_EQUAL "5.7.31920") + target_compile_definitions(rccl PRIVATE HIP_UNCACHED_MEMORY) + message(STATUS "HIP_UNCACHED_MEMORY enabled") + else() +-- +2.43.0 +