Adds MIOpen to the project. (#27)

MIOpen carries a lot of deps and is a complicated project. This leaves a
couple of the optional deps off just to get things going.

Adds the following on the path to MIOpen:

* base/half
* math-libs/hipBLAS
* math-libs/rocRAND
* third-party/FunctionalPlus
* third-party/boost (just atomics, filesystem, system as static PIC
libraries)
* third-party/eigen
* third-party/frugally-deep
* third-party/nlohmann-json

Re-organized fetch_sources.py to allow controlling fetch of math-libs
and ml-frameworks.

Because we build with disaggregated libraries (vs glomming them all into
one prefix directory), this fuzzed what seems to be CMake config errors
in several of our libraries. I've patches around them vs fixing upstream
for the moment. See ml-frameworks/pre_hook_MIOpen.cmake for TODOs.

CMakeLists.txt

@@ -11,7 +11,10 @@ endif()
 
 project(THEROCK)
 
+cmake_policy(SET CMP0135 NEW)
+
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+include(CMakeDependentOption)
 include(ExternalProject)
 include(therock_subproject)
 include(therock_job_pools)
@@ -31,6 +34,7 @@ message(STATUS "ROCM_GIT_DIR is set to: ${ROCM_GIT_DIR}")
 # Library specific enable flags.
 option(THEROCK_ENABLE_RCCL "Enable the comm_libs/rccl sub-project" ON)
 option(THEROCK_ENABLE_MATH_LIBS "Enable building of math libraries" ON)
+cmake_dependent_option(THEROCK_ENABLE_ML_FRAMEWORKS "Enables building of ML frameworks" ON THEROCK_ENABLE_MATH_LIBS OFF)
 
 # Initialize the install directory.
 if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
@@ -75,13 +79,17 @@ endif()
 
 # Add subdirectories in dependency DAG order (which happens to be semi-alpha:
 # don't be fooled).
+add_subdirectory(third-party)
 add_subdirectory(base)
 add_subdirectory(compiler)
 add_subdirectory(core)
 add_subdirectory(comm-libs)
 if(THEROCK_ENABLE_MATH_LIBS)
   add_subdirectory(math-libs)
 endif()
+if(THEROCK_ENABLE_ML_FRAMEWORKS)
+  add_subdirectory(ml-frameworks)
+endif()
 
 # ################################################################################
 # # Testing

base/CMakeLists.txt

@@ -6,7 +6,9 @@ therock_cmake_subproject_declare(rocm-cmake
   EXTERNAL_SOURCE_DIR "rocm-cmake"
 )
 therock_cmake_subproject_provide_package(rocm-cmake
-  ROCM share/rocmcmakebuildtools/cmake)
+  ROCmCMakeBuildToolsConfig share/rocmcmakebuildtools/cmake)
+therock_cmake_subproject_provide_package(rocm-cmake
+  ROCM share/rocm/cmake)
 therock_cmake_subproject_activate(rocm-cmake)
 
 
@@ -63,3 +65,19 @@ therock_cmake_subproject_glob_c_sources(rocprofiler-register
 therock_cmake_subproject_provide_package(rocprofiler-register
   rocprofiler-register lib/cmake/rocprofiler-register)
 therock_cmake_subproject_activate(rocprofiler-register)
+
+
+################################################################################
+# rocm-half
+################################################################################
+
+therock_cmake_subproject_declare(rocm-half
+  EXTERNAL_SOURCE_DIR "half"
+  BUILD_DEPS
+    rocm-cmake
+)
+therock_cmake_subproject_glob_c_sources(rocm-half
+  SUBDIRS
+    include
+)
+therock_cmake_subproject_activate(rocm-half)

base/half

@@ -0,0 +1 @@
+../sources/half

build_tools/fetch_sources.py

@@ -20,6 +20,15 @@ def exec(args: list[str | Path], cwd: Path):
     subprocess.check_call(args, cwd=str(cwd), stdin=subprocess.DEVNULL)
 
 
+def get_enabled_projects(args) -> list[str]:
+    projects = list(args.projects)
+    if args.include_math_libs:
+        projects.extend(args.math_lib_projects)
+    if args.include_ml_frameworks:
+        projects.extend(args.ml_framework_projects)
+    return projects
+
+
 def run(args):
     repo_dir: Path = args.dir
     print(f"Setting up repo in {repo_dir}")
@@ -41,7 +50,7 @@ def run(args):
         repo_args,
         cwd=repo_dir,
     )
-    exec(["repo", "sync", "-j16"] + args.projects, cwd=repo_dir)
+    exec(["repo", "sync", "-j16"] + get_enabled_projects(args), cwd=repo_dir)
 
     populate_ancillary_sources(args)
     apply_patches(args)
@@ -62,7 +71,7 @@ def apply_patches(args):
         patch_files = list(patch_project_dir.glob("*.patch"))
         patch_files.sort()
         print(f"Applying {len(patch_files)} patches")
-        exec(["git", "am"] + patch_files, cwd=project_dir)
+        exec(["git", "am", "--whitespace=nowarn"] + patch_files, cwd=project_dir)
 
 
 def populate_ancillary_sources(args):
@@ -113,12 +122,25 @@ def main(argv):
     parser.add_argument(
         "--depth", type=int, help="Git depth to pass to repo", default=None
     )
+    parser.add_argument(
+        "--include-math-libs",
+        default=True,
+        action=argparse.BooleanOptionalAction,
+        help="Include supported math libraries",
+    )
+    parser.add_argument(
+        "--include-ml-frameworks",
+        default=True,
+        action=argparse.BooleanOptionalAction,
+        help="Include machine learning frameworks that are part of ROCM",
+    )
     parser.add_argument(
         "--projects",
         nargs="+",
         type=str,
         default=[
             "clr",
+            "half",
             "HIP",
             "HIPIFY",
             "llvm-project",
@@ -129,10 +151,26 @@ def main(argv):
             "rocminfo",
             "rocprofiler-register",
             "ROCR-Runtime",
-            # Math Libraries
+        ],
+    )
+    parser.add_argument(
+        "--math-lib-projects",
+        nargs="+",
+        type=str,
+        default=[
             "hipBLAS-common",
+            "hipBLAS",
             "hipBLASLt",
             "rocBLAS",
+            "rocRAND",
+        ],
+    )
+    parser.add_argument(
+        "--ml-framework-projects",
+        nargs="+",
+        type=str,
+        default=[
+            "MIOpen",
         ],
     )
     args = parser.parse_args(argv)

cmake/therock_subproject.cmake

@@ -5,6 +5,8 @@
 # of the subprojects are expected to be modified as part of the super-project
 # development flow.
 
+include(ExternalProject)
+
 # Global properties.
 # THEROCK_DEFAULT_CMAKE_VARS:
 # List of CMake variables that will be injected by default into the
@@ -16,6 +18,7 @@ set_property(GLOBAL PROPERTY THEROCK_DEFAULT_CMAKE_VARS
   Python3_EXECUTABLE
   Python3_FIND_VIRTUALENV
   THEROCK_SOURCE_DIR
+  THEROCK_BINARY_DIR
   ROCM_GIT_DIR
   AMDGPU_TARGETS
 )
@@ -29,6 +32,61 @@ if(CMAKE_CXX_VISIBILITY_PRESET)
   list(APPEND THEROCK_DEFAULT_CMAKE_VARS ${CMAKE_CXX_VISIBILITY_PRESET})
 endif()
 
+# therock_subproject_fetch
+# Fetches arbitrary content. This mostly defers to ExternalProject_Add to get
+# content but it performs no actual building.
+# All unrecognized options are passed to ExternalProject_Add.
+# This can interoperate with therock_cmake_subproject_declare by adding the
+# CMAKE_PROJECT option, which makes the CMakeLists.txt in the archive visible
+# to CMake (which the subproject depends on). Additional touch byproducts
+# can be generated with TOUCH.
+function(therock_subproject_fetch target_name)
+  cmake_parse_arguments(
+    PARSE_ARGV 1 ARG
+    "CMAKE_PROJECT"
+    "SOURCE_DIR;EXCLUDE_FROM_ALL;PREFIX"
+    "TOUCH"
+  )
+
+  if(NOT DEFINED ARG_EXCLUDE_FROM_ALL)
+    set(ARG_EXCLUDE_FROM_ALL TRUE)
+  endif()
+  if(NOT DEFINED ARG_PREFIX)
+    set(ARG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/${target_name}_fetch")
+  endif()
+  if(NOT DEFINED ARG_SOURCE_DIR)
+    set(ARG_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/source")
+  endif()
+
+  set(_extra)
+  # In order to interop with therock_cmake_subproject_declare, the CMakeLists.txt
+  # file must exist so we mark this as a by-product. This serves as the dependency
+  # anchor and causes proper ordering of fetch->configure.
+  if(ARG_CMAKE_PROJECT)
+    list(APPEND ARG_TOUCH "${ARG_SOURCE_DIR}/CMakeLists.txt")
+  endif()
+  if(ARG_TOUCH)
+    list(APPEND _extra
+      INSTALL_COMMAND "${CMAKE_COMMAND}" -E touch ${ARG_TOUCH}
+      INSTALL_BYPRODUCTS ${ARG_TOUCH}
+    )
+  else()
+    list(APPEND _extra "INSTALL_COMMAND" "")
+  endif()
+
+  ExternalProject_Add(
+    "${target_name}"
+    EXCLUDE_FROM_ALL "${ARG_EXCLUDE_FROM_ALL}"
+    PREFIX "${ARG_PREFIX}"
+    SOURCE_DIR "${ARG_SOURCE_DIR}"
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    TEST_COMMAND ""
+    ${_extra}
+    ${ARG_UNPARSED_ARGUMENTS}
+  )
+endfunction()
+
 # therock_cmake_subproject_declare
 # This declares a cmake based subproject by setting a number of key properties
 # and setting up boiler-plate targets.
@@ -68,12 +126,13 @@ endif()
 # allows some overlapping of work (controlled by THEROCK_BACKGROUND_BUILD_JOBS).
 # CMAKE_LISTS_RELPATH: Relative path within the source directory to the
 # CMakeLists.txt.
+# EXTRA_DEPENDS: Extra target dependencies to add to the configure command.
 function(therock_cmake_subproject_declare target_name)
   cmake_parse_arguments(
     PARSE_ARGV 1 ARG
     "ACTIVATE;EXCLUDE_FROM_ALL;BACKGROUND_BUILD"
     "EXTERNAL_SOURCE_DIR;BINARY_DIR;DIR_PREFIX;INSTALL_DESTINATION;COMPILER_TOOLCHAIN;INTERFACE_PROGRAM_DIRS;CMAKE_LISTS_RELPATH"
-    "BUILD_DEPS;RUNTIME_DEPS;CMAKE_ARGS;INTERFACE_LINK_DIRS;IGNORE_PACKAGES"
+    "BUILD_DEPS;RUNTIME_DEPS;CMAKE_ARGS;INTERFACE_LINK_DIRS;IGNORE_PACKAGES;EXTRA_DEPENDS"
   )
   if(TARGET "${target_name}")
     message(FATAL_ERROR "Cannot declare subproject '${target_name}': a target with that name already exists")
@@ -179,6 +238,7 @@ function(therock_cmake_subproject_declare target_name)
     THEROCK_COMPILER_TOOLCHAIN "${ARG_COMPILER_TOOLCHAIN}"
     # Any extra depend files that must be added to the configure phase of dependents.
     THEROCK_INTERFACE_CONFIGURE_DEPEND_FILES "${_transitive_configure_depend_files}"
+    THEROCK_EXTRA_DEPENDS "${ARG_EXTRA_DEPENDS}"
   )
 
   if(ARG_ACTIVATE)
@@ -222,6 +282,7 @@ function(therock_cmake_subproject_activate target_name)
   get_target_property(_cmake_source_dir "${target_name}" THEROCK_CMAKE_SOURCE_DIR)
   get_target_property(_exclude_from_all "${target_name}" THEROCK_EXCLUDE_FROM_ALL)
   get_target_property(_external_source_dir "${target_name}" THEROCK_EXTERNAL_SOURCE_DIR)
+  get_target_property(_extra_depends "${target_name}" THEROCK_EXTRA_DEPENDS)
   get_target_property(_ignore_packages "${target_name}" THEROCK_IGNORE_PACKAGES)
   get_target_property(_install_destination "${target_name}" THEROCK_INSTALL_DESTINATION)
   get_target_property(_private_link_dirs "${target_name}" THEROCK_PRIVATE_LINK_DIRS)
@@ -338,7 +399,6 @@ function(therock_cmake_subproject_activate target_name)
       "-G${CMAKE_GENERATOR}"
       "-B${_binary_dir}"
       "-S${_cmake_source_dir}"
-      "-DCPACK_PACKAGING_INSTALL_PREFIX=${STAGING_INSTALL_DIR}"
       "-DCMAKE_INSTALL_PREFIX=${_stage_destination_dir}"
       "-DCMAKE_TOOLCHAIN_FILE=${_cmake_project_toolchain_file}"
       "-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=${_cmake_project_init_file}"
@@ -360,6 +420,7 @@ function(therock_cmake_subproject_activate target_name)
       "${_cmake_project_toolchain_file}"
       "${_cmake_project_init_file}"
       "${_injected_file}"
+      ${_extra_depends}
       ${_dep_provider_file}
       ${_configure_dep_stamps}
       ${_pre_hook_path}

cmake/therock_subproject_dep_provider.cmake

@@ -9,8 +9,11 @@
 #     THEROCK_PROVIDED_PACKAGES, falling back to the system resolver.
 # See: _therock_cmake_subproject_setup_deps which assembles these variables
 macro(therock_dependency_provider method package_name)
+  cmake_policy(PUSH)
+  cmake_policy(SET CMP0057 NEW)
   if("${package_name}" IN_LIST THEROCK_PROVIDED_PACKAGES AND NOT
      "${package_name}" IN_LIST THEROCK_IGNORE_PACKAGES)
+    cmake_policy(POP)
     # It is quite hard to completely neuter find_package so that for an
     # arbitrary signature it will only attempt to find from one specified path.
     # This is important because it "latches" and if any find_package manages
@@ -26,6 +29,7 @@ macro(therock_dependency_provider method package_name)
       "${THEROCK_PACKAGE_DIR_${package_name}}" "${package_name}" ${ARGN})
     find_package(${_therock_rewritten_superproject_find_package_sig})
   else()
+    cmake_policy(POP)
     message(STATUS "Resolving system find_package(${package_name}) (not found in super-project ${THEROCK_PROVIDED_PACKAGES})")
     find_package(${package_name} ${ARGN} BYPASS_PROVIDER)
   endif()

core/CMakeLists.txt

@@ -48,7 +48,11 @@ therock_cmake_subproject_declare(hip-clr
 )
 therock_cmake_subproject_glob_c_sources(hip-clr SUBDIRS .)
 therock_cmake_subproject_provide_package(hip-clr hip lib/cmake/hip)
+# TODO: Some projects resolve "hip" vs "HIP" so we advertise both, but this isn't
+# great.
 therock_cmake_subproject_provide_package(hip-clr HIP lib/cmake/hip)
+therock_cmake_subproject_provide_package(hip-clr hip-lang lib/cmake/hip-lang)
+therock_cmake_subproject_provide_package(hip-clr hiprtc lib/cmake/hiprtc)
 therock_cmake_subproject_activate(hip-clr)
 
 

math-libs/CMakeLists.txt

@@ -98,4 +98,53 @@ therock_cmake_subproject_glob_c_sources(rocBLAS
   SUBDIRS
     .
 )
+therock_cmake_subproject_provide_package(rocBLAS rocblas lib/cmake/rocblas)
 therock_cmake_subproject_activate(rocBLAS)
+
+#################################################################################
+# rocRAND
+#################################################################################
+
+therock_cmake_subproject_declare(rocRAND
+  EXTERNAL_SOURCE_DIR "rocRAND"
+  CMAKE_ARGS
+    -DHIP_PLATFORM=amd
+    -DROCM_PATH=
+    -DROCM_DIR=
+  COMPILER_TOOLCHAIN
+    amd-hip
+  RUNTIME_DEPS
+    hip-clr
+)
+therock_cmake_subproject_glob_c_sources(rocRAND
+  SUBDIRS
+    .
+)
+therock_cmake_subproject_provide_package(rocRAND rocrand lib/cmake/rocrand)
+therock_cmake_subproject_activate(rocRAND)
+
+#################################################################################
+# hipBLAS
+#################################################################################
+
+therock_cmake_subproject_declare(hipBLAS
+  EXTERNAL_SOURCE_DIR "hipBLAS"
+  CMAKE_ARGS
+    -DHIP_PLATFORM=amd
+    -DROCM_PATH=
+    -DROCM_DIR=
+    -DBUILD_WITH_SOLVER=OFF  # TODO: add/enable rocSOLVER
+  COMPILER_TOOLCHAIN
+    amd-hip
+  BUILD_DEPS
+    hipBLAS-common
+  RUNTIME_DEPS
+    hip-clr
+    rocBLAS
+)
+therock_cmake_subproject_glob_c_sources(hipBLAS
+  SUBDIRS
+    .
+)
+therock_cmake_subproject_provide_package(hipBLAS hipblas lib/cmake/hipblas)
+therock_cmake_subproject_activate(hipBLAS)

math-libs/hipBLAS

@@ -0,0 +1 @@
+../sources/hipBLAS

math-libs/post_hook_hipBLAS.cmake

@@ -0,0 +1,9 @@
+list(APPEND CMAKE_MODULE_PATH "${THEROCK_SOURCE_DIR}/cmake")
+include(therock_rpath)
+
+therock_set_install_rpath(
+  TARGETS
+    hipblas
+  PATHS
+    .
+)

math-libs/post_hook_rocRAND.cmake

@@ -0,0 +1,9 @@
+list(APPEND CMAKE_MODULE_PATH "${THEROCK_SOURCE_DIR}/cmake")
+include(therock_rpath)
+
+therock_set_install_rpath(
+  TARGETS
+    rocrand
+  PATHS
+    .
+)

math-libs/rocRAND

@@ -0,0 +1 @@
+../sources/rocRAND