From 9c5b57a8b9e6981e300df02c41a296bd49e07c99 Mon Sep 17 00:00:00 2001 From: Stella Laurenzo Date: Wed, 23 Oct 2024 22:57:45 -0700 Subject: [PATCH] Use FetchContent for both pybind11 and nanobind. (#18872) This avoids various pinning problems and ensures that the version that IREE specifies at the top of a build is used consistently throughout. The nanobind incantation was taken from shortfin. The pybind11 incantation was adapted from [a comment](https://github.com/pybind/pybind11/issues/2817#issuecomment-766392922) and extended to use find_package integration. The latter is sufficient for MLIR's pybind11 finding to delegate to the one set at the top level. The MLIR code for finding pybind11 is ancient and should be modernized to use FetchContent and find_package integration with a pinned version. This would ensure consistent interop with the rest of the ecosystem. --------- Signed-off-by: Stella Laurenzo Co-authored-by: Marius Brehler --- .gitmodules | 4 -- CMakeLists.txt | 37 ++++++++++++------- compiler/pyproject.toml | 5 --- runtime/bindings/python/CMakeLists.txt | 23 ++++-------- .../iree/runtime/build_requirements.txt | 5 --- runtime/pyproject.toml | 1 - third_party/pybind11 | 1 - 7 files changed, 32 insertions(+), 44 deletions(-) delete mode 160000 third_party/pybind11 diff --git a/.gitmodules b/.gitmodules index 58e22edefcec..2c6c117e9c57 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,10 +7,6 @@ [submodule "third_party/vulkan_headers"] path = third_party/vulkan_headers url = https://github.com/KhronosGroup/Vulkan-Headers.git -[submodule "third_party/pybind11"] - path = third_party/pybind11 - url = https://github.com/pybind/pybind11.git - branch = stable [submodule "third_party/benchmark"] path = third_party/benchmark url = https://github.com/google/benchmark.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e34ee8b05bd..a8a9b70bfa44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -772,6 +772,30 @@ endif() # MLIR/LLVM Dependency #------------------------------------------------------------------------------- +# Both the IREE and MLIR Python bindings require pybind11. We initialize it here +# at the top level so that everything uses ours consistently. +if(IREE_BUILD_PYTHON_BINDINGS AND IREE_BUILD_COMPILER) + set(pybind11_VERSION 2.13.6) + include(FetchContent) + FetchContent_Declare( + pybind11 + GIT_REPOSITORY https://github.com/pybind/pybind11 + GIT_TAG v${pybind11_VERSION} + ) + set(PYBIND11_FINDPYTHON ON) + FetchContent_MakeAvailable(pybind11) + # pybind11 source fetches do not include find_package integration, which is + # a shame since sub-projects can require that to work. If we were using + # CMake 3.24, we could just add OVERRIDE_FIND_PACKAGE to the + # FetchContent_Declare call above and it would take care of doing the + # following to let subsequent sub-project find_package calls to resolve + # successfully. + set(pybind11_DIR "${pybind11_BINARY_DIR}") + file(WRITE "${pybind11_BINARY_DIR}/pybind11Config.cmake" "") + file(WRITE "${pybind11_BINARY_DIR}/pybind11ConfigVersion.cmake" + "set(PACKAGE_VERSION ${pybind11_VERSION})\nset(PACKAGE_VERSION_COMPATIBLE TRUE)") +endif() + if(NOT IREE_BUILD_COMPILER) message(STATUS "Not adding LLVM/MLIR because the configuration does not require it") else() @@ -921,19 +945,6 @@ if(IREE_BUILD_TESTS) include(iree_configure_testing) endif() -if(IREE_BUILD_PYTHON_BINDINGS) - # The compiler uses pybind11 - if(IREE_BUILD_COMPILER) - if(NOT TARGET pybind11::module) - message(STATUS "Using bundled pybind11") - set(PYBIND11_FINDPYTHON ON) - add_subdirectory(third_party/pybind11 EXCLUDE_FROM_ALL) - else() - message(STATUS "Not including bundled pybind11 (already configured)") - endif() - endif() -endif() - if(IREE_TARGET_BACKEND_METAL_SPIRV) # SPIRV-Cross is needed to cross compile SPIR-V into MSL source code. iree_set_spirv_cross_cmake_options() diff --git a/compiler/pyproject.toml b/compiler/pyproject.toml index 5a07bd90cc5e..b7a4fd4de382 100644 --- a/compiler/pyproject.toml +++ b/compiler/pyproject.toml @@ -3,15 +3,10 @@ requires = [ "setuptools>=42", "wheel", "cmake", - # Note that the compiler wheel does not presently need nanobind, but - # it's build is enabled by the same flag which enables the runtime - # configuration, which does. - "nanobind==2.2.0", "ninja", # MLIR build depends. "numpy", "packaging", - "pybind11==2.13.6", "sympy", ] build-backend = "setuptools.build_meta" diff --git a/runtime/bindings/python/CMakeLists.txt b/runtime/bindings/python/CMakeLists.txt index 5ba7b84fbcf9..35a85d190a27 100644 --- a/runtime/bindings/python/CMakeLists.txt +++ b/runtime/bindings/python/CMakeLists.txt @@ -4,21 +4,14 @@ # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -if(NOT nanobind_FOUND) - find_package(nanobind CONFIG QUIET) - if(NOT nanobind_FOUND) - execute_process( - COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir - OUTPUT_STRIP_TRAILING_WHITESPACE - OUTPUT_VARIABLE NB_DIR - RESULT_VARIABLE RC) - if(RC AND NOT RC EQUAL 0) - message(WARNING "Probing for nanobind failed. Please install the project's Python dependencies or '${Python_EXECUTABLE} -m pip install nanobind'") - endif() - list(APPEND CMAKE_PREFIX_PATH "${NB_DIR}") - endif() - find_package(nanobind CONFIG REQUIRED) -endif() +# nanobind +include(FetchContent) +FetchContent_Declare( + nanobind + GIT_REPOSITORY https://github.com/wjakob/nanobind.git + GIT_TAG 784efa2a0358a4dc5432c74f5685ee026e20f2b6 # 2.2.0 +) +FetchContent_MakeAvailable(nanobind) set(_EXTRA_INSTALL_TOOL_TARGETS) set(_TRACY_ENABLED OFF) diff --git a/runtime/bindings/python/iree/runtime/build_requirements.txt b/runtime/bindings/python/iree/runtime/build_requirements.txt index e3f07b20f0bb..0a8dca312c7d 100644 --- a/runtime/bindings/python/iree/runtime/build_requirements.txt +++ b/runtime/bindings/python/iree/runtime/build_requirements.txt @@ -5,12 +5,7 @@ pip>=21.3 setuptools>=62.4.0 -nanobind==2.2.0 numpy>=2.0.0b1 requests>=2.28.0 wheel>=0.36.2 sympy==1.12.1 - -# TODO: nanobind is used in the runtime but the compiler uses pybind and -# removing this breaks CI bots; remove this. -pybind11==2.13.6 diff --git a/runtime/pyproject.toml b/runtime/pyproject.toml index 3259f736c8db..16567a601072 100644 --- a/runtime/pyproject.toml +++ b/runtime/pyproject.toml @@ -3,7 +3,6 @@ requires = [ "setuptools>=42", "wheel", "cmake", - "nanobind==2.2.0", "ninja", "numpy>=2.0.0b1", "packaging", diff --git a/third_party/pybind11 b/third_party/pybind11 deleted file mode 160000 index a2e59f0e7065..000000000000 --- a/third_party/pybind11 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a2e59f0e7065404b44dfe92a28aca47ba1378dc4