Opt into free-threaded Python (iree-org#18770)

With this patch the compiler and runtime opt into free-threaded Python
(aka. "nogil") which was added with Python 3.13. For more information
about free-threaded Python see https://py-free-threading.github.io/.

The patch
* Bumps the pybind11 submodule to version 2.13.6 and pins the pip
installed version
* Pins nanobind to version 2.2.0
* Enables building with free-threading support

CMakeLists.txt

@@ -645,7 +645,7 @@ if(IREE_BUILD_PYTHON_BINDINGS)
   # Some parts of the build use FindPython instead of FindPython3. Why? No
   # one knows, but they are different. So make sure to bootstrap this one too.
   # Not doing this here risks them diverging, which on multi-Python systems,
-  # can be troublesome. Note that nanobind requires FindPython.
+  # can be troublesome. Note that pybind11 and nanobind require FindPython.
   set(Python_EXECUTABLE "${Python3_EXECUTABLE}")
   find_package(Python 3.9 COMPONENTS Interpreter Development.Module NumPy REQUIRED)
 elseif(IREE_BUILD_COMPILER OR IREE_BUILD_TESTS)
@@ -926,6 +926,7 @@ if(IREE_BUILD_PYTHON_BINDINGS)
   if(IREE_BUILD_COMPILER)
     if(NOT TARGET pybind11::module)
       message(STATUS "Using bundled pybind11")
+      set(PYBIND11_FINDPYTHON ON)
       add_subdirectory(third_party/pybind11 EXCLUDE_FROM_ALL)
     else()
       message(STATUS "Not including bundled pybind11 (already configured)")

compiler/bindings/python/IREECompilerRegistration.cpp

@@ -21,7 +21,7 @@ class GlobalInitializer {
 
 } // namespace
 
-PYBIND11_MODULE(_site_initialize_0, m) {
+PYBIND11_MODULE(_site_initialize_0, m, py::mod_gil_not_used()) {
   m.doc() = "iree-compile registration";
 
   // Make sure that GlobalInitialize and GlobalShutdown are called with module

compiler/pyproject.toml

@@ -6,12 +6,12 @@ requires = [
     # Note that the compiler wheel does not presently need nanobind, but
     # it's build is enabled by the same flag which enables the runtime
     # configuration, which does.
-    "nanobind==2.0",
+    "nanobind==2.2.0",
     "ninja",
     # MLIR build depends.
     "numpy",
     "packaging",
-    "pybind11>=2.10.1",
+    "pybind11==2.13.6",
     "sympy",
 ]
 build-backend = "setuptools.build_meta"

llvm-external-projects/iree-dialects/python/IREEDialectsModule.cpp

@@ -14,7 +14,7 @@
 namespace py = pybind11;
 using namespace mlir::python::adaptors;
 
-PYBIND11_MODULE(_ireeDialects, m) {
+PYBIND11_MODULE(_ireeDialects, m, py::mod_gil_not_used()) {
   m.doc() = "iree-dialects main python extension";
 
   auto irModule = py::module::import(MAKE_MLIR_PYTHON_QUALNAME("ir"));

runtime/bindings/python/CMakeLists.txt

@@ -59,7 +59,7 @@ iree_select_compiler_opts(_RTTI_AND_EXCEPTION_COPTS
 )
 
 nanobind_add_module(iree_runtime_bindings_python_PyExtRt
-  NB_STATIC LTO
+  NB_STATIC LTO FREE_THREADED
   "binding.h"
   "initialize_module.cc"
   "invoke.h"

runtime/bindings/python/iree/runtime/build_requirements.txt

@@ -5,12 +5,12 @@
 
 pip>=21.3
 setuptools>=62.4.0
-nanobind==2.0
+nanobind==2.2.0
 numpy>=2.0.0b1
 requests>=2.28.0
 wheel>=0.36.2
 sympy==1.12.1
 
 # TODO: nanobind is used in the runtime but the compiler uses pybind and
 # removing this breaks CI bots; remove this.
-pybind11>=2.10.1
+pybind11==2.13.6

runtime/pyproject.toml

@@ -3,7 +3,7 @@ requires = [
     "setuptools>=42",
     "wheel",
     "cmake",
-    "nanobind==2.0",
+    "nanobind==2.2.0",
     "ninja",
     "numpy>=2.0.0b1",
     "packaging",