Demo: Testing pybind11-stubgen + mypy

.appveyor.yml

@@ -17,7 +17,7 @@ install:
     if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" }
     $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH"
     python -W ignore -m pip install --upgrade pip wheel
-    python -W ignore -m pip install pytest numpy --no-warn-script-location pytest-timeout
+    python -W ignore -m pip install --no-warn-script-location -r tests/requirements.txt
 - ps: |
     Start-FileDownload 'https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.zip'
     7z x eigen-3.3.7.zip -y > $null

.github/workflows/ci.yml

@@ -343,8 +343,17 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Add wget and python3
-      run: apt-get update && apt-get install -y python3-dev python3-numpy python3-pytest libeigen3-dev
+    - name: Add python3
+      run: apt-get update && apt-get install -y python3-dev libeigen3-dev
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        activate-environment: true
+        enable-cache: true
+
+    - name: Prepare env
+      run: uv pip install -r tests/requirements.txt
 
     - name: Configure
       shell: bash
@@ -380,7 +389,15 @@ jobs:
 
     # tzdata will try to ask for the timezone, so set the DEBIAN_FRONTEND
     - name: Install 🐍 3
-      run: apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y cmake git python3-dev python3-pytest python3-numpy
+      run: apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y cmake git python3-dev
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
+
+    - name: Prepare env
+      run: uv pip install --python=python3 --system -r tests/requirements.txt
 
     - name: Configure
       run: cmake -S . -B build -DPYBIND11_CUDA_TESTS=ON -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON
@@ -455,11 +472,11 @@ jobs:
     - name: Install 🐍 3 & NVHPC
       run: |
         sudo apt-get update -y && \
-        sudo apt-get install -y cmake environment-modules git python3-dev python3-pip python3-numpy && \
+        sudo apt-get install -y cmake environment-modules git python3-dev python3-pip && \
         sudo apt-get install -y --no-install-recommends nvhpc-23-5 && \
         sudo rm -rf /var/lib/apt/lists/*
         python3 -m pip install --upgrade pip
-        python3 -m pip install --upgrade pytest
+        python3 -m pip install -r tests/requirements.txt
 
     # On some systems, you many need further workarounds:
     # https://github.com/pybind/pybind11/pull/2475
@@ -506,10 +523,16 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Add Python 3
-      run: apt-get update; apt-get install -y python3-dev python3-numpy python3-pytest python3-pip libeigen3-dev
+      run: apt-get update; apt-get install -y python3-dev libeigen3-dev
 
-    - name: Update pip
-      run: python3 -m pip install --upgrade pip
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        activate-environment: true
+        enable-cache: true
+
+    - name: Prepare env
+      run: uv pip install -r tests/requirements.txt
 
     - name: Update CMake
       uses: jwlawson/[email protected]
@@ -724,7 +747,7 @@ jobs:
       run: |
         apt-get update
         apt-get install -y git make cmake g++ libeigen3-dev python3-dev python3-pip
-        pip3 install "pytest==6.*"
+        pip3 install -r tests/requirements.txt
 
     - name: Configure for install
       run: >
@@ -992,6 +1015,10 @@ jobs:
 
     - uses: actions/checkout@v4
 
+    - name: Prepare env
+      run: python -m pip install -r tests/requirements.txt
+
+
     - name: Configure C++11
       # LTO leads to many undefined reference like
       # `pybind11::detail::function_call::function_call(pybind11::detail::function_call&&)

tests/CMakeLists.txt

@@ -169,6 +169,8 @@ set(PYBIND11_TEST_FILES
     test_smart_ptr
     test_stl
     test_stl_binders
+    test_stubgen
+    test_stubgen_error
     test_tagbased_polymorphic
     test_thread
     test_type_caster_pyobject_ptr

tests/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "scikit_build_core.build"
 [project]
 name = "pybind11_tests"
 version = "0.0.1"
-dependencies = ["pytest", "pytest-timeout", "numpy", "scipy"]
+dependencies = ["pytest", "pytest-timeout", "numpy", "scipy", "pybind11-stubgen", "mypy"]
 
 [tool.scikit-build.cmake.define]
 PYBIND11_FINDPYTHON = true

tests/requirements.txt

@@ -1,6 +1,7 @@
 --extra-index-url=https://www.graalvm.org/python/wheels
 --only-binary=:all:
 build>=1
+mypy
 numpy~=1.23.0; python_version=="3.8" and platform_python_implementation=="PyPy"
 numpy~=1.25.0; python_version=="3.9" and platform_python_implementation=="PyPy"
 numpy~=2.2.0; python_version=="3.10" and platform_python_implementation=="PyPy"
@@ -9,6 +10,7 @@ numpy~=1.21.5; platform_python_implementation=="CPython" and python_version>="3.
 numpy~=1.22.2; platform_python_implementation=="CPython" and python_version=="3.10"
 numpy~=1.26.0; platform_python_implementation=="CPython" and python_version>="3.11" and python_version<"3.13"
 numpy~=2.2.0; platform_python_implementation=="CPython" and python_version=="3.13"
+pybind11-stubgen
 pytest>=6
 pytest-timeout
 scipy~=1.5.4; platform_python_implementation=="CPython" and python_version<"3.10"

tests/test_stubgen.cpp

@@ -0,0 +1,5 @@
+#include "pybind11_tests.h"
+
+TEST_SUBMODULE(stubgen, m) {
+    m.def("add_int", [](int a, int b) { return a + b; }, "a"_a, "b"_a);
+}

tests/test_stubgen.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pybind11_stubgen
+import pytest
+from mypy import api
+
+from pybind11_tests import stubgen as m
+
+
+@pytest.mark.xfail(
+    sys.version_info >= (3, 14), reason="mypy does not support Python 3.14+ yet"
+)
+def test_stubgen(tmp_path: Path) -> None:
+    assert m.add_int(1, 2) == 3
+    # Generate stub into temporary directory
+    pybind11_stubgen.main(
+        [
+            "pybind11_tests.stubgen",
+            "-o",
+            tmp_path.as_posix(),
+        ]
+    )
+    # Check stub file is generated and contains expected content
+    stub_file = tmp_path / "pybind11_tests" / "stubgen.pyi"
+    assert stub_file.exists()
+    stub_content = stub_file.read_text()
+    assert (
+        "def add_int(a: typing.SupportsInt, b: typing.SupportsInt) -> int:"
+        in stub_content
+    )
+    # Run mypy on the generated stub file
+    normal_report, error_report, exit_status = api.run([stub_file.as_posix()])
+    print("Normal report:")
+    print(normal_report)
+    print("Error report:")
+    print(error_report)
+    assert exit_status == 0
+    assert "Success: no issues found in 1 source file" in normal_report

tests/test_stubgen_error.cpp

@@ -0,0 +1,5 @@
+#include "pybind11_tests.h"
+
+TEST_SUBMODULE(stubgen_error, m) {
+    m.def("identity_capsule", [](py::capsule c) { return c; }, "c"_a);
+}

tests/test_stubgen_error.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pybind11_stubgen
+import pytest
+from mypy import api
+
+from pybind11_tests import stubgen_error as m
+
+
+@pytest.mark.skipif(
+    sys.version_info >= (3, 13), reason="CapsuleType available in 3.13+"
+)
+def test_stubgen(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
+    """Show stubgen/mypy errors for CapsuleType (not available in Python < 3.13)."""
+    assert hasattr(m, "identity_capsule")
+    # Generate stub into temporary directory
+    pybind11_stubgen.main(
+        [
+            "pybind11_tests.stubgen_error",
+            "-o",
+            tmp_path.as_posix(),
+        ]
+    )
+    # Errors are reported using logging
+    assert "Can't find/import 'types.CapsuleType'" in caplog.text
+    # Stub file is still generated if error is not fatal
+    stub_file = tmp_path / "pybind11_tests" / "stubgen_error.pyi"
+    assert stub_file.exists()
+    stub_content = stub_file.read_text()
+    assert (
+        "def identity_capsule(c: types.CapsuleType) -> types.CapsuleType:"
+        in stub_content
+    )
+    # Run mypy on the generated stub file
+    # normal_report -> stdout, error_report -> stderr
+    # Type errors seem to go into normal_report
+    normal_report, error_report, exit_status = api.run(
+        [stub_file.as_posix(), "--no-color-output"]
+    )
+    print("Normal report:")
+    print(normal_report)
+    print("Error report:")
+    print(error_report)
+    assert exit_status == 1
+    assert 'error: Name "types" is not defined  [name-defined]' in normal_report