Skip to content

shared_ptrs to Python-derived instances outlive the associated Python object #1546

Closed
@cdyson37

Description

@cdyson37

Issue description

Storing shared_ptrs to Python-derived instances for later execution causes virtual functions to fail if the Python instance has subsequently been destroyed.

Tested on master: e7761e3 / Python 3.6 / gcc 7.2

Reproducible example code

C++

#include <pybind11/pybind11.h>

namespace py = pybind11;

struct Base
{
  virtual void f (int i) const = 0;
  virtual ~Base () { };
};

struct Trampoline : Base
{
  void f (int i) const override
  {
    PYBIND11_OVERLOAD_PURE_NAME (
      void,
      Base,
      "f",
      impl,
      /* args */
      i);
  }
};

struct Holder
{
  Holder (std::shared_ptr<Base> ptr)
  : ptr (std::move (ptr))
  { }

  std::shared_ptr<Base> ptr;
};

void takes_holder (const Holder &h)
{
  h.ptr->f (42);
}

PYBIND11_MODULE (testmodule, m)
{
  py::class_<Base, Trampoline, std::shared_ptr<Base>>
    base_wrap (m, "Base");

  base_wrap.def (py::init<> ());

  py::class_<Holder> holder_wrap (m, "Holder");
  holder_wrap.def (py::init<std::shared_ptr<Base>> ());

  m.def ("takes_holder", &takes_holder);
}

Python

import testmodule

class PythonDerived (testmodule.Base):
    def __init__ (self):
        testmodule.Base.__init__ (self)

    def f (self, i):
        print ("Called with: {}".format (i))


pd = PythonDerived ()
holder = testmodule.Holder (pd)
del pd # Comment this out and it works
testmodule.takes_holder (holder)

Output

Traceback (most recent call last):
  File "./test.py", line 16, in <module>
    testmodule.takes_holder (holder)
RuntimeError: Tried to call pure virtual function "Base::f"

Output (with 'del' removed)

Called with: 42

Workaround:

Add the following (tweaked by @iwanders - thanks)

namespace pybind11::detail {

  template<>
  struct type_caster<std::shared_ptr<Base>>
  {
    PYBIND11_TYPE_CASTER (std::shared_ptr<Base>, _("Base"));

    using BaseCaster = copyable_holder_caster<Base, std::shared_ptr<Base>>;

    bool load (pybind11::handle src, bool b)
    {
      BaseCaster bc;
      bool success = bc.load (src, b);
      if (!success)
      {
        return false;
      }

      auto py_obj = py::reinterpret_borrow<py::object> (src);
      auto base_ptr = static_cast<std::shared_ptr<Base>> (bc);

      // Construct a shared_ptr to the py::object
      auto py_obj_ptr = std::shared_ptr<object>{
          new object{py_obj},
          [](auto py_object_ptr) {
              // It's possible that when the shared_ptr dies we won't have the
              // gil (if the last holder is in a non-Python thread), so we
              // make sure to acquire it in the deleter.
              gil_scoped_acquire gil;
              delete py_object_ptr;
         }
      };

      value = std::shared_ptr<Base> (py_obj_ptr, base_ptr.get ());
      return true;
    }

    static handle cast (std::shared_ptr<Base> base,
                        return_value_policy rvp,
                        handle h)
    {
      return BaseCaster::cast (base, rvp, h);
    }
  };

  template <>
  struct is_holder_type<Base, std::shared_ptr<Base>> : std::true_type {};
}

This replaces the default shared_ptr caster with one that keeps the Python object alive, using the aliasing constructor of shared_ptr.

Comment

Ideally, whenever there's a shared_ptr to a Python-derived object, that shared_ptr should keep the Python side of the object alive. The above seems to do that, but there may be a more sensible approach!

I also tried a workaround using

  python_wrap.def ("__init__", [] (py::detail::value_and_holder &v_h) {

and constructing a std::shared_ptr in a similar fashion, but I found that I couldn't capture "self" without passing it in as an additional argument (i.e. testmodule.Base.__init__ (self, self)).

I've also tried patching cast.h (below) as a proof-of-concept, and that appears to work (thanks @iwanders for the tweak)

diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index 5529768..e976129 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -10,6 +10,7 @@
 
 #pragma once
 
+#include "pybind11.h"
 #include "pytypes.h"
 #include "detail/typeid.h"
 #include "detail/descr.h"
@@ -1496,9 +1497,55 @@ protected:
     holder_type holder;
 };
 
-/// Specialize for the common std::shared_ptr, so users don't need to
+/// Specialize type_caster for std::shared_ptr<T>.
+/// This is the same as copyable_holder_caster, except that when casting to C++
+/// we keep the Python object alive through the shared_ptr as e.g. virtual
+/// functions and derived state might be defined there.
 template <typename T>
-class type_caster<std::shared_ptr<T>> : public copyable_holder_caster<T, std::shared_ptr<T>> { };
+class type_caster<std::shared_ptr<T>>
+{
+    PYBIND11_TYPE_CASTER(std::shared_ptr<T>, _(PYBIND11_STRING_NAME));
+
+    // Re-use copyable_holder_caster
+    using BaseCaster = copyable_holder_caster<T, std::shared_ptr<T>>;
+
+    bool load(pybind11::handle src, bool b)
+    {
+        BaseCaster bc;
+        bool success = bc.load(src, b);
+        if (!success) {
+            return false;
+        }
+
+        // Get src as a py::object
+        auto py_obj = reinterpret_borrow<object>(src);
+
+        // Construct a shared_ptr to the py::object
+        auto py_obj_ptr = std::shared_ptr<object>{
+            new object{py_obj},
+            [](auto py_object_ptr) {
+                // It's possible that when the shared_ptr dies we won't have the
+                // gil (if the last holder is in a non-Python thread), so we
+                // make sure to acquire it in the deleter.
+                gil_scoped_acquire gil;
+                delete py_object_ptr;
+           }
+        };
+
+        // * Use BaseCaster to get it as the shared_ptr<T>
+        // * Use this to make an aliased shared_ptr<T> that keeps the py::object alive
+        auto base_ptr = static_cast<std::shared_ptr<T>>(bc);
+        value = std::shared_ptr<T>(py_obj_ptr, base_ptr.get());
+        return true;
+    }
+
+    static handle cast(std::shared_ptr<T> sp,
+                       return_value_policy rvp,
+                       handle h)
+    {
+        return BaseCaster::cast(sp, rvp, h);
+    }
+};
 
 template <typename type, typename holder_type>
 struct move_only_holder_caster {
@@ -1540,6 +1587,9 @@ template <typename base, typename holder> struct is_holder_type :
 template <typename base, typename deleter> struct is_holder_type<base, std::unique_ptr<base, deleter>> :
     std::true_type {};
 
+template <typename T>
+struct is_holder_type<T, std::shared_ptr<T>> : std::true_type {};
+
 template <typename T> struct handle_type_name { static constexpr auto name = _<T>(); };
 template <> struct handle_type_name<bytes> { static constexpr auto name = _(PYBIND11_BYTES_NAME); };
 template <> struct handle_type_name<args> { static constexpr auto name = _("*args"); };
diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h
index fa5ed7c..fe88fcc 100644
--- a/include/pybind11/pytypes.h
+++ b/include/pybind11/pytypes.h
@@ -20,6 +20,8 @@ NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 class handle; class object;
 class str; class iterator;
 struct arg; struct arg_v;
+class gil_scoped_acquire;
+class gil_scoped_release;
 
 NAMESPACE_BEGIN(detail)
 class args_proxy;

Metadata

Metadata

Assignees

No one assigned

    Labels

    holdersIssues and PRs regarding holders

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions