Skip to content

Commit bfd48c3

Browse files
p-wysockip-wysocki
authored
[PyOV][Free threaded] Always attach PyThreadState in multithreading contexts (#32988)
Backport of #32853 ### Details: - `ConditionalGILScopedAcquire` and similar were implemented under an assumption that no op is required when we're using GIL-free Python - Actually `py::gil_scoped_acquire` is necessary, because when using free threaded Python it ignores GIL, but it's also responsible for attaching Python's `PyThreadState` to threads - `py::gil_scoped_acquire` name is misleading - `PyThreadState` is required for thread interaction (sort of like an interface) - Without it, whenever a C++ process would interact with a Python process' resources, it would segfault - Removing `std::monostate` usage allows for proper Python thread management - This PR fixes everything that doesn't work with free threaded Python, but an additional PR will soon be opened with cleanup of the rest of the codebase ### Tickets: - CVS-171534 --------- Signed-off-by: p-wysocki <[email protected]>
1 parent 66fb447 commit bfd48c3

File tree

16 files changed

+63
-85
lines changed

16 files changed

+63
-85
lines changed

‎src/bindings/python/src/pyopenvino/core/async_infer_queue.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ class AsyncInferQueue {
4646

4747
bool _is_ready() {
4848
// Check if any request has finished already
49-
ConditionalGILScopedRelease release;
49+
py::gil_scoped_release release;
5050
// acquire the mutex to access m_errors and m_idle_handles
5151
std::lock_guard<std::mutex> lock(m_mutex);
5252
if (m_errors.size() > 0)
@@ -57,7 +57,7 @@ class AsyncInferQueue {
5757
size_t get_idle_request_id() {
5858
// Wait for any request to complete and return its id
5959
// release GIL to avoid deadlock on python callback
60-
ConditionalGILScopedRelease release;
60+
py::gil_scoped_release release;
6161
// acquire the mutex to access m_errors and m_idle_handles
6262
std::unique_lock<std::mutex> lock(m_mutex);
6363
m_cv.wait(lock, [this] {
@@ -74,7 +74,7 @@ class AsyncInferQueue {
7474
void wait_all() {
7575
// Wait for all request to complete
7676
// release GIL to avoid deadlock on python callback
77-
ConditionalGILScopedRelease release;
77+
py::gil_scoped_release release;
7878
for (auto&& request : m_requests) {
7979
request.m_request->wait();
8080
}
@@ -118,8 +118,8 @@ class AsyncInferQueue {
118118
m_requests[handle].m_request->set_callback([this, callback_sp, handle](std::exception_ptr exception_ptr) {
119119
*m_requests[handle].m_end_time = Time::now();
120120
if (exception_ptr == nullptr) {
121-
// Acquire GIL, execute Python function
122-
ConditionalGILScopedAcquire acquire;
121+
// For free-threaded Python, gil_scoped_acquire still ensures thread is attached
122+
py::gil_scoped_acquire acquire;
123123
try {
124124
(*callback_sp)(m_requests[handle], m_user_ids[handle]);
125125
} catch (const py::error_already_set& py_error) {
@@ -200,7 +200,7 @@ void regclass_AsyncInferQueue(py::module m) {
200200
self.m_requests[handle].m_request->set_input_tensor(inputs);
201201
// Now GIL can be released - we are NOT working with Python objects in this block
202202
{
203-
ConditionalGILScopedRelease release;
203+
py::gil_scoped_release release;
204204
*self.m_requests[handle].m_start_time = Time::now();
205205
// Start InferRequest in asynchronus mode
206206
self.m_requests[handle].m_request->start_async();
@@ -246,7 +246,7 @@ void regclass_AsyncInferQueue(py::module m) {
246246
Common::set_request_tensors(*self.m_requests[handle].m_request, inputs);
247247
// Now GIL can be released - we are NOT working with Python objects in this block
248248
{
249-
ConditionalGILScopedRelease release;
249+
py::gil_scoped_release release;
250250
*self.m_requests[handle].m_start_time = Time::now();
251251
// Start InferRequest in asynchronus mode
252252
self.m_requests[handle].m_request->start_async();

‎src/bindings/python/src/pyopenvino/core/common.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -419,7 +419,7 @@ std::shared_ptr<ov::SharedBuffer<py::array>> get_shared_memory(py::array& array)
419419
array.ndim() == 0 ? array.itemsize() : array.nbytes(),
420420
array);
421421
std::shared_ptr<ov::SharedBuffer<py::array>> memory(buffer, [](ov::SharedBuffer<py::array>* buffer) {
422-
ConditionalGILScopedAcquire acquire;
422+
py::gil_scoped_acquire acquire;
423423
delete buffer;
424424
});
425425
return memory;

‎src/bindings/python/src/pyopenvino/core/compiled_model.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ void regclass_CompiledModel(py::module m) {
3030
// Create temporary ov::InferRequest and move it to actual wrapper class.
3131
ov::InferRequest request;
3232
{
33-
ConditionalGILScopedRelease release;
33+
py::gil_scoped_release release;
3434
request = self.create_infer_request();
3535
}
3636
return std::make_shared<InferRequestWrapper>(std::move(request), self.inputs(), self.outputs());
@@ -171,7 +171,7 @@ void regclass_CompiledModel(py::module m) {
171171

172172
cls.def("get_runtime_model",
173173
&ov::CompiledModel::get_runtime_model,
174-
CallGuardConditionalGILRelease(),
174+
py::call_guard<py::gil_scoped_release>(),
175175
R"(
176176
Gets runtime model information from a device.
177177
@@ -186,7 +186,7 @@ void regclass_CompiledModel(py::module m) {
186186

187187
cls.def("release_memory",
188188
&ov::CompiledModel::release_memory,
189-
CallGuardConditionalGILRelease(),
189+
py::call_guard<py::gil_scoped_release>(),
190190
R"(
191191
Release intermediate memory.
192192

‎src/bindings/python/src/pyopenvino/core/core.cpp‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ void regclass_Core(py::module m) {
157157
const std::string& device_name,
158158
const std::map<std::string, py::object>& properties) {
159159
auto _properties = Common::utils::properties_to_any_map(properties);
160-
ConditionalGILScopedRelease release;
160+
py::gil_scoped_release release;
161161
return self.compile_model(model, device_name, _properties);
162162
},
163163
py::arg("model"),
@@ -186,7 +186,7 @@ void regclass_Core(py::module m) {
186186
const std::shared_ptr<const ov::Model>& model,
187187
const std::map<std::string, py::object>& properties) {
188188
auto _properties = Common::utils::properties_to_any_map(properties);
189-
ConditionalGILScopedRelease release;
189+
py::gil_scoped_release release;
190190
return self.compile_model(model, _properties);
191191
},
192192
py::arg("model"),
@@ -214,7 +214,7 @@ void regclass_Core(py::module m) {
214214
const std::map<std::string, py::object>& properties) {
215215
auto _properties = Common::utils::properties_to_any_map(properties);
216216
std::string path = Common::utils::convert_path_to_string(model_path);
217-
ConditionalGILScopedRelease release;
217+
py::gil_scoped_release release;
218218
return self.compile_model(path, device_name, _properties);
219219
},
220220
py::arg("model_path"),
@@ -259,7 +259,7 @@ void regclass_Core(py::module m) {
259259
tensor = ov::Tensor(ov::element::Type_t::u8, {bin_size});
260260
}
261261
auto _properties = Common::utils::properties_to_any_map(properties);
262-
ConditionalGILScopedRelease release;
262+
py::gil_scoped_release release;
263263
return self.compile_model(model.cast<std::string>(), tensor, device_name, _properties);
264264
},
265265
py::arg("model_buffer"),
@@ -290,7 +290,7 @@ void regclass_Core(py::module m) {
290290
[](ov::Core& self, const py::object& model_path, const std::map<std::string, py::object>& properties) {
291291
auto _properties = Common::utils::properties_to_any_map(properties);
292292
std::string path = Common::utils::convert_path_to_string(model_path);
293-
ConditionalGILScopedRelease release;
293+
py::gil_scoped_release release;
294294
return self.compile_model(path, _properties);
295295
},
296296
py::arg("model_path"),
@@ -317,7 +317,7 @@ void regclass_Core(py::module m) {
317317
const RemoteContextWrapper& context,
318318
const std::map<std::string, py::object>& properties) {
319319
auto _properties = Common::utils::properties_to_any_map(properties);
320-
ConditionalGILScopedRelease release;
320+
py::gil_scoped_release release;
321321
return self.compile_model(model, context.context, _properties);
322322
},
323323
py::arg("model"),
@@ -398,7 +398,7 @@ void regclass_Core(py::module m) {
398398
const uint8_t* bin = reinterpret_cast<const uint8_t*>(info.ptr);
399399
std::memcpy(tensor.data(), bin, bin_size);
400400
}
401-
ConditionalGILScopedRelease release;
401+
py::gil_scoped_release release;
402402
return self.read_model(ir, tensor);
403403
},
404404
py::arg("model"),
@@ -423,7 +423,7 @@ void regclass_Core(py::module m) {
423423
const std::string& weight_path,
424424
const std::map<std::string, py::object>& config) {
425425
const auto any_map = Common::utils::properties_to_any_map(config);
426-
ConditionalGILScopedRelease release;
426+
py::gil_scoped_release release;
427427
return self.read_model(model_path, weight_path, any_map);
428428
},
429429
py::arg("model"),
@@ -453,7 +453,7 @@ void regclass_Core(py::module m) {
453453
cls.def(
454454
"read_model",
455455
(std::shared_ptr<ov::Model>(ov::Core::*)(const std::string&, const ov::Tensor&) const) & ov::Core::read_model,
456-
CallGuardConditionalGILRelease(),
456+
py::call_guard<py::gil_scoped_release>(),
457457
py::arg("model"),
458458
py::arg("weights"),
459459
R"(
@@ -492,7 +492,7 @@ void regclass_Core(py::module m) {
492492
const uint8_t* bin = reinterpret_cast<const uint8_t*>(info.ptr);
493493
std::memcpy(tensor.data(), bin, bin_size);
494494
}
495-
ConditionalGILScopedRelease release;
495+
py::gil_scoped_release release;
496496
return self.read_model(std::string(static_cast<char*>(buffer_info.ptr), buffer_info.size), tensor);
497497
} else if (py::isinstance(model_path, py::module_::import("pathlib").attr("Path")) ||
498498
py::isinstance<py::str>(model_path)) {
@@ -502,7 +502,7 @@ void regclass_Core(py::module m) {
502502
weights_path_cpp = py::str(weights_path);
503503
}
504504
const auto any_map = Common::utils::properties_to_any_map(config);
505-
ConditionalGILScopedRelease release;
505+
py::gil_scoped_release release;
506506
return self.read_model(model_path_cpp, weights_path_cpp, any_map);
507507
}
508508

@@ -540,7 +540,7 @@ void regclass_Core(py::module m) {
540540
const std::string& device_name,
541541
const std::map<std::string, py::object>& properties) {
542542
const auto _properties = Common::utils::properties_to_any_map(properties);
543-
ConditionalGILScopedRelease release;
543+
py::gil_scoped_release release;
544544
return self.import_model(exported_blob, device_name, _properties);
545545
},
546546
py::arg("tensor"),
@@ -587,7 +587,7 @@ void regclass_Core(py::module m) {
587587
ov::SharedStreamBuffer mb{info.ptr, static_cast<size_t>(info.size)};
588588
std::istream stream{&mb};
589589

590-
ConditionalGILScopedRelease release;
590+
py::gil_scoped_release release;
591591
return self.import_model(stream, device_name, _properties);
592592
},
593593
py::arg("model_stream"),
@@ -701,7 +701,7 @@ void regclass_Core(py::module m) {
701701
const std::string& device_name,
702702
const std::map<std::string, py::object>& properties) -> std::map<std::string, std::string> {
703703
auto _properties = Common::utils::properties_to_any_map(properties);
704-
ConditionalGILScopedRelease release;
704+
py::gil_scoped_release release;
705705
return self.query_model(model, device_name, _properties);
706706
},
707707
py::arg("model"),
@@ -768,7 +768,7 @@ void regclass_Core(py::module m) {
768768

769769
cls.def("get_available_devices",
770770
&ov::Core::get_available_devices,
771-
CallGuardConditionalGILRelease(),
771+
py::call_guard<py::gil_scoped_release>(),
772772
R"(
773773
Returns devices available for inference Core objects goes over all registered plugins.
774774
@@ -783,7 +783,7 @@ void regclass_Core(py::module m) {
783783

784784
cls.def_property_readonly("available_devices",
785785
&ov::Core::get_available_devices,
786-
CallGuardConditionalGILRelease(),
786+
py::call_guard<py::gil_scoped_release>(),
787787
R"(
788788
Returns devices available for inference Core objects goes over all registered plugins.
789789

‎src/bindings/python/src/pyopenvino/core/infer_request.cpp‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ PYBIND11_MAKE_OPAQUE(ov::TensorVector);
1919

2020
inline py::object run_sync_infer(InferRequestWrapper& self, bool share_outputs, bool decode_strings) {
2121
{
22-
ConditionalGILScopedRelease release;
22+
py::gil_scoped_release release;
2323
*self.m_start_time = Time::now();
2424
self.m_request->infer();
2525
*self.m_end_time = Time::now();
@@ -320,7 +320,7 @@ void regclass_InferRequest(py::module m) {
320320
PyErr_WarnEx(PyExc_RuntimeWarning, "There is no callback function to pass `userdata` into!", 1);
321321
}
322322
}
323-
ConditionalGILScopedRelease release;
323+
py::gil_scoped_release release;
324324
*self.m_start_time = Time::now();
325325
self.m_request->start_async();
326326
},
@@ -359,7 +359,7 @@ void regclass_InferRequest(py::module m) {
359359
PyErr_WarnEx(PyExc_RuntimeWarning, "There is no callback function!", 1);
360360
}
361361
}
362-
ConditionalGILScopedRelease release;
362+
py::gil_scoped_release release;
363363
*self.m_start_time = Time::now();
364364
self.m_request->start_async();
365365
},
@@ -392,7 +392,7 @@ void regclass_InferRequest(py::module m) {
392392
cls.def(
393393
"wait",
394394
[](InferRequestWrapper& self) {
395-
ConditionalGILScopedRelease release;
395+
py::gil_scoped_release release;
396396
self.m_request->wait();
397397
},
398398
R"(
@@ -405,7 +405,7 @@ void regclass_InferRequest(py::module m) {
405405
cls.def(
406406
"wait_for",
407407
[](InferRequestWrapper& self, const int timeout) {
408-
ConditionalGILScopedRelease release;
408+
py::gil_scoped_release release;
409409
return self.m_request->wait_for(std::chrono::milliseconds(timeout));
410410
},
411411
py::arg("timeout"),
@@ -440,8 +440,8 @@ void regclass_InferRequest(py::module m) {
440440
} catch (const std::exception& e) {
441441
OPENVINO_THROW("Caught exception: ", e.what());
442442
}
443-
// Acquire GIL, execute Python function
444-
ConditionalGILScopedAcquire acquire;
443+
// For free-threaded Python, gil_scoped_acquire still ensures thread is attached
444+
py::gil_scoped_acquire acquire;
445445
(*callback_sp)(self.m_userdata);
446446
});
447447
},
@@ -698,7 +698,7 @@ void regclass_InferRequest(py::module m) {
698698
[](InferRequestWrapper& self) {
699699
return self.m_request->get_profiling_info();
700700
},
701-
CallGuardConditionalGILRelease(),
701+
py::call_guard<py::gil_scoped_release>(),
702702
R"(
703703
Queries performance is measured per layer to get feedback on what
704704
is the most time-consuming operation, not all plugins provide
@@ -715,7 +715,7 @@ void regclass_InferRequest(py::module m) {
715715
[](InferRequestWrapper& self) {
716716
return self.m_request->query_state();
717717
},
718-
CallGuardConditionalGILRelease(),
718+
py::call_guard<py::gil_scoped_release>(),
719719
R"(
720720
Gets state control interface for given infer request.
721721
@@ -819,7 +819,7 @@ void regclass_InferRequest(py::module m) {
819819
[](InferRequestWrapper& self) {
820820
return self.m_request->get_profiling_info();
821821
},
822-
CallGuardConditionalGILRelease(),
822+
py::call_guard<py::gil_scoped_release>(),
823823
R"(
824824
Performance is measured per layer to get feedback on the most time-consuming operation.
825825
Not all plugins provide meaningful data!

‎src/bindings/python/src/pyopenvino/core/remote_context.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ void regclass_RemoteContext(py::module m) {
5151
const ov::Shape& shape,
5252
const std::map<std::string, py::object>& properties) {
5353
auto _properties = Common::utils::properties_to_any_map(properties);
54-
ConditionalGILScopedRelease release;
54+
py::gil_scoped_release release;
5555
return RemoteTensorWrapper(self.context.create_tensor(type, shape, _properties));
5656
},
5757
py::arg("type"),
@@ -79,7 +79,7 @@ void regclass_RemoteContext(py::module m) {
7979
[](RemoteContextWrapper& self, const ov::element::Type& type, const ov::Shape& shape) {
8080
return self.context.create_host_tensor(type, shape);
8181
},
82-
CallGuardConditionalGILRelease(),
82+
py::call_guard<py::gil_scoped_release>(),
8383
py::arg("type"),
8484
py::arg("shape"),
8585
R"(
@@ -132,7 +132,7 @@ void regclass_VAContext(py::module m) {
132132
[](VAContextWrapper& self, const size_t height, const size_t width, const uint32_t nv12_surface) {
133133
ov::RemoteTensor y_tensor, uv_tensor;
134134
{
135-
ConditionalGILScopedRelease release;
135+
py::gil_scoped_release release;
136136
ov::AnyMap tensor_params = {
137137
{ov::intel_gpu::shared_mem_type.name(), ov::intel_gpu::SharedMemType::VA_SURFACE},
138138
{ov::intel_gpu::dev_object_handle.name(), nv12_surface},
@@ -174,7 +174,7 @@ void regclass_VAContext(py::module m) {
174174
{ov::intel_gpu::va_plane.name(), plane}};
175175
return VASurfaceTensorWrapper(self.context.create_tensor(type, shape, params));
176176
},
177-
CallGuardConditionalGILRelease(),
177+
py::call_guard<py::gil_scoped_release>(),
178178
py::arg("type"),
179179
py::arg("shape"),
180180
py::arg("surface"),

‎src/bindings/python/src/pyopenvino/frontend/extension.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,15 @@ void regclass_frontend_TelemetryExtension(py::module m) {
4040
const std::string& action,
4141
const std::string& label,
4242
int value) {
43-
ConditionalGILScopedAcquire acquire;
43+
py::gil_scoped_acquire acquire;
4444
(*send_event_sp)(category, action, label, value);
4545
},
4646
[send_error_sp](const std::string& category, const std::string& error_message) {
47-
ConditionalGILScopedAcquire acquire;
47+
py::gil_scoped_acquire acquire;
4848
(*send_error_sp)(category, error_message);
4949
},
5050
[send_stack_trace_sp](const std::string& category, const std::string& error_message) {
51-
ConditionalGILScopedAcquire acquire;
51+
py::gil_scoped_acquire acquire;
5252
(*send_stack_trace_sp)(category, error_message);
5353
});
5454
}));
@@ -125,7 +125,7 @@ void regclass_frontend_ProgressReporterExtension(py::module m) {
125125

126126
ext.def(py::init([](py::function& callback) {
127127
return std::make_shared<ProgressReporterExtension>([callback](float a, unsigned int b, unsigned int c) {
128-
ConditionalGILScopedAcquire acquire;
128+
py::gil_scoped_acquire acquire;
129129
callback(a, b, c);
130130
});
131131
}));

0 commit comments

Comments
 (0)