Skip to content

Commit 73beb3f

Browse files
ShimuuarShimuuar
authored
Merge pull request #26 from Shimuuar/nested-runPyInMain-deadlocks
Fix many deadlock in runPyInMain
2 parents d3aae1b + 406ffa2 commit 73beb3f

File tree

5 files changed

+100
-49
lines changed

5 files changed

+100
-49
lines changed

‎ChangeLog.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
0.1.1 [2025.02.13]
2+
------------------
3+
* Number of deadlocks in `runPyInMain` fixed:
4+
- It no longer deadlocks is exception is thrown
5+
- Nested calls no longer deadlock.
6+
- Calling it from python callback.
7+
* `ToPy` instance added for `Py b`, `a -> Py b`, `a1 -> a2 -> Py b`
8+
9+
10+
0.1 [2025.01.18]
11+
----------------
12+
Initial release

‎inline-python.cabal‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ Cabal-Version: 3.0
22
Build-Type: Simple
33

44
Name: inline-python
5-
Version: 0.1
5+
Version: 0.1.1
66
Synopsis: Python interpreter embedded into haskell.
77
Description:
88
This package embeds python interpreter into haskell program and

‎src/Python/Internal/Eval.hs‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -457,23 +457,46 @@ runPyInMain :: Py a -> IO a
457457
-- See NOTE: [Python and threading]
458458
runPyInMain py
459459
-- Multithreaded RTS
460-
| rtsSupportsBoundThreads = bracket acquireMain releaseMain evalMain
460+
| rtsSupportsBoundThreads = do
461+
tid <- myThreadId
462+
bracket (acquireMain tid) fst snd
461463
-- Single-threaded RTS
462464
| otherwise = runPy py
463465
where
464-
acquireMain = atomically $ readTVar globalPyState >>= \case
466+
acquireMain tid = atomically $ readTVar globalPyState >>= \case
465467
NotInitialized -> throwSTM PythonNotInitialized
466468
InitFailed -> throwSTM PyInitializationFailed
467469
Finalized -> throwSTM PythonIsFinalized
468470
InInitialization -> retry
469471
InFinalization -> retry
470472
Running1 -> throwSTM $ PyInternalError "runPyInMain: Running1"
471-
RunningN _ eval tid_main _ -> do
472-
acquireLock tid_main
473-
pure (tid_main, eval)
473+
RunningN _ eval tid_main _ -> readTVar globalPyLock >>= \case
474+
LockUninialized -> throwSTM PythonNotInitialized
475+
LockFinalized -> throwSTM PythonIsFinalized
476+
LockedByGC -> retry
477+
-- We need to send closure to main python thread when we're grabbing lock.
478+
LockUnlocked -> do
479+
writeTVar globalPyLock $ Locked tid_main []
480+
pure ( atomically (releaseLock tid_main)
481+
, evalInOtherThread tid_main eval
482+
)
483+
-- If we can grab lock and main thread taken lock we're
484+
-- already executing on main thread. We can simply execute code
485+
Locked t ts
486+
| t /= tid
487+
-> retry
488+
| t == tid_main || (tid_main `elem` ts) -> do
489+
writeTVar globalPyLock $ Locked t (t : ts)
490+
pure ( atomically (releaseLock t)
491+
, unsafeRunPy $ ensureGIL py
492+
)
493+
| otherwise -> do
494+
writeTVar globalPyLock $ Locked tid_main (t : ts)
495+
pure ( atomically (releaseLock tid_main)
496+
, evalInOtherThread tid_main eval
497+
)
474498
--
475-
releaseMain (tid_main, _ ) = atomically (releaseLock tid_main)
476-
evalMain (tid_main, eval) = do
499+
evalInOtherThread tid_main eval = do
477500
r <- mask_ $ do resp <- newEmptyMVar
478501
putMVar eval $ EvalReq py resp
479502
takeMVar resp `onException` throwTo tid_main InterruptMain

‎test/TST/Callbacks.hs‎

Lines changed: 56 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -69,44 +69,59 @@ tests = testGroup "Callbacks"
6969
let foo :: Int -> IO Int
7070
foo y = pure $ 10 `div` y
7171
throwsPy [py_| foo_hs(0) |]
72-
, testCase "Haskell exception in callback(arity=2)" $ runPy $ do
73-
let foo :: Int -> Int -> IO Int
74-
foo x y = pure $ x `div` y
75-
throwsPy [py_| foo_hs(1, 0) |]
76-
----------------------------------------
77-
, testCase "Call python in callback (arity=1)" $ runPy $ do
78-
let foo :: Int -> IO Int
79-
foo x = do Just x' <- runPy $ fromPy =<< [pye| 100 // x_hs |]
80-
pure x'
81-
[py_|
82-
assert foo_hs(5) == 20
83-
|]
84-
, testCase "Call python in callback (arity=2" $ runPy $ do
85-
let foo :: Int -> Int -> IO Int
86-
foo x y = do Just x' <- runPy $ fromPy =<< [pye| x_hs // y_hs |]
87-
pure x'
88-
[py_|
89-
assert foo_hs(100,5) == 20
90-
|]
91-
----------------------------------------
92-
, testCase "No leaks (arity=1)" $ runPy $ do
93-
let foo :: Int -> IO Int
94-
foo y = pure $ 10 * y
95-
[py_|
96-
import sys
97-
x = 123456
98-
old_refcount = sys.getrefcount(x)
99-
foo_hs(x)
100-
assert old_refcount == sys.getrefcount(x)
101-
|]
102-
, testCase "No leaks (arity=2)" $ runPy $ do
103-
let foo :: Int -> Int -> IO Int
104-
foo x y = pure $ x * y
105-
[py_|
106-
import sys
107-
x = 123456
108-
old_refcount = sys.getrefcount(x)
109-
foo_hs(1,x)
110-
assert old_refcount == sys.getrefcount(x)
111-
|]
112-
]
72+
, testCase "Haskell exception in callback(arity=2)" $ runPy $ do
73+
let foo :: Int -> Int -> IO Int
74+
foo x y = pure $ x `div` y
75+
throwsPy [py_| foo_hs(1, 0) |]
76+
----------------------------------------
77+
, testCase "Call python in callback (arity=1)" $ runPy $ do
78+
let foo :: Int -> IO Int
79+
foo x = do Just x' <- runPy $ fromPy =<< [pye| 100 // x_hs |]
80+
pure x'
81+
[py_|
82+
assert foo_hs(5) == 20
83+
|]
84+
, testCase "Call python in callback (arity=2" $ runPy $ do
85+
let foo :: Int -> Int -> IO Int
86+
foo x y = do Just x' <- runPy $ fromPy =<< [pye| x_hs // y_hs |]
87+
pure x'
88+
[py_|
89+
assert foo_hs(100,5) == 20
90+
|]
91+
----------------------------------------
92+
, testCase "runPyInMain in runPyInMain (arity=1)" $ do
93+
let foo :: Int -> IO Int
94+
foo x = do Just x' <- runPyInMain $ fromPy =<< [pye| 100 // x_hs |]
95+
pure x'
96+
runPyInMain [py_|
97+
assert foo_hs(5) == 20
98+
|]
99+
, testCase "runPyInMain in runPy (arity=1)" $ do
100+
let foo :: Int -> IO Int
101+
foo x = do Just x' <- runPyInMain $ fromPy =<< [pye| 100 // x_hs |]
102+
pure x'
103+
runPy [py_|
104+
assert foo_hs(5) == 20
105+
|]
106+
----------------------------------------
107+
, testCase "No leaks (arity=1)" $ runPy $ do
108+
let foo :: Int -> IO Int
109+
foo y = pure $ 10 * y
110+
[py_|
111+
import sys
112+
x = 123456
113+
old_refcount = sys.getrefcount(x)
114+
foo_hs(x)
115+
assert old_refcount == sys.getrefcount(x)
116+
|]
117+
, testCase "No leaks (arity=2)" $ runPy $ do
118+
let foo :: Int -> Int -> IO Int
119+
foo x y = pure $ x * y
120+
[py_|
121+
import sys
122+
x = 123456
123+
old_refcount = sys.getrefcount(x)
124+
foo_hs(1,x)
125+
assert old_refcount == sys.getrefcount(x)
126+
|]
127+
]

‎test/TST/Run.hs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ tests = testGroup "Run python"
1717
[ testCase "Empty QQ" $ runPy [py_| |]
1818
, testCase "Second init is noop" $ initializePython
1919
, testCase "Nested runPy" $ runPy $ liftIO $ runPy $ pure ()
20+
, testCase "Nested runPyInMain" $ runPyInMain $ liftIO $ runPyInMain $ pure ()
2021
, testCase "runPyInMain" $ runPyInMain $ [py_|
2122
import threading
2223
assert threading.main_thread() == threading.current_thread()

0 commit comments

Comments
 (0)