force multiprocessing start method to be "spawn" at the start of extension,

Author: andrii-i

jupyter_scheduler/extension.py

@@ -1,4 +1,5 @@
 import asyncio
+import multiprocessing
 
 from jupyter_core.paths import jupyter_data_dir
 from jupyter_server.extension.application import ExtensionApp
@@ -69,6 +70,15 @@ def _db_url_default(self):
     )
 
     def initialize_settings(self):
+        # Forces new processes to not be forked on Linux.
+        # This is necessary because `asyncio.get_event_loop()` is bugged in
+        # forked processes in Python versions below 3.12. This method is
+        # called by `jupyter_core` by `nbconvert` in the default executor.
+
+        # See: https://github.com/python/cpython/issues/66285
+        # See also: https://github.com/jupyter/jupyter_core/pull/362
+        multiprocessing.set_start_method("spawn", force=True)
+
         super().initialize_settings()
 
         create_tables(self.db_url, self.drop_tables)

jupyter_scheduler/job_files_manager.py

@@ -1,7 +1,8 @@
+from multiprocessing import Process
 import os
 import random
 import tarfile
-import multiprocessing as mp
+
 from typing import Dict, List, Optional, Type
 
 import fsspec
@@ -23,15 +24,7 @@ async def copy_from_staging(self, job_id: str, redownload: Optional[bool] = Fals
         output_filenames = self.scheduler.get_job_filenames(job)
         output_dir = self.scheduler.get_local_output_path(model=job, root_dir_relative=True)
 
-        # The MP context forces new processes to not be forked on Linux.
-        # This is necessary because `asyncio.get_event_loop()` is bugged in
-        # forked processes in Python versions below 3.12. This method is
-        # called by `jupyter_core` by `nbconvert` in the default executor.
-        #
-        # See: https://github.com/python/cpython/issues/66285
-        # See also: https://github.com/jupyter/jupyter_core/pull/362
-        mp_ctx = mp.get_context("spawn")
-        p = mp_ctx.Process(
+        p = Process(
             target=Downloader(
                 output_formats=job.output_formats,
                 output_filenames=output_filenames,

jupyter_scheduler/scheduler.py

@@ -1,4 +1,4 @@
-import multiprocessing as mp
+from multiprocessing import Process
 import os
 import random
 import shutil
@@ -481,15 +481,7 @@ def create_job(self, model: CreateJob) -> str:
             else:
                 self.copy_input_file(model.input_uri, staging_paths["input"])
 
-            # The MP context forces new processes to not be forked on Linux.
-            # This is necessary because `asyncio.get_event_loop()` is bugged in
-            # forked processes in Python versions below 3.12. This method is
-            # called by `jupyter_core` by `nbconvert` in the default executor.
-            #
-            # See: https://github.com/python/cpython/issues/66285
-            # See also: https://github.com/jupyter/jupyter_core/pull/362
-            mp_ctx = mp.get_context("spawn")
-            p = mp_ctx.Process(
+            p = Process(
                 target=self.execution_manager_class(
                     job_id=job.job_id,
                     staging_paths=staging_paths,