adds code to enable profiling of Celery tasks

* as a workaround for multiprocessing refusal to fork daemon processes,
  incorporates changes to allow importing of Process and Pipe from
  billiard, the Celery project's fork of the multiprocessing module if
  MEMPROF_PREFERS_BILLIARD envvar is set to true

Author: FredAtLandMetrics

memory_profiler.py

@@ -18,13 +18,38 @@
 import logging
 import traceback
 from signal import SIGKILL
-
+import importlib
 
 # TODO: provide alternative when multiprocessing is not available
-try:
-    from multiprocessing import Process, Pipe
-except ImportError:
-    from multiprocessing.dummy import Process, Pipe
+
+# import Process and Pipe from a multiprocessing library in the following order:
+#
+#   billiard - if and only if MEMPROF_PREFERS_BILLIARD envvar is set and true
+#   multiprocessing
+#   multiprocessing.dummy
+#
+MULTILIBS = (
+    ("billiard", (lambda: True  if "MEMPROF_PREFERS_BILLIARD" in os.environ and
+                                    os.environ["MEMPROF_PREFERS_BILLIARD"] in ["True", "true", "TRUE", "1"]
+                                else False)),
+    ("multiprocessing", None,),
+    ("multiprocessing.dummy", None,)
+)
+for module_tuple in MULTILIBS:
+    multi_module_name, use_if_test = module_tuple
+    try:
+        multi_module = importlib.import_module(multi_module_name)
+    except ImportError:
+        if multi_module_name == "multiprocessing.dummy":
+            raise
+        continue
+    if use_if_test is None or use_if_test():
+        try:
+            Process = multi_module.Process
+            Pipe = multi_module.Pipe
+            break
+        except AttributeError:
+            continue
 
 try:
     from IPython.core.magic import Magics, line_cell_magic, magics_class