🐛 Set all threads to be daemon=True

Addresses issue of hanging tests due to threads still running, this includes the QueuedDispatcher subclass.

Author: kzscisoft

simvue/client.py

@@ -642,7 +642,9 @@ def get_artifacts_as_files(
             Artifact.from_run(run_id=run_id, category=category)
         )
 
-        with ThreadPoolExecutor(CONCURRENT_DOWNLOADS) as executor:
+        with ThreadPoolExecutor(
+            CONCURRENT_DOWNLOADS, thread_name_prefix=f"get_artifacts_run_{run_id}"
+        ) as executor:
             futures = [
                 executor.submit(_download_artifact_to_file, artifact, output_dir)
                 for _, artifact in _artifacts

simvue/executor.py

@@ -81,6 +81,7 @@ def trigger_check(
             target=trigger_check,
             args=(completion_callback, completion_trigger, _result),
             daemon=True,
+            name=f"{proc_id}_Thread",
         )
         thread_out.start()
 

simvue/factory/dispatch/__init__.py

@@ -23,6 +23,7 @@ def Dispatcher(
     callback: typing.Callable[[list[typing.Any], str, dict[str, typing.Any]], None],
     object_types: list[str],
     termination_trigger: "Event",
+    name: str | None = None,
     **kwargs,
 ) -> "DispatcherBaseClass":
     """Returns instance of dispatcher based on configuration
@@ -43,6 +44,8 @@ def Dispatcher(
         categories, this is mainly used for creation of queues in a QueueDispatcher
     termination_trigger : Event
         event which triggers termination of the dispatcher
+    name : str | None, optional
+        name for the underlying thread, default None
 
     Returns
     -------
@@ -63,5 +66,6 @@ def Dispatcher(
             callback=callback,
             object_types=object_types,
             termination_trigger=termination_trigger,
+            name=name,
             **kwargs,
         )

simvue/factory/dispatch/queued.py

@@ -36,6 +36,7 @@ def __init__(
         callback: typing.Callable[[list[typing.Any], str], None],
         object_types: list[str],
         termination_trigger: threading.Event,
+        name: str | None = None,
         max_buffer_size: int = MAX_BUFFER_SIZE,
         max_read_rate: float = MAX_REQUESTS_PER_SECOND,
     ) -> None:
@@ -51,6 +52,8 @@ def __init__(
         termination_trigger : threading.Event
             a threading event which when set declares that the dispatcher
             should terminate
+        name : str | None, optional
+            name for underlying thread, default None
         max_buffer_size : int
             maximum number of items allowed in created buffer.
         max_read_rate : float
@@ -62,7 +65,7 @@ def __init__(
             object_types=object_types,
             termination_trigger=termination_trigger,
         )
-        super().__init__()
+        super().__init__(name=name, daemon=True)
 
         self._termination_trigger = termination_trigger
         self._callback = callback

simvue/run.py

@@ -539,7 +539,9 @@ def _start(self) -> bool:
             )
 
             self._heartbeat_thread = threading.Thread(
-                target=self._create_heartbeat_callback(), daemon=True
+                target=self._create_heartbeat_callback(),
+                daemon=True,
+                name=f"{self.id}_heartbeat",
             )
 
         except RuntimeError as e:

simvue/sender.py

@@ -201,7 +201,9 @@ def sender(
             for file_path in _offline_files:
                 upload_cached_file(cache_dir, _obj_type, file_path, _id_mapping, _lock)
         else:
-            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            with ThreadPoolExecutor(
+                max_workers=max_workers, thread_name_prefix="sender_session_upload"
+            ) as executor:
                 _results = executor.map(
                     lambda file_path: upload_cached_file(
                         cache_dir=cache_dir,
@@ -230,7 +232,9 @@ def sender(
                 ),
             )
     else:
-        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        with ThreadPoolExecutor(
+            max_workers=max_workers, thread_name_prefix="sender_heartbeat"
+        ) as executor:
             _results = executor.map(
                 lambda _heartbeat_file: send_heartbeat(
                     file_path=_heartbeat_file,

tests/conftest.py

@@ -310,3 +310,4 @@ def setup_test_run(run: sv_run.Run, create_objects: bool, request: pytest.Fixtur
     TEST_DATA["alert_ids"] = _alert_ids
 
     return TEST_DATA
+

tests/functional/test_dispatch.py

@@ -4,6 +4,7 @@
 import time
 from threading import Event, Thread
 from queue import Queue
+from concurrent.futures import ThreadPoolExecutor
 
 
 from simvue.factory.dispatch.queued import QueuedDispatcher
@@ -12,17 +13,16 @@
 
 # FIXME: Update the layout of these tests
 
+
 @pytest.mark.dispatch
+@pytest.mark.parametrize("overload_buffer", (True, False), ids=("overload", "normal"))
 @pytest.mark.parametrize(
-    "overload_buffer", (True, False),
-    ids=("overload", "normal")
-)
-@pytest.mark.parametrize(
-    "append_during_dispatch", (True, False),
-    ids=("pre_append", "append")
+    "append_during_dispatch", (True, False), ids=("pre_append", "append")
 )
 @pytest.mark.parametrize("multiple", (True, False), ids=("multiple", "single"))
-def test_queued_dispatcher(overload_buffer: bool, multiple: bool, append_during_dispatch: bool) -> None:
+def test_queued_dispatcher(
+    overload_buffer: bool, multiple: bool, append_during_dispatch: bool
+) -> None:
     buffer_size: int = 10
     n_elements: int = 2 * buffer_size if overload_buffer else buffer_size - 1
     max_read_rate: float = 0.2
@@ -42,24 +42,39 @@ def test_queued_dispatcher(overload_buffer: bool, multiple: bool, append_during_
 
     for variable in variables:
         check_dict[variable] = {"counter": 0}
-        def callback(___: list[typing.Any], _: str, args=check_dict, var=variable) -> None:
+
+        def callback(
+            ___: list[typing.Any], _: str, args=check_dict, var=variable
+        ) -> None:
             args[var]["counter"] += 1
+
         dispatchers.append(
-            QueuedDispatcher(callback, [variable], event, max_buffer_size=buffer_size, max_read_rate=max_read_rate)
+            QueuedDispatcher(
+                callback,
+                [variable],
+                event,
+                max_buffer_size=buffer_size,
+                max_read_rate=max_read_rate,
+                name=f"Queued_Dispatcher_{variable}"
+            )
         )
 
     if not append_during_dispatch:
         for i in range(n_elements):
-            for variable, dispatcher in zip(variables, dispatchers):  
-                dispatcher.add_item({string.ascii_uppercase[i % 26]: i}, variable, False)
+            for variable, dispatcher in zip(variables, dispatchers):
+                dispatcher.add_item(
+                    {string.ascii_uppercase[i % 26]: i}, variable, False
+                )
 
     for dispatcher in dispatchers:
         dispatcher.start()
 
     if append_during_dispatch:
         for i in range(n_elements):
-            for variable, dispatcher in zip(variables, dispatchers):  
-                dispatcher.add_item({string.ascii_uppercase[i % 26]: i}, variable, False)
+            for variable, dispatcher in zip(variables, dispatchers):
+                dispatcher.add_item(
+                    {string.ascii_uppercase[i % 26]: i}, variable, False
+                )
 
     while not dispatcher.empty:
         time.sleep(0.1)
@@ -70,7 +85,9 @@ def callback(___: list[typing.Any], _: str, args=check_dict, var=variable) -> No
     time.sleep(0.1)
 
     for variable in variables:
-        assert check_dict[variable]["counter"] >= (2 if overload_buffer else 1), f"Check of counter for dispatcher '{variable}' failed with count = {check_dict[variable]['counter']}"
+        assert check_dict[variable]["counter"] >= (2 if overload_buffer else 1), (
+            f"Check of counter for dispatcher '{variable}' failed with count = {check_dict[variable]['counter']}"
+        )
     assert time.time() - start_time < time_threshold
 
 
@@ -86,33 +103,48 @@ def test_nested_queued_dispatch(multi_queue: bool) -> None:
     result_queue = Queue()
 
     event = Event()
+
     def create_callback(index):
-        def callback(___: list[typing.Any], _: str, check_dict=check_dict[index]) -> None:
+        def callback(
+            ___: list[typing.Any], _: str, check_dict=check_dict[index]
+        ) -> None:
             check_dict["counter"] += 1
+
         return callback
-    def _main(res_queue, index, dispatch_callback=create_callback, term_event=event, variable=variable) -> bool:
 
+    def _main(
+        res_queue,
+        index,
+        dispatch_callback=create_callback,
+        term_event=event,
+        variable=variable,
+    ) -> bool:
         term_event = Event()
         dispatcher = QueuedDispatcher(
             dispatch_callback(index),
             [variable] if isinstance(variable, str) else variable,
             term_event,
             max_buffer_size=buffer_size,
-            max_read_rate=max_read_rate
+            max_read_rate=max_read_rate,
+            name=f"test_nested_queued_dispatch"
         )
 
         dispatcher.start()
 
         try:
             for i in range(n_elements):
                 if isinstance(variable, str):
-                    dispatcher.add_item({string.ascii_uppercase[i % 26]: i}, variable, False)
+                    dispatcher.add_item(
+                        {string.ascii_uppercase[i % 26]: i}, variable, False
+                    )
                 else:
                     for var in variable:
-                        dispatcher.add_item({string.ascii_uppercase[i % 26]: i}, var, False)
-        except(RuntimeError):
+                        dispatcher.add_item(
+                            {string.ascii_uppercase[i % 26]: i}, var, False
+                        )
+        except RuntimeError:
             res_queue.put("AARGHGHGHGHAHSHGHSDHFSEDHSE")
- 
+
         time.sleep(0.1)
 
         while not dispatcher.empty:
@@ -127,18 +159,29 @@ def _main(res_queue, index, dispatch_callback=create_callback, term_event=event,
     threads = []
 
     for i in range(3):
-        _thread = Thread(target=_main, args=(result_queue, i,), daemon=True)
+        _thread = Thread(
+            target=_main,
+            args=(
+                result_queue,
+                i,
+            ),
+            daemon=True,
+            name=f"nested_queue_dispatch_{i}_Thread",
+        )
         _thread.start()
         threads.append(_thread)
-    
+
     for i in range(3):
         threads[i].join()
 
     if not result_queue.empty():
         assert False
 
     for i in range(3):
-        assert check_dict[i]["counter"] >= 2, f"Check of counter for dispatcher '{variable}' failed with count = {check_dict[i]['counter']}"
+        assert check_dict[i]["counter"] >= 2, (
+            f"Check of counter for dispatcher '{variable}' failed with count = {check_dict[i]['counter']}"
+        )
+
 
 def test_queued_dispatch_error_adding_item_after_termination() -> None:
     trigger = Event()
@@ -148,7 +191,8 @@ def test_queued_dispatch_error_adding_item_after_termination() -> None:
         object_types=["q"],
         termination_trigger=trigger,
         max_buffer_size=5,
-        max_read_rate=2
+        max_read_rate=2,
+        name="test_queued_dispatch_error_adding_item_after_termination"
     )
     dispatcher.start()
 
@@ -157,14 +201,16 @@ def test_queued_dispatch_error_adding_item_after_termination() -> None:
     with pytest.raises(RuntimeError):
         dispatcher.add_item("blah", "q", False)
 
+
 def test_queued_dispatch_error_attempting_to_use_non_existent_queue() -> None:
     trigger = Event()
     dispatcher = QueuedDispatcher(
         callback=lambda *_: None,
         object_types=["q"],
         termination_trigger=trigger,
         max_buffer_size=5,
-        max_read_rate=2
+        max_read_rate=2,
+        name="test_queued_dispatch_error_attempting_to_use_non_existent_queue"
     )
     dispatcher.start()
 
@@ -194,18 +240,22 @@ def test_direct_dispatcher(multiple: bool) -> None:
 
     for variable in variables:
         check_dict[variable] = {"counter": 0}
-        def callback(___: list[typing.Any], _: str, args=check_dict, var=variable) -> None:
+
+        def callback(
+            ___: list[typing.Any], _: str, args=check_dict, var=variable
+        ) -> None:
             args[var]["counter"] += 1
-        dispatchers.append(
-            DirectDispatcher(callback, [variable], event)
-        )
+
+        dispatchers.append(DirectDispatcher(callback, [variable], event))
 
     for i in range(n_elements):
-        for variable, dispatcher in zip(variables, dispatchers):  
+        for variable, dispatcher in zip(variables, dispatchers):
             dispatcher.add_item({string.ascii_uppercase[i % 26]: i}, variable)
 
     event.set()
 
     for variable in variables:
-        assert check_dict[variable]["counter"] >= 1, f"Check of counter for dispatcher '{variable}' failed with count = {check_dict[variable]['counter']}"
+        assert check_dict[variable]["counter"] >= 1, (
+            f"Check of counter for dispatcher '{variable}' failed with count = {check_dict[variable]['counter']}"
+        )
     assert time.time() - start_time < time_threshold

tests/functional/test_run_class.py

@@ -464,6 +464,7 @@ def test_update_metadata_offline(
 
 
 @pytest.mark.run
+@pytest.mark.scenario
 @pytest.mark.parametrize("multi_threaded", (True, False), ids=("multi", "single"))
 def test_runs_multiple_parallel(
     multi_threaded: bool, request: pytest.FixtureRequest
@@ -491,7 +492,7 @@ def thread_func(index: int) -> tuple[int, list[dict[str, typing.Any]], str]:
                     run.log_metrics(metric)
             return index, metrics, run.id
 
-        with concurrent.futures.ThreadPoolExecutor(max_workers=N_RUNS) as executor:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=N_RUNS, thread_name_prefix="test_runs_multiple_parallel") as executor:
             futures = [executor.submit(thread_func, i) for i in range(N_RUNS)]
 
             time.sleep(1)
@@ -561,6 +562,7 @@ def thread_func(index: int) -> tuple[int, list[dict[str, typing.Any]], str]:
 
 
 @pytest.mark.run
+@pytest.mark.scenario
 def test_runs_multiple_series(request: pytest.FixtureRequest) -> None:
     N_RUNS: int = 2