Merge pull request #414 from simvue-io/hotfix/kill-all-on-exception

Add runtime error handling for Remote class to prevent heartbeat thread hang

Author: kzscisoft

simvue/run.py

@@ -168,12 +168,16 @@ def __exit__(
                 self.set_status("completed")
             else:
                 if self._active:
-                    self.log_event(f"{exc_type.__name__}: {value}")
+                    # If the dispatcher has already been aborted then this will
+                    # fail so just continue without the event
+                    with contextlib.suppress(RuntimeError):
+                        self.log_event(f"{exc_type.__name__}: {value}")
                 if exc_type.__name__ in ("KeyboardInterrupt",) and self._active:
                     self.set_status("terminated")
                 else:
                     if traceback and self._active:
-                        self.log_event(f"Traceback: {traceback}")
+                        with contextlib.suppress(RuntimeError):
+                            self.log_event(f"Traceback: {traceback}")
                         self.set_status("failed")
         else:
             if self._shutdown_event is not None:
@@ -984,7 +988,12 @@ def update_tags(self, tags: list[str]) -> bool:
 
         if not self._simvue:
             return False
-        current_tags: list[str] = self._simvue.list_tags() or []
+
+        try:
+            current_tags: list[str] = self._simvue.list_tags() or []
+        except RuntimeError as e:
+            self._error(f"{e.args[0]}")
+            return False
 
         try:
             self.set_tags(list(set(current_tags + tags)))
@@ -1012,6 +1021,9 @@ def log_event(self, message: str, timestamp: typing.Optional[str] = None) -> boo
         bool
             whether event log was successful
         """
+        if self._aborted:
+            return False
+
         if self._mode == "disabled":
             self._error("Cannot log events in 'disabled' state")
             return True
@@ -1169,7 +1181,13 @@ def save_object(
         }
 
         # Register file
-        return self._simvue is not None and self._simvue.save_file(data) is not None
+        try:
+            file_save = self._simvue.save_file(data)
+        except RuntimeError as e:
+            self._error(f"{e.args[0]}")
+            return False
+
+        return self._simvue is not None and file_save is not None
 
     @skip_if_failed("_aborted", "_suppress_errors", False)
     @check_run_initialised
@@ -1256,7 +1274,11 @@ def save_file(
             return True
 
         # Register file
-        return self._simvue.save_file(data) is not None
+        try:
+            return self._simvue.save_file(data) is not None
+        except RuntimeError as e:
+            self._error(f"{e.args[0]}")
+            return False
 
     @skip_if_failed("_aborted", "_suppress_errors", False)
     @check_run_initialised
@@ -1386,7 +1408,13 @@ def set_status(
         data: dict[str, str] = {"name": self._name, "status": status}
         self._status = status
 
-        if self._simvue and self._simvue.update(data):
+        try:
+            updated = self._simvue.update(data)
+        except RuntimeError as e:
+            self._error(f"{e.args[0]}")
+            return False
+
+        if self._simvue and updated:
             return True
 
         return False
@@ -1496,8 +1524,12 @@ def set_folder_details(
         if description:
             data["description"] = description
 
-        if self._simvue.set_folder_details(data):
-            return True
+        try:
+            if self._simvue.set_folder_details(data):
+                return True
+        except RuntimeError as e:
+            self._error(f"{e.args[0]}")
+            return False
 
         return False
 
@@ -1531,10 +1563,14 @@ def add_alerts(
         names = names or []
 
         if names and not ids:
-            if alerts := self._simvue.list_alerts():
-                for alert in alerts:
-                    if alert["name"] in names:
-                        ids.append(alert["id"])
+            try:
+                if alerts := self._simvue.list_alerts():
+                    for alert in alerts:
+                        if alert["name"] in names:
+                            ids.append(alert["id"])
+            except RuntimeError as e:
+                self._error(f"{e.args[0]}")
+                return False
             else:
                 self._error("No existing alerts")
                 return False
@@ -1544,8 +1580,12 @@ def add_alerts(
 
         data: dict[str, typing.Any] = {"id": self._id, "alerts": ids}
 
-        if self._simvue.update(data):
-            return True
+        try:
+            if self._simvue.update(data):
+                return True
+        except RuntimeError as e:
+            self._error(f"{e.args[0]}")
+            return False
 
         return False
 
@@ -1689,7 +1729,12 @@ def create_alert(
 
         # Check if the alert already exists
         alert_id: typing.Optional[str] = None
-        alerts = self._simvue.list_alerts()
+        try:
+            alerts = self._simvue.list_alerts()
+        except RuntimeError as e:
+            self._error(f"{e.args[0]}")
+            return alerts
+
         if alerts:
             for existing_alert in alerts:
                 if existing_alert["name"] == alert["name"]:
@@ -1699,7 +1744,11 @@ def create_alert(
                         break
 
         if not alert_id:
-            response = self._simvue.add_alert(alert)
+            try:
+                response = self._simvue.add_alert(alert)
+            except RuntimeError as e:
+                self._error(f"{e.args[0]}")
+                return False
             if response:
                 if "id" in response:
                     alert_id = response["id"]
@@ -1710,7 +1759,12 @@ def create_alert(
         if alert_id:
             # TODO: What if we keep existing alerts/add a new one later?
             data = {"id": self._id, "alerts": [alert_id], "abort": trigger_abort}
-            self._simvue.update(data)
+
+            try:
+                self._simvue.update(data)
+            except RuntimeError as e:
+                self._error(f"{e.args[0]}")
+                return False
 
         return alert_id
 
@@ -1740,6 +1794,11 @@ def log_alert(
         if not self._simvue:
             self._error("Cannot log alert, run not initialised")
             return False
-        self._simvue.set_alert_state(identifier, state)
+
+        try:
+            self._simvue.set_alert_state(identifier, state)
+        except RuntimeError as e:
+            self._error(f"{e.args[0]}")
+            return False
 
         return True