Fix deadlock in graceful shutdown on error

src/feo/src/agent/direct/primary.rs

@@ -14,6 +14,7 @@
 //! Implementation of the primary agent for direct scheduler-to-worker signalling
 
 use crate::activity::ActivityIdAndBuilder;
+use crate::agent::register_sigterm_handler;
 use crate::agent::NodeAddress;
 use crate::debug_fmt::ScoreDebugDebug;
 use crate::error::Error;
@@ -29,7 +30,7 @@ use alloc::sync::Arc;
 use alloc::vec::Vec;
 use core::sync::atomic::AtomicBool;
 use feo_time::Duration;
-use score_log::{error, info};
+use score_log::error;
 use std::collections::HashMap;
 use std::thread::{self, JoinHandle};
 
@@ -134,12 +135,7 @@ impl Primary {
 
         // Create a shared flag to signal shutdown from an OS signal (e.g., Ctrl-C).
         let shutdown_requested = Arc::new(AtomicBool::new(false));
-        let shutdown_clone = shutdown_requested.clone();
-        ctrlc::set_handler(move || {
-            info!("Ctrl-C detected. Requesting graceful shutdown...");
-            shutdown_clone.store(true, core::sync::atomic::Ordering::Relaxed);
-        })
-        .expect("Error setting Ctrl-C handler");
+        register_sigterm_handler(shutdown_requested.clone());
 
         let scheduler = Scheduler::new(
             config.id,

src/feo/src/agent/direct/primary_mpsc.rs

@@ -14,6 +14,7 @@
 //! Implementation of the primary agent for mpsc-only signalling
 
 use crate::activity::ActivityIdAndBuilder;
+use crate::agent::register_sigterm_handler;
 use crate::debug_fmt::ScoreDebugDebug;
 use crate::error::Error;
 use crate::ids::{ActivityId, AgentId, WorkerId};
@@ -27,7 +28,7 @@ use alloc::sync::Arc;
 use alloc::vec::Vec;
 use core::sync::atomic::AtomicBool;
 use feo_time::Duration;
-use score_log::{debug, error, info};
+use score_log::{debug, error};
 use std::collections::HashMap;
 use std::thread::{self, JoinHandle};
 
@@ -103,12 +104,7 @@ impl Primary {
 
         // Create a shared flag to signal shutdown from an OS signal (e.g., Ctrl-C).
         let shutdown_requested = Arc::new(AtomicBool::new(false));
-        let shutdown_clone = shutdown_requested.clone();
-        ctrlc::set_handler(move || {
-            info!("Ctrl-C detected. Requesting graceful shutdown...");
-            shutdown_clone.store(true, core::sync::atomic::Ordering::Relaxed);
-        })
-        .expect("Error setting Ctrl-C handler");
+        register_sigterm_handler(shutdown_requested.clone());
 
         let scheduler = Scheduler::new(
             config.id,

src/feo/src/agent/mod.rs

@@ -17,7 +17,11 @@
 //! agents. The primary agent is responsible for triggering the execution of all activities distributed
 //! across all agents.
 
+use alloc::sync::Arc;
 use core::net::SocketAddr;
+use core::sync::atomic::AtomicBool;
+use core::sync::atomic::Ordering;
+use score_log::info;
 use std::path::PathBuf;
 
 pub mod com_init;
@@ -30,3 +34,16 @@ pub enum NodeAddress {
     Tcp(SocketAddr),
     UnixSocket(PathBuf),
 }
+
+fn register_sigterm_handler(shutdown: Arc<AtomicBool>) {
+    ctrlc::set_handler(move || {
+        if shutdown.load(Ordering::Relaxed) {
+            info!("Terminate triggered, exiting...");
+            std::process::exit(1);
+        } else {
+            info!("Ctrl-C detected. Requesting graceful shutdown...");
+            shutdown.store(true, core::sync::atomic::Ordering::Relaxed);
+        }
+    })
+    .expect("Error setting Ctrl-C handler")
+}

src/feo/src/agent/relayed/primary.rs

@@ -14,6 +14,7 @@
 //! Implementation of the primary agent for mixed signalling using sockets and mpsc channels
 
 use crate::activity::ActivityIdAndBuilder;
+use crate::agent::register_sigterm_handler;
 use crate::agent::NodeAddress;
 use crate::error::Error;
 use crate::ids::{ActivityId, AgentId, WorkerId};
@@ -27,7 +28,7 @@ use alloc::sync::Arc;
 use alloc::vec::Vec;
 use core::sync::atomic::AtomicBool;
 use feo_time::Duration;
-use score_log::{debug, info};
+use score_log::debug;
 use std::collections::HashMap;
 use std::thread::{self, JoinHandle};
 
@@ -144,12 +145,7 @@ impl Primary {
 
         // Create a shared flag to signal shutdown from an OS signal (e.g., Ctrl-C).
         let shutdown_requested = Arc::new(AtomicBool::new(false));
-        let shutdown_clone = shutdown_requested.clone();
-        ctrlc::set_handler(move || {
-            info!("Ctrl-C detected. Requesting graceful shutdown...");
-            shutdown_clone.store(true, core::sync::atomic::Ordering::Relaxed);
-        })
-        .expect("Error setting Ctrl-C handler");
+        register_sigterm_handler(shutdown_requested.clone());
 
         let scheduler = Scheduler::new(
             id,