feat: Adding signal and external error detection & output (#3856)

- output error message & context reliably in the log, even if the stderr get lost or used for another reason,
- be sure to detect any kernel / system allocator errors and add the stack-trace of these errors,
- factorize them with external tools / scripts, thus highlighting which are the source rank(s) of the issue.
- also has the effect to prevent the stacktrace to be cut by other ranks message, which could previously happen on a signal.

Author: MelReyCG

src/coreComponents/common/CMakeLists.txt

@@ -31,7 +31,6 @@ set( common_headers
      format/LogPart.hpp
      format/Format.hpp
      format/StringUtilities.hpp
-     logger/Logger.hpp
      BufferAllocator.hpp
      DataLayouts.hpp
      DataTypes.hpp
@@ -41,6 +40,7 @@ set( common_headers
      MemoryInfos.hpp
      logger/Logger.hpp
      logger/ErrorHandling.hpp
+     logger/ExternalErrorHandler.hpp
      MpiWrapper.hpp
      Path.hpp
      Span.hpp
@@ -77,6 +77,7 @@ set( common_sources
      format/StringUtilities.cpp
      logger/Logger.cpp
      logger/ErrorHandling.cpp
+     logger/ExternalErrorHandler.cpp
      BufferAllocator.cpp
      MemoryInfos.cpp
      MpiWrapper.cpp

src/coreComponents/common/initializeEnvironment.cpp

@@ -20,6 +20,8 @@
 #include "LvArray/src/system.hpp"
 #include "common/LifoStorageCommon.hpp"
 #include "common/MemoryInfos.hpp"
+#include "logger/ErrorHandling.hpp"
+#include "logger/ExternalErrorHandler.hpp"
 #include <umpire/TypedAllocator.hpp>
 // TPL includes
 #include <umpire/ResourceManager.hpp>
@@ -66,6 +68,88 @@ void setupLogger()
 #else
   logger::InitializeLogger();
 #endif
+
+  { // setup error handling (using LvArray helper system functions)
+    using ErrorContext = ErrorLogger::ErrorContext;
+
+    ///// set Post-Handled Error behaviour /////
+    LvArray::system::setErrorHandler( []()
+    {
+  #if defined( GEOS_USE_MPI )
+      int mpi = 0;
+      MPI_Initialized( &mpi );
+      if( mpi )
+      {
+        MPI_Abort( MPI_COMM_WORLD, EXIT_FAILURE );
+      }
+  #endif
+      std::abort();
+    } );
+
+    ///// set external error handling behaviour /////
+    ExternalErrorHandler::instance().setErrorHandling( []( string_view errorMsg,
+                                                           string_view detectionLocation )
+    {
+      std::string const stackHistory = LvArray::system::stackTrace( true );
+
+      GEOS_LOG( GEOS_FMT( "***** ERROR\n"
+                          "***** LOCATION: (external error, detected {})\n"
+                          "{}\n{}",
+                          detectionLocation, errorMsg, stackHistory ) );
+      if( ErrorLogger::global().isOutputFileEnabled() )
+      {
+        ErrorLogger::ErrorMsg error;
+        error.setType( ErrorLogger::MsgType::Error );
+        error.addToMsg( errorMsg );
+        error.addRank( ::geos::logger::internal::g_rank );
+        error.addCallStackInfo( stackHistory );
+        error.addContextInfo(
+          ErrorContext{ { { ErrorContext::Attribute::DetectionLoc, string( detectionLocation ) } } } );
+
+        ErrorLogger::global().flushErrorMsg( error );
+      }
+
+      // we do not terminate the program as 1. the error could be non-fatal, 2. there may be more messages to output.
+    } );
+    ExternalErrorHandler::instance().enableStderrPipeDeviation( true );
+
+    ///// set signal handling behaviour /////
+    LvArray::system::setSignalHandling( []( int const signal )
+    {
+      // Disable signal handling to prevent catching exit signal (infinite loop)
+      LvArray::system::setSignalHandling( nullptr );
+
+      // first of all, external error can await to be output, we must output them
+      ExternalErrorHandler::instance().flush( "before signal error output" );
+
+      // error message output
+      std::string const stackHistory = LvArray::system::stackTrace( true );
+      ErrorLogger::ErrorMsg error;
+      error.addSignalToMsg( signal );
+
+      GEOS_LOG( GEOS_FMT( "***** ERROR\n"
+                          "***** SIGNAL: {}\n"
+                          "***** LOCATION: (external error, captured by signal handler)\n"
+                          "{}\n{}",
+                          signal, error.m_msg, stackHistory ) );
+
+      if( ErrorLogger::global().isOutputFileEnabled() )
+      {
+        error.setType( ErrorLogger::MsgType::Error );
+        error.addRank( ::geos::logger::internal::g_rank );
+        error.addCallStackInfo( stackHistory );
+        error.addContextInfo(
+          ErrorContext{ { { ErrorContext::Attribute::Signal, std::to_string( signal ) } }, 1 },
+          ErrorContext{ { { ErrorContext::Attribute::DetectionLoc, string( "signal handler" ) } }, 0 } );
+
+        ErrorLogger::global().flushErrorMsg( error );
+      }
+
+      // call program termination
+      LvArray::system::callErrorHandler();
+    } );
+
+  }
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -77,21 +161,6 @@ void finalizeLogger()
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 void setupLvArray()
 {
-  LvArray::system::setErrorHandler( []()
-  {
-  #if defined( GEOS_USE_MPI )
-    int mpi = 0;
-    MPI_Initialized( &mpi );
-    if( mpi )
-    {
-      MPI_Abort( MPI_COMM_WORLD, EXIT_FAILURE );
-    }
-  #endif
-    std::abort();
-  } );
-
-  LvArray::system::setSignalHandling( []( int const signal ) { LvArray::system::stackTraceHandler( signal, true ); } );
-
 #if defined(GEOS_USE_FPE)
   LvArray::system::setFPE();
 #else

src/coreComponents/common/logger/ErrorHandling.cpp

@@ -25,6 +25,11 @@
 #include <regex>
 #include <string_view>
 
+// signal management
+#include <csignal>
+#include <cfenv>
+#include <cstring>
+
 namespace geos
 {
 static constexpr std::string_view g_level1Start = "  - ";
@@ -68,7 +73,9 @@ std::string ErrorLogger::ErrorContext::attributeToString( ErrorLogger::ErrorCont
     case ErrorLogger::ErrorContext::Attribute::InputFile: return "inputFile";
     case ErrorLogger::ErrorContext::Attribute::InputLine: return "inputLine";
     case ErrorLogger::ErrorContext::Attribute::DataPath: return "dataPath";
-    default: return "Unknown";
+    case ErrorLogger::ErrorContext::Attribute::DetectionLoc: return "detectionLocation";
+    case ErrorLogger::ErrorContext::Attribute::Signal: return "signal";
+    default: return "unknown";
   }
 }
 
@@ -98,6 +105,39 @@ ErrorLogger::ErrorMsg & ErrorLogger::ErrorMsg::addToMsg( std::string_view errorM
   return *this;
 }
 
+ErrorLogger::ErrorMsg & ErrorLogger::ErrorMsg::addSignalToMsg( int sig, bool toEnd )
+{
+  if( sig == SIGFPE )
+  {
+    std::string errorMsg = "Floating point error encountered: \n";
+
+    if( std::fetestexcept( FE_DIVBYZERO ) )
+      errorMsg += "- Division by zero operation.\n";
+
+    if( std::fetestexcept( FE_INEXACT ) )
+      errorMsg += "- Inexact result.\n";
+
+    if( std::fetestexcept( FE_INVALID ) )
+      errorMsg += "- Domain error occurred in an earlier floating-point operation.\n";
+
+    if( std::fetestexcept( FE_OVERFLOW ) )
+      errorMsg += "- The result of the earlier floating-point operation was too large to be representable.\n";
+
+    if( std::fetestexcept( FE_UNDERFLOW ) )
+      errorMsg += "- The result of the earlier floating-point operation was subnormal with a loss of precision.\n";
+
+    return addToMsg( errorMsg,
+                     toEnd );
+  }
+  else
+  {
+    // standard messages
+    return addToMsg( GEOS_FMT( "Signal no. {} encountered: {}\n",
+                               sig, ::strsignal( sig ) ),
+                     toEnd );
+  }
+}
+
 ErrorLogger::ErrorMsg & ErrorLogger::ErrorMsg::setCodeLocation( std::string_view msgFile, integer msgLine )
 {
   m_file = msgFile;
@@ -230,10 +270,12 @@ void ErrorLogger::flushErrorMsg( ErrorLogger::ErrorMsg & errorMsg )
     }
 
     // Location of the error in the code
-    yamlFile << g_level1Next << "sourceLocation:\n";
-    yamlFile << g_level2Next << "file: " << errorMsg.m_file << "\n";
-    yamlFile << g_level2Next << "line: " << errorMsg.m_line << "\n";
-
+    if( !errorMsg.m_file.empty() )
+    {
+      yamlFile << g_level1Next << "sourceLocation:\n";
+      yamlFile << g_level2Next << "file: " << errorMsg.m_file << "\n";
+      yamlFile << g_level2Next << "line: " << errorMsg.m_line << "\n";
+    }
     // Information about the stack trace
     if( !errorMsg.m_sourceCallStack.empty() )
     {
@@ -249,7 +291,7 @@ void ErrorLogger::flushErrorMsg( ErrorLogger::ErrorMsg & errorMsg )
     yamlFile << "\n";
     yamlFile.flush();
     errorMsg = ErrorMsg();
-    GEOS_LOG_RANK( GEOS_FMT( "The error file {} was appended.", m_filename ) );
+    GEOS_LOG_RANK( GEOS_FMT( "The error file {} has been appended.\n", m_filename ) );
   }
   else
   {

src/coreComponents/common/logger/ErrorHandling.hpp

@@ -61,7 +61,9 @@ class ErrorLogger
     {
       InputFile,
       InputLine,
-      DataPath
+      DataPath,
+      DetectionLoc,
+      Signal,
     };
 
     /// The map contains contextual information about the error
@@ -157,6 +159,16 @@ class ErrorLogger
      */
     ErrorMsg & addToMsg( std::string_view msg, bool toEnd = false );
 
+    /**
+     * @brief Add text to the error msg that occured according to the specified signal.
+     *        - the signal can be one of the main error signals.
+     *        - if the signal is SIGFPE, the nature of floating point error will be interpreted.
+     * @param signal The signal, from ISO C99 or POSIX standard.
+     * @param toEnd adds the message to the end if true, at the start otherwise.
+     * @return The instance, for builder pattern.
+     */
+    ErrorMsg & addSignalToMsg( int signal, bool toEnd = false );
+
     /**
      * @brief Set the source code location values (file and line where the error is detected)
      * @param msgFile Name of the source file location to add