Merge pull request #425 from SCOREC/apw/nompi

PCU: Add nompi stub

Author: cwsmith

.github/workflows/cmake.yml

@@ -15,6 +15,7 @@ jobs:
           - { name: GNU, CC: gcc-10, CXX: g++-10 }
           - { name: LLVM, CC: clang, CXX: clang++ }
         build_type: [Debug, Release]
+        no_mpi: [OFF, ON]
 
     steps:
     - uses: actions/checkout@v4
@@ -30,7 +31,7 @@ jobs:
       env:
         MPICH_CXX: ${{matrix.compiler.CXX}}
         MPICH_CC: ${{matrix.compiler.CC}}
-      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install
+      run: cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_CXX_COMPILER=mpicxx -DCMAKE_C_COMPILER=mpicc -DCMAKE_VERBOSE_MAKEFILE=ON -DMESHES=${{github.workspace}}/pumi-meshes -DIS_TESTING=ON -DSCOREC_CXX_WARNINGS=ON -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install -DSCOREC_NO_MPI=${{matrix.no_mpi}}
 
     - name: Build
       env:
@@ -54,3 +55,20 @@ jobs:
       run: |
         cmake -S ${{github.workspace}}/doc -B ${{github.workspace}}/buildExample -DCMAKE_CXX_COMPILER=mpicxx -DSCOREC_PREFIX=${{github.workspace}}/build/install
         cmake --build ${{github.workspace}}/buildExample
+
+    - name: Build MPI-NoMPI Example
+      # Test if a SCOREC_NO_MPI build works with MPI applications.
+      if: >-
+        matrix.compiler.name == 'GNU' && matrix.build_type == 'Release' &&
+        matrix.no_mpi == 'ON'
+      env:
+        MPICH_CXX: ${{matrix.compiler.CXX}}
+        MPICH_CC: ${{matrix.compiler.CC}}
+      run: >
+        cmake -S ${{github.workspace}}/example/mpi-nompi
+        -B ${{github.workspace}}/example/mpi-nompi/build
+        -DCMAKE_CXX_COMPILER=mpicxx
+        -DSCOREC_PREFIX=${{github.workspace}}/build/install ;
+        cmake --build ${{github.workspace}}/example/mpi-nompi/build ;
+        ctest --test-dir ${{github.workspace}}/example/mpi-nompi/build
+        --output-on-failure

CMakeLists.txt

@@ -6,7 +6,7 @@ endif()
 # This is the top level CMake file for the SCOREC build
 cmake_minimum_required(VERSION 3.8)
 
-project(SCOREC VERSION 3.0.3 LANGUAGES CXX C)
+project(SCOREC VERSION 4.0.0 LANGUAGES CXX C)
 
 include(cmake/bob.cmake)
 include(cmake/xsdk.cmake)
@@ -36,6 +36,15 @@ elseif(ENABLE_OMEGA_H)
   bob_set_cxx_standard(17)
 endif()
 
+#
+# option related to MPI
+#
+option(SCOREC_NO_MPI "Disable MPI support.  When enabled only serial execution is supported." OFF)
+message(STATUS "SCOREC_NO_MPI: ${SCOREC_NO_MPI}")
+if(SCOREC_NO_MPI AND ENABLE_CGNS)
+  message(FATAL_ERROR "SCOREC_NO_MPI is incompatible with CGNS.")
+endif()
+
 # Set some default compiler flags that should always be used
 if(NOT USE_XSDK_DEFAULTS)
   bob_set_shared_libs()
@@ -137,7 +146,10 @@ endif()
 
 macro(scorec_export_library target)
 bob_export_target(${target})
-install(FILES ${HEADERS} DESTINATION include)
+install(FILES ${HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
+target_include_directories(${target} INTERFACE
+  "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
+)
 endmacro(scorec_export_library)
 
 if(ENABLE_SIMMETRIX)
@@ -168,6 +180,10 @@ if(ENABLE_CGNS)
   add_definitions(-DHAVE_CGNS)
 endif()
 
+configure_file(SCOREC_config.h.in SCOREC_config.h)
+install(FILES "${CMAKE_BINARY_DIR}/SCOREC_config.h" DESTINATION include)
+include_directories(PUBLIC "$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}>")
+
 # Include the SCOREC project packages
 add_subdirectory(lion)
 add_subdirectory(pcu)

SCOREC_config.h.in

@@ -0,0 +1,7 @@
+#ifndef SCOREC_CONFIG_H
+#define SCOREC_CONFIG_H
+
+#cmakedefine SCOREC_NO_MPI
+
+#endif // SCOREC_CONFIG_H
+

apf/CMakeLists.txt

@@ -93,7 +93,6 @@ add_library(apf ${SOURCES})
 # Include directories
 target_include_directories(apf PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 # Link this library to these others

apf/apfCGNS.cc

@@ -545,8 +545,9 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe
         cgp_error_exit();
 
       std::vector<int> allNumbersForThisType(m->getPCU()->Peers(), 0);
-      MPI_Allgather(&numbersByElementType[o], 1, MPI_INT, allNumbersForThisType.data(), 1,
-                    MPI_INT, m->getPCU()->GetMPIComm());
+      m->getPCU()->Allgather(
+        &numbersByElementType[o], allNumbersForThisType.data(), 1
+      );
 
       cgsize_t num = 0;
       for (int i = 0; i < m->getPCU()->Self(); i++)
@@ -656,8 +657,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
         }
 
         std::vector<int> allNumbersForThisType(m->getPCU()->Peers(), 0);
-        MPI_Allgather(&number, 1, MPI_INT, allNumbersForThisType.data(), 1,
-                      MPI_INT, m->getPCU()->GetMPIComm());
+        m->getPCU()->Allgather(&number, allNumbersForThisType.data(), 1);
 
         cgsize_t num = 0;
         for (int i = 0; i < m->getPCU()->Self(); i++)
@@ -684,19 +684,16 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
       }
     }
     std::vector<int> cacheStarts(m->getPCU()->Peers(), 0);
-    MPI_Allgather(&cacheStart, 1, MPI_INT, cacheStarts.data(), 1,
-                  MPI_INT, m->getPCU()->GetMPIComm());
+    m->getPCU()->Allgather(&cacheStart, cacheStarts.data(), 1);
     std::vector<int> cacheEnds(m->getPCU()->Peers(), 0);
-    MPI_Allgather(&cacheEnd, 1, MPI_INT, cacheEnds.data(), 1,
-                  MPI_INT, m->getPCU()->GetMPIComm());
+    m->getPCU()->Allgather(&cacheEnd, cacheEnds.data(), 1);
     return std::make_pair(cacheStarts, cacheEnds);
   };
 
   const auto globalElementList = [&m](const std::vector<cgsize_t> &bcList, std::vector<cgsize_t> &allElements) {
     std::vector<int> sizes(m->getPCU()->Peers(), 0); // important initialiser
     const int l = bcList.size();
-    MPI_Allgather(&l, 1, MPI_INT, sizes.data(), 1,
-                  MPI_INT, m->getPCU()->GetMPIComm());
+    m->getPCU()->Allgather(&l, sizes.data(), 1);
 
     int totalLength = 0;
     for (const auto &i : sizes)
@@ -708,9 +705,15 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults,
       displacement[i] = displacement[i - 1] + sizes[i - 1];
 
     allElements.resize(totalLength);
+    #ifndef SCOREC_NO_MPI
+    PCU_Comm comm;
+    m->getPCU()->DupComm(&comm);
     MPI_Allgatherv(bcList.data(), bcList.size(), MPI_INT, allElements.data(),
-                   sizes.data(), displacement.data(), MPI_INT,
-                   m->getPCU()->GetMPIComm());
+                   sizes.data(), displacement.data(), MPI_INT, comm);
+    MPI_Comm_free(&comm);
+    #else
+    std::copy(bcList.begin(), bcList.end(), allElements.begin());
+    #endif
   };
 
   const auto doVertexBC = [&](const auto &iter) {
@@ -1046,7 +1049,8 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap
   sizes[2] = 0;                 // nodes are unsorted, as defined by api
 
   // Copy communicator
-  auto communicator = m->getPCU()->GetMPIComm();
+  PCU_Comm communicator;
+  m->getPCU()->DupComm(&communicator);
   cgp_mpi_comm(communicator);
   //
   cgp_pio_mode(CGP_INDEPENDENT);
@@ -1134,6 +1138,9 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap
   destroyGlobalNumbering(gcn);
   //
   cgp_close(cgns.index);
+  #ifndef SCOREC_NO_MPI
+  MPI_Comm_free(&communicator);
+  #endif
 }
 } // namespace
 

apf_cap/CMakeLists.txt

@@ -39,7 +39,6 @@ include(GNUInstallDirs)
 
 target_include_directories(apf_cap PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-  $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )
 
 scorec_export_library(apf_cap)

apf_sim/CMakeLists.txt

@@ -19,7 +19,6 @@ target_link_libraries(apf_sim PUBLIC ${SIMMODSUITE_LIBS})
 target_include_directories(gmi_sim PUBLIC ${SIMMODSUITE_INCLUDE_DIR})
 target_include_directories(apf_sim PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 option(ENABLE_FIELDSIM "Enable use of FieldSim from Simmetrix SimModSuite" FALSE)

can/CMakeLists.txt

@@ -11,7 +11,6 @@ add_library(can INTERFACE)
 # Include directories
 target_include_directories(can INTERFACE
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 scorec_export_library(can)

crv/CMakeLists.txt

@@ -33,7 +33,6 @@ add_library(crv ${SOURCES})
 # Include directories
 target_include_directories(crv INTERFACE
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    $<INSTALL_INTERFACE:include>
     )
 
 # Link this package to these libraries

doc/CMakeLists.txt

@@ -16,13 +16,13 @@ set(SCOREC_PREFIX "" CACHE STRING "Directory where SCOREC is installed")
 # If SCOREC_PREFIX was specified, only link to that directory,
 # i.e. don't link to another installation in /usr/lib by mistake
 if (SCOREC_PREFIX)
-  find_package(SCOREC 3.0.0 REQUIRED CONFIG PATHS ${SCOREC_PREFIX} NO_DEFAULT_PATH)
+  find_package(SCOREC 4 REQUIRED CONFIG PATHS ${SCOREC_PREFIX} NO_DEFAULT_PATH)
 else()
 # IF SCOREC_PREFIX was not specified, look in typical system directories,
 # and also in CMAKE_PREFIX_PATH (environment variable)
   find_package(
       SCOREC #package name, has to be SCOREC
-      3.0.0  #version. can be omitted, and will match any installed version
+      4      #version. can be omitted, and will match any installed version
              #greater than or equal to this one, as long as the major number
              #is the same
       REQUIRED #indicate that SCOREC is really needed to compile