.github/workflows/MPI.yml

@@ -0,0 +1,261 @@
+# Perform builds with supported MPI versions
+name: MPI
+
+on:
+  # On pull_requests which mutate this CI workflow, using pr rather than push:branch to ensure that the merged state will be OK, as that is what is important here.
+  pull_request:
+    paths:
+      - ".github/workflows/MPI.yml"
+      - "tests/test_cases/simulation/test_mpi_ensemble.cu"
+      - "include/flamegpu/simulation/detail/MPISimRunner.h"
+      - "include/flamegpu/simulation/detail/AbstractSimRunner.h"
+      - "src/flamegpu/simulation/detail/MPISimRunner.cu"
+      - "src/flamegpu/simulation/detail/AbstractSimRunner.cu"
+      - "src/flamegpu/simulation/CUDAEnsemble.cu"
+      - "**mpi**"
+      - "**MPI**"
+  # Or trigger on manual dispatch. 
+  workflow_dispatch:
+
+defaults:
+  run:
+    # Default to using bash regardless of OS unless otherwise specified.
+    shell: bash
+
+jobs:
+  build-ubuntu:
+    runs-on: ${{ matrix.cudacxx.os }}
+    strategy:
+      fail-fast: false
+      # Multiplicative build matrix
+      # optional exclude: can be partial, include: must be specific
+      matrix:
+        # CUDA_ARCH values are reduced compared to wheels due to CI memory issues while compiling the test suite.
+        cudacxx:
+          - cuda: "12.0"
+            cuda_arch: "50-real;"
+            hostcxx: gcc-11
+            os: ubuntu-22.04
+        python:
+          - "3.8"
+        mpi:
+          - lib: "openmpi"
+            version: "apt" # MPI 3.1
+          # - lib: "openmpi"
+          #   version: "4.1.6" # MPI 3.1
+          # - lib: "openmpi"
+          #   version: "4.0.0" # MPI 3.1
+          # - lib: "openmpi"
+          #   version: "3.0.0" # MPI 3.1
+          # - lib: "openmpi"
+          #   version: "2.0.0" # MPI 3.1
+          - lib: "openmpi"
+            version: "1.10.7" # MPI 3.0
+          - lib: "mpich"
+            version: "apt" # MPI 4.0
+          # - lib: "mpich"
+          #   version: "4.1.2" # MPI 4.0
+          # - lib: "mpich"
+          #   version: "4.0" # MPI 4.0
+          # - lib: "mpich"
+          #   version: "3.4.3" # MPI 3.1
+          - lib: "mpich"
+            version: "3.3" # MPI 3.1
+        config:
+          - name: "Release"
+            config: "Release"
+            SEATBELTS: "ON"
+        VISUALISATION: 
+          - "OFF"
+
+    # Name the job based on matrix/env options
+    name: "build-ubuntu-mpi (${{ matrix.mpi.lib }}, ${{ matrix.mpi.version }}, ${{ matrix.cudacxx.cuda }}, ${{matrix.python}}, ${{ matrix.VISUALISATION }}, ${{ matrix.config.name }}, ${{ matrix.cudacxx.os }})"
+
+    # Define job-wide env constants, and promote matrix elements to env constants for portable steps.
+    env:
+      # Define constants
+      BUILD_DIR: "build"
+      FLAMEGPU_BUILD_TESTS: "ON"
+      # Conditional based on matrix via awkward almost ternary
+      FLAMEGPU_BUILD_PYTHON: ${{ fromJSON('{true:"ON",false:"OFF"}')[matrix.python != ''] }}
+      # Port matrix options to environment, for more portability.
+      CUDA: ${{ matrix.cudacxx.cuda }}
+      CUDA_ARCH: ${{ matrix.cudacxx.cuda_arch }}
+      HOSTCXX: ${{ matrix.cudacxx.hostcxx }}
+      OS: ${{ matrix.cudacxx.os }}
+      CONFIG: ${{ matrix.config.config }}
+      FLAMEGPU_SEATBELTS: ${{ matrix.config.SEATBELTS }}
+      PYTHON: ${{ matrix.python }}
+      MPI_LIB: ${{ matrix.mpi.lib }}
+      MPI_VERSION: ${{ matrix.mpi.version }}
+      VISUALISATION: ${{ matrix.VISUALISATION }}
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Install CUDA
+      if: ${{ startswith(env.OS, 'ubuntu') && env.CUDA != '' }}
+      env:
+        cuda: ${{ env.CUDA }}
+      run: .github/scripts/install_cuda_ubuntu.sh
+
+    - name: Install/Select gcc and g++
+      if: ${{ startsWith(env.HOSTCXX, 'gcc-') }}
+      run: |
+        gcc_version=${HOSTCXX//gcc-/}
+        sudo apt-get install -y gcc-${gcc_version} g++-${gcc_version}
+        echo "CC=/usr/bin/gcc-${gcc_version}" >> $GITHUB_ENV
+        echo "CXX=/usr/bin/g++-${gcc_version}" >> $GITHUB_ENV
+        echo "CUDAHOSTCXX=/usr/bin/g++-${gcc_version}" >> $GITHUB_ENV
+    
+    - name: Install MPI from apt
+      if: ${{ env.MPI_VERSION == 'apt' }}
+      working-directory: ${{ runner.temp }}
+      run: |
+        sudo apt-get install lib${{ env.MPI_LIB }}-dev
+
+    - name: Install OpenMPI from source
+      if: ${{ env.MPI_VERSION != 'apt' && env.MPI_LIB == 'openmpi' }}
+      working-directory: ${{ runner.temp }}
+      run: |
+        # Note: using download.open-mpi.org as gh tags aren't pre configured
+        MPI_VERISON_MAJOR_MINOR=$(cut -d '.' -f 1,2 <<< "${{ env.MPI_VERSION}}")
+        echo "https://download.open-mpi.org/release/open-mpi/v${MPI_VERISON_MAJOR_MINOR}/openmpi-${{ env.MPI_VERSION}}.tar.gz"
+        wget -q https://download.open-mpi.org/release/open-mpi/v${MPI_VERISON_MAJOR_MINOR}/openmpi-${{ env.MPI_VERSION}}.tar.gz --output-document openmpi-${{ env.MPI_VERSION }}.tar.gz || (echo "An Error occurred while downloading OpenMPI '${{ env.MPI_VERSION }}'. Is it a valid version of OpenMPI?" && exit 1)
+        tar -zxvf openmpi-${{ env.MPI_VERSION }}.tar.gz
+        cd openmpi-${{ env.MPI_VERSION}}
+        ./configure --prefix="${{ runner.temp }}/mpi"
+        make -j `nproc`
+        make install -j `nproc`
+        echo "${{ runner.temp }}/mpi/bin" >> $GITHUB_PATH
+        echo "LD_LIBRARY_PATH=${{ runner.temp }}/mpi/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+        echo "LD_RUN_PATH=${{ runner.temp }}/mpi/lib:${LD_RUN_PATH}" >> $GITHUB_ENV
+
+    # This will only work for mpich >= 3.3:
+    #   3.0-3.2 doesn't appear compatible with default gcc in 22.04. 
+    #   1.x is named mpich2 so requires handling differently
+    # Uses the ch3 interface, as ch4 isn't available pre 3.4, but one must be specified for some versions
+    - name: Install MPICH from source
+      if: ${{ env.MPI_VERSION != 'apt' && env.MPI_LIB == 'mpich' }}
+      working-directory: ${{ runner.temp }}
+      run: |
+        MPI_MAJOR=$(cut -d '.' -f 1 <<< "${{ env.MPI_VERSION}}")
+        MPI_MINOR=$(cut -d '.' -f 2 <<< "${{ env.MPI_VERSION}}")
+        [[ ${MPI_MAJOR} < 3 ]] && echo "MPICH must be >= 3.0" && exit 1
+        echo "https://www.mpich.org/static/downloads/${{ env.MPI_VERSION }}/mpich-${{ env.MPI_VERSION}}.tar.gz"
+        wget -q https://www.mpich.org/static/downloads/${{ env.MPI_VERSION }}/mpich-${{ env.MPI_VERSION}}.tar.gz --output-document mpich-${{ env.MPI_VERSION }}.tar.gz || (echo "An Error occurred while downloading MPICH '${{ env.MPI_VERSION }}'. Is it a valid version of MPICH?" && exit 1)
+        tar -zxvf mpich-${{ env.MPI_VERSION }}.tar.gz
+        cd mpich-${{ env.MPI_VERSION}}
+        DISABLE_FORTRAN_FLAGS=""
+        if (( ${MPI_MAJOR} >= 4 )) || ( ((${MPI_MAJOR} >= 3)) && ((${MPI_MINOR} >= 2)) ); then
+            # MPICH >= 3.2 has --disable-fortran
+            DISABLE_FORTRAN_FLAGS="--disable-fortran" 
+        else
+            DISABLE_FORTRAN_FLAGS="--disable-f77 --disable-fc"
+        fi
+        ./configure --prefix="${{ runner.temp }}/mpi" --with-device=ch3 ${DISABLE_FORTRAN_FLAGS}
+        make -j `nproc`
+        make install -j `nproc`
+        echo "${{ runner.temp }}/mpi/bin" >> $GITHUB_PATH
+        echo "LD_LIBRARY_PATH=${{ runner.temp }}/mpi/lib:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+        echo "LD_RUN_PATH=${{ runner.temp }}/mpi/lib:${LD_RUN_PATH}" >> $GITHUB_ENV
+    
+    - name: Select Python
+      if: ${{ env.PYTHON != '' && env.FLAMEGPU_BUILD_PYTHON == 'ON' }} 
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PYTHON }}
+
+    - name: Install python dependencies
+      if: ${{ env.PYTHON != '' && env.FLAMEGPU_BUILD_PYTHON == 'ON' }}
+      run: |
+        sudo apt-get install python3-venv
+        python3 -m pip install --upgrade wheel build setuptools
+
+    - name: Install Visualisation Dependencies
+      if: ${{ startswith(env.OS, 'ubuntu') && env.VISUALISATION == 'ON' }}
+      run: |
+        # Install ubuntu-20.04 packages
+        if [ "$OS" == 'ubuntu-20.04' ]; then 
+          sudo apt-get install -y libglew-dev libfontconfig1-dev libsdl2-dev libdevil-dev libfreetype-dev
+        fi
+        # Install Ubuntu 18.04 packages
+        if [ "$OS" == 'ubuntu-18.04' ]; then 
+          sudo apt-get install -y libglew-dev libfontconfig1-dev libsdl2-dev libdevil-dev libfreetype6-dev libgl1-mesa-dev
+        fi
+    
+    - name: Install Swig >= 4.0.2
+      run: |
+        # Remove existing swig install, so CMake finds the correct swig
+        if [ "$OS" == 'ubuntu-20.04' ]; then 
+          sudo apt-get remove -y swig swig4.0
+        fi
+        # Install Ubuntu 18.04 packages
+        if [ "$OS" == 'ubuntu-18.04' ]; then 
+          sudo apt-get remove -y swig
+        fi
+        # Install additional apt-based dependencies required to build swig 4.0.2
+        sudo apt-get install -y bison
+        # Create a local directory to build swig in.
+        mkdir -p swig-from-source && cd swig-from-source
+        # Install SWIG building from source dependencies
+        wget https://github.com/swig/swig/archive/refs/tags/v4.0.2.tar.gz
+        tar -zxf v4.0.2.tar.gz
+        cd swig-4.0.2/
+        ./autogen.sh
+        ./configure
+        make
+        sudo make install
+
+    # This pre-emptively patches a bug where ManyLinux didn't generate buildnumber as git dir was owned by diff user
+    - name: Enable git safe-directory
+      run: git config --global --add safe.directory $GITHUB_WORKSPACE
+
+    - name: Configure cmake
+      run: >
+        cmake . -B "${{ env.BUILD_DIR }}"
+        -DCMAKE_BUILD_TYPE="${{ env.CONFIG }}"
+        -Werror=dev
+        -DCMAKE_WARN_DEPRECATED="OFF" 
+        -DFLAMEGPU_WARNINGS_AS_ERRORS="ON"
+        -DCMAKE_CUDA_ARCHITECTURES="${{ env.CUDA_ARCH }}"
+        -DFLAMEGPU_BUILD_TESTS="${{ env.FLAMEGPU_BUILD_TESTS }}"
+        -DFLAMEGPU_BUILD_PYTHON="${{ env.FLAMEGPU_BUILD_PYTHON }}"
+        -DPYTHON3_EXACT_VERSION="${{ env.PYTHON }}"
+        -DFLAMEGPU_VISUALISATION="${{ env.VISUALISATION }}"
+        -DFLAMEGPU_ENABLE_MPI="ON"
+        -DFLAMEGPU_ENABLE_NVTX="ON"
+        ${MPI_OVERRIDE_CXX_OPTIONS}
+
+    - name: Reconfigure cmake fixing MPICH from apt
+      if: ${{ env.MPI_VERSION == 'apt' && env.MPI_LIB == 'mpich' }}
+      run: >
+        cmake . -B "${{ env.BUILD_DIR }}"
+        -DMPI_CXX_COMPILE_OPTIONS=""
+
+    - name: Build static library
+      working-directory: ${{ env.BUILD_DIR }}
+      run: cmake --build . --target flamegpu --verbose -j `nproc`
+
+    - name: Build python wheel
+      if: ${{ env.FLAMEGPU_BUILD_PYTHON == 'ON' }}
+      working-directory: ${{ env.BUILD_DIR }}
+      run: cmake --build . --target pyflamegpu --verbose -j `nproc`
+
+    - name: Build tests
+      if: ${{ env.FLAMEGPU_BUILD_TESTS == 'ON' }}
+      working-directory: ${{ env.BUILD_DIR }}
+      run: cmake --build . --target tests --verbose -j `nproc`
+
+    - name: Build tests_mpi
+      if: ${{ env.FLAMEGPU_BUILD_TESTS == 'ON' }}
+      working-directory: ${{ env.BUILD_DIR }}
+      run: cmake --build . --target tests_mpi --verbose -j `nproc`
+
+    - name: Build ensemble example
+      working-directory: ${{ env.BUILD_DIR }}
+      run: cmake --build . --target ensemble --verbose -j `nproc`
+
+    - name: Build all remaining targets
+      working-directory: ${{ env.BUILD_DIR }}
+      run: cmake --build . --target all --verbose -j `nproc`

.github/workflows/Ubuntu.yml

@@ -186,6 +186,11 @@ jobs:
         -DFLAMEGPU_VISUALISATION="${{ env.VISUALISATION }}"
         -DFLAMEGPU_ENABLE_NVTX="ON"
 
+    # Check for bugs when cmake is reconfigured, i.e. fetch content patching
+    - name: Re-configure cmake
+      run: >
+        cmake . -B "${{ env.BUILD_DIR }}"
+
     - name: Build static library
       working-directory: ${{ env.BUILD_DIR }}
       run: cmake --build . --target flamegpu --verbose -j `nproc`

.github/workflows/Windows.yml

@@ -146,6 +146,11 @@ jobs:
         -DFLAMEGPU_VISUALISATION="${{ env.VISUALISATION }}"
         -DFLAMEGPU_ENABLE_NVTX="ON"
 
+    # Check for bugs when cmake is reconfigured, i.e. fetch content patching
+    - name: Re-configure cmake
+      run: >
+        cmake . -B "${{ env.BUILD_DIR }}"
+
     - name: Build static library
       working-directory: ${{ env.BUILD_DIR }}
       run: cmake --build . --config ${{ env.CONFIG }} --target flamegpu --verbose -j `nproc`

.gitignore

@@ -3,6 +3,7 @@
 *.json
 *.gv
 *.bin
+*test_rtcfunc_file
 
 # GoogleTest directories (handled by CMake)
 googletest-build/