diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 4f0b638b..814268b4 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -67,6 +67,12 @@ jobs: DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 HOST_ARCH: ppc64le RUNS_ON: streak2 + - name: Pangea 4 (centos 8, gcc 12.1.0, hpcx 2.20.0, mkl 2023.2.0) + DOCKER_REPOSITORY: geosx/pangea4-gcc12.1-hpcxompi2.20.0-onemkl2023.2.0 + TPL_DOCKERFILE: docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile + DOCKER_ROOT_IMAGE: onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 + INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p4/install/tpl + RUNS_ON: ubuntu-latest - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11 TPL_DOCKERFILE: docker/Stanford/Dockerfile diff --git a/docker/TotalEnergies/Dockerfile b/docker/TotalEnergies/Dockerfile deleted file mode 100644 index 2c796542..00000000 --- a/docker/TotalEnergies/Dockerfile +++ /dev/null @@ -1,92 +0,0 @@ -# NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments -ARG TMP_DIR=/tmp -ARG SRC_DIR=$TMP_DIR/thirdPartyLibs -ARG BLD_DIR=$TMP_DIR/build - -# The docker base image can be pecan or pangea. -ARG DOCKER_ROOT_IMAGE -FROM $DOCKER_ROOT_IMAGE as tpl_toolchain_intersect_geosx_toolchain -ARG SRC_DIR - -ARG INSTALL_DIR -ENV GEOSX_TPL_DIR=$INSTALL_DIR - -RUN yum install --disablerepo=intel-mkl-repo -y \ - ca-certificates \ - curl \ - python3 \ - zlib-devel \ -# Additional spack dependencies - python3-pip \ - pkgconfig \ - # xz \ - unzip \ - bzip2 \ - && pip3 install virtualenv - -# Install clingo for Spack -RUN python3 -m pip install --upgrade pip && \ - python3 -m pip install clingo - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh - -FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain -ARG SRC_DIR -ARG BLD_DIR - -RUN yum install --disablerepo=intel-mkl-repo -y \ - make \ - bc \ - file \ - bison \ - flex \ - patch \ - ca-certificates \ - autoconf \ - automake \ - git - -ARG SPEC - -# Run uberenv -# Have to create install directory first for uberenv -# -k flag is to ignore SSL errors -RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ - mkdir -p ${GEOSX_TPL_DIR} && \ - ./scripts/uberenv/uberenv.py \ - # spack spec defined per image - --spec "${SPEC}" \ - --spack-env-file=${SRC_DIR}/docker/total-spack.yaml \ - --project-json=.uberenv_config.json \ - --prefix ${GEOSX_TPL_DIR} \ - -k && \ -# Remove host-config generated for LvArray - rm lvarray* && \ -# Rename and copy spack-generated host-config to root directory - cp *.cmake /spack-generated.cmake && \ -# Remove extraneous spack files - cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ - -FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain -ARG SRC_DIR - -COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR - -# Extract the generated host-config -COPY --from=tpl_toolchain /spack-generated.cmake / - -RUN yum install --disablerepo=intel-mkl-repo -y \ - openssh-clients \ - ca-certificates \ - curl \ - python3 \ - texlive \ - graphviz \ - libxml2 \ - git - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh - -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh -ENV SCCACHE=/opt/sccache/bin/sccache diff --git a/docker/TotalEnergies/Pangea3.Dockerfile b/docker/TotalEnergies/Pangea3.Dockerfile index bfad0260..513eda54 100644 --- a/docker/TotalEnergies/Pangea3.Dockerfile +++ b/docker/TotalEnergies/Pangea3.Dockerfile @@ -49,7 +49,7 @@ RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ # Run uberenv ./scripts/uberenv/uberenv.py \ --spec "%gcc@9.4.0+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ - --spack-env-file=${SRC_DIR}/docker/pangea-spack.yaml \ + --spack-env-file=${SRC_DIR}/docker/pangea3-spack.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ -k && \ diff --git a/docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile b/docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile new file mode 100644 index 00000000..5085401a --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-gcc-hpcxompi-onemkl.Dockerfile @@ -0,0 +1,81 @@ +####################################### +# Pangea 4 image : gcc - hpcxompi - onemkl +####################################### +# +# Installs : +# - hpcx = 2.20.0 +# - intel-oneapi-mkl = 2023.2.0 +# +####################################### +# +# Description : +# - the image is based on onetechssc/pangea4:gcc12.1_v1.0 built from pangea4-gcc.Dockerfile +# - this image is deployed as onetechssc/pangea4:gcc12.1-hpcx2.20.0-onemkl2023.2.0_v1.0 +# - oneAPI MKL is installed via spack (as done for Pangea 4) +# - hpcx is installed by extracting the tarball (as done for Pangea 4) +# - gcc wrappers are created to mimic the Cray environment (cc, CC, ftn) +# +# Usage : +# build the image: +# - podman build --format docker -t onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 -f pangea4-gcc-hpcxompi-onemkl.Dockerfile -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem . +# run the image: +# - podman run -it --detach --privileged --name pangea4_gcc_hpcxompi_onemkl -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# - podman exec -it pangea4_gcc_hpcxompi_onemkl /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 docker://docker.io/onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# +####################################### + +# ------------------------------------- +# PANGEA4 - GCC-HPCX-MKL +FROM docker.io/onetechssc/pangea4:gcc12.1_v1.0 AS pangea4 +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python - hpcx - mkl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG ONEAPI_MKL_VERSION=2023.2.0 +ARG HPCX_VERSION=2.20 +ARG HPCX_FULL_NAME="hpcx-v$HPCX_VERSION-gcc-mlnx_ofed-redhat8-cuda12-x86_64" +ARG HPCX_TARBALL="$HPCX_FULL_NAME.tbz" +ARG HPCX_URL="http://www.mellanox.com/downloads/hpc/hpc-x/v$HPCX_VERSION/$HPCX_TARBALL" +ARG CRAY_WRAPPERS_DIR=/sw/cray-wrappers +# ------ +# INSTALL +# spack config (make sure you survive our wonderfull proxy) +RUN spack config --scope defaults add config:connect_timeout:120 +RUN spack config --scope defaults add config:url_fetch_method:curl +# intel-oneapi-mkl +RUN spack install intel-oneapi-mkl@$ONEAPI_MKL_VERSION %gcc@$GCC_VERSION +# hpcx not available in spack -> download and untar in /sw directory +RUN mkdir -p /sw && \ + spack load wget && \ + wget --ca-certificate=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem $HPCX_URL -O /tmp/$HPCX_TARBALL && \ + tar -xvf /tmp/$HPCX_TARBALL -C /sw && \ + rm -f /tmp/$HPCX_TARBALL +ENV HPCX_HOME=/sw/$HPCX_FULL_NAME +# numpy with mkl +RUN spack install py-numpy %gcc@$GCC_VERSION ^python@$PYTHON_VERSION ^intel-oneapi-mkl@$ONEAPI_MKL_VERSION +# ------ +# ENV +# create wrappers for gcc +RUN mkdir -p $CRAY_WRAPPERS_DIR && \ + spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION && \ + GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) && \ + ln -s ${GCC_INSTALL_DIR}/bin/gcc $CRAY_WRAPPERS_DIR/cc && \ + ln -s ${GCC_INSTALL_DIR}/bin/g++ $CRAY_WRAPPERS_DIR/CC && \ + ln -s ${GCC_INSTALL_DIR}/bin/gfortran $CRAY_WRAPPERS_DIR/ftn +# create env script +RUN < /root/.setup_env.sh +. /opt/spack/share/spack/setup-env.sh +spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION intel-oneapi-mkl@$ONEAPI_MKL_VERSION py-numpy +source ${HPCX_HOME}/hpcx-init.sh +hpcx_load +export PATH=$CRAY_WRAPPERS_DIR:\$PATH +export GCC_INSTALL_DIR=$(spack location -i gcc@$GCC_VERSION) +export GOMP_ROOT=\${GCC_INSTALL_DIR}/lib64 +EOF +RUN chmod +x /root/.setup_env.sh diff --git a/docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile b/docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile new file mode 100644 index 00000000..920162fd --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-gcc.Dockerfile @@ -0,0 +1,67 @@ +# syntax=docker/dockerfile:1.4 + +####################################### +# Pangea 4 image : gcc - cmake - python +####################################### +# +# Installs : +# - gcc = 12.1 +# - cmake = 3.27.9 +# - python = 3.11 +# +####################################### +# +# Description : +# - the image is based on spack/centos-stream:latest (CentOS Stream 8 image with spack installed) +# - this image is deployed as onetechssc/pangea4:gcc12.1_v1.0 +# - gcc, cmake, python and wget are installed via spack (as done for Pangea 4) +# +# Usage : +# build the image: +# - podman build --format docker -t onetechssc/pangea4:gcc12.1_v1.0 -f pangea4-gcc.Dockerfile -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem . +# run the image: +# - podman run -it --detach --privileged --name pangea4_gcc -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea4:gcc12.1_v1.0 +# - podman exec -it pangea4_gcc /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/pangea4:gcc12.1_v1.0 docker://docker.io/onetechssc/pangea4:gcc12.1_v1.0 +# +####################################### + +# ------------------------------------- +# PANGEA4 - BASE [GCC - CMAKE - PYTHON] +# install tools via spack for linux/x86_64 architecture and redhat8 platform +FROM --platform=linux/x86_64 docker.io/spack/centos-stream:latest AS pangea4-base +# ------ +# LABELS +LABEL description="Pangea 4 image : gcc - cmake - python" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG GCC_VERSION=12.1 +ARG CMAKE_VERSION=3.27.9 +ARG PYTHON_VERSION=3.11 +# ------ +# INSTALL +# spack config (make sure you survive our wonderfull proxy) +RUN spack config --scope defaults add config:connect_timeout:120 +RUN spack config --scope defaults add config:url_fetch_method:curl +# gcc +RUN spack install gcc@$GCC_VERSION +RUN . /opt/spack/share/spack/setup-env.sh &&\ + spack load gcc@$GCC_VERSION &&\ + spack compiler find +# python +RUN spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION +# cmake (openssl error when using curl) +RUN spack config --scope defaults add config:url_fetch_method:urllib &&\ + spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION &&\ + spack config --scope defaults add config:url_fetch_method:curl +# wget +RUN spack install wget %gcc@$GCC_VERSION +# ------ +# ENV +ENV GCC_VERSION=${GCC_VERSION} +ENV CMAKE_VERSION=${CMAKE_VERSION} +ENV PYTHON_VERSION=${PYTHON_VERSION} \ No newline at end of file diff --git a/docker/TotalEnergies/pangea-4/pangea4-spack.yaml b/docker/TotalEnergies/pangea-4/pangea4-spack.yaml new file mode 100644 index 00000000..a16e4acf --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-spack.yaml @@ -0,0 +1,207 @@ +# This is a Spack Environment file for Pangea-4 docker image. +# - replicate Pangea-4 environement as close as possible +# while targeting x86_64 for ci/cd. +# - used with onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# docker image +# +# It describes a set of packages to be installed, along with +# configuration settings. +# +# Run command from the top-level of the repository: +# ./scripts/uberenv/uberenv.py \ +# --spec "%gcc@12.2.1 ~openmp~pygeosx" \ +# --spack-env-file=docker/TotalEnergies/pangea-4/pangea4-spack.yaml \ +# --project-json=.uberenv_config.json \ +# --prefix ${GEOS_TPL_DIR} +spack: + config: + # make sure you survive our wonderfull proxy when testing on prem + connect_timeout: 120 + url_fetch_method: curl + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + build_stage:: + - $spack/../build_stage + + # Regular TPLs do not need views + view: false + + ############# + # COMPILERS # + ############# + + compilers: + - compiler: + spec: gcc@=12.2.1 + paths: + cc: /opt/spack/opt/spack/linux-centos8-zen/gcc-8.5.0/gcc-12.1.0-74x6yognzvkzjoh6zvmkimhhsuijh237/bin/gcc + cxx: /opt/spack/opt/spack/linux-centos8-zen/gcc-8.5.0/gcc-12.1.0-74x6yognzvkzjoh6zvmkimhhsuijh237/bin/g++ + f77: /opt/spack/opt/spack/linux-centos8-zen/gcc-8.5.0/gcc-12.1.0-74x6yognzvkzjoh6zvmkimhhsuijh237/bin/gfortran + fc: /opt/spack/opt/spack/linux-centos8-zen/gcc-8.5.0/gcc-12.1.0-74x6yognzvkzjoh6zvmkimhhsuijh237/bin/gfortran + flags: + cflags: -march=native -mtune=native + cxxflags: -march=native -mtune=native + operating_system: centos8 + target: x86_64 + modules: [] + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: [] + + ############# + # PACKAGES # + ############# + + packages: + all: + target: [x86_64] + compiler: [gcc] + providers: + blas: [intel-oneapi-mkl] + lapack: [intel-oneapi-mkl] + mpi: [hpcx-mpi] + + #### + # make sure spack doesn't rebuild mpi, blas and lapack libs + # without this it will try to build mpich and openblas + mpi: + buildable: false + blas: + buildable: false + lapack: + buildable: false + + #### + # spec of packages to build + + # v0.6.2 + blt: + require: "@git.9ff77344f0b2a6ee345e452bddd6bfd46cbbfa35=develop" + # master - 10/18/24 + hypre: + require: "@git.c893886d15eb57e87dd36efec23693ece3ddc88e" + # v2024.07.0 + chai: + require: "@git.df7741f1dbbdc5fff5f7d626151fdf1904e62b19=develop" + # v2024.07.0 + umpire: + require: "@git.abd729f40064175e999a83d11d6b073dac4c01d2=develop" + # v2024.07.0 + raja: + require: "@git.4d7fcba55ebc7cb972b7cc9f6778b48e43792ea1=develop" + # v2024.07.0 + camp: + require: "@git.0f07de4240c42e0b38a8d872a20440cb4b33d9f5=main" + # v2.12.0 + caliper: + require: "@git.287b7f3ad2d12f520aad04268d44f353cd05403c" + # v0.9.2 + conduit: + require: "@git.ad86e316ad56a75c099d30ca5ce75cff275b5924=develop" + # master - 04/12/20 + uncrustify: + require: "@git.401a4098bce9dcc47e024987403f2d59d9ba7bd2" + # master - 04/26/20 + superlu-dist: + require: "@git.0f6efc377df2440c235452d13d28d2c717f832a1" + + #### + # spec of spack packages to reuse + + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2023.2.0 threads=openmp + prefix: /opt/spack/opt/spack/linux-centos8-zen3/gcc-12.1.0/intel-oneapi-mkl-2023.2.0-os7byfaou43rtdoutsio7cqfta67uvpb + buildable: False + python: + externals: + - spec: python@3.11.7 + prefix: /opt/spack/opt/spack/linux-centos8-zen3/gcc-12.1.0/python-3.11.7-gtr54gwtvgrt7rcvecnjwbzo5mjvhlhr + buildable: false + cmake: + externals: + - spec: cmake@3.27.9 + prefix: /opt/spack/opt/spack/linux-centos8-zen3/gcc-12.1.0/cmake-3.27.9-qlycuifsyg46wiqljnglszevjl34seux + buildable: false + berkeley-db: + externals: + - spec: berkeley-db@18.1.40 + prefix: /opt/spack/opt/spack/linux-centos8-zen/gcc-8.5.0/berkeley-db-18.1.40-x4zdnigaawimrbhtaw3enreohi54ljwe + buildable: false + perl: + externals: + - spec: perl@5.38.0 + prefix: /opt/spack/opt/spack/linux-centos8-zen/gcc-8.5.0/perl-5.38.0-b7ny3twfj25hgukovwifhdbobv4zn3nw + buildable: false + mpfr: + externals: + - spec: mpfr@4.2.1 + prefix: /opt/spack/opt/spack/linux-centos8-zen/gcc-8.5.0/mpfr-4.2.1-6adatrj5k6lpmk4ozvmet7lwfdbibrca + buildable: false + gmp: + externals: + - spec: gmp@6.2.1 + prefix: /opt/spack/opt/spack/linux-centos8-zen/gcc-8.5.0/gmp-6.2.1-lflaucke4mt4xox245ahvqdehbr4k6i2 + buildable: false + gcc-runtime: + externals: + - spec: gcc-runtime@12.2.1 + prefix: /opt/spack/opt/spack/linux-centos8-zen/gcc-12.1.0/gcc-runtime-12.1.0-dt4sq2ct65tsyhuv2f5l4r2sstfs7yuz + buildable: false + + #### + # spec of system packages to reuse + + # bin + flex: + externals: + - spec: flex@2.6.1+lex + prefix: /usr + buildable: false + git: + externals: + - spec: git@2.43.0 + prefix: /usr + buildable: false + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + buildable: false + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + buildable: false + # libs + glibc: + externals: + - spec: glibc@2.28 + prefix: /usr + buildable: false + hpcx-mpi: + externals: + - spec: hpcx-mpi@2.20.0 + prefix: /sw/hpcx-v2.20-gcc-mlnx_ofed-redhat8-cuda12-x86_64/ompi + buildable: false + pkgconf: + externals: + - spec: pkgconf@3.0.0 + prefix: /usr + buildable: false + readline: + buildable: false + externals: + - spec: readline@7.0 + prefix: /usr + zlib: + externals: + - spec: zlib@1.2.11 + prefix: /usr + buildable: false \ No newline at end of file diff --git a/docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile b/docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile new file mode 100644 index 00000000..28de13be --- /dev/null +++ b/docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile @@ -0,0 +1,102 @@ +####################################### +# Pangea 4 tpl image +# +# Description : +# - generic image for building geos tpls on Pangea 4 environments +# - the docker base image must match the spack environment +# - tools of the base image are expected to be sourced in the set_env.sh script +# +# Usage : +# build the image (from the root of the repository!): +# - podman build --format docker --progress=plain \ +# --build-arg DOCKER_ROOT_IMAGE=onetechssc/pangea4:gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 \ +# --build-arg INSTALL_DIR=/workrd/SCR/NUM/GEOS_environment/p4/install/tpl/install \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -t onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 \ +# -f docker/TotalEnergies/pangea-4/pangea4-tpl.Dockerfile . +# run the image: +# - podman run -it --detach --privileged --name pangea4_tpl -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# - podman exec -it pangea4_tpl /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 docker://docker.io/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 +# build geos from the image (from the root of geos reopsitory!): +# - podman run --name pangea4_geos \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -v .:/tmp/geos \ +# localhost/onetechssc/geos:tpl_gcc12.1-hpcxompi2.20.0-onemkl2023.2.0_v1.0 \ +# /tmp/geos/scripts/ci_build_and_test_in_container.sh \ +# --host-config /spack-generated.cmake \ +# --repository /tmp/geos --cmake-build-type Release --install-dir /tmp/install --build-exe-only +####################################### + +# ------------------------------------- +# PANGEA4 - TPL BASE +ARG DOCKER_ROOT_IMAGE +FROM $DOCKER_ROOT_IMAGE AS pangea4_tpl_base +# ------ +# LABELS +LABEL description="Pangea 4 image : geos_tpl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG INSTALL_DIR +# ------ +# ENV +ENV GEOS_TPL_DIR=$INSTALL_DIR +ENV GEOSX_TPL_DIR=$GEOS_TPL_DIR + +# ------------------------------------- +# PANGEA4 - TPL BUILDER +FROM pangea4_tpl_base AS pangea4_tpl_builder +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/src +# ------ +# INSTALL +# Run uberenv +# Have to create install directory first for uberenv +# -k flag is to ignore SSL errors +RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ + mkdir -p ${GEOS_TPL_DIR} && \ +# - run uberenv + ./scripts/uberenv/uberenv.py \ + --spec "%gcc@12.2.1 ~openmp~pygeosx" \ + --spack-env-file=${SRC_DIR}/docker/TotalEnergies/pangea-4/pangea4-spack.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOS_TPL_DIR} \ + -k && \ +# - remove host-config generated for LvArray + rm lvarray* && \ +# - rename and copy spack-generated host-config to root directory + cp *.cmake /spack-generated.cmake && \ +# - remove extraneous spack files + cd ${GEOS_TPL_DIR} && \ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + +# ------------------------------------- +# PANGEA4 - TPL TOOLCHAIN +FROM pangea4_tpl_base AS pangea4_tpl_toolchain +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/src +# ------ +# INSTALL +# - copy TPLs install directory +COPY --from=pangea4_tpl_builder $GEOS_TPL_DIR $GEOS_TPL_DIR +# - copy the generated host-config +COPY --from=pangea4_tpl_builder /spack-generated.cmake / +# - install ninja for geos build +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh +# - install `sccache` binaries to speed up the build of `geos` +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +# ------ +# ENV +ENV SCCACHE=/opt/sccache/bin/sccache +# ------ +# ENTRYPOINT +# set entry point for geos ci build script +ENTRYPOINT ["/bin/bash", "-c", "source /root/.setup_env.sh && exec \"$@\"", "--"] \ No newline at end of file diff --git a/docker/TotalEnergies/pecan-CPU.cmake b/docker/TotalEnergies/pecan-CPU.cmake deleted file mode 100644 index 722ea2b4..00000000 --- a/docker/TotalEnergies/pecan-CPU.cmake +++ /dev/null @@ -1,31 +0,0 @@ -set(COMPILER_HOME "/apps/gcc/8.2.0/x86_64") -set(MPI_HOME "/hrtc/apps/mpi/openmpi/4.0.1/RDHPC/gcc/8.2.0") - -set(CMAKE_C_COMPILER ${COMPILER_HOME}/bin/gcc CACHE PATH "" FORCE) -set(CMAKE_CXX_COMPILER ${COMPILER_HOME}/bin/g++ CACHE PATH "" FORCE) -set(CMAKE_Fortran_COMPILER ${COMPILER_HOME}/bin/gfortran CACHE PATH "" FORCE) -set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE) - -set(ENABLE_MPI ON CACHE PATH "" FORCE) -set(ENABLE_OPENMP ON CACHE PATH "" FORCE) - -set(MPI_C_COMPILER "${MPI_HOME}/bin/mpicc" CACHE PATH "" FORCE) -set(MPI_CXX_COMPILER "${MPI_HOME}/bin/mpicxx" CACHE PATH "" FORCE) -set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpifort" CACHE PATH "" FORCE) -set(MPIEXEC_EXECUTABLE "${MPI_HOME}/bin/mpirun" CACHE PATH "" FORCE) -#set(MPIEXEC_EXECUTABLE /apps/slurm/x86/20.02.0/bin/srun CACHE PATH "" FORCE) -#set(MPIEXEC_NUMPROC_FLAG "-p pecan -n" CACHE STRING "") -set(ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC ON CACHE BOOL "") - -set(ENABLE_GTEST_DEATH_TESTS ON CACHE BOOL "" FORCE) -set(ENABLE_CALIPER ON CACHE BOOL "") - -set(ENABLE_MKL ON CACHE BOOL "") -set(INTEL_ROOT "/apps/intel/2019/u5/compilers_and_libraries_2019.5.281/linux" ) -set(MKL_ROOT "${INTEL_ROOT}/mkl" ) -set(MKL_INCLUDE_DIRS ${MKL_ROOT}/include CACHE STRING "") -set(MKL_LIBRARIES ${MKL_ROOT}/lib/intel64/libmkl_intel_lp64.so - ${MKL_ROOT}/lib/intel64/libmkl_intel_thread.so - ${MKL_ROOT}/lib/intel64/libmkl_core.so - ${INTEL_ROOT}/compiler/lib/intel64_lin/libiomp5.so - CACHE STRING "") diff --git a/docker/TotalEnergies/pecan-GPU.cmake b/docker/TotalEnergies/pecan-GPU.cmake deleted file mode 100644 index ef6e4457..00000000 --- a/docker/TotalEnergies/pecan-GPU.cmake +++ /dev/null @@ -1,24 +0,0 @@ -# Retrieve the compilers, standard libraries... from the CPU configuration -include(${CMAKE_CURRENT_LIST_DIR}/pecan-CPU.cmake) - -# Now let's add what's dedicated to GPU. -set(ENABLE_CUDA ON CACHE PATH "" FORCE) -set(CUDA_TOOLKIT_ROOT_DIR /hrtc/apps/cuda/11.5.119/x86_64/centos7 CACHE PATH "") -set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "") -set(CMAKE_CUDA_COMPILER ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc CACHE STRING "") - -set(CUDA_ARCH sm_75 CACHE STRING "") -set(CMAKE_CUDA_ARCHITECTURES 75 CACHE STRING "") -set(CMAKE_CUDA_FLAGS "-restrict -arch ${CUDA_ARCH} --expt-relaxed-constexpr --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" CACHE STRING "") -set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3" CACHE STRING "") -set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}" CACHE STRING "") -set(CMAKE_CUDA_FLAGS_DEBUG "-g -G -O0 -Xcompiler -O0" CACHE STRING "") - -# Current version of hypre does not build with GPU support inside of docker. -# Hypre's build system awaits to be embedded into a CUDA environment. -# This is a bit tedious to reproduce in docker environment. -# And since most recent version of hypre do build without this constraint. -# Let's wait for an upgrade on our side. -# In the mean time, if you need the GPU support for hypre, -# simply install the classical way, with some `module load cuda`. -#set(ENABLE_HYPRE_CUDA ON CACHE BOOL "" FORCE) diff --git a/docker/TotalEnergies/pecan.Dockerfile b/docker/TotalEnergies/pecan.Dockerfile deleted file mode 100644 index 3e19fbef..00000000 --- a/docker/TotalEnergies/pecan.Dockerfile +++ /dev/null @@ -1,193 +0,0 @@ -ARG GCC_VERSION=8.2.0 -ARG OPENMPI_VERSION=4.0.1 -ARG UCX_VERSION=1.3.0 -# Note that you need to define both SLURM_TARBALL and SLURM_HOME because one cannot be deduced from the other smoothly. -# It would require (heavy?) text processing for little benefit. -ARG SLURM_TARBALL=slurm-20-02-0-1.tar.gz -ARG SLURM_HOME=/apps/slurm/x86/20.02.0 - -ARG GCC_HOME=/apps/gcc/${GCC_VERSION}/x86_64 -ARG UCX_HOME=/hrtc/apps/devtools/ucx/${UCX_VERSION}/x86_64_nocuda/gcc/${GCC_VERSION} -ARG OPENMPI_HOME=/hrtc/apps/mpi/openmpi/${OPENMPI_VERSION}/RDHPC/gcc/${GCC_VERSION} -# While the installation directory is defined here, the patches and exact versions are still defined in the CUDA stage. -ARG CUDA_HOME=/hrtc/apps/cuda/11.5.119/x86_64/centos7 - -FROM centos:7.7.1908 AS shared_components - -RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ - sed -i s/^#.*baseurl=http/baseurl=https/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo - -RUN yum update -y && \ - yum install -y \ - glibc-devel - -FROM shared_components AS gcc_stage - -ARG GCC_VERSION -ARG GCC_HOME - -# FIXME wget could be replaced by curl in the contrib/download_prerequisites script. To be challenged. -RUN yum install -y \ - make \ - gcc \ - gcc-c++ \ - wget \ - bzip2 \ - zlib-devel - -WORKDIR /tmp/src -RUN curl -s https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.gz | tar --strip-components=1 -xzf - -RUN ./contrib/download_prerequisites -RUN ./configure \ - --prefix=${GCC_HOME} \ - --disable-multilib \ - --with-system-zlib \ - --enable-threads=posix \ - --enable-languages=c,c++,fortran -RUN make -j $(nproc) && make install-strip - -FROM shared_components AS ucx_stage - -ARG UCX_VERSION -ARG UCX_HOME -ARG GCC_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} - -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -RUN yum install -y \ - make \ - numactl-devel - -WORKDIR /tmp/src -RUN curl -fsSL https://github.com/openucx/ucx/releases/download/v${UCX_VERSION}/ucx-${UCX_VERSION}.tar.gz | tar --strip-components=1 -xzf - -RUN ./configure \ - --prefix=${UCX_HOME} \ - --enable-mt \ - --disable-optimizations \ - --disable-logging \ - --disable-debug \ - --disable-assertions -RUN make -j $(nproc) && make install - -FROM shared_components AS slurm_stage - -ARG SLURM_TARBALL -ARG SLURM_HOME -ARG GCC_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} - -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -RUN yum install -y perl python3 file make - -WORKDIR /tmp/src -RUN curl -fsSL https://github.com/SchedMD/slurm/archive/${SLURM_TARBALL} | tar --strip-components=1 -xzf - -RUN ./configure --prefix=${SLURM_HOME} -RUN make -j $(nproc) && make install - -FROM shared_components AS openmpi_stage - -ARG SLURM_HOME -ARG UCX_HOME -ARG GCC_HOME -# FIXME so we do not use SLURM? -ARG OPENMPI_VERSION -ARG OPENMPI_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} -COPY --from=ucx_stage ${UCX_HOME} ${UCX_HOME} -COPY --from=slurm_stage ${SLURM_HOME} ${SLURM_HOME} - -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -RUN yum install -y \ - perl \ - make \ - zlib-devel \ - numactl-devel -# FIXME deal with the devel probably too much - -WORKDIR /tmp/src -RUN curl -fsSL https://download.open-mpi.org/release/open-mpi/v${OPENMPI_VERSION%.[0-9]*}/openmpi-${OPENMPI_VERSION}.tar.gz | tar --strip-components=1 -xzf - -RUN ./configure CC=$CC FC=$FC CXX=$CXX \ - --prefix=${OPENMPI_HOME} \ - --enable-static \ - --enable-smp-locks \ - --enable-mpi-thread-multiple \ - --with-slurm \ - --with-ucx=${UCX_HOME} \ - --with-ucx-libdir=${UCX_HOME}/lib \ - --with-io-romio-flags=--with-file-system=testfs+ufs+lustre -RUN make -j $(nproc) && make install - -FROM shared_components AS cuda_stage - -ARG GCC_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} - -ARG CUDA_HOME - -# FIXME Why glibc-devel?!?!? -RUN yum install -y which glibc-devel - -ENV PATH=${GCC_HOME}/bin:${PATH} \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64 - -WORKDIR /tmp/src -ADD https://developer.download.nvidia.com/compute/cuda/11.5.0/local_installers/cuda_11.5.0_495.29.05_linux.run . -RUN mkdir -p ${CUDA_HOME} -RUN sh cuda_11.5.0_495.29.05_linux.run --silent --toolkit --no-man-page --installpath=${CUDA_HOME} - -FROM shared_components AS intel_stage - -ARG SLURM_HOME -ARG UCX_HOME -ARG GCC_HOME -ARG OPENMPI_HOME -ARG CUDA_HOME - -COPY --from=gcc_stage ${GCC_HOME} ${GCC_HOME} -COPY --from=ucx_stage ${UCX_HOME} ${UCX_HOME} -COPY --from=slurm_stage ${SLURM_HOME} ${SLURM_HOME} -COPY --from=openmpi_stage ${OPENMPI_HOME} ${OPENMPI_HOME} -COPY --from=cuda_stage ${CUDA_HOME} ${CUDA_HOME} - -RUN yum install -y \ - numactl-devel -# FIXME maybe only numactl-libs? - -RUN rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB -RUN yum-config-manager \ - --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo -RUN yum install -y \ - intel-mkl-2019.5-075.x86_64 && \ - yum clean all - -ARG PECAN_INTEL_HOME=/apps/intel/2019/u5 -ARG INTEL_DIR_NAME=compilers_and_libraries_2019.5.281 -RUN mkdir -p /apps/intel/2019/u5 && \ - ln -s /opt/intel/${INTEL_DIR_NAME} ${PECAN_INTEL_HOME}/${INTEL_DIR_NAME} - -# Exposing quite everything, making future modularization more complicated. -# Most likely there will be no future modularization! -ENV CC=${GCC_HOME}/bin/gcc \ - CXX=${GCC_HOME}/bin/g++ \ - FC=${GCC_HOME}/bin/gfortran \ - MPICC=${OPENMPI_HOME}/bin/mpicc \ - MPICXX=${OPENMPI_HOME}/bin/mpicxx \ - MPIFC=${OPENMPI_HOME}/bin/mpifort \ - LD_LIBRARY_PATH=${GCC_HOME}/lib64:${PECAN_INTEL_HOME}/${INTEL_DIR_NAME}/linux/mkl/lib/intel64:${PECAN_INTEL_HOME}/${INTEL_DIR_NAME}/compiler/lib/intel64:${OPENMPI_HOME}/lib diff --git a/docker/pangea-spack.yaml b/docker/pangea3-spack.yaml similarity index 100% rename from docker/pangea-spack.yaml rename to docker/pangea3-spack.yaml diff --git a/docker/pecan-spack.yaml b/docker/pecan-spack.yaml deleted file mode 100644 index d2eb2c96..00000000 --- a/docker/pecan-spack.yaml +++ /dev/null @@ -1,182 +0,0 @@ -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage - - # Regular TPLs do not need views - view: false - - compilers:: - # Pecan CPU/GPU compiler - - compiler: - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - extra_rpaths: [] - flags: - cflags: -pthread - cxxflags: -pthread - modules: [] - operating_system: centos7 - paths: - cc: /apps/gcc/8.2.0/x86_64/bin/gcc - cxx: /apps/gcc/8.2.0/x86_64/bin/g++ - f77: /apps/gcc/8.2.0/x86_64/bin/gfortran - fc: /apps/gcc/8.2.0/x86_64/bin/gfortran - spec: gcc@8.2.0 - target: x86_64 - - packages: - all: - target: [x86_64] - compiler: [gcc, clang] - providers: - blas: [intel-mkl] - lapack: [intel-mkl] - mpi: [openmpi] - - - # v0.6.2 - blt: - require: "@git.9ff77344f0b2a6ee345e452bddd6bfd46cbbfa35=develop" - - # master - 10/18/24 - hypre: - require: "@git.c893886d15eb57e87dd36efec23693ece3ddc88e" - - # v2024.07.0 - chai: - require: "@git.df7741f1dbbdc5fff5f7d626151fdf1904e62b19=develop" - - # v2024.07.0 - umpire: - require: "@git.abd729f40064175e999a83d11d6b073dac4c01d2=develop" - - # v2024.07.0 - raja: - require: "@git.4d7fcba55ebc7cb972b7cc9f6778b48e43792ea1=develop" - - # v2024.07.0 - camp: - require: "@git.0f07de4240c42e0b38a8d872a20440cb4b33d9f5=main" - - # v2.12.0 - caliper: - require: "@git.287b7f3ad2d12f520aad04268d44f353cd05403c" - - # v0.9.2 - conduit: - require: "@git.ad86e316ad56a75c099d30ca5ce75cff275b5924=develop" - - uncrustify: - require: "@git.401a4098bce9dcc47e024987403f2d59d9ba7bd2" - - superlu-dist: - require: "@git.0f6efc377df2440c235452d13d28d2c717f832a1" - - autoconf: - version: [2.71] - buildable: false - externals: - - spec: autoconf@2.71 - prefix: /usr - automake: - version: [1.16.5] - buildable: false - externals: - - spec: automake@1.16.5 - prefix: /usr - bison: - version: [3.8.2] - buildable: false - externals: - - spec: bison@3.8.2 - prefix: /usr - cmake: - version: [3.28.3] - buildable: false - externals: - - spec: cmake@3.28.3 - prefix: /usr/local - cuda: - buildable: False - externals: - - spec: cuda@11.5.1 +allow-unsupported-compilers - prefix: /hrtc/apps/cuda/11.5.119/x86_64/centos7 - flex: - version: [2.6.4] - buildable: false - externals: - - spec: flex@2.6.4 - prefix: /usr - - # Lock down blas, lapack implementation - intel-mkl: - buildable: False - externals: - # Pecan CPU/GPU - - spec: intel-mkl@2019.5.281%gcc@8.2.0 threads=openmp - prefix: /opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/ - blas: - buildable: false - - lapack: - buildable: false - - m4: - buildable: false - externals: - - spec: m4@1.4.18 - prefix: /usr - - # Spack may grab for mpi & we don't want to use them - mpi: - buildable: false - - openmpi: - externals: - # Pecan CPU/GPU - - spec: openmpi@4.0.1%gcc@8.2.0 - prefix: /hrtc/apps/mpi/openmpi/4.0.1/RDHPC/gcc/8.2.0 - perl: - buildable: false - externals: - - spec: perl@5.34.0 - prefix: /usr - pkg-config: - buildable: false - externals: - - spec: pkg-config@0.29.2 - prefix: /usr - py-sphinx: - buildable: false - externals: - - spec: py-sphinx@4.3.2 - prefix: /usr - python: - buildable: false - externals: - - spec: python@3.10.12 - prefix: /usr - tar: - buildable: false - externals: - - spec: tar@1.34 - prefix: /usr - unzip: - buildable: false - externals: - - spec: unzip@6.0 - prefix: /usr - xz: - buildable: false - externals: - - spec: xz@5.2.5 - prefix: /usr