diff --git a/.environment-scripts/install_call_py_fort.sh b/.environment-scripts/install_call_py_fort.sh new file mode 100644 index 0000000000..70c758f472 --- /dev/null +++ b/.environment-scripts/install_call_py_fort.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +PREFIX=$1 + +URL=https://github.com/nbren12/call_py_fort.git +BRANCH=v0.2.0 +git clone $URL --branch=$BRANCH $PREFIX +cd $PREFIX +make +make install +ldconfig diff --git a/.environment-scripts/install_esmf.sh b/.environment-scripts/install_esmf.sh new file mode 100644 index 0000000000..bd23bc6e8e --- /dev/null +++ b/.environment-scripts/install_esmf.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +CLONE_PREFIX=$1 +INSTALL_PREFIX=$2 +ESMF_OS=$3 +ESMF_COMPILER=$4 +ESMF_SITE=$5 +NETCDF_INCLUDE=$6 + +export ESMF_DIR=$CLONE_PREFIX +export ESMF_INSTALL_PREFIX=$INSTALL_PREFIX +export ESMF_INSTALL_MODDIR=$ESMF_INSTALL_PREFIX/include +export ESMF_INSTALL_HEADERDIR=$ESMF_INSTALL_PREFIX/include +export ESMF_INSTALL_LIBDIR=$ESMF_INSTALL_PREFIX/lib +export ESMF_INSTALL_BINDIR=$ESMF_INSTALL_PREFIX/bin +export ESMF_NETCDF_INCLUDE=$NETCDF_INCLUDE +export ESMF_NETCDF_LIBS="-lnetcdf -lnetcdff" +export ESMF_BOPT=O3 + +# These flags set how the build is configured depending on the platform: +# https://github.com/esmf-org/esmf/tree/develop/build_config +export ESMF_OS=$ESMF_OS +export ESMF_COMPILER=$ESMF_COMPILER +export ESMF_SITE=$ESMF_SITE + +# We may want to upgrade to ESMF_8_1_0 to make things easier on other platforms. +git clone -b ESMF_8_0_0 --depth 1 https://github.com/esmf-org/esmf.git $ESMF_DIR +cd $ESMF_DIR +make lib -j24 +make install +make installcheck diff --git a/.environment-scripts/install_fms.sh b/.environment-scripts/install_fms.sh new file mode 100644 index 0000000000..0843c2ff56 --- /dev/null +++ b/.environment-scripts/install_fms.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +INSTALL_PREFIX=$1 + +cd $INSTALL_PREFIX +autoreconf -f --install +./configure +make -j8 +mv $INSTALL_PREFIX/*/*.o $INSTALL_PREFIX/*/*.h $INSTALL_PREFIX/*/*.mod $INSTALL_PREFIX/ diff --git a/.environment-scripts/install_fv3gfs_fortran.sh b/.environment-scripts/install_fv3gfs_fortran.sh new file mode 100644 index 0000000000..edfffdb8b4 --- /dev/null +++ b/.environment-scripts/install_fv3gfs_fortran.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +PREFIX=$1 +PLATFORM=$2 +INSTALL_PREFIX=$3 + +cd $PREFIX +./configure $PLATFORM +make -j 8 +PREFIX=$INSTALL_PREFIX make install diff --git a/.environment-scripts/install_fv3net_python_dependencies.sh b/.environment-scripts/install_fv3net_python_dependencies.sh new file mode 100644 index 0000000000..51094d36bb --- /dev/null +++ b/.environment-scripts/install_fv3net_python_dependencies.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +REQUIREMENTS_PATH=$1 +LOCAL_DEPENDENCIES=${@:2} + +pip install --no-cache-dir -r $REQUIREMENTS_PATH +for dependency in $LOCAL_DEPENDENCIES +do + pip install --no-cache-dir --no-dependencies -e $dependency +done diff --git a/.environment-scripts/install_nceplibs.sh b/.environment-scripts/install_nceplibs.sh new file mode 100644 index 0000000000..994c917d3b --- /dev/null +++ b/.environment-scripts/install_nceplibs.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +CLONE_PREFIX=$1 +INSTALL_PREFIX=$2 +PLATFORM=$3 +COMPILER=$4 + +URL=https://github.com/NCAR/NCEPlibs.git +SHA=3da51e139d5cd731c9fc27f39d88cb4e1328212b + +git clone $URL $CLONE_PREFIX +cd $CLONE_PREFIX +git checkout $SHA +mkdir -p $INSTALL_PREFIX +echo "y" | bash make_ncep_libs.sh -s $PLATFORM -c $COMPILER -d $INSTALL_PREFIX -o 1 diff --git a/.environment-scripts/install_python_wrapper.sh b/.environment-scripts/install_python_wrapper.sh new file mode 100644 index 0000000000..cfbb47364e --- /dev/null +++ b/.environment-scripts/install_python_wrapper.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +FV3_DIR=$1 + +make -C $FV3_DIR wrapper_build +pip install --no-dependencies $FV3_DIR/wrapper/dist/*.whl diff --git a/Makefile b/Makefile index aa46ec33cc..3d3fcd3a6e 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ CACHE_TAG =latest BEAM_VERSION = 2.37.0 UBUNTU_IMAGE = ubuntu@sha256:9101220a875cee98b016668342c489ff0674f247f6ca20dfc91b91c0f28581ae # prognostic base image is updated manually, not on every commit -PROGNOSTIC_BASE_VERSION = 1.0.0 +PROGNOSTIC_BASE_VERSION = 1.0.1 DOCKER_INTERACTIVE_ARGS = \ --tty \ --interactive \ diff --git a/docker/prognostic_run/Dockerfile b/docker/prognostic_run/Dockerfile index 23e5de3847..38b9af8c89 100644 --- a/docker/prognostic_run/Dockerfile +++ b/docker/prognostic_run/Dockerfile @@ -4,44 +4,39 @@ FROM ${BASE_IMAGE} as bld # build/install the fortran model COPY external/fv3gfs-fortran/ /tmp/fortran-build -RUN cd /tmp/fortran-build/FV3 && \ - ./configure gnu_docker && \ - sed -i "33i CALLPYFORT=Y" conf/configure.fv3 && \ - make -j 8 && \ - PREFIX=/usr/local make install - -# Python Stuff Here -# -# Equivalent to passing --no-cache-dir to every pip install -ENV PIP_NO_CACHE_DIR=off - -# Install dependencies -COPY docker/prognostic_run/requirements.txt /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -# compile and install the wrapper. Then import fv3gfs.wrapper as a minimal test -# that it works. -RUN make -C /tmp/fortran-build/FV3 wrapper_build &&\ - pip3 install --no-dependencies /tmp/fortran-build/FV3/wrapper/dist/*.whl && \ - python3 -c 'import fv3gfs.wrapper' +COPY .environment-scripts/install_fv3gfs_fortran.sh . +RUN CALLPYFORT=Y bash install_fv3gfs_fortran.sh /tmp/fortran-build/FV3 gnu_docker /usr/local +# Install the Python dependencies +COPY docker/prognostic_run/requirements.txt /fv3net/docker/prognostic_run/requirements.txt COPY external/vcm /fv3net/external/vcm -RUN pip3 install --no-dependencies -e /fv3net/external/vcm - COPY external/artifacts /fv3net/external/artifacts -RUN pip3 install --no-dependencies -e /fv3net/external/artifacts - COPY external/loaders /fv3net/external/loaders -RUN pip3 install --no-dependencies -e /fv3net/external/loaders - COPY external/fv3fit /fv3net/external/fv3fit -RUN pip3 install --no-dependencies -e /fv3net/external/fv3fit - COPY external/fv3kube /fv3net/external/fv3kube -RUN pip3 install --no-dependencies -e /fv3net/external/fv3kube - COPY workflows/post_process_run /fv3net/workflows/post_process_run -RUN pip3 install --no-dependencies -e /fv3net/workflows/post_process_run +COPY workflows/prognostic_c48_run/ /fv3net/workflows/prognostic_c48_run +COPY external/emulation /fv3net/external/emulation +COPY external/radiation /fv3net/external/radiation + +COPY .environment-scripts/install_fv3net_python_dependencies.sh . +RUN bash install_fv3net_python_dependencies.sh \ + /fv3net/docker/prognostic_run/requirements.txt \ + /fv3net/external/vcm \ + /fv3net/external/artifacts \ + /fv3net/external/loaders \ + /fv3net/external/fv3fit \ + /fv3net/external/fv3kube \ + /fv3net/workflows/post_process_run \ + /fv3net/workflows/prognostic_c48_run \ + /fv3net/external/emulation \ + /fv3net/external/radiation + +# compile and install the wrapper. Then import fv3gfs.wrapper as a minimal test +# that it works. +COPY .environment-scripts/install_python_wrapper.sh . +RUN CALLPYFORT=Y bash install_python_wrapper.sh /tmp/fortran-build/FV3 +RUN python3 -c 'import fv3gfs.wrapper' RUN echo "ulimit -s unlimited" >> /etc/bash.bashrc && \ mkdir /outdir && \ @@ -52,12 +47,7 @@ RUN echo "ulimit -s unlimited" >> /etc/bash.bashrc && \ ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 -COPY workflows/prognostic_c48_run/ /fv3net/workflows/prognostic_c48_run -RUN pip install --no-dependencies -e /fv3net/workflows/prognostic_c48_run - # Override microphysics emulation -COPY external/emulation /fv3net/external/emulation -RUN pip install --no-dependencies -e /fv3net/external/emulation ENV VAR_META_PATH=/fv3net/external/emulation/microphysics_parameter_metadata.yaml ENV OUTPUT_FREQ_SEC=18000 @@ -66,10 +56,6 @@ COPY projects/microphysics/scripts /fv3net/projects/microphysics/scripts RUN chmod +x /fv3net/projects/microphysics/scripts/* ENV PATH=/fv3net/projects/microphysics/scripts:${PATH} -# Add python radiation port -COPY external/radiation /fv3net/external/radiation -RUN pip install --no-dependencies -e /fv3net/external/radiation - ENV PYTHONPATH=/fv3net/workflows/prognostic_c48_run:/fv3net/external/fv3fit:/fv3net/external/emulation:/fv3net/external/vcm:/fv3net/external/artifacts:/fv3net/external/loaders:/fv3net/external/fv3kube:/fv3net/workflows/post_process_run:/fv3net/external/radiation:${PYTHONPATH} WORKDIR /fv3net/workflows/prognostic_c48_run CMD ["bash"] diff --git a/docker/prognostic_run/base.Dockerfile b/docker/prognostic_run/base.Dockerfile index 2e7539639a..56b17d05a8 100644 --- a/docker/prognostic_run/base.Dockerfile +++ b/docker/prognostic_run/base.Dockerfile @@ -47,22 +47,28 @@ RUN apt-get update && \ rsync \ wget -COPY docker/prognostic_run/scripts/install_esmf.sh install_esmf.sh -RUN bash install_esmf.sh /usr/local/esmf +COPY .environment-scripts/install_esmf.sh . +RUN bash install_esmf.sh /esmf /usr/local/esmf Linux gfortran default -COPY docker/prognostic_run/scripts/install_fms.sh install_fms.sh +COPY .environment-scripts/install_fms.sh . COPY external/fv3gfs-fortran/FMS /FMS -RUN bash install_fms.sh /FMS +RUN CC=/usr/bin/mpicc \ + FC=/usr/bin/mpif90 \ + LDFLAGS='-L/usr/lib' \ + LOG_DRIVER_FLAGS='--comments' \ + CPPFLAGS='-I/usr/include -Duse_LARGEFILE -DMAXFIELDMETHODS_=500 -DGFS_PHYS' \ + FCFLAGS='-fcray-pointer -Waliasing -ffree-line-length-none -fno-range-check -fdefault-real-8 -fdefault-double-8 -fopenmp' \ + bash install_fms.sh /FMS -COPY docker/prognostic_run/scripts/install_nceplibs.sh . -RUN bash install_nceplibs.sh /opt/NCEPlibs +COPY .environment-scripts/install_nceplibs.sh . +RUN bash install_nceplibs.sh /NCEPlibs /opt/NCEPlibs linux gnu ENV ESMF_DIR=/usr/local/esmf ENV CALLPY_DIR=/usr/local ENV FMS_DIR=/FMS ENV FV3GFS_FORTRAN_DIR=/external/fv3gfs-fortran -ENV ESMF_INC="-I${ESMF_DIR}/include -I${ESMF_DIR}/mod/modO3/Linux.gfortran.64.mpiuni.default/" +ENV ESMF_INC="-I${ESMF_DIR}/include" ENV FMS_LIB=${FMS_DIR}/libFMS/.libs/ ENV ESMF_LIB=${ESMF_DIR}/lib @@ -70,10 +76,10 @@ ENV CALLPYFORT_LIB=${CALLPY_DIR}/lib ENV CALLPYFORT_INCL=${CALLPY_DIR}/include ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${ESMF_LIB}:${FMS_LIB}:${CALLPYFORT_LIB} -RUN cd /opt && git clone https://github.com/nbren12/call_py_fort.git --branch=v0.2.0 ENV CALLPY=/opt/call_py_fort \ PYTHONPATH=${CALLPY}/src/:$PYTHONPATH -RUN cd ${CALLPY} && make && make install && ldconfig +COPY .environment-scripts/install_call_py_fort.sh . +RUN bash install_call_py_fort.sh $CALLPY # Install gcloud RUN apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl gettext && \ diff --git a/docker/prognostic_run/scripts/install_esmf.sh b/docker/prognostic_run/scripts/install_esmf.sh deleted file mode 100644 index f69455ad3f..0000000000 --- a/docker/prognostic_run/scripts/install_esmf.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -e - -PREFIX="$1" - -export ESMF_DIR=/esmf -export ESMF_INSTALL_PREFIX=$PREFIX -export ESMF_INSTALL_MODDIR=$PREFIX/include -export ESMF_INSTALL_HEADERDIR=$PREFIX/include -export ESMF_INSTALL_LIBDIR=$PREFIX/lib -export ESMF_INSTALL_BINDIR=$PREFIX/bin -export ESMF_NETCDF_INCLUDE=/usr/include -export ESMF_NETCDF_LIBS="-lnetcdf -lnetcdff" -export ESMF_BOPT=O3 - -git clone -b ESMF_8_0_0 --depth 1 https://github.com/esmf-org/esmf.git $ESMF_DIR -cd $ESMF_DIR -make lib -j24 -make install -make installcheck \ No newline at end of file diff --git a/docker/prognostic_run/scripts/install_fms.sh b/docker/prognostic_run/scripts/install_fms.sh deleted file mode 100644 index c7a67ab991..0000000000 --- a/docker/prognostic_run/scripts/install_fms.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -src="$1" - -export CC=/usr/bin/mpicc -export FC=/usr/bin/mpif90 -export LDFLAGS="-L/usr/lib" -export LOG_DRIVER_FLAGS="--comments" -export CPPFLAGS="-I/usr/include -Duse_LARGEFILE -DMAXFIELDMETHODS_=500 -DGFS_PHYS" -export FCFLAGS="-fcray-pointer -Waliasing -ffree-line-length-none -fno-range-check -fdefault-real-8 -fdefault-double-8 -fopenmp" - -cd $src -autoreconf --install -./configure -make -j8 -mv $src/*/*.mod $src/*/*.o $src/*/*.h $src/ diff --git a/docker/prognostic_run/scripts/install_nceplibs.sh b/docker/prognostic_run/scripts/install_nceplibs.sh deleted file mode 100644 index 1b75ba354d..0000000000 --- a/docker/prognostic_run/scripts/install_nceplibs.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -e - -PREFIX="$1" - -curl -L https://github.com/NCAR/NCEPlibs/archive/3da51e139d5cd731c9fc27f39d88cb4e1328212b.tar.gz | tar xz -mkdir -p "$PREFIX" -cd NCEPlibs-* -echo "y" | ./make_ncep_libs.sh -s linux -c gnu -d "$PREFIX" -o 1 \ No newline at end of file