hpc4cmb · tskisner · Sep 17, 2020 · Sep 17, 2020 · Sep 17, 2020 · Sep 17, 2020
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,6 +1,6 @@
 # Test action that attempts to generate new docker containers on push to master
 
-name:  CMBenv Docker Test
+name:  CMBenv Test
 
 on:
   push:
@@ -9,18 +9,6 @@ on:
     branches: [ master ]
 
 jobs:
-  py35:
-    name: Docker Py3.5 Debian
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Create Dockerfile
-        run: ./cmbenv -c docker-py3.5-debian -p /usr/local
-      - name: Build Docker Image
-        run: docker build -t hpc4cmb/cmbenv:temp_py35 -f Dockerfile_docker-py3.5-debian .
-      - name: Test Docker Image
-        run: docker run hpc4cmb/cmbenv:temp_py35 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py35 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
   py36:
     name: Docker Py3.6 Debian
     runs-on: ubuntu-latest
@@ -57,3 +45,23 @@ jobs:
         run: docker build -t hpc4cmb/cmbenv:temp_py38 -f Dockerfile_docker-py3.8-debian .
       - name: Test Docker Image
         run: docker run hpc4cmb/cmbenv:temp_py38 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py38 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
+  macos-venv-clang:
+    name: MacOS Virtualenv Clang
+    runs-on: macos-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Install Script
+        run: ./cmbenv -c macos-venv-clang -p ~/cmbenv
+      - name: Install Packages
+        run:  mkdir build; cd build; ../install_macos-venv-clang.sh; cd ..; . ~/cmbenv/cmbenv_init.sh; mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
+  macos-venv-gcc:
+    name: MacOS Virtualenv GCC
+    runs-on: macos-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Install Script
+        run: ./cmbenv -c macos-venv-gcc -p ~/cmbenv
+      - name: Install Packages
+        run:  mkdir build; cd build; ../install_macos-venv-gcc.sh; cd ..; . ~/cmbenv/cmbenv_init.sh; mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
diff --git a/README.md b/README.md
@@ -21,37 +21,57 @@ If you are installing on a Linux or OS X workstation you may be able to use one
 
     Linux:  Is your system python3 fairly recent (>= 3.6.0)?
         - YES.  Do you want to use Anaconda python for some reason?
-            - YES.  Use the linux-conda config.
-            - NO.  Use the linux-venv config.
-        - NO.  You should use the linux-conda config.
-    OS X:  Are you currently using macports or homebrew to get python3?
+            - YES.  Do you want to use MKL for your own compiled python extensions?
+                - Yes.  Use the linux-conda-nomkl config, so that numpy uses OpenBLAS.
+                - No.  Use the linux-conda config.
+            - NO.  Use the linux-venv config.  See MKL notes below.
+        - NO.  You should use the linux-conda or linux-conda-nomkl config.
+    OS X:  Are you currently using macports or homebrew to get a newer python3?
         - YES.  Do you want to use Fortran packages (libmadam)?
             - YES.  Install GNU compilers with macports or homebrew.
-                    Use the osx-venv-gcc config.
-            - NO.  Use the osx-venv-clang config.
+                    Use the macos-venv-gcc config.
+            - NO.  Use the macos-venv-clang config.
         - NO.  Do you want to use Fortran packages (libmadam)?
             - YES.  Install GNU compilers with macports or homebrew.
-                    Use the osx-conda-gcc config.
-            - NO.  Use the osx-conda-clang config.
+                    Use the macos-conda-gcc config.
+            - NO.  Use the macos-conda-clang config.
 
 ### Special Notes for OS X
 
 Apple's operating system can be challenging to work with when building larger compiled packages that do not use X Code.  The native compilers do not support some languages and features, and installing external compilers (gcc) require care to ensure that all dependencies are also built with the same compiler.  Here are some recommendations:
 
   - Whenever you update your OS, also ensure that X Code and the commandline tools have been updated.
 
-  - If using macports or homebrew, wipe and reinstall these when you upgrade your OS to a new major version.
+  - If using macports or homebrew, wipe and reinstall these when you upgrade your OS to a new version.
 
-If you have trouble with python or gcc on OS X, try testing the "osx-conda-clang" config.  This should be the most stand-alone solution since it uses only the system clang compiler.  Downsides of this are the lack of fortran support and perhaps other features like OpenMP threading.
+If you have trouble with python or gcc on OS X, try testing the "macos-conda-clang" config.  This should be the most stand-alone solution since it uses only the system clang compiler.  Downsides of this are the lack of fortran support and perhaps other features like OpenMP threading.
+
+### MKL Warning
+
+Some versions of numpy use a bundled version of MKL and the Intel threading libraries for linear algebra.  Some packages in `cmbenv` have compiled extensions that also link to external BLAS / LAPACK libraries.  The linear algebra libraries specified in the `BLAS` and `LAPACK` config variables must either be *different* from those used by numpy or **identical** to those used by numpy.
+
+Since it is very challenging to control what upstream MKL is used by numpy, we take a different approach:
+
+  - If you want to use the Intel compilers and MKL when building packages with `cmbenv`, please ensure that your numpy is NOT using MKL (for example, but choosing the "conda nomkl" option for the python package in the config files).
+
+  - If you are using OpenBLAS (external or built by `cmbenv`) or the Apple accelerate framework, then you should not have any conflicts regardless of what numpy is using.
+
+More details:  If you have a compiled extension that is linking to MKL and numpy is also
+linked to a (different) MKL, then only one MKL version will be dlopen'ed by the dynamic
+library loader.  So the order of python imports will determine which one gets loaded.
+Even if the MKL versions are binary compatible, they may link to different threading
+interface libraries.  For example, numpy ships with a bundled version of the Intel
+threading library.  If your extension is built with gcc and linked to MKL, it will link
+to the GNU interface library.
 
 ### Custom Configurations
 
-Create or edit a file in the "configs" subdirectory that is named whatever you
-like.  This file will define compilers, flags, etc. Optionally create files
-with the same name and the ".module" and ".sh" suffixes.  These optional files
-should contain any modulefile and shell commands needed to set up the
-environment prior to building the tools or loading them later.  See existing
-files for examples.
+Create or edit a file in the "configs" subdirectory that is named whatever you like.
+This file will define compilers, flags, etc. Optionally create files with the same name
+and the ".module" and ".sh" suffixes.  These optional files should contain any
+modulefile and shell commands needed to set up the environment prior to building the
+tools or loading them later.  The ".pkgs" file defines the packages to build for your
+system.  See existing files for examples.  
 
 To create a config for a docker image, the config file must be prefixed
 with "docker-".  You should not have any "*.module" or "*.sh" files for
@@ -107,9 +127,10 @@ it here.
 
 It is recommended to tag the container with the hash of the cmbenv git
 repository. In the following snippets, I assume that username is the same on
-local machine, Github, DockerHub and NERSC. Build it with:
+local machine, Github, DockerHub and NERSC. Build it with (for example):
 
-    %> docker build . -t $USER/cmbenv:$(git rev-parse --short HEAD)
+    %> docker build -t $USER/cmbenv:$(git rev-parse --short HEAD) \
+       -f Dockerfile_docker-py3.8-debian .
 
 ### Push the Docker container to Docker Hub
 
@@ -139,11 +160,12 @@ To use this image in a slurm batch script, just add:
 
     #SBATCH --image=docker:YOURUSERNAME/cmbenv:xxxxx
 
-In the SLURM header to set the image. Then prepend `shifter` to all running
-commands (after `srun` and all its options).  For example, to run a script in the container:
+In the SLURM header to set the image. Then prepend `shifter` to all running commands
+(after `srun` and all its options).  For example, to run a script in the container:
 
     srun -n 1 -N 1 shifter myscript.py
 
-If you are running any command in the container, make sure to prepend the shifter command.  For example:
+If you are running any command in the container, make sure to prepend the shifter
+command.  For example:
 
     shifter which myscript.py
diff --git a/configs/cori-gcc b/configs/cori-gcc
@@ -1,5 +1,5 @@
 
-# OS type (linux OR osx)
+# OS type (linux OR macos)
 OSTYPE = linux
 
 # Specify the template to use

diff --git a/configs/cori-gcc-gpu b/configs/cori-gcc-gpu
@@ -1,5 +1,5 @@
 
-# OS type (linux OR osx)
+# OS type (linux OR macos)
 OSTYPE = linux
 
 # Specify the template to use

diff --git a/configs/cori-intel b/configs/cori-intel
@@ -1,5 +1,5 @@
 
-# OS type (linux OR osx)
+# OS type (linux OR macos)
 OSTYPE = linux
 
 # Specify the template to use

diff --git a/configs/docker-py3.5-alpine3.11 b/configs/docker-py3.5-alpine3.11
@@ -1,5 +1,5 @@
 
-# OS type (linux OR osx)
+# OS type (linux OR macos)
 OSTYPE = linux
 
 # Specify the template to use

diff --git a/configs/docker-py3.5-debian b/configs/docker-py3.5-debian
@@ -1,5 +1,5 @@
 
-# OS type (linux OR osx)
+# OS type (linux OR macos)
 OSTYPE = linux
 
 # Specify the template to use

diff --git a/configs/docker-py3.6-alpine3.11 b/configs/docker-py3.6-alpine3.11
@@ -1,5 +1,5 @@
 
-# OS type (linux OR osx)
+# OS type (linux OR macos)
 OSTYPE = linux
 
 # Specify the template to use

diff --git a/configs/docker-py3.6-alpine3.11.pkgs b/configs/docker-py3.6-alpine3.11.pkgs
diff --git a/configs/docker-py3.6-alpine3.11.pkgs b/configs/docker-py3.6-alpine3.11.pkgs
@@ -0,0 +1,103 @@
+# This is the full set of packages that can be installed.  This can be
+# used as a starting point for the package list for a particular config.
+# Some packages accept additional options which can be added after a colon ":".
+# These extra options are documented in the shell snippet for each package
+# in the top-level pkgs directory.
+#
+# Autotools.  If you already have recent versions installed with your OS
+# package manager, then comment these out.
+#m4
+#libtool
+#autoconf
+#automake
+#
+# CMake.
+#cmake
+#
+# Install zlib.  This is needed on some systems where the OS-provided zlib is
+# very ancient.
+#zlib
+#
+# Install OpenBLAS.  Comment out if you are using MKL with the Intel compilers
+# or some other vendor BLAS / LAPACK solution.
+#openblas
+#
+# Python 3.  Uncomment one of the following lines.  You should only use the
+# "default" option for Docker images.  All other configs should use either
+# a virtualenv or a conda environment.  This will also install a set of
+# common base packages.
+#python:virtualenv
+#python:conda
+python:default
+#
+# Install MPICH.  This step is optional and only needed in 2 cases:
+#   1.  You don't have MPI available through system packages that is
+#       compatible with the serial compilers you are using.
+#   2.  You are building a docker container for use at NERSC with shifter
+#       and need an MPICH compatible with the Cray one.
+mpich
+#
+# Install mpi4py.  This will use the MPI compilers specified in your config.
+mpi4py
+#
+# Install CFITSIO.
+cfitsio
+#
+# Install FFTW3.
+#fftw
+#
+# Install libbz2.  This is required to enable some features in the boost
+# build which are in turn needed by spt3g.
+bzip2
+#
+# Install Boost.
+#boost
+#
+# Install HDF5 (serial version).
+hdf5
+#
+# Install h5py (using our HDF5).
+h5py
+#
+# Install healpy
+healpy
+#
+# Install quaternionarray
+qarray
+#
+# Install fitsio
+fitsio
+#
+# Install aatm.  Needed for some features of atmosphere simulations.
+aatm
+#
+# Install suitesparse
+#suitesparse
+#
+# Install libconviqt.  4-PI a_lm space beam convolution.
+conviqt
+#
+# Install libsharp.  Distributed harmonic transforms.
+libsharp
+#
+# Install libmadam.  Destriping map-maker.
+madam
+#
+# Install FLAC, needed by spt3g
+#libflac
+#
+# Install SPT3G.  Frame based data format.
+#spt3g
+#
+# Install TIDAS.  HDF5-based timestream data format.
+tidas
+#
+# Install PySM.  Python Sky Model.
+pysm
+#
+# Install PyMPIT for environment testing.  Useful for testing working
+# mpi4py stack.
+pympit
+#
+# Install TOAST.
+toast
diff --git a/configs/docker-py3.6-debian b/configs/docker-py3.6-debian
@@ -1,5 +1,5 @@
 
-# OS type (linux OR osx)
+# OS type (linux OR macos)
 OSTYPE = linux
 
 # Specify the template to use

diff --git a/configs/docker-py3.7-alpine3.11 b/configs/docker-py3.7-alpine3.11
@@ -1,5 +1,5 @@
 
-# OS type (linux OR osx)
+# OS type (linux OR macos)
 OSTYPE = linux
 
 # Specify the template to use

diff --git a/configs/docker-py3.7-alpine3.11.pkgs b/configs/docker-py3.7-alpine3.11.pkgs