From ddcac277d5f5ca60c1c9165c729a47d0e2329025 Mon Sep 17 00:00:00 2001
From: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date: Mon, 27 Jan 2020 11:30:04 -0500
Subject: [PATCH] Squashed commit of the following:

commit 649629ceba98b79df61186ef7490018ce4f1cc95
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Mon Jan 27 03:30:35 2020 -0500

    Add !nose to SAGE_CHECK_PACKAGES

commit 3fad10896ad67d689650320bb02c02f28da9fce6
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Mon Jan 27 00:25:15 2020 -0500

    Add openblas on Fedora

commit 2076fbc7074a29de0edc08aa4a108d48deee071f
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Mon Jan 27 00:14:25 2020 -0500

    fixes for Fedora 26, Debian Jessie

commit 648a6074421cbe080e73d8c66cfc76cd30939f45
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 23:57:30 2020 -0500

    build/pkgs/fedora-bootstrap.txt: Add pkg-config

commit bd905613611ba5b1b4f5bf66d99c92c9b72e3757
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 23:33:47 2020 -0500

    disable R testsuite

commit 67ec8c7ad37df4b07815f18194316911f4d4aae2
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 23:13:15 2020 -0500

    fixup makeflags

commit 344cd95ad1b6c97b4c5f31caebbcdbeee30af4c6
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 22:15:15 2020 -0500

    fixup

commit 8ad26f30a05cda7e3cf636d285f3f2c125cdbce4
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 22:05:13 2020 -0500

    fixup

commit 9f37768149c844bc5a644d98cb7d8f0afa270776
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 21:37:39 2020 -0500

    fixup

commit 31b90cc33ac33911e25e454e6494ebbfd6e479c6
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 21:04:24 2020 -0500

    fix up use of condarc

commit 4b2a5323d242aae9f5991edd713cc33e7160d5c6
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 20:37:18 2020 -0500

    Better handling of IGNORE_MISSING_SYSTEM_PACKAGES

commit 26538a11ec749cd64ffeb622fd72417c496e9281
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 20:36:27 2020 -0500

    tox.ini: Fix commands syntax

commit eeb5ec8aec7b3d4519416c08445d26956926695b
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 20:36:13 2020 -0500

    tox.ini: debian-sid does not need IGNORE_MISSING_SYSTEM_PACKAGES

commit 4eecb3a82dd9100c1b33e80c947757e0d0766abf
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 20:34:00 2020 -0500

    conda-bootstrap.txt: Add pkg-config

commit 87cde57cbe1fca0f0fd87788877954ed559ea9a9
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 18:09:36 2020 -0500

    IGNORE_MISSING_SYSTEM_PACKAGES

commit ad12b08feaf7d9e2bdb98e6284922176a9ad8fa9
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 17:11:20 2020 -0500

    fixup

commit 8f7cd6e3bffbf503877dbd7e8c0b88bf0787c5d2
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 17:08:03 2020 -0500

    fixup

commit 4a89a095c754eec8348a8a4743d366a6869fb30b
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 16:58:59 2020 -0500

    Add conda

commit 8e8ed1d157bf6758e58d7f8eeb6114bc1198b61d
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 16:21:48 2020 -0500

    Add condarc.yml

commit c5eb59e0c96a2dd12cb45e4c6e461dfb86128c18
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 14:11:54 2020 -0500

    build/bin/sage-spkg: Explain to user what will be done with the system package

commit 8b15253db7aa6b445b49d93c23eee94676ecf7a2
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 14:11:28 2020 -0500

    build/bin/write-dockerfile.sh: Only install a package if it has spkg-configure.m4

commit b7e8a61914f4bcaa85cafc513f85e1cdd7ff6bef
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 13:03:26 2020 -0500

    fixup

commit 81b7c62dbe07fed9ecaae105a3b484e784c16c73
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 12:48:16 2020 -0500

    Improve log handling

commit ea660983b4076e4036d03b702bae3a31f2213530
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 12:16:14 2020 -0500

    build/pkgs/debian.txt: python3-minimal->python3

commit c834523c2180ae175c5654aa46aa4acb2850bb9c
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 12:10:20 2020 -0500

    build/pkgs/debian.txt: add g++

commit 2f272cd6fd1c9014d8c53c87d276baa940fc4438
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 12:04:57 2020 -0500

    build/pkgs/arch.txt: Add gcc

commit 41ff83d03441169d7af7706b8fc7142191516868
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 11:56:57 2020 -0500

    remove nonexisting fedora package

commit da1282f444e9ef8169aa60a7a1fd49a1466bad79
Author: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date:   Sun Jan 26 11:47:19 2020 -0500

    build/pkgs/debian-bootstrap.txt: Add pkg-config
---
 build/bin/write-dockerfile.sh          | 85 ++++++++++++++++++++++----
 build/pkgs/conda-bootstrap.txt         |  2 +-
 build/pkgs/fedora-bootstrap.txt        |  2 +
 build/pkgs/fedora.txt                  |  6 ++
 build/pkgs/openblas/distros/fedora.txt |  1 +
 condarc.yml                            |  6 ++
 tox.ini                                | 27 +++++++-
 7 files changed, 113 insertions(+), 16 deletions(-)
 create mode 100644 build/pkgs/openblas/distros/fedora.txt
 create mode 100644 condarc.yml

diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh
index c12af2c6686..17812a2ec0d 100755
--- a/build/bin/write-dockerfile.sh
+++ b/build/bin/write-dockerfile.sh
@@ -7,6 +7,7 @@ SYSTEM="${1:-debian}"
 shopt -s extglob
 TYPE_PATTERN="${2:-standard}"
 WITH_SYSTEM_SPKG="${3:-yes}"
+IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
 #
 STRIP_COMMENTS="sed s/#.*//;"
 SAGE_ROOT=.
@@ -32,36 +33,92 @@ for PKG_SCRIPTS in build/pkgs/*; do
         fi
     fi
 done
+echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"
+echo "# the :comments: separate the generated file into sections"
+echo "# to simplify writing scripts that customize this file"
 case $SYSTEM in
     debian*|ubuntu*)
         cat <<EOF
 ARG BASE_IMAGE=ubuntu:latest
 FROM \${BASE_IMAGE}
-RUN apt-get update &&  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes $SYSTEM_PACKAGES
 EOF
+        UPDATE="apt-get update &&"
+        INSTALL="DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes"
+        CLEAN="&& apt-get clean"
         ;;
     fedora*|redhat*|centos*)
         cat <<EOF
 ARG BASE_IMAGE=fedora:latest
 FROM \${BASE_IMAGE}
-RUN yum install -y $SYSTEM_PACKAGES
 EOF
+        INSTALL="yum install -y"
         ;;
     arch*)
         # https://hub.docker.com/_/archlinux/
         cat <<EOF
 ARG BASE_IMAGE=archlinux:latest
 FROM \${BASE_IMAGE}
-RUN pacman -Syu --noconfirm $SYSTEM_PACKAGES
 EOF
+        INSTALL="pacman -Syu --noconfirm"
+        ;;
+    conda*)
+        cat <<EOF
+ARG BASE_IMAGE=continuumio/miniconda3:latest
+FROM \${BASE_IMAGE}
+ARG USE_CONDARC=condarc.yml
+ADD *condarc*.yml /tmp/
+RUN echo \${CONDARC}; cd /tmp && conda config --stdin < \${USE_CONDARC}
+RUN conda update -n base conda
+EOF
+        INSTALL="conda install --update-all --quiet --yes"
+        EXISTS="2>/dev/null >/dev/null conda search -f"
+        #EXISTS="conda search -f"
         ;;
     *)
-        echo "Not implemented: package installation for SYSTEM=$SYSTEM"
+        echo "Not implemented: package installation for SYSTEM=$SYSTEM" >&2
         exit 1
         ;;
 esac
 cat <<EOF
-# Bootstrapping
+#:packages:
+ENV PACKAGES="$SYSTEM_PACKAGES"
+EOF
+case "$IGNORE_MISSING_SYSTEM_PACKAGES" in
+    no)
+        cat <<EOF
+RUN $UPDATE $INSTALL $SYSTEM_PACKAGES $CLEAN
+EOF
+        ;;
+    yes)
+        if [ -n "$EXISTS" ]; then
+            # Filter by existing packages, try to install these in one shot; fall back to one by one.
+            cat <<EOF
+RUN $UPDATE EXISTING_PACKAGES=""; for pkg in \$PACKAGES; do echo -n .; if $EXISTS \$pkg; then EXISTING_PACKAGES="\$EXISTING_PACKAGES \$pkg"; echo -n "\$pkg"; fi; done; $INSTALL \$EXISTING_PACKAGES || (echo "Trying again one by one:"; for pkg in \$EXISTING_PACKAGES; do echo "Trying to install \$pkg"; $INSTALL \$pkg || echo "(ignoring error)"; done); : $CLEAN
+EOF
+        else
+            # Try in one shot, fall back to one by one.  Separate "RUN" commands
+            # for caching by docker.
+            cat <<EOF
+RUN $UPDATE $INSTALL \${PACKAGES} || echo "(ignoring error)"
+EOF
+            for pkg in $SYSTEM_PACKAGES; do
+                cat <<EOF
+RUN $INSTALL $pkg || echo "(ignoring error)"
+EOF
+            done
+            if [ -n "$CLEAN" ]; then
+                cat <<EOF
+RUN : $CLEAN
+EOF
+            fi
+        fi
+        ;;
+    *)
+        echo "Argument IGNORE_MISSING_SYSTEM_PACKAGES must be yes or no"
+        ;;
+esac
+cat <<EOF
+#:bootstrapping:
 RUN mkdir -p /sage
 WORKDIR /sage
 ADD Makefile VERSION.txt README.md bootstrap configure.ac sage ./
@@ -69,11 +126,11 @@ ADD m4 ./m4
 ADD build ./build
 ADD src/bin/sage-version.sh src/bin/sage-version.sh
 RUN ./bootstrap
-# Configuring
+#:configuring:
 ADD src/ext src/ext
 ADD src/bin src/bin
 ADD src/Makefile.in src/Makefile.in
-ARG EXTRA_CONFIGURE_ARGS
+ARG EXTRA_CONFIGURE_ARGS=""
 EOF
 if [ ${WITH_SYSTEM_SPKG} = "force" ]; then
     cat <<EOF
@@ -88,10 +145,14 @@ cat <<EOF
 # We first compile base-toolchain because otherwise lots of packages are missing their dependency on 'patch'
 ARG NUMPROC=8
 ENV MAKE="make -j\${NUMPROC}"
-RUN make base-toolchain
+ARG USE_MAKEFLAGS="-k"
+#:toolchain:
+RUN make \${USE_MAKEFLAGS} base-toolchain
+#:make:
 # Avoid running the lengthy testsuite of the following.
-RUN make cython
-# Compile something tricky: Everything that uses BLAS.
-ARG TARGETS="scipy cbc csdp fflas_ffpack gsl iml numpy r suitesparse cvxopt"
-RUN SAGE_CHECK=yes MAKE="make -j4" make -k \${TARGETS}
+RUN make \${USE_MAKEFLAGS} cython
+# By default, compile something tricky but that does not take too long. scipy uses BLAS.
+ARG TARGETS="scipy"
+RUN SAGE_CHECK=yes SAGE_CHECK_PACKAGES="!r,!python3,!python2,!nose" make \${USE_MAKEFLAGS} \${TARGETS}
+#:end:
 EOF
diff --git a/build/pkgs/conda-bootstrap.txt b/build/pkgs/conda-bootstrap.txt
index 5405f70cf3d..55f45bfd276 100644
--- a/build/pkgs/conda-bootstrap.txt
+++ b/build/pkgs/conda-bootstrap.txt
@@ -1,2 +1,2 @@
 # Packages needed for ./bootstrap
-gettext autoconf automake libtool
+gettext autoconf automake libtool pkg-config
diff --git a/build/pkgs/fedora-bootstrap.txt b/build/pkgs/fedora-bootstrap.txt
index 5cd20fd529c..97f16705319 100644
--- a/build/pkgs/fedora-bootstrap.txt
+++ b/build/pkgs/fedora-bootstrap.txt
@@ -1,2 +1,4 @@
 # Packages needed for ./bootstrap
 gettext-devel autoconf automake libtool
+# Fedora 26 needs:
+pkg-config
diff --git a/build/pkgs/fedora.txt b/build/pkgs/fedora.txt
index 5004324c836..6f1404b2849 100644
--- a/build/pkgs/fedora.txt
+++ b/build/pkgs/fedora.txt
@@ -18,3 +18,9 @@ m4
 perl
 perl-ExtUtils-MakeMaker
 tar
+gcc
+# On Fedora 26, need C++ even to survive 'configure'. Otherwise:
+# checking how to run the C++ preprocessor... /lib/cpp
+# configure: error: in `/sage':
+# configure: error: C++ preprocessor "/lib/cpp" fails sanity check
+gcc-c++
diff --git a/build/pkgs/openblas/distros/fedora.txt b/build/pkgs/openblas/distros/fedora.txt
new file mode 100644
index 00000000000..52e4a9f78f4
--- /dev/null
+++ b/build/pkgs/openblas/distros/fedora.txt
@@ -0,0 +1 @@
+openblas-devel
diff --git a/condarc.yml b/condarc.yml
new file mode 100644
index 00000000000..9538902cacd
--- /dev/null
+++ b/condarc.yml
@@ -0,0 +1,6 @@
+# For the environment.yml to reliably use the conda-forge packages,
+# need to set this option.  See https://github.com/conda-forge/sagelib-feedstock/issues/87
+channel_priority: strict
+channels:
+  - conda-forge
+  - defaults
diff --git a/tox.ini b/tox.ini
index 696a131f3ea..6c3025aec0f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,7 +11,8 @@ envlist =
     docker-{ubuntu-{trusty,xenial,bionic,latest,eoan,rolling,focal,devel},
             debian-{sid,jessie,buster,bullseye},
             fedora-{26,27,28,29,30,31,32},
-            arch-latest
+            arch-latest,
+            conda-forge,conda-anaconda3
             }-{minimal,standard,maximal}
 skipsdist = true
 
@@ -22,6 +23,9 @@ setenv =
     # Set this to 'force' instead of 'yes' to make it an error if an spkg with spkg-configure and system package
     # is not recognized.
     WITH_SYSTEM_SPKG=yes
+    # Set this to 'yes' instead of 'no' to ignore missing system packages - by installing them one by one
+    # and ignoring errors.  We use that to take care of old versions of distributions.
+    IGNORE_MISSING_SYSTEM_PACKAGES=no
     # What system packages should be installed. Default: All standard packages with spkg-configure.
     # These are bash extglob patterns.
     TYPE_PATTERN=standard
@@ -37,7 +41,9 @@ setenv =
     #
     ubuntu:         SYSTEM=debian
     ubuntu-trusty:  BASE_IMAGE=ubuntu:trusty
+    ubuntu-trusty:                             IGNORE_MISSING_SYSTEM_PACKAGES=yes
     ubuntu-xenial:  BASE_IMAGE=ubuntu:xenial
+    ubuntu-xenial:                             IGNORE_MISSING_SYSTEM_PACKAGES=yes
     ubuntu-bionic:  BASE_IMAGE=ubuntu:bionic
     ubuntu-latest:  BASE_IMAGE=ubuntu:latest
     ubuntu-eoan:    BASE_IMAGE=ubuntu:eoan
@@ -50,6 +56,7 @@ setenv =
     debian:          SYSTEM=debian
     debian-sid:      BASE_IMAGE=debian:sid
     debian-jessie:   BASE_IMAGE=debian:jessie
+    debian-jessie:                               IGNORE_MISSING_SYSTEM_PACKAGES=yes
     debian-buster:   BASE_IMAGE=debian:buster
     debian-bullseye: BASE_IMAGE=debian:bullseye
     #
@@ -57,6 +64,7 @@ setenv =
     #
     fedora:          SYSTEM=fedora
     fedora-26:       BASE_IMAGE=fedora:26
+    fedora-26:                                 IGNORE_MISSING_SYSTEM_PACKAGES=yes
     fedora-27:       BASE_IMAGE=fedora:27
     fedora-28:       BASE_IMAGE=fedora:28
     fedora-29:       BASE_IMAGE=fedora:29
@@ -68,10 +76,23 @@ setenv =
     #
     archlinux:        SYSTEM=arch
     archlinux-latest: BASE_IMAGE=archlinux:latest
+    #
+    # https://hub.docker.com/r/continuumio
+    #
+    conda:            SYSTEM=conda
+    conda-forge:      BASE_IMAGE=continuumio/miniconda3:latest
+    conda-forge:                                                CONDARC=condarc.yml
+    conda-anaconda3:  BASE_IMAGE=continuumio/anaconda3:latest
+    conda-anaconda3:                                            CONDARC=/dev/null
+    conda-anaconda3:                                                                  IGNORE_MISSING_SYSTEM_PACKAGES=yes
 
 whitelist_externals =
     bash
     docker
 commands =
-    bash -c 'build/bin/write-dockerfile.sh {env:SYSTEM} "{env:TYPE_PATTERN:}" {env:WITH_SYSTEM_SPKG} > {envdir}/Dockerfile'
-    docker build . -f {envdir}/Dockerfile --build-arg EXTRA_CONFIGURE_ARGS="{env:EXTRA_CONFIGURE_ARGS:}" {env:EXTRA_DOCKER_BUILD_ARGS:} --build-arg BASE_IMAGE={env:BASE_IMAGE}
+    docker:        bash -c 'build/bin/write-dockerfile.sh {env:SYSTEM} "{env:TYPE_PATTERN:}" {env:WITH_SYSTEM_SPKG} {env:IGNORE_MISSING_SYSTEM_PACKAGES} > {envdir}/Dockerfile'
+    docker:        docker build . -f {envdir}/Dockerfile                          \
+    docker:        --build-arg EXTRA_CONFIGURE_ARGS="{env:EXTRA_CONFIGURE_ARGS:}" \
+    docker:        --build-arg BASE_IMAGE={env:BASE_IMAGE}                        \
+    docker-conda:  --build-arg USE_CONDARC="{env:CONDARC}"                        \
+    docker:        {env:EXTRA_DOCKER_BUILD_ARGS:}