Use latest stable release of Debian in the reproducible environment (#…

…186)

* Use latest stable release of Debian in the reproducible environment

* Updates to CI setup

* pytest-pep8 unmaintained, will need to find a replacement

* Fix the new internal skipif function

* CI: try to evict cache

* Try to specify extras to pip when installing from file

* Skip native in some test runs for now

* Handle sundials including "klu.h"

* Use sundials-5.5.0

* debian:buster -> debian:bullseye

* more verbose test output from docker based script

* bump dev version

* remove unused import

.drone.yml

@@ -3,46 +3,47 @@ pipeline:
     image: drillster/drone-volume-cache
     restore: true
     mount:
-      - ./ci-cache/pyusrb
-      - ./ci-cache/conda_packages
-      - ./ci-cache/pip_cache
+      - ./ci_cache/pyusrb
+      - ./ci_cache/conda_packages
+      - ./ci_cache/pip_cache
     volumes:
       - /tmp/cache:/cache
     ttl: 90  # liftetime in days
 
   install:
-    image: bjodah/bjodahimg20dev:v1.2.3
+    image: bjodah/bjodahimg20dot:2.1.0.3
     environment:
       - CC=gcc-10
       - CXX=g++-10
-      - CPLUS_INCLUDE_PATH=/opt/boost_1_73_p/include
-      - SUNDBASE=/opt/sundials-5.3.0-klu-lapack
+      - CPLUS_INCLUDE_PATH=/opt/boost_1_74_p/include
+      - SUNDBASE=/opt/sundials-5.4.0-klu-lapack
       - CPATH=/usr/include/suitesparse  # sunlinsol_klu.h includes "klu.h"
     commands:
-      - export PYTHONUSERBASE=$(pwd)/ci-cache/pyusrb
+      - export PYTHONUSERBASE=$(pwd)/ci_cache/pyusrb
       - if [ ! -d $PYTHONUSERBASE ]; then mkdir -p $PYTHONUSERBASE; fi
       - export CPATH=$SUNDBASE/include:$CPATH
       - export LIBRARY_PATH=$SUNDBASE/lib
       - export LD_LIBRARY_PATH=$SUNDBASE/lib
-      - python3 -m pip install --cache-dir ./ci-cache/pip_cache --user -e .[all]
-      - python3 -c "import pycvodes"  # debug this CI config
+      - python3 -m pip install --cache-dir ./ci_cache/pip_cache --user -e .[all]
+      - python3 -c "import pycvodes; import pyodesys; import pygslodeiv2"  # debug this CI config
       - git fetch -tq
       - python3 setup.py sdist                    # test pip installable sdist (checks MANIFEST.in)
       - git archive -o dist/chempy-head.zip HEAD  # test pip installable zip (symlinks break)
       - mkdir -p deploy/public_html/branches/${DRONE_BRANCH}
       - cp dist/chempy-* deploy/public_html/branches/${DRONE_BRANCH}/
 
   test-suite:
-    image: bjodah/bjodahimg20dev:v1.2.3
+    image: bjodah/bjodahimg20dot:2.1.0.3
     group: testing
     environment:
       - CC=gcc-10
       - CXX=g++-10
-      - CPLUS_INCLUDE_PATH=/opt/boost_1_73_p/include
-      - SUNDBASE=/opt/sundials-5.3.0-klu-lapack
+      - CPLUS_INCLUDE_PATH=/opt/boost_1_74_p/include
+      - SUNDBASE=/opt/sundials-5.4.0-klu-lapack
       - CPATH=/usr/include/suitesparse  # sunlinsol_klu.h includes "klu.h"
+      - CHEMPY_SKIP_NO_TESTS=1
     commands:
-      - export PYTHONUSERBASE=$(pwd)/ci-cache/pyusrb
+      - export PYTHONUSERBASE=$(pwd)/ci_cache/pyusrb
       - export CPATH=$SUNDBASE/include:$CPATH
       - export LIBRARY_PATH=$SUNDBASE/lib
       - export LD_LIBRARY_PATH=$SUNDBASE/lib
@@ -55,20 +56,22 @@ pipeline:
       - python3 -m virtualenv /tmp/test_sdist
       - python3 -m virtualenv /tmp/test_git_archive
       - cd deploy/public_html/branches/${DRONE_BRANCH}
-      - bash -c "source /tmp/test_sdist/bin/activate; pip install --cache-dir ./ci-cache/pip_cache $(eval ls chempy-*.tar.gz) pytest; pytest --pyargs chempy"
-      - bash -c "source /tmp/test_git_archive/bin/activate; pip install --cache-dir ./ci-cache/pip_cache chempy-head.zip pytest; pytest --pyargs chempy"
+      - unset CHEMPY_SKIP_NO_TESTS  # I can't get pip to install extras when using local file...
+      - bash -c "source /tmp/test_sdist/bin/activate; pip install --cache-dir ./ci_cache/pip_cache file://$(realpath $(eval ls chempy-*.tar.gz))#chempy[all] pytest; pytest --pyargs chempy"
+      - bash -c "source /tmp/test_git_archive/bin/activate; pip install --cache-dir ./ci_cache/pip_cache file://$(realpath chempy-head.zip)#chempy[all] pytest; pytest --pyargs chempy"
 
   render-notebooks:
-    image: bjodah/bjodahimg20dev:v1.2.3
+    image: bjodah/bjodahimg20dot:2.1.0.3
     group: testing
     environment:
       - CHEMPY_DEPRECATION_FILTER=ignore
-      - SUNDBASE=/opt/sundials-5.3.0-klu-lapack
+      - SUNDBASE=/opt/sundials-5.4.0-klu-lapack
       - CC=gcc-10
       - CXX=g++-10
+      - CPATH=/usr/include/suitesparse  # sunlinsol_klu.h includes "klu.h"
     commands:
-      - export PYTHONUSERBASE=$(pwd)/ci-cache/pyusrb
-      - export CPATH=$SUNDBASE/include
+      - export PYTHONUSERBASE=$(pwd)/ci_cache/pyusrb
+      - export CPATH=$SUNDBASE/include:$CPATH
       - export LIBRARY_PATH=$SUNDBASE/lib
       - export LD_LIBRARY_PATH=$SUNDBASE/lib
       - ./scripts/render_notebooks.sh
@@ -78,22 +81,22 @@ pipeline:
       - cp -r index.* examples/ "deploy/public_html/branches/${DRONE_BRANCH}"
 
 #  conda-recipe:
-#    image: bjodah/bjodahimg20dev:v1.2.3
+#    image: bjodah/bjodahimg20dot:2.1.0.3
 #    group: testing
 #    commands:
-#      - export CONDA_PKGS_DIRS=$(pwd)/ci-cache/conda_packages
+#      - export CONDA_PKGS_DIRS=$(pwd)/ci_cache/conda_packages
 #      - git fetch -tq
 #      - PATH=/opt/miniconda3/bin:$PATH conda config --add channels bjodah  # sym, pyodesys, pyneqsys
 #      - PATH=/opt/miniconda3/bin:$PATH conda build --output-folder "deploy/public_html/branches/${DRONE_BRANCH}" conda-recipe
 #      - (cd $CONDA_PKGS_DIRS; find . -maxdepth 1 -type d -not -path . -not -path .. | xargs rm -r)
 
   compile-documentation:
-    image: bjodah/bjodahimg20dev:v1.2.3
+    image: bjodah/bjodahimg20dot:2.1.0.3
     environment:
       - CHEMPY_DEPRECATION_FILTER=ignore
-      - SUNDBASE=/opt/sundials-5.3.0-klu-lapack
+      - SUNDBASE=/opt/sundials-5.4.0-klu-lapack
     commands:
-      - export PYTHONUSERBASE=$(pwd)/ci-cache/pyusrb
+      - export PYTHONUSERBASE=$(pwd)/ci_cache/pyusrb
       - export CPATH=$SUNDBASE/include
       - export LIBRARY_PATH=$SUNDBASE/lib
       - export LD_LIBRARY_PATH=$SUNDBASE/lib
@@ -105,10 +108,10 @@ pipeline:
     image: drillster/drone-volume-cache
     rebuild: true
     mount:
-#      - ./ci-cache/sund-3.2.1
-      - ./ci-cache/pyusrb
-      - ./ci-cache/conda_packages
-      - ./ci-cache/pip_cache
+#      - ./ci_cache/sund-3.2.1
+      - ./ci_cache/pyusrb
+      - ./ci_cache/conda_packages
+      - ./ci_cache/pip_cache
     volumes:
       - /tmp/cache:/cache
 

chempy/_release.py

@@ -1 +1 @@
-__version__ = "0.8.0.dev0+git"
+__version__ = "0.8.1.dev0+git"

chempy/util/testing.py

@@ -3,6 +3,7 @@
 
 from pkg_resources import parse_requirements, parse_version
 
+import os
 from operator import lt, le, eq, ne, ge, gt
 import pytest
 
@@ -55,4 +56,11 @@ def __call__(self, cb):
             r += " Missing modules: %s." % ', '.join(self.missing)
         if self.incomp:
             r += " Incomp versions: %s." % ', '.join(self.incomp)
-        return pytest.mark.skipif(self.missing or self.incomp, reason=r)(cb)
+        return skipif(self.missing or self.incomp, reason=r)(cb)
+
+
+def skipif(predicate, *, reason):
+    if os.environ.get('CHEMPY_SKIP_NO_TESTS', '0') == '1':
+        return pytest.mark.skipif(False, reason=reason)
+    else:
+        return pytest.mark.skipif(predicate, reason=reason)

chempy/util/tests/test_graph.py

@@ -7,11 +7,9 @@
 import shutil
 import tempfile
 
-import pytest
-
 from chempy import Reaction, ReactionSystem, Substance
 from ..graph import rsys2dot, rsys2graph
-from ..testing import requires
+from ..testing import requires, skipif
 
 try:
     dot_missing = subprocess.call(['dot', '-?']) != 0
@@ -28,7 +26,7 @@ def _get_rsys():
 
 
 @requires('numpy')
-@pytest.mark.skipif(dot_missing, reason='graphviz not installed? (dot command missing)')
+@skipif(dot_missing, reason='graphviz not installed? (dot command missing)')
 def test_rsys2dot():
     rsys = _get_rsys()
     assert list(map(str.strip, rsys2dot(rsys))) == [
@@ -46,7 +44,7 @@ def test_rsys2dot():
 
 
 @requires('numpy')
-@pytest.mark.skipif(dot_missing, reason='graphviz not installed? (dot command missing)')
+@skipif(dot_missing, reason='graphviz not installed? (dot command missing)')
 def test_rsys2graph():
     rsys = _get_rsys()
     tempdir = tempfile.mkdtemp()

chempy/util/tests/test_table.py

@@ -11,6 +11,7 @@
     rsys2tablines, rsys2table, rsys2pdf_table
 )
 from .test_graph import _get_rsys
+from ..testing import skipif
 
 
 try:
@@ -44,7 +45,7 @@ def test_rsys2table():
 
 
 @pytest.mark.parametrize('longtable', (True, False))
-@pytest.mark.skipif(pdflatex_missing, reason='latex not installed? (pdflatex command missing)')
+@skipif(pdflatex_missing, reason='latex not installed? (pdflatex command missing)')
 def test_rsys2pdf_table(longtable):
     rsys = _get_rsys()
     tempdir = tempfile.mkdtemp()

scripts/environment/Dockerfile

@@ -1,22 +1,38 @@
-FROM debian:stretch
+FROM debian:bullseye
 
 MAINTAINER Björn Dahlgren <[email protected]>
 
 ENV LANG C.UTF-8
 
-# This dockerfile is designed to run on binder (mybinder.org)
 RUN apt-get update && \
-    apt-get --quiet --assume-yes install curl git g++-6 libgmp-dev binutils-dev bzip2 make cmake sudo \
-    python3-dev python3-pip libgsl-dev liblapack-dev graphviz && \
-    apt-get clean && \
-    curl -LOs http://computation.llnl.gov/projects/sundials/download/sundials-3.1.2.tar.gz && \
-    tar xzf sundials-3.1.2.tar.gz && mkdir build/ && cd build/ && \
-    cmake -DCMAKE_PREFIX_PATH=/usr/local -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release \
-    -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=OFF -DEXAMPLES_ENABLE=OFF -DEXAMPLES_INSTALL=OFF \
-    ../sundials*/ && make install && cd - && rm -r build/ sundials* && \
-    python3 -m pip install --upgrade pip && \
-    curl -LOs http://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2 && \
-    tar xjf boost_*.tar.bz2 && cd boost* && ./bootstrap.sh && ./b2 -j 2 --prefix=/usr/local/ install && cd -
+    apt-get --quiet --assume-yes install curl git g++-10 gfortran-10 libgmp-dev binutils-dev bzip2 make cmake sudo \
+    python3-dev python3-pip libboost-dev libgsl-dev liblapack-dev libsuitesparse-dev graphviz && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN mkdir /tmp/sundials-5.5.0-build && \
+    curl -Ls https://github.com/LLNL/sundials/releases/download/v5.5.0/sundials-5.5.0.tar.gz | tar xz -C /tmp && \
+    FC=gfortran-10 cmake \
+        -S /tmp/sundials-5.5.0 \
+        -B /tmp/sundials-5.5.0-build \
+        -DCMAKE_INSTALL_PREFIX=/usr/local \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DBUILD_SHARED_LIBS=ON \
+        -DBUILD_STATIC_LIBS=OFF \
+        -DEXAMPLES_ENABLE_C=OFF \
+        -DEXAMPLES_INSTALL=OFF \
+        -DENABLE_LAPACK=ON \
+        -DSUNDIALS_INDEX_SIZE=32 \
+        -DENABLE_KLU=ON \
+        -DKLU_INCLUDE_DIR=/usr/include/suitesparse \
+        -DKLU_LIBRARY_DIR=/usr/lib/x86_64-linux-gnu && \
+    cmake --build /tmp/sundials-5.5.0-build && \
+    cmake --build /tmp/sundials-5.5.0-build --target install && \
+    rm -r /tmp/sundials-5.5.0*/ && \
+    python3 -m pip install --upgrade-strategy=eager --upgrade pip && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+
+# http://computation.llnl.gov/projects/sundials/download/sundials-5.5.0.tar.gz
 
 # At this point the system should be able to pip-install the package and all of its dependencies. We'll do so
 # when running the image using the ``host-jupyter-using-docker.sh`` script. Installed packages are cached.

scripts/host-jupyter-using-docker.sh

@@ -1,14 +1,17 @@
 #!/bin/bash -ue
 #
-# Usage:
+# This script requires that Docker is installed.
+#
+# Arguments: mount-path, port-number, Dockerfile-path
+#
+# To host a local jupyter notebook server rung e.g.:
 #
 #  $ ./scripts/host-jupyter-using-docker.sh
 #  $ ./scripts/host-jupyter-using-docker.sh . 8888 ./scripts/environment
-#  $ ./scripts/host-jupyter-using-docker.sh . 0 ./scripts/environment
 #
-# Arguments: mount-path, port-number, Dockerfile-path
+# To instead run the test suite, specify "0" as the port number:
 #
-# If port == 0: the test suite is run
+#  $ ./scripts/host-jupyter-using-docker.sh . 0 ./scripts/environment
 #
 MOUNT=${1:-.}
 PORT=${2:-8888}
@@ -26,19 +29,18 @@ if [[ "$DOCKERIMAGE" == ./* ]]; then
     DOCKERIMAGE=$(sudo docker build $DOCKERIMAGE | tee /dev/tty | tail -1 | cut -d' ' -f3)
 fi
 if [[ "$PORT" == "0" ]]; then
-    LOCALCMD="pytest -rs --pyargs $PKG"
+    LOCALCMD="pytest -sv -ra --pyargs $PKG"
     PORTFWD=""
 else
     LOCALCMD="jupyter notebook --no-browser --port $PORT --ip=* index.ipynb"
-    PORTFWD="-p 127.0.0.1:$PORT:$PORT"
+    PORTFWD="-p ${4:-127.0.0.1}:$PORT:$PORT"
 fi
 MYCMD="groupadd -f --gid \$HOST_GID \$HOST_WHOAMI; \
 useradd --uid \$HOST_UID --gid \$HOST_GID --home /mount \$HOST_WHOAMI; \
-sudo --preserve-env --login -u \$HOST_WHOAMI python3 -m pip install --user symcxx quantities; \
-sudo --preserve-env --login -u \$HOST_WHOAMI python3 -m pip install --user -e .[all]; \
-sudo --preserve-env --login -u \$HOST_WHOAMI /mount/.local/bin/jupyter-nbextension enable --user --py widgetsnbextension; \
-sudo --preserve-env --login -u \$HOST_WHOAMI LD_LIBRARY_PATH=/usr/local/lib MPLBACKEND=Agg /mount/.local/bin/$LOCALCMD"
+sudo --login -u \$HOST_WHOAMI PYCVODES_NO_LAPACK=1 PYCVODES_NO_KLU=1 python3 -m pip install --user -e .[all]; \
+sudo --login -u \$HOST_WHOAMI /mount/.local/bin/jupyter-nbextension enable --user --py widgetsnbextension; \
+sudo --login -u \$HOST_WHOAMI LD_LIBRARY_PATH=/usr/local/lib MPLBACKEND=Agg /mount/.local/bin/$LOCALCMD"
 set -x
 sudo docker run --rm --name "${PKG}_nb_${PORT}" $PORTFWD \
  -e HOST_WHOAMI=${HOST_USER} -e HOST_UID=$(id -u ${HOST_USER}) -e HOST_GID=$(id -g ${HOST_USER})\
- -v $MOUNT:/mount -w /mount -it $DOCKERIMAGE /usr/bin/env bash -x -c "$MYCMD"
+ -v $MOUNT:/mount -w /mount -it $DOCKERIMAGE /bin/bash -x -c "$MYCMD"

setup.cfg

@@ -1,10 +1,10 @@
 [tool:pytest]
-norecursedirs = .* _* build dist conda-recipe scripts benchmarks doc deploy venv *cache*
-pep8maxlinelength=119
-pep8ignore =
-    doc/conf.py ALL  # conf.py is a generated file
-    test_water_viscosity_korson_1969.py E222  # multiple spaces after operator
-    chempy/kinetics/tests/test_rates.py E221 E222 E226 E251
+testpaths = chempy
+# pep8maxlinelength=119
+# pep8ignore =
+#     doc/conf.py ALL  # conf.py is a generated file
+#     test_water_viscosity_korson_1969.py E222  # multiple spaces after operator
+#     chempy/kinetics/tests/test_rates.py E221 E222 E226 E251
 flakes-ignore =
     doc/conf.py ALL  # conf.py is a generated file
     __init__.py UnusedImport ImportStarUsed

setup.py

@@ -75,6 +75,7 @@ def _path_under_setup(*args):
     'Programming Language :: Python :: 3',
     'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
+    'Programming Language :: Python :: 3.9',
 ]
 
 with io.open(_path_under_setup(pkg_name, '__init__.py'), 'rt', encoding='utf-8') as f: