Environment pinning (#985)

Pin environment definitions with [`conda-lock`](https://github.com/conda/conda-lock)

* Add conda environment definitions

* Add multi-platform lockfiles produced by `conda-lock lock`

* Add single-platform explicit lockfiles from `conda-lock render`

* Convert azure pipeline to use conda-lock lockfiles

* Pin PETSc on macOS

  Linkage between petsc 3.20.2_*_102 and scotch 7.0 is broken.
  conda-forge/petsc-feedstock#180

* Document different triggers

* Document use of lockfiles

Author: guyer

.azure/pipelines.yml

@@ -3,11 +3,14 @@
 # Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
 # https://docs.microsoft.com/en-us/azure/devops/pipelines/ecosystems/anaconda
 
+# test every commit (implicit branches and explicit tags) for breaking
+# changes in commit
 trigger:
   tags:
     include:
     - '*'
 
+# test master nightly for breaking changes in dependencies
 schedules:
 - cron: "0 0 * * *"
   displayName: Daily midnight build
@@ -34,13 +37,8 @@ stages:
       steps:
       - template: templates/install.yml
         parameters:
-          python_version: 3
           conda_packages: 'hunspell'
-
-      - bash: |
-          source activate myEnvironment
-          pip install pyspelling
-        displayName: Install pip packages
+          pip_packages: pyspelling
 
       - bash: |
           wget -O en_US.aff  https://cgit.freedesktop.org/libreoffice/dictionaries/plain/en/en_US.aff?id=a4473e06b56bfe35187e302754f6baaa8d75e54f
@@ -62,13 +60,8 @@ stages:
       steps:
       - template: templates/install.yml
         parameters:
-          python_version: 3
           conda_packages: 'numpy'
-
-      - bash: |
-          source activate myEnvironment
-          pip install pylint
-        displayName: Install pip packages
+          pip_packages: pylint
 
       - bash: |
           source activate myEnvironment
@@ -85,13 +78,8 @@ stages:
       steps:
       - template: templates/install.yml
         parameters:
-          python_version: 3
           conda_packages: 'numpy'
-
-      - bash: |
-          source activate myEnvironment
-          pip install flake8
-        displayName: Install pip packages
+          pip_packages: flake8
 
       - bash: |
           source activate myEnvironment
@@ -106,74 +94,50 @@ stages:
         matrix:
           linux-py3k-petsc:
             image: ubuntu-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mpich mpi4py petsc4py mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: petsc
             MPIRUN:
           linux-py3k-petsc-parallel:
             image: ubuntu-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mpich mpi4py petsc4py mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: petsc
             MPIRUN: 'mpirun -np 2'
           linux-py3k-scipy:
             image: ubuntu-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: scipy
             MPIRUN:
           linux-py3k-trilinos:
             image: ubuntu-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mpich mpi4py pytrilinos mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: trilinos
             MPIRUN:
           linux-py3k-trilinos-parallel:
             image: ubuntu-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mpich mpi4py pytrilinos mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: trilinos
             MPIRUN: 'mpirun -np 2'
           macos-py27-pysparse:
             image: macos-latest
-            python_version: 2.7
-            conda_packages: 'numpy scipy matplotlib-base future packaging pysparse mayavi "traitsui<7.0.0" "gmsh<4.0"'
             FIPY_SOLVERS: pysparse
             MPIRUN:
           macos-py3k-petsc:
             image: macos-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mpich mpi4py petsc4py mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: petsc
             MPIRUN:
           macos-py3k-petsc-parallel:
             image: macos-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mpich mpi4py petsc4py mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: petsc
             MPIRUN: 'mpirun -np 2'
           macos-py3k-scipy:
             image: macos-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: scipy
             MPIRUN:
           macos-py3k-trilinos:
             image: macos-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mpich mpi4py pytrilinos mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: trilinos
             MPIRUN:
           macos-py3k-trilinos-parallel:
             image: macos-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mpich mpi4py pytrilinos mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: trilinos
             MPIRUN: 'mpirun -np 2'
           windows-py3k-scipy:
             image: windows-latest
-            python_version: 3
-            conda_packages: 'numpy scipy matplotlib-base future packaging mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: scipy
             MPIRUN:
 
@@ -183,12 +147,13 @@ stages:
       steps:
       - template: templates/install.yml
         parameters:
-          python_version: $(python_version)
-          conda_packages: $(conda_packages)
+          solver: $(FIPY_SOLVERS)
 
       - bash: |
-          conda env export --name myEnvironment
-        displayName: Environment
+          source activate myEnvironment
+          # OK if this fails
+          pip install scikit-fmm || true
+        displayName: Install pip packages
 
       - bash: |
           source activate myEnvironment
@@ -209,14 +174,10 @@ stages:
         matrix:
           unix:
             image: ubuntu-latest
-            python_version: 3.10
-            conda_packages: 'numpy scipy matplotlib-base future packaging mpich mpi4py petsc4py mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: petsc
             MPIRUN:
           windows:
             image: windows-latest
-            python_version: 3.10
-            conda_packages: 'numpy scipy matplotlib-base future packaging mayavi "gmsh <4.0|>=4.5.2"'
             FIPY_SOLVERS: scipy
             MPIRUN:
 
@@ -226,8 +187,7 @@ stages:
       steps:
       - template: templates/install.yml
         parameters:
-          python_version: $(python_version)
-          conda_packages: $(conda_packages)
+          solver: $(FIPY_SOLVERS)
 
       - bash: git fetch origin --tags --force
         displayName: Fetch tags

.azure/templates/install.yml

@@ -1,10 +1,13 @@
 parameters:
-- name: python_version
-  type: string
-  default: 3.9
 - name: conda_packages
   type: string
   default: ''
+- name: pip_packages
+  type: string
+  default: ''
+- name: solver
+  type: string
+  default: scipy
 
 steps:
   - bash: echo "##vso[task.prependpath]$CONDA/bin"
@@ -32,16 +35,42 @@ steps:
       conda config --set always_yes yes --set changeps1 no
       conda config --remove channels defaults
       conda config --set solver libmamba
-    displayName: Create Anaconda environment
+    displayName: Configure Anaconda
+
+  # Most triggered builds use lockfile to ensure failures are
+  # due to commit and not due to dependencies
+  - bash: |
+      conda install --channel=conda-forge --name=base conda-lock
+      conda-lock install --name myEnvironment \
+        environments/locks/conda-${{ parameters.solver }}-lock.yml
+    displayName: Create Anaconda environment from lockfile
+    condition: not(eq(variables['Build.CronSchedule.DisplayName'], 'Daily midnight build'))
 
+  # Daily midnight build uses environment.yml files to catch regressions
+  # due to updates in dependencies
   - bash: |
-      conda create --yes --quiet --name myEnvironment --channel conda-forge python=${{ parameters.python_version }} ${{ parameters.conda_packages }}
+      conda env create --name myEnvironment \
+        --file environments/${{ parameters.solver }}-environment.yml
+    displayName: Create Anaconda environment from environments
+    condition: eq(variables['Build.CronSchedule.DisplayName'], 'Daily midnight build')
+
+  - bash: |
+      if [[ "${{ parameters.conda_packages }}" != "" ]]; then
+        source activate myEnvironment
+        # "Libmamba only supports a subset of the MatchSpec interface for
+        # now.  You can only use ('name', 'version', 'build', 'channel',
+        # 'subdir'), but you tried to use ('md5',)."
+        conda install --solver classic --channel conda-forge ${{ parameters.conda_packages }}
+      fi
     displayName: Install Anaconda packages
 
   - bash: |
-      if [[ ${{ parameters.python_version }} != "2.7" ]]; then
+      if [[ "${{ parameters.pip_packages }}" != "" ]]; then
         source activate myEnvironment
-        pip install scikit-fmm
-        pip install packaging
+        python -m pip install ${{ parameters.pip_packages }}
       fi
-    displayName: Install pip packages
+    displayName: Install Pip packages
+
+  - bash: |
+      conda env export --name myEnvironment
+    displayName: Environment

.en-custom.txt

@@ -183,6 +183,7 @@ linearize
 linearizing
 liquidus
 Lobkovsky
+lockfiles
 LSMLIB
 MacBook
 Malalasekera