cleanup testing matrix (#132)

- Requires more test runners, but they boot fast and we can be patient. - Filters applied in corresponding YAML to ignore tests that are irrelevant. - (Hopefully) stop codecov from failing status checks. Code coverage is nice, but not required.
svenevs · Dec 24, 2021 · cc17b2a · cc17b2a
1 parent 16e88b7
commit cc17b2a
Show file tree

Hide file tree

Showing 4 changed files with 101 additions and 105 deletions.
diff --git a/.github/workflows/test_extras.yaml b/.github/workflows/test_extras.yaml
@@ -1,15 +1,38 @@
-name: Test Extras
+name: Extras
+
 on:
   push:
     branches:
       - master
+    paths:
+      - 'exhale'
+      - 'testing'
+      - 'docs'
+      - 'README.rst'
+      - 'setup.py'
+      - 'setup.cfg'
+      - 'MANIFEST.in'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - '.github/workflows/test_extras.yaml'
   pull_request:
     branches:
       - master
+    paths:
+      - 'exhale'
+      - 'testing'
+      - 'docs'
+      - 'README.rst'
+      - 'setup.py'
+      - 'setup.cfg'
+      - 'MANIFEST.in'
+      - 'pyproject.toml'
+      - 'tox.ini'
+      - '.github/workflows/test_extras.yaml'
 
 jobs:
-  build_docs:
-    name: Docs
+  build_docs_linkcheck:
+    name: Docs / Linkcheck
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -22,31 +45,18 @@ jobs:
           python-version: '3.9'
       - name: Install Tools
         run: |
-          pip install -U tox
           sudo apt-get install -y graphviz
+          pip install -U tox
+      # NOTE: the RTD PR build is enabled to view docs, this tests with nitpicky
+      # flags that will catch non-critical warnings.
       - name: Test Docs
         run: |
           tox -e docs
-
-  build_linkcheck:
-    name: Linkcheck
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Use Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
-      - name: Install Tools
-        run: |
-          pip install -U tox
       - name: Test Linkcheck
         run: |
           tox -e linkcheck
 
+  # TODO: this will become `lint` which will have more checks.
   build_flake8:
     name: Flake8
     runs-on: ubuntu-latest

diff --git a/.github/workflows/test_python.yaml b/.github/workflows/test_python.yaml
@@ -1,15 +1,26 @@
-name: Test Python
+name: Python
+
 on:
   push:
     branches:
       - master
+    paths:
+      - 'exhale'
+      - 'testing'
+      - 'tox.ini'
+      - '.github/workflows/test_python.yaml'
   pull_request:
     branches:
       - master
+    paths:
+      - 'exhale'
+      - 'testing'
+      - 'tox.ini'
+      - '.github/workflows/test_python.yaml'
 
 jobs:
   build:
-    name: ${{ matrix.os }}
+    name: ${{ matrix.os }} / ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -18,6 +29,7 @@ jobs:
         # TODO: either obtain doxygen 1.8.20 on mac, or find out why everything
         # breaks in doxygen 1.9.2.
         os: [ubuntu-latest, windows-latest]
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
     steps:
       - uses: actions/checkout@v2
       ##########################################################################
@@ -41,98 +53,37 @@ jobs:
       - name: Doxygen Version Dump
         run: doxygen --version
       ##########################################################################
-      - name: Use Python 3.6
+      - name: Use Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: ${{ matrix.python-version }}
       - name: Install Python Tools
-        run: pip install -U tox codecov coverage
-      - name: Test Python 3.6
-        run: tox -e py -- --cov-report xml:coverage.xml --cov
-      - name: Upload Code Coverage for Python 3.6
-        run: |
-          codecov --required -X gcov -f coverage.xml --name "[GHA] ${{ matrix.os }}-py3.6"
-      - name: Cleanup Python 3.6
         run: |
-          mv .gitignore nolongerignored
-          git clean -n
-          git clean -f
-          git reset --hard
-      ##########################################################################
-      - name: Use Python 3.7
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Install Python Tools
-        run: pip install -U tox codecov coverage
-      - name: Test Python 3.7
-        run: tox -e py -- --cov-report xml:coverage.xml --cov
-      - name: Upload Code Coverage for Python 3.7
-        run: |
-          codecov --required -X gcov -f coverage.xml --name "[GHA] ${{ matrix.os }}-py3.7"
-      - name: Cleanup Python 3.7
-        run: |
-          mv .gitignore nolongerignored
-          git clean -n
-          git clean -f
-          git reset --hard
-      ##########################################################################
-      - name: Use Python 3.8
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.8'
-      - name: Install Python Tools
-        run: pip install -U tox codecov coverage
-      - name: Test Python 3.8
+          pip install -U tox codecov coverage
+      - name: Test Python ${{ matrix.python-version }} / sphinx==3.0.0
         env:
+          # NOTE: in the future, we may want to use a larger matrix with more
+          # combinations of sphinx and breathe, but for now it is not necessary.
           SPHINX_VERSION: '==3.0.0'
-        run: tox -e py -- --cov-report xml:coverage.xml --cov
-      - name: Upload Code Coverage for Python 3.8
         run: |
-          codecov --required -X gcov -f coverage.xml --name "[GHA] ${{ matrix.os }}-py3.8"
-      - name: Cleanup Python 3.8
+          tox -e py -- --cov-report xml:coverage.xml --cov
+      - name: Upload Code Coverage for Python ${{ matrix.python-version }} / sphinx==3.0.0
         run: |
-          mv .gitignore nolongerignored
-          git clean -n
-          git clean -f
-          git reset --hard
-      ##########################################################################
-      - name: Use Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
-      - name: Install Python Tools
-        run: pip install -U tox codecov coverage
-      - name: Test Python 3.9
-        env:
-          SPHINX_VERSION: '>=4.0.0'
-        run: tox -e py -- --cov-report xml:coverage.xml --cov
-      - name: Upload Code Coverage for Python 3.9
-        run: |
-          codecov --required -X gcov -f coverage.xml --name "[GHA] ${{ matrix.os }}-py3.9"
-      - name: Cleanup Python 3.9
+          codecov --required -X gcov -f coverage.xml --name "[GHA] ${{ matrix.os }}-py${{ matrix.python-version }}-sphinx3.0.0"
+      # Don't let code coverage utilities share anything, force clean it all.
+      - name: Cleanup Python ${{ matrix.python-version }} / sphinx==3.0.0 Artifacts
         run: |
           mv .gitignore nolongerignored
           git clean -n
           git clean -f
           git reset --hard
-      ##########################################################################
-      - name: Use Python 3.10
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.10'
-      - name: Install Python Tools
-        run: pip install -U tox codecov coverage
-      - name: Test Python 3.10
+      - name: Test Python ${{ matrix.python-version }} / sphinx 4.x
         env:
-          SPHINX_VERSION: '>=4.0.0'
-        run: tox -e py -- --cov-report xml:coverage.xml --cov
-      - name: Upload Code Coverage for Python 3.10
+          # NOTE: in the future, we may want to use a larger matrix with more
+          # combinations of sphinx and breathe, but for now it is not necessary.
+          SPHINX_VERSION: '>=4<5'
         run: |
-          codecov --required -X gcov -f coverage.xml --name "[GHA] ${{ matrix.os }}-py3.10"
-      - name: Cleanup Python 3.10
+          tox -e py -- --cov-report xml:coverage.xml --cov
+      - name: Upload Code Coverage for Python ${{ matrix.python-version }} / sphinx 4.x
         run: |
-          mv .gitignore nolongerignored
-          git clean -n
-          git clean -f
-          git reset --hard
+          codecov --required -X gcov -f coverage.xml --name "[GHA] ${{ matrix.os }}-py${{ matrix.python-version }}-sphinx4.x"
diff --git a/.github/workflows/testing_projects.yaml b/.github/workflows/testing_projects.yaml
@@ -1,11 +1,20 @@
-name: Testing Projects
+name: Projects
+
 on:
   push:
     branches:
       - master
+    paths:
+      - 'exhale'
+      - 'testing'
+      - '.github/workflows/testing_projects.yaml'
   pull_request:
     branches:
       - master
+    paths:
+      - 'exhale'
+      - 'testing'
+      - '.github/workflows/testing_projects.yaml'
 
 jobs:
   build_linux:
@@ -20,7 +29,7 @@ jobs:
       - name: Use Python 3.8
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: '3.8'
 
       - name: Install Build Tools
         run: |
@@ -65,7 +74,7 @@ jobs:
       - name: Use Python 3.8
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: '3.8'
 
       - name: Install Build Tools
         run: |

diff --git a/codecov.yml b/codecov.yml
@@ -1,2 +1,28 @@
-# https://docs.codecov.io/v4.3.6/docs/pull-request-comments
+# .github/workflows/*.yaml conditionally enable / disable tests.
+# https://docs.codecov.com/docs/commit-status#excluding-tests-example
+coverage:
+  status:
+    project:
+      default: false
+      tests:
+        # At this time we don't care about coverage changes as far as status
+        # blockers go.  Code coverage is nice, but not a requirement, reviewers
+        # should use their best judgement...
+        target: auto
+        threshold: 99%
+        paths:
+          - "exhale/"
+          - "testing/*.py"
+          - "testing/tests/*.py"
+          - "!testing/projects/"
+      projects:
+        target: auto
+        threshold: 99%
+        paths:
+          - "!exhale/"
+          - "!testing/*.py"
+          - "!testing/tests/*.py"
+          - "testing/projects/"
+
+# https://docs.codecov.com/docs/pull-request-comments
 comment: false