TEST PR: GitHub ci use newer python versions cleanup 2.7 3.6

.github/workflows/main.yml

@@ -28,10 +28,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: '3.10'
-          os: ubuntu-22.04
         - python-version: '3.11'
           os: ubuntu-22.04
+        - python-version: '3.12'
+          os: ubuntu-22.04
+        - python-version: '3.13'
+          os: ubuntu-22.04
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -42,38 +44,18 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Python 2.7 from Ubuntu 20.04 using apt-get install
-        if: ${{ matrix.os == 'ubuntu-20.04' }}
-        run: sudo apt-get update && sudo apt-get install -y python2-dev
-
       - name: Install missing cpio in containers of nektos/act
         if: ${{ github.actor == 'nektos/act'}}
         run: apt-get update && apt-get install -y cpio
 
       - name: Run of tox on ubuntu-latest
-        if: ${{ startsWith(matrix.python-version, '3.') && matrix.python-version != 3.6 }}
         run: |
           pip install 'virtualenv<20.22' 'tox==4.5.1' tox-gh-actions
           tox --workdir .github/workflows/.tox --recreate
 
-      # tox >= 4.0.0 is needed for using optional-dependencies from pyproject.toml, which is
-      # is not available for python <= 3.6, so use the python3.8 of Ubuntu-20.04 to install it:
-      - name: Run tox for 3.6 and 3.8 on ${{ matrix.os }}'s 3.8 to get 'extras' from pyproject.toml)
-        if: ${{ matrix.python-version == 2.7 || matrix.python-version == 3.6 }}
-        run: |
-          set -xv;curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-          python3.8 get-pip.py
-          # The alternative is installing python3-pip but we don't need full pip function for now:
-          # sudo apt-get update && sudo apt-get install -y python3-pip
-          # Let tox-gh-actions get the environment(s) to run tests with from tox.ini:
-          # Use tox==4.5.1: tox>=4 is needed for reading the extras from pyproject.toml
-          # Warning: tox>=4.5.2 depends on virutalenv>=20.23, which breaks Python 2.7:
-          python3.8 -m pip install 'virtualenv<20.22' 'tox==4.5.1' tox-gh-actions
-          tox --workdir .github/workflows/.tox --recreate
-
       - name: Select the coverage file for upload
         if: |
-          ( matrix.python-version == '3.6' || matrix.python-version == '3.11' ) &&
+          matrix.python-version == '3.11' &&
           ( !cancelled() && github.actor != 'nektos/act' )
         id: coverage
         run: mv $( ls -t .github/workflows/.tox/*/log/.coverage | head -1 ) .coverage

CONTRIBUTING.md

@@ -1,3 +1,14 @@
+# Development setup
+
+## Create a virtual environment with the test dependencies
+
+```bash
+python -m venv .venv
+. .venv/bin/activate
+pip install pip-tools==7.3.0
+pip-compile --extra=test,mypy,pyright,pytype,tox -o - pyproject.toml | pip install -r /dev/stdin
+```
+
 ## Development setup on Fedora 37
 
 On Fedora 37, the `tox` rpm installs all Python versions.
@@ -6,9 +17,26 @@ But this `tox` is older, so install `tox==4.5.1` using `pip` (see below)
 ```bash
 sudo dnf install tox;sudo rpm -e tox
 ```
+But preferably use `tox` from the virtual environment instead.
+
+## Development setup on Ubuntu 24.04
+
+Prefer the virtual environment. Alternatively, an option is to use `pipx`:
+
+```bash
+sudo apt install pipx
+pipx install tox; pipx install 'pytest<7';pipx install pylint
+pipx inject pytest pytest-{forked,localftpserver,pythonpath,subprocess,timeout} pyfakefs pytest_httpserver six mock
+pipx inject pylint pyfakefs six mock pytest{,_forked,-localftpserver}
+```
+
+Use the `deadsnakes` ppa to install Python versions like 3.8 and 3.11 (see below)
 
 ## Development setup on Ubuntu 22.04
 
+Usage of <https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa> to install
+other python versions.
+
 ```bash
 sudo apt update
 sudo apt install software-properties-common python{2,3}-dev
@@ -77,7 +105,7 @@ Using pip-tools, you can extract the requirements and extras from `pyptoject.tom
 
 ```bash
 PYTHON=python3.10
-EXTRAS=.,test,mypy,pytype,tox
+EXTRAS=.,test,mypy,pyright,pytype,tox
 PFLAGS="--no-warn-conflicts"
 $PYTHON -m pip install pip-tools==7.3.0
 $PYTHON -m piptools compile --extra=$EXTRAS -o - pyproject.toml |

pyproject.toml

@@ -111,11 +111,12 @@ show_error_context = true
 error_summary = true
 files = ["xcp", "tests/test_*.py"]
 mypy_path = "stubs"
-python_version = "3.10"
+python_version = "3.11"
 warn_redundant_casts = true
 warn_return_any = true
 warn_unreachable = true
 warn_unused_configs = true
+explicit_package_bases = true
 disallow_any_unimported = true
 disallow_any_explicit = false
 disallow_any_generics = true
@@ -206,7 +207,7 @@ stubPath = "stubs"
 inputs = ["xcp", "tests", "./*.py"]
 keepgoing = true
 platform = "linux"
-python_version = "3.10"
+python_version = "3.11"
 pythonpath = ".:stubs"
 disable = ["ignored-type-comment"]
 overriding_parameter_count_checks = true

pytest.ini

@@ -8,10 +8,9 @@
 # These are the most of the needed pytest plugins, unfortunately this list does
 # not support ;python_version<=3.0 or ;python_version>3.0. Therefore, it can
 # only list plugins available for all tested python versions (2.7, 3.6 ... 3.11):
-# pytest-localftpserver is also used, but its installation is not checked
-# to to its installation not being detected on Ubuntu 24.04:
 required_plugins =
     pytest_httpserver
+    pytest_localftpserver
     pytest-forked
     pytest-pythonpath
     pytest-subprocess

tests/test_ifrename_dynamic.py

@@ -20,8 +20,8 @@
         openLog(self.logbuf, logging.NOTSET)
 
     def tearDown(self):
-        closeLogs()
         self.logbuf.close()
+        closeLogs()
 
     def test_null(self):
         self.assertLoadDynamicRules("")

tests/test_ifrename_static.py

@@ -18,9 +18,8 @@ def setUp(self):
         openLog(self.logbuf, logging.NOTSET)
 
     def tearDown(self):
-
-        closeLogs()
         self.logbuf.close()
+        closeLogs()
 
     def test_null(self):
         sr = StaticRules()

tests/test_logger.py

@@ -10,8 +10,16 @@
 from xcp.logger import openLog
 
 
-def test_openLog_mock_open():
-    """Cover xcp.logger.openLog.open_with_codec_handling and check the arguments used for open()"""
+def test_openLog_mock_open(fs):
+    # type(FakeFilesystem) -> None
+    """
+    - Covers xcp.logger.openLog.open_with_codec_handling and
+    - checks the arguments it uses for open()
+
+    Because needs to call openLog() which creates a logfile, this test uses
+    the pytest pyfakefs fixture 'fs' which wraps creating it in a virtual fs.
+    With it, this tests does not create a log file on the filesystem of the host.
+    """
     fh = StringIO()
     with patch("xcp.compat.open", mock_open()) as open_mock:
         open_mock.return_value = fh

tox.ini

@@ -1,11 +1,17 @@
 [tox]
-# This is the order how tox runs the tests when used interactively during development.
-# Run the tests which uncover issues most often first! For example:
-# 1. python 2.7 and 3.10 coverage test for changed, but not covered lines and mypy check
-# 2. python 3.6 test and pylint warnings from changed lines
-# 3. pytype (needs Python 3.8 for best results)
-# 4. pyright checks, pytest test report as markdown for GitHub Actions summary
-envlist = py38-covcombine-check, py311-lint-test, py310-pytype, py311-pyright-mdreport
+# Set the envlist:
+# Defines the environments that tox runs by default (when -e <environment> is not used)
+#
+# Default order of execution when using `tox` without `-e <environment>`:
+#
+# 1. pytest using python 3.11 for code coverage with diff-cover and mypy check
+# 2. pytype (needs Python 3.10 or newer)
+# 3. pylint and pyright checks (any modern Python version is fine)
+#
+# .github/workflows/main.yml is set up to test with 3.11, 3.12 and 3.13 in parallel.
+# Therefore, use three environments: One with 3.11, one with 3.12 and one with 3.13:
+#
+envlist = py311-covcp-check-mdreport, py312-cov-pytype, py313-cov-lint-pyright
 isolated_build = true
 skip_missing_interpreters = true
 requires =
@@ -125,7 +131,7 @@ commands    =
     coverage html -d {envlogdir}/htmlcov
     coverage html -d {envlogdir}/htmlcov-tests --fail-under {env:TESTS_COV_MIN:96}    \
                       --include="tests/*"
-    diff-cover --compare-branch=origin/master --include-untracked                     \
+    diff-cover --compare-branch=origin/master                                         \
       {env:PY3_DIFFCOVER_OPTIONS} --fail-under {env:DIFF_COV_MIN:92}                  \
       --html-report  {envlogdir}/coverage-diff.html                                   \
                      {envlogdir}/coverage.xml
@@ -186,6 +192,8 @@ python =
     3.9: py39
     3.10: py310
     3.11: py311
+    3.12: py312
+    3.13: py313
 
 [check]
 extras = mypy
@@ -207,7 +215,7 @@ commands =
 deps = pytype
     pandas
 commands =
-    python3.10 -V # Needs python <= 3.10, and 3.10 is needed to parse new "|" syntax
+    # Needs python >= 3.10: Needed to parse the newer syntax for "Type2 | Type2"
     pytype --version
     # Runs pytype -j auto -k --config .github/workflows/pytype.cfg and parses the output:
     python3 pytype_runner.py # When switching versions, update .github/workflows/pytype.cfg too!