Merge pull request #1213 from moreati/spring-clean-2025

Spring clean 2025

Author: moreati

ansible_mitogen/mixins.py

@@ -103,13 +103,10 @@ def __init__(self, task, connection, *args, **kwargs):
 
         # required for python interpreter discovery
         connection.templar = self._templar
-        self._finding_python_interpreter = False
-        self._rediscovered_python = False
-        # redeclaring interpreter discovery vars here in case running ansible < 2.8.0
-        self._discovered_interpreter_key = None
-        self._discovered_interpreter = False
-        self._discovery_deprecation_warnings = []
-        self._discovery_warnings = []
+
+        self._mitogen_discovering_interpreter = False
+        self._mitogen_interpreter_candidate = None
+        self._mitogen_rediscovered_interpreter = False
 
     def run(self, tmp=None, task_vars=None):
         """
@@ -402,7 +399,7 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None,
             # only cache discovered_interpreter if we're not running a rediscovery
             # rediscovery happens in places like docker connections that could have different
             # python interpreters than the main host
-            if not self._rediscovered_python:
+            if not self._mitogen_rediscovered_interpreter:
                 result['ansible_facts'][self._discovered_interpreter_key] = self._discovered_interpreter
 
         if self._discovery_warnings:
@@ -462,7 +459,7 @@ def _low_level_execute_command(self, cmd, sudoable=True, in_data=None,
         # calling exec_command until we run into the right python we'll use
         # chicken-and-egg issue, mitogen needs a python to run low_level_execute_command
         # which is required by Ansible's discover_interpreter function
-        if self._finding_python_interpreter:
+        if self._mitogen_discovering_interpreter:
             possible_pythons = [
                 '/usr/bin/python',
                 'python3',
@@ -479,32 +476,27 @@ def _low_level_execute_command(self, cmd, sudoable=True, in_data=None,
             # not used, just adding a filler value
             possible_pythons = ['python']
 
-        def _run_cmd():
-            return self._connection.exec_command(
-                cmd=cmd,
-                in_data=in_data,
-                sudoable=sudoable,
-                mitogen_chdir=chdir,
-            )
-
         for possible_python in possible_pythons:
             try:
-                self._possible_python_interpreter = possible_python
-                rc, stdout, stderr = _run_cmd()
+                self._mitogen_interpreter_candidate = possible_python
+                rc, stdout, stderr = self._connection.exec_command(
+                    cmd, in_data, sudoable, mitogen_chdir=chdir,
+                )
             # TODO: what exception is thrown?
             except:
                 # we've reached the last python attempted and failed
-                # TODO: could use enumerate(), need to check which version of python first had it though
-                if possible_python == 'python':
+                if possible_python == possible_pythons[-1]:
                     raise
                 else:
                     continue
 
         stdout_text = to_text(stdout, errors=encoding_errors)
+        stderr_text = to_text(stderr, errors=encoding_errors)
 
         return {
             'rc': rc,
             'stdout': stdout_text,
             'stdout_lines': stdout_text.splitlines(),
-            'stderr': stderr,
+            'stderr': stderr_text,
+            'stderr_lines': stderr_text.splitlines(),
         }

ansible_mitogen/transport_config.py

@@ -87,8 +87,8 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
     it could be different than what's ran on the host
     """
     # keep trying different interpreters until we don't error
-    if action._finding_python_interpreter:
-        return action._possible_python_interpreter
+    if action._mitogen_discovering_interpreter:
+        return action._mitogen_interpreter_candidate
 
     if s in ['auto', 'auto_legacy', 'auto_silent', 'auto_legacy_silent']:
         # python is the only supported interpreter_name as of Ansible 2.8.8
@@ -102,13 +102,13 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
             # if we're rediscovering python then chances are we're running something like a docker connection
             # this will handle scenarios like running a playbook that does stuff + then dynamically creates a docker container,
             # then runs the rest of the playbook inside that container, and then rerunning the playbook again
-            action._rediscovered_python = True
+            action._mitogen_rediscovered_interpreter = True
 
             # blow away the discovered_interpreter_config cache and rediscover
             del task_vars['ansible_facts'][discovered_interpreter_config]
 
         if discovered_interpreter_config not in task_vars['ansible_facts']:
-            action._finding_python_interpreter = True
+            action._mitogen_discovering_interpreter = True
             # fake pipelining so discover_interpreter can be happy
             action._connection.has_pipelining = True
             s = ansible.executor.interpreter_discovery.discover_interpreter(
@@ -128,7 +128,7 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
         action._discovered_interpreter_key = discovered_interpreter_config
         action._discovered_interpreter = s
 
-    action._finding_python_interpreter = False
+    action._mitogen_discovering_interpreter = False
     return s
 
 

docs/changelog.rst

@@ -21,6 +21,19 @@ To avail of fixes in an unreleased version, please download a ZIP file
 In progress (unreleased)
 ------------------------
 
+* :gh:issue:`1213` tests: Enable default Python warnings
+* :gh:issue:`1111` :mod:`mitogen`: Replace uses of deprecated
+  :py:func:`pkgutil.find_loader`
+* :gh:issue:`1213` :mod:`mitogen`: Fix unclosed file in first stage
+* :gh:issue:`1213` tests: Fix unclosed file in fd_check script
+* :gh:issue:`1213` :mod:`ansible_mitogen`: Don't redeclare Ansible interpreter
+  discovery attributes
+* :gh:issue:`1213` :mod:`ansible_mitogen`: Rename Mitogen interpreter discovery
+  attributes
+* :gh:issue:`1213` :mod:`ansible_mitogen`: Decouple possible_pythons order &
+  error handling
+* :gh:issue:`1213` :mod:`ansible_mitogen`: Return ``stderr_lines`` from
+  ``_low_level_execute_command()``
 
 
 v0.3.21 (2025-01-20)

mitogen/master.py

@@ -54,21 +54,33 @@
     import importlib.machinery
     import importlib.util
     from _imp import is_builtin as _is_builtin
+
+    def _find_loader(fullname):
+        try:
+            maybe_spec = importlib.util.find_spec(fullname)
+        except (ImportError, AttributeError, TypeError, ValueError):
+            exc = sys.exc_info()[1]
+            raise ImportError(*exc.args)
+        try:
+            return maybe_spec.loader
+        except AttributeError:
+            return None
 except ImportError:
     # Python < 3.4, PEP 302 Import Hooks
     import imp
     from imp import is_builtin as _is_builtin
 
+    try:
+        from pkgutil import find_loader as _find_loader
+    except ImportError:
+        # Python < 2.5
+        from mitogen.compat.pkgutil import find_loader as _find_loader
+
 try:
     import sysconfig
 except ImportError:
     sysconfig = None
 
-if not hasattr(pkgutil, 'find_loader'):
-    # find_loader() was new in >=2.5, but the modern pkgutil.py syntax has
-    # been kept intentionally 2.3 compatible so we can reuse it.
-    from mitogen.compat import pkgutil
-
 import mitogen
 import mitogen.core
 import mitogen.minify
@@ -175,7 +187,7 @@ def get_child_modules(path, fullname):
         return [to_text(name) for _, name, _ in pkgutil.iter_modules([mod_path])]
     else:
         # we loaded some weird package in memory, so we'll see if it has a custom loader we can use
-        loader = pkgutil.find_loader(fullname)
+        loader = _find_loader(fullname)
         return [to_text(name) for name, _ in loader.iter_modules(None)] if loader else []
 
 
@@ -528,7 +540,7 @@ def find(self, fullname):
             # then the containing package is imported.
             # Pre-'import spec' this returned None, in Python3.6 it raises
             # ImportError.
-            loader = pkgutil.find_loader(fullname)
+            loader = _find_loader(fullname)
         except ImportError:
             e = sys.exc_info()[1]
             LOG.debug('%r: find_loader(%r) failed: %s', self, fullname, e)

mitogen/parent.py

@@ -1429,7 +1429,9 @@ def _first_stage():
             os.environ['ARGV0']=sys.executable
             os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')
         os.write(1,'MITO000\n'.encode())
-        C=zlib.decompress(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN))
+        fp=os.fdopen(0,'rb')
+        C=zlib.decompress(fp.read(PREAMBLE_COMPRESSED_LEN))
+        fp.close()
         fp=os.fdopen(W,'wb',0)
         fp.write(C)
         fp.close()

tests/ansible/ansible.cfg

@@ -10,6 +10,7 @@ callbacks_enabled =
 callback_whitelist =
     profile_tasks,
     timer
+duplicate_dict_key = error
 inventory = hosts
 gathering = explicit
 strategy_plugins = ../../ansible_mitogen/plugins/strategy
@@ -46,9 +47,13 @@ timeout = 30
 host_key_checking = False
 
 [inventory]
+# Fatal error if any inventory source is unparsed by every available plugin.
 any_unparsed_is_failed = true
+# Fatal error if no inventory sources have a match for a host pattern.
 host_pattern_mismatch = error
 ignore_extensions = ~, .bak, .disabled
+# Fatal error if no inventory sources are successfully parsed.
+unparsed_is_failed = true
 
 [callback_profile_tasks]
 task_output_limit = 10

tests/ansible/integration/runner/_reset_conn.yml

@@ -45,7 +45,9 @@ def make_connection(self):
         conn = self.klass(play_context, new_stdin=False)
         # conn functions don't fetch ActionModuleMixin objs from _get_task_vars()
         # through the usual walk-the-stack approach so we'll not run interpreter discovery here
-        conn._action = mock.MagicMock(_possible_python_interpreter=testlib.base_executable())
+        conn._action = mock.MagicMock(
+            _mitogen_interpreter_candidate=testlib.base_executable(),
+        )
         conn.on_action_run(
             task_vars={},
             delegate_to_hostname=None,

tests/ansible/tests/connection_test.py

@@ -26,6 +26,7 @@ def controlling_tty():
         return None
 
 
+out_path = sys.argv[1]
 fd = int(sys.argv[2])
 st = os.fstat(fd)
 
@@ -35,12 +36,29 @@ def controlling_tty():
 else:
     buf = os.read(fd, 4).decode()
 
-open(sys.argv[1], 'w').write(repr({
+output = repr({
     'buf': buf,
     'flags': fcntl.fcntl(fd, fcntl.F_GETFL),
     'st_mode': st.st_mode,
     'st_dev': st.st_dev,
     'st_ino': st.st_ino,
     'ttyname': ttyname(fd),
     'controlling_tty': controlling_tty(),
-}))
+})
+
+try:
+    out_f = open(out_path, 'w')
+except Exception:
+    exc = sys.exc_info()[1]
+    sys.stderr.write("Failed to open %r: %r" % (out_path, exc))
+    sys.exit(1)
+
+try:
+    out_f.write(output)
+except Exception:
+    out_f.close()
+    exc = sys.exc_info()[1]
+    sys.stderr.write("Failed to write to %r: %r" % (out_path, exc))
+    sys.exit(2)
+
+out_f.close()

tests/data/fd_check.py

@@ -105,6 +105,8 @@ setenv =
     NOCOVERAGE_ERASE = 1
     NOCOVERAGE_REPORT = 1
     PIP_CONSTRAINT={toxinidir}/tests/constraints.txt
+    # Print warning on the first occurence at each module:linenno in Mitogen. Available Python 2.7, 3.2+.
+    PYTHONWARNINGS=default:::ansible_mitogen,default:::mitogen
     # Ansible 6 - 8 (ansible-core 2.13 - 2.15) require Python 2.7 or >= 3.5 on targets
     ansible6: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004
     ansible7: MITOGEN_TEST_DISTRO_SPECS=centos7 centos8 debian9 debian10 debian11 ubuntu1604 ubuntu1804 ubuntu2004
@@ -140,7 +142,6 @@ allowlist_externals =
     # Added: Tox 3.18: Tox 4.0+
     *_install.py
     *_tests.py
-    aws
     docker
     docker-credential-secretservice
     echo
@@ -150,7 +151,6 @@ whitelist_externals =
     # Deprecated: Tox 3.18+; Removed: Tox 4.0
     *_install.py
     *_tests.py
-    aws
     docker
     docker-credential-secretservice
     echo