Merge branch 'develop' into system-with-gcccore-dep

Author: Flamefire

.github/workflows/linting.yml

@@ -29,7 +29,7 @@ jobs:
     - name: install Python packages
       run: |
         pip install --upgrade pip
-        pip install --upgrade flake8
+        pip install --upgrade flake8 flake8-comprehensions
 
     - name: Run flake8 to verify PEP8-compliance of Python code
       run: |

easybuild/base/fancylogger.py

@@ -435,7 +435,7 @@ def thread_name():
     """
     returns the current threads name
     """
-    return threading.currentThread().getName()
+    return threading.current_thread().name
 
 
 def getLogger(name=None, fname=False, clsname=False, fancyrecord=None):
@@ -577,8 +577,8 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
         'mode': 'a',
         'maxBytes': max_bytes,
         'backupCount': backup_count,
+        'encoding': 'utf-8',
     }
-    handleropts['encoding'] = 'utf-8'
     # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed
     directory = os.path.dirname(filename)
     if not os.path.exists(directory):
@@ -783,7 +783,7 @@ def getAllExistingLoggers():
     """
     # not-so-well documented manager (in 2.6 and later)
     # return list of (name,logger) tuple
-    return [x for x in logging.Logger.manager.loggerDict.items()] + [(logging.root.name, logging.root)]
+    return list(logging.Logger.manager.loggerDict.items()) + [(logging.root.name, logging.root)]
 
 
 def getAllNonFancyloggers():

easybuild/base/testing.py

@@ -224,6 +224,16 @@ def mocked_stdout_stderr(self, mock_stdout=True, mock_stderr=True):
             if mock_stderr:
                 self.mock_stderr(False)
 
+    @contextmanager
+    def saved_env(self):
+        """Context manager to reset environment to state when it was entered"""
+        orig_env = os.environ.copy()
+        try:
+            yield
+        finally:
+            os.environ.clear()
+            os.environ.update(orig_env)
+
     def tearDown(self):
         """Cleanup after running a test."""
         self.mock_stdout(False)

easybuild/base/wrapper.py

@@ -39,7 +39,7 @@ def proxy(self, *args):  # pylint:disable=unused-argument
         # create proxies for wrapped object's double-underscore attributes
         type.__init__(cls, name, bases, dct)
         if cls.__wraps__:
-            ignore = set("__%s__" % n for n in cls.__ignore__.split())
+            ignore = {"__%s__" % n for n in cls.__ignore__.split()}
             for name in dir(cls.__wraps__):
                 if name.startswith("__"):
                     if name not in ignore and name not in dct:

easybuild/framework/easyconfig/easyconfig.py

@@ -261,7 +261,7 @@ def det_subtoolchain_version(current_tc, subtoolchain_names, optional_toolchains
 
     for subtoolchain_name in subtoolchain_names:
 
-        uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name])
+        uniq_subtc_versions = {subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name}
 
         # system toolchain: bottom of the hierarchy
         if is_system_toolchain(subtoolchain_name):
@@ -318,8 +318,8 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
     # obtain list of all possible subtoolchains
     _, all_tc_classes = search_toolchain('')
     subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes}
-    optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False))
-    composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1)
+    optional_toolchains = {tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)}
+    composite_toolchains = {tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1}
 
     # the parent toolchain is at the top of the hierarchy,
     # we need a copy so that adding capabilities (below) doesn't affect the original object
@@ -1219,14 +1219,16 @@ def dependencies(self, build_only=False, runtime_only=False):
 
         return retained_deps
 
-    def dependency_names(self, build_only=False):
+    def dependency_names(self, build_only=False, runtime_only=False):
         """
         Return a set of names of all (direct) dependencies after filtering.
         Iterable builddependencies are flattened when not iterating.
 
         :param build_only: only return build dependencies, discard others
+        :param runtime_only: only return runtime dependencies, discard others
         """
-        return {dep['name'] for dep in self.dependencies(build_only=build_only) if dep['name']}
+        return {dep['name'] for dep in self.dependencies(build_only=build_only, runtime_only=runtime_only)
+                if dep['name']}
 
     def builddependencies(self):
         """

easybuild/framework/easyconfig/templates.py

@@ -50,9 +50,12 @@
     'nameletter': 'First letter of software name',
     'toolchain_name': 'Toolchain name',
     'toolchain_version': 'Toolchain version',
+    'version_major_minor_patch': "Major.Minor.Patch version",
     'version_major_minor': "Major.Minor version",
     'version_major': 'Major version',
+    'version_minor_patch': 'Minor.Patch version',
     'version_minor': 'Minor version',
+    'version_patch': 'Patch version',
 }
 # derived from EasyConfig._config
 TEMPLATE_NAMES_CONFIG = [
@@ -204,9 +207,12 @@
     'r_short_ver': 'rshortver',
     'r_ver': 'rver',
     'toolchain_ver': 'toolchain_version',
+    'ver_maj_min_patch': 'version_major_minor_patch',
     'ver_maj_min': 'version_major_minor',
     'ver_maj': 'version_major',
+    'ver_min_patch': 'version_minor_patch',
     'ver_min': 'version_minor',
+    'ver_patch': 'version_patch',
     'version_prefix': 'versionprefix',
     'version_suffix': 'versionsuffix',
 }
@@ -343,11 +349,16 @@ def template_constant_dict(config, ignore=None, toolchain=None):
                     minor = version[1]
                     template_values['version_minor'] = minor
                     template_values['version_major_minor'] = '.'.join([major, minor])
+                    if len(version) > 2:
+                        patch = version[2]
+                        template_values['version_patch'] = patch
+                        template_values['version_minor_patch'] = '.'.join([minor, patch])
+                        template_values['version_major_minor_patch'] = '.'.join([major, minor, patch])
                 except IndexError:
                     # if there is no minor version, skip it
                     pass
                 # only go through this once
-                ignore.extend(['version_major', 'version_minor', 'version_major_minor'])
+                ignore.extend(name for name in TEMPLATE_NAMES_EASYCONFIG if name.startswith('version_'))
 
         elif name.endswith('letter'):
             # parse first letters

easybuild/framework/easyconfig/types.py

@@ -541,7 +541,7 @@ def _to_checksum(checksum, list_level=0, allow_dict=True):
                 # When we already are in a tuple no further recursion is allowed -> set list_level very high
                 return tuple(_to_checksum(x, list_level=99, allow_dict=allow_dict) for x in checksum)
             else:
-                return list(_to_checksum(x, list_level=list_level+1, allow_dict=allow_dict) for x in checksum)
+                return [_to_checksum(x, list_level=list_level+1, allow_dict=allow_dict) for x in checksum]
     elif isinstance(checksum, dict) and allow_dict:
         return {key: _to_checksum(value, allow_dict=False) for key, value in checksum.items()}
 

easybuild/tools/docs.py

@@ -1468,7 +1468,7 @@ def gen_easyblock_doc_section_md(eb_class, path_to_examples, common_params, doc_
 
         table_titles = ['easyconfig parameter', 'description']
         table_values = [
-            [opt for opt in common_params[classname]],
+            common_params[classname],
             [DEFAULT_CONFIG[opt][1] for opt in common_params[classname]],
         ]
 
@@ -1556,7 +1556,7 @@ def gen_easyblock_doc_section_rst(eb_class, path_to_examples, common_params, doc
 
         table_titles = ['easyconfig parameter', 'description']
         table_values = [
-            [opt for opt in common_params[classname]],
+            common_params[classname],
             [DEFAULT_CONFIG[opt][1] for opt in common_params[classname]],
         ]
 

easybuild/tools/filetools.py

@@ -786,7 +786,7 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header
             if urlpat in urlpat_headers.keys():
                 urlpat_headers[urlpat].append(argline)  # add headers to the list
             else:
-                urlpat_headers[urlpat] = list([argline])  # new list headers for this urlpat
+                urlpat_headers[urlpat] = [argline]  # new list headers for this urlpat
         else:
             _log.warning("Non-empty argument to http-header-fields-urlpat ignored (missing URL pattern)")
 
@@ -838,7 +838,7 @@ def download_file(filename, url, path, forced=False, trace=True, max_attempts=No
     # parse option HTTP header fields for URLs containing a pattern
     http_header_fields_urlpat = build_option('http_header_fields_urlpat')
     # compile a dict full of {urlpat: [header, list]}
-    urlpat_headers = dict()
+    urlpat_headers = {}
     if http_header_fields_urlpat is not None:
         # there may be multiple options given, parse them all, while updating urlpat_headers
         for arg in http_header_fields_urlpat:
@@ -2368,7 +2368,7 @@ def encode_string(name):
     """
 
     # do the character remapping, return same char by default
-    result = ''.join(map(lambda x: STRING_ENCODING_CHARMAP.get(x, x), name))
+    result = ''.join(STRING_ENCODING_CHARMAP.get(x, x) for x in name)
     return result
 
 

easybuild/tools/modules.py

@@ -1095,6 +1095,8 @@ def load(self, modules, mod_paths=None, purge=False, init_env=None, allow_reload
         :param init_env: original environment to restore after running 'module purge'
         :param allow_reload: allow reloading an already loaded module
         """
+        if not any((modules, mod_paths, purge)):
+            return  # Avoid costly module paths if nothing to do
         if mod_paths is None:
             mod_paths = []
 
@@ -1123,10 +1125,10 @@ def load(self, modules, mod_paths=None, purge=False, init_env=None, allow_reload
             if os.path.exists(full_mod_path):
                 self.prepend_module_path(full_mod_path, priority=priority)
 
-        loaded_modules = self.loaded_modules()
+        if not allow_reload:
+            modules = set(modules) - set(self.loaded_modules())
         for mod in modules:
-            if allow_reload or mod not in loaded_modules:
-                self.run_module('load', mod)
+            self.run_module('load', mod)
 
     def unload(self, modules, log_changes=True):
         """
@@ -1289,7 +1291,7 @@ def run_module(self, *args, **kwargs):
             # keep track of current values of select env vars, so we can correct the adjusted values below
             # Identical to `{key: os.environ.get(key, '').split(os.pathsep)[::-1] for key in LD_ENV_VAR_KEYS}`
             # but Python 2 treats that as a local function and refused the `exec` below
-            prev_ld_values = dict([(key, os.environ.get(key, '').split(os.pathsep)[::-1]) for key in LD_ENV_VAR_KEYS])
+            prev_ld_values = {key: os.environ.get(key, '').split(os.pathsep)[::-1] for key in LD_ENV_VAR_KEYS}
 
             # Change the environment
             try: