Load dependencies of SYSTEM toolchain by full name

easybuild/base/testing.py

@@ -139,6 +139,7 @@ def setUp(self):
 
         self.orig_sys_stdout = sys.stdout
         self.orig_sys_stderr = sys.stderr
+        self.stdout_std_err_buffers = []
 
     def convert_exception_to_str(self, err):
         """Convert an Exception instance to a string."""
@@ -182,6 +183,7 @@ def mock_stdout(self, enable):
         sys.stdout.flush()
         if enable:
             sys.stdout = StringIO()
+            self.stdout_std_err_buffers.append(sys.stdout)
         else:
             sys.stdout = self.orig_sys_stdout
 
@@ -190,6 +192,7 @@ def mock_stderr(self, enable):
         sys.stderr.flush()
         if enable:
             sys.stderr = StringIO()
+            self.stdout_std_err_buffers.append(sys.stderr)
         else:
             sys.stderr = self.orig_sys_stderr
 
@@ -235,4 +238,6 @@ def tearDown(self):
         """Cleanup after running a test."""
         self.mock_stdout(False)
         self.mock_stderr(False)
+        for buf in self.stdout_std_err_buffers:
+            buf.close()
         super().tearDown()

easybuild/tools/run.py

@@ -573,8 +573,10 @@ def to_cmd_str(cmd):
         for line in iter(proc.stdout.readline, b''):
             _log.debug(f"Captured stdout: {line.decode(errors='ignore').rstrip()}")
             stdout += line
+        proc.stdout.close()
         if split_stderr:
             stderr += proc.stderr.read() or b''
+            proc.stderr.close()
     else:
         (stdout, stderr) = proc.communicate(input=stdin)
 

easybuild/tools/toolchain/toolchain.py

@@ -658,7 +658,15 @@ def _load_toolchain_module(self, silent=False):
 
     def _load_dependencies_modules(self, silent=False):
         """Load modules for dependencies, and handle special cases like external modules."""
-        dep_mods = [dep['short_mod_name'] for dep in self.dependencies]
+        # For SYSTEM software we allow using dependencies from any toolchain so we need to use the full
+        # module name to allow loading them in the hierarchical MNS
+        is_system_toolchain = self.is_system_toolchain()
+
+        def get_module_name(dep):
+            key = 'full_mod_name' if is_system_toolchain and not dep[SYSTEM_TOOLCHAIN_NAME] else 'short_mod_name'
+            return dep[key]
+
+        dep_mods = [get_module_name(dep) for dep in self.dependencies]
 
         if self.dry_run:
             dry_run_msg("\nLoading modules for dependencies...\n", silent=silent)
@@ -667,7 +675,7 @@ def _load_dependencies_modules(self, silent=False):
 
             # load available modules for dependencies, simulate load for others
             for dep, dep_mod_exists in zip(self.dependencies, mods_exist):
-                mod_name = dep['short_mod_name']
+                mod_name = get_module_name(dep)
                 if dep_mod_exists:
                     self.modules_tool.load([mod_name])
                     dry_run_msg("module load %s" % mod_name, silent=silent)
@@ -683,15 +691,15 @@ def _load_dependencies_modules(self, silent=False):
             self.modules_tool.load(dep_mods)
 
             if self.dependencies:
-                build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']]
+                build_dep_mods = [get_module_name(dep) for dep in self.dependencies if dep['build_only']]
                 if build_dep_mods:
                     trace_msg("loading modules for build dependencies:")
                     for dep_mod in build_dep_mods:
                         trace_msg(' * ' + dep_mod)
                 else:
                     trace_msg("(no build dependencies specified)")
 
-                run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']]
+                run_dep_mods = [get_module_name(dep) for dep in self.dependencies if not dep['build_only']]
                 if run_dep_mods:
                     trace_msg("loading modules for (runtime) dependencies:")
                     for dep_mod in run_dep_mods:
@@ -703,7 +711,7 @@ def _load_dependencies_modules(self, silent=False):
         self.modules.extend(dep_mods)
 
         # define $EBROOT* and $EBVERSION* for external modules, if metadata is available
-        for dep in [d for d in self.dependencies if d['external_module']]:
+        for dep in (d for d in self.dependencies if d['external_module']):
             mod_name = dep['full_mod_name']
             metadata = dep['external_module_metadata']
             self.log.debug("Metadata for external module %s: %s", mod_name, metadata)
@@ -752,7 +760,7 @@ def _verify_toolchain(self):
         self.log.debug("List of toolchain dependencies from toolchain module: %s", self.toolchain_dep_mods)
 
         # only retain names of toolchain elements, excluding toolchain name
-        toolchain_definition = {e for es in self.definition().values() for e in es if not e == self.name}
+        toolchain_definition = {e for es in self.definition().values() for e in es if e != self.name}
 
         # filter out optional toolchain elements if they're not used in the module
         for elem_name in toolchain_definition.copy():
@@ -862,7 +870,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
         if deps is None:
             deps = []
         self.dependencies = self._check_dependencies(deps, check_modules=loadmod)
-        if not len(deps) == len(self.dependencies):
+        if len(deps) != len(self.dependencies):
             self.log.debug("dep %s (%s)" % (len(deps), deps))
             self.log.debug("tc.dep %s (%s)" % (len(self.dependencies), self.dependencies))
             raise EasyBuildError('Not all dependencies have a matching toolchain version')

test/framework/modules.py

@@ -54,7 +54,7 @@
 
 
 # number of modules included for testing purposes
-TEST_MODULES_COUNT = 118
+TEST_MODULES_COUNT = 119
 
 
 class ModulesTest(EnhancedTestCase):

test/framework/modules/ncurses/6.4-GCCcore-12.3.0

@@ -0,0 +1,36 @@
+#%Module
+proc ModulesHelp { } {
+    puts stderr {
+
+Description
+===========
+The Ncurses (new curses) library is a free software emulation of curses in
+ System V Release 4.0, and more. It uses Terminfo format, supports pads and
+ color and multiple highlights and forms characters and function-key mapping,
+ and has all the other SYSV-curses enhancements over BSD Curses.
+
+
+More information
+================
+ - Homepage: https://www.gnu.org/software/ncurses/
+    }
+}
+
+module-whatis {Description:
+ The Ncurses (new curses) library is a free software emulation of curses in
+ System V Release 4.0, and more. It uses Terminfo format, supports pads and
+ color and multiple highlights and forms characters and function-key mapping,
+ and has all the other SYSV-curses enhancements over BSD Curses.
+}
+module-whatis {Homepage: https://www.gnu.org/software/ncurses/}
+module-whatis {URL: https://www.gnu.org/software/ncurses/}
+
+set root /tmp/software/ncurses/6.4-GCCcore-12.3.0
+
+conflict ncurses
+
+setenv	EBROOTNCURSES		"$root"
+setenv	EBVERSIONNCURSES		"6.4"
+setenv	EBDEVELNCURSES		"$root/easybuild/Compiler-GCCcore-12.3.0-ncurses-6.4-easybuild-devel"
+
+# Built with EasyBuild version 5.1.2.dev0-r5819168038d7bd74810f38f75acfced9fb41870e