diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index ccef36a1ac..8acc604a11 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -139,6 +139,7 @@ def setUp(self): self.orig_sys_stdout = sys.stdout self.orig_sys_stderr = sys.stderr + self.stdout_std_err_buffers = [] def convert_exception_to_str(self, err): """Convert an Exception instance to a string.""" @@ -182,6 +183,7 @@ def mock_stdout(self, enable): sys.stdout.flush() if enable: sys.stdout = StringIO() + self.stdout_std_err_buffers.append(sys.stdout) else: sys.stdout = self.orig_sys_stdout @@ -190,6 +192,7 @@ def mock_stderr(self, enable): sys.stderr.flush() if enable: sys.stderr = StringIO() + self.stdout_std_err_buffers.append(sys.stderr) else: sys.stderr = self.orig_sys_stderr @@ -235,4 +238,6 @@ def tearDown(self): """Cleanup after running a test.""" self.mock_stdout(False) self.mock_stderr(False) + for buf in self.stdout_std_err_buffers: + buf.close() super().tearDown() diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 0e9aa23cef..0778e937f8 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -573,8 +573,10 @@ def to_cmd_str(cmd): for line in iter(proc.stdout.readline, b''): _log.debug(f"Captured stdout: {line.decode(errors='ignore').rstrip()}") stdout += line + proc.stdout.close() if split_stderr: stderr += proc.stderr.read() or b'' + proc.stderr.close() else: (stdout, stderr) = proc.communicate(input=stdin) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index 6201699237..d7980b4e8e 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -658,7 +658,15 @@ def _load_toolchain_module(self, silent=False): def _load_dependencies_modules(self, silent=False): """Load modules for dependencies, and handle special cases like external modules.""" - dep_mods = [dep['short_mod_name'] for dep in self.dependencies] + # For SYSTEM software we allow using dependencies from any toolchain so we need to use the full + # module name to allow loading them in the hierarchical MNS + is_system_toolchain = self.is_system_toolchain() + + def get_module_name(dep): + key = 'full_mod_name' if is_system_toolchain and not dep[SYSTEM_TOOLCHAIN_NAME] else 'short_mod_name' + return dep[key] + + dep_mods = [get_module_name(dep) for dep in self.dependencies] if self.dry_run: dry_run_msg("\nLoading modules for dependencies...\n", silent=silent) @@ -667,7 +675,7 @@ def _load_dependencies_modules(self, silent=False): # load available modules for dependencies, simulate load for others for dep, dep_mod_exists in zip(self.dependencies, mods_exist): - mod_name = dep['short_mod_name'] + mod_name = get_module_name(dep) if dep_mod_exists: self.modules_tool.load([mod_name]) dry_run_msg("module load %s" % mod_name, silent=silent) @@ -683,7 +691,7 @@ def _load_dependencies_modules(self, silent=False): self.modules_tool.load(dep_mods) if self.dependencies: - build_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if dep['build_only']] + build_dep_mods = [get_module_name(dep) for dep in self.dependencies if dep['build_only']] if build_dep_mods: trace_msg("loading modules for build dependencies:") for dep_mod in build_dep_mods: @@ -691,7 +699,7 @@ def _load_dependencies_modules(self, silent=False): else: trace_msg("(no build dependencies specified)") - run_dep_mods = [dep['short_mod_name'] for dep in self.dependencies if not dep['build_only']] + run_dep_mods = [get_module_name(dep) for dep in self.dependencies if not dep['build_only']] if run_dep_mods: trace_msg("loading modules for (runtime) dependencies:") for dep_mod in run_dep_mods: @@ -703,7 +711,7 @@ def _load_dependencies_modules(self, silent=False): self.modules.extend(dep_mods) # define $EBROOT* and $EBVERSION* for external modules, if metadata is available - for dep in [d for d in self.dependencies if d['external_module']]: + for dep in (d for d in self.dependencies if d['external_module']): mod_name = dep['full_mod_name'] metadata = dep['external_module_metadata'] self.log.debug("Metadata for external module %s: %s", mod_name, metadata) @@ -752,7 +760,7 @@ def _verify_toolchain(self): self.log.debug("List of toolchain dependencies from toolchain module: %s", self.toolchain_dep_mods) # only retain names of toolchain elements, excluding toolchain name - toolchain_definition = {e for es in self.definition().values() for e in es if not e == self.name} + toolchain_definition = {e for es in self.definition().values() for e in es if e != self.name} # filter out optional toolchain elements if they're not used in the module for elem_name in toolchain_definition.copy(): @@ -862,7 +870,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True, if deps is None: deps = [] self.dependencies = self._check_dependencies(deps, check_modules=loadmod) - if not len(deps) == len(self.dependencies): + if len(deps) != len(self.dependencies): self.log.debug("dep %s (%s)" % (len(deps), deps)) self.log.debug("tc.dep %s (%s)" % (len(self.dependencies), self.dependencies)) raise EasyBuildError('Not all dependencies have a matching toolchain version') diff --git a/test/framework/modules.py b/test/framework/modules.py index b69bb69c3e..ca4a55bc21 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -54,7 +54,7 @@ # number of modules included for testing purposes -TEST_MODULES_COUNT = 118 +TEST_MODULES_COUNT = 119 class ModulesTest(EnhancedTestCase): diff --git a/test/framework/modules/ncurses/6.4-GCCcore-12.3.0 b/test/framework/modules/ncurses/6.4-GCCcore-12.3.0 new file mode 100644 index 0000000000..b1c469d008 --- /dev/null +++ b/test/framework/modules/ncurses/6.4-GCCcore-12.3.0 @@ -0,0 +1,36 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The Ncurses (new curses) library is a free software emulation of curses in + System V Release 4.0, and more. It uses Terminfo format, supports pads and + color and multiple highlights and forms characters and function-key mapping, + and has all the other SYSV-curses enhancements over BSD Curses. + + +More information +================ + - Homepage: https://www.gnu.org/software/ncurses/ + } +} + +module-whatis {Description: + The Ncurses (new curses) library is a free software emulation of curses in + System V Release 4.0, and more. It uses Terminfo format, supports pads and + color and multiple highlights and forms characters and function-key mapping, + and has all the other SYSV-curses enhancements over BSD Curses. +} +module-whatis {Homepage: https://www.gnu.org/software/ncurses/} +module-whatis {URL: https://www.gnu.org/software/ncurses/} + +set root /tmp/software/ncurses/6.4-GCCcore-12.3.0 + +conflict ncurses + +setenv EBROOTNCURSES "$root" +setenv EBVERSIONNCURSES "6.4" +setenv EBDEVELNCURSES "$root/easybuild/Compiler-GCCcore-12.3.0-ncurses-6.4-easybuild-devel" + +# Built with EasyBuild version 5.1.2.dev0-r5819168038d7bd74810f38f75acfced9fb41870e diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 973f76ac03..1f982af09d 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -63,6 +63,11 @@ from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_ECS_DIR = os.path.join(TEST_DIR, 'easyconfigs', 'test_ecs') +TOY_EC = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0.eb') +TOY_EC_TXT: str = read_file(TOY_EC) + class ToyBuildTest(EnhancedTestCase): """Toy build unit test.""" @@ -165,13 +170,13 @@ def _test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=Tru extra_args = [] test_readme = False if ec_file is None: - ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + ec_file = TOY_EC test_readme = True full_ver = '0.0%s' % versionsuffix args = [ ec_file, '--unittest-file=%s' % self.logfile, - '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), + '--robot=%s' % os.pathsep.join([self.test_buildpath, TEST_DIR]), ] if debug: args.append('--debug') @@ -259,8 +264,7 @@ def test_toy_broken(self): """Test deliberately broken toy build.""" tmpdir = tempfile.mkdtemp() broken_toy_ec = os.path.join(tmpdir, "toy-broken.eb") - toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - broken_toy_ec_txt = read_file(toy_ec_file) + broken_toy_ec_txt = TOY_EC_TXT broken_toy_ec_txt += "checksums = ['clearywrongSHA256checksumoflength64-0123456789012345678901234567']" write_file(broken_toy_ec, broken_toy_ec_txt) error_regex = "Checksum verification .* failed" @@ -288,13 +292,10 @@ def test_toy_broken_copy_log_build_dir(self): Test whether log files and the build directory are copied to a permanent location after a failed installation. """ - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec_txt = re.sub( r'toy-0\.0_fix-silly-typo-in-printf-statement\.patch', r'toy-0.0_add-bug.patch', - toy_ec_txt + TOY_EC_TXT ) test_ec = os.path.join(self.test_prefix, 'toy-0.0-buggy.eb') write_file(test_ec, test_ec_txt) @@ -361,9 +362,8 @@ def test_toy_broken_copy_log_build_dir(self): def test_toy_tweaked(self): """Test toy build with tweaked easyconfig, for testing extra easyconfig parameters.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') ec_file = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') - shutil.copy2(os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb'), ec_file) + shutil.copy2(TOY_EC, ec_file) modloadmsg = 'THANKS FOR LOADING ME\\nI AM %(name)s v%(version)s' modloadmsg_regex_tcl = r'THANKS.*\n\s*I AM toy v0.0\n\s*"' @@ -462,9 +462,8 @@ def test_toy_tweaked(self): def test_toy_buggy_easyblock(self): """Test build using a buggy/broken easyblock, make sure a traceback is reported.""" - ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') kwargs = { - 'ec_file': ec_file, + 'ec_file': TOY_EC, 'extra_args': ['--easyblock=EB_toy_buggy'], 'raise_error': True, 'verify': False, @@ -477,14 +476,14 @@ def test_toy_build_formatv2(self): """Perform a toy build (format v2).""" # set $MODULEPATH such that modules for specified dependencies are found modulepath = os.environ.get('MODULEPATH') - os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) + os.environ['MODULEPATH'] = os.path.join(TEST_DIR, 'modules') args = [ - os.path.join(os.path.dirname(__file__), 'easyconfigs', 'v2.0', 'toy.eb'), + os.path.join(TEST_DIR, 'easyconfigs', 'v2.0', 'toy.eb'), '--debug', '--unittest-file=%s' % self.logfile, '--force', - '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), + '--robot=%s' % os.pathsep.join([self.test_buildpath, TEST_DIR]), '--software-version=0.0', '--toolchain=system,system', '--experimental', @@ -508,8 +507,7 @@ def test_toy_build_with_blocks(self): # note get_paths_for expects easybuild/easyconfigs subdir ecs_path = os.path.join(tmpdir, "easybuild", "easyconfigs") os.makedirs(ecs_path) - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - shutil.copy2(os.path.join(test_ecs, 't', 'toy', 'toy-0.0-multiple.eb'), ecs_path) + shutil.copy2(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-multiple.eb'), ecs_path) sys.path.append(tmpdir) args = [ @@ -546,11 +544,11 @@ def test_toy_build_formatv2_sections(self): for version, specs in versions.items(): args = [ - os.path.join(os.path.dirname(__file__), 'easyconfigs', 'v2.0', 'toy-with-sections.eb'), + os.path.join(TEST_DIR, 'easyconfigs', 'v2.0', 'toy-with-sections.eb'), '--debug', '--unittest-file=%s' % self.logfile, '--force', - '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), + '--robot=%s' % os.pathsep.join([self.test_buildpath, TEST_DIR]), '--software-version=%s' % version, '--toolchain=system,system', '--experimental', @@ -566,9 +564,8 @@ def test_toy_download_sources(self): """Test toy build with sources that still need to be 'downloaded'.""" tmpdir = tempfile.mkdtemp() # copy toy easyconfig file, and append source_urls to it - topdir = os.path.dirname(os.path.abspath(__file__)) - shutil.copy2(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), tmpdir) - source_url = os.path.join(topdir, 'sandbox', 'sources', 'toy') + shutil.copy2(TOY_EC, tmpdir) + source_url = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy') ec_file = os.path.join(tmpdir, 'toy-0.0.eb') write_file(ec_file, '\nsource_urls = ["file://%s"]\n' % source_url, append=True) @@ -594,8 +591,7 @@ def test_toy_download_sources(self): def test_toy_permissions(self): """Test toy build with custom umask settings.""" - toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_ec_txt = read_file(toy_ec_file) + test_ec_txt = TOY_EC_TXT # remove exec perms on bin subdirectory for others, to check whether correct dir permissions are set test_ec_txt += "\npostinstallcmds += ['chmod o-x %(installdir)s/bin']" @@ -703,8 +699,7 @@ def test_toy_permissions_installdir(self): # set umask hard to verify default reliably orig_umask = os.umask(0o022) - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT # take away read permissions, to check whether they are correctly restored by EasyBuild after installation test_ec_txt += "\npostinstallcmds += ['chmod -R og-r %(installdir)s']" @@ -830,7 +825,6 @@ def test_toy_group_check(self): self.assertEqual(res.exit_code, 0, "Failed to select group to use in test") group_name = res.output.split(' ')[0].strip() - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') args = [ test_ec, @@ -841,9 +835,9 @@ def test_toy_group_check(self): for group in [group_name, (group_name, "Hey, you're not in the '%s' group!" % group_name)]: if isinstance(group, str): - write_file(test_ec, read_file(toy_ec) + "\ngroup = '%s'\n" % group) + write_file(test_ec, TOY_EC_TXT + "\ngroup = '%s'\n" % group) else: - write_file(test_ec, read_file(toy_ec) + "\ngroup = %s\n" % str(group)) + write_file(test_ec, TOY_EC_TXT + "\ngroup = %s\n" % str(group)) self.mock_stdout(True) outtxt = self.eb_main(args, logfile=dummylogfn, do_build=True, raise_error=True, raise_systemexit=True) @@ -895,7 +889,7 @@ def test_toy_group_check(self): else: self.fail("Unknown module syntax: %s" % get_module_syntax()) - write_file(test_ec, read_file(toy_ec) + "\ngroup = ('%s', 'custom message', 'extra item')\n" % group_name) + write_file(test_ec, TOY_EC_TXT + "\ngroup = ('%s', 'custom message', 'extra item')\n" % group_name) self.assertErrorRegex(SystemExit, '.*', self.eb_main, args, do_build=True, raise_error=True, raise_systemexit=True) @@ -903,8 +897,7 @@ def test_allow_system_deps(self): """Test allow_system_deps easyconfig parameter.""" tmpdir = tempfile.mkdtemp() # copy toy easyconfig file, and append source_urls to it - topdir = os.path.dirname(os.path.abspath(__file__)) - shutil.copy2(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), tmpdir) + shutil.copy2(TOY_EC, tmpdir) ec_file = os.path.join(tmpdir, 'toy-0.0.eb') write_file(ec_file, "\nallow_system_deps = [('Python', SYS_PYTHON_VERSION)]\n", append=True) with self.mocked_stdout_stderr(): @@ -913,17 +906,15 @@ def test_allow_system_deps(self): def test_toy_hierarchical(self): """Test toy build under example hierarchical module naming scheme.""" - - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') self.setup_hierarchical_modules() mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') args = [ - os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb'), + os.path.join(TOY_EC), '--debug', '--unittest-file=%s' % self.logfile, '--force', - '--robot=%s' % test_easyconfigs, + '--robot=%s' % TEST_ECS_DIR, '--module-naming-scheme=HierarchicalMNS', ] @@ -1077,7 +1068,7 @@ def test_toy_hierarchical(self): if get_module_syntax() == 'Lua': gompi_module_path += '.lua' - args[0] = os.path.join(test_easyconfigs, 'g', 'gompi', 'gompi-2018a.eb') + args[0] = os.path.join(TEST_ECS_DIR, 'g', 'gompi', 'gompi-2018a.eb') self.modtool.purge() with self.mocked_stdout_stderr(): self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) @@ -1093,7 +1084,6 @@ def test_toy_hierarchical_subdir_user_modules(self): mkdir(home) os.environ['HOME'] = home - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') self.setup_hierarchical_modules() mod_prefix = os.path.join(self.test_installpath, 'modules', 'all') @@ -1119,7 +1109,7 @@ def test_toy_hierarchical_subdir_user_modules(self): write_file(openmpi_mod, extra_modtxt, append=True) args = [ - os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), + os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), '--installpath=%s' % home, '--unittest-file=%s' % self.logfile, '--force', @@ -1201,9 +1191,8 @@ def test_toy_hierarchical_subdir_user_modules(self): def test_toy_advanced(self): """Test toy build with extensions and non-system toolchain.""" - test_dir = os.path.abspath(os.path.dirname(__file__)) - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') - test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + os.environ['MODULEPATH'] = os.path.join(TEST_DIR, 'modules') + test_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') with self.mocked_stdout_stderr(): self._test_toy_build(ec_file=test_ec, versionsuffix='-gompi-2018a-test', extra_args=['--debug']) @@ -1232,9 +1221,8 @@ def test_toy_advanced_filter_deps(self): """Test toy build with extensions, and filtered build dependency.""" # test case for bug https://github.com/easybuilders/easybuild-framework/pull/2515 - test_dir = os.path.abspath(os.path.dirname(__file__)) - os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') - toy_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + os.environ['MODULEPATH'] = os.path.join(TEST_DIR, 'modules') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') toy_ec_txt = read_file(toy_ec) # add FFTW as build dependency, just to filter it out again @@ -1253,10 +1241,8 @@ def test_toy_advanced_filter_deps(self): def test_toy_hidden_cmdline(self): """Test installing a hidden module using the '--hidden' command line option.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') with self.mocked_stdout_stderr(): - self._test_toy_build(ec_file=ec_file, extra_args=['--hidden'], verify=False) + self._test_toy_build(ec_file=TOY_EC, extra_args=['--hidden'], verify=False) # module file is hidden toy_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '.0.0') if get_module_syntax() == 'Lua': @@ -1269,9 +1255,7 @@ def test_toy_hidden_cmdline(self): def test_toy_hidden_easyconfig(self): """Test installing a hidden module using the 'hidden = True' easyconfig parameter.""" # copy toy easyconfig file, and add hiding option to it - topdir = os.path.dirname(os.path.abspath(__file__)) - ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - shutil.copy2(ec_file, self.test_prefix) + shutil.copy2(TOY_EC, self.test_prefix) ec_file = os.path.join(self.test_prefix, 'toy-0.0.eb') write_file(ec_file, "\nhidden = True\n", append=True) with self.mocked_stdout_stderr(): @@ -1290,10 +1274,8 @@ def test_module_filepath_tweaking(self): mns_path = "easybuild.tools.module_naming_scheme.test_module_naming_scheme" __import__(mns_path, globals(), locals(), ['']) - topdir = os.path.dirname(os.path.abspath(__file__)) - eb_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') args = [ - eb_file, + TOY_EC, '--force', '--debug', '--suffix-modules-path=foobarbaz', @@ -1348,15 +1330,11 @@ def test_toy_patches(self): def test_toy_extension_patches_postinstallcmds(self): """Test install toy that includes extensions with patches and postinstallcmds.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - # create file that we'll copy via 'patches' write_file(os.path.join(self.test_prefix, 'test.txt'), 'test123') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = f"{toy_ec_txt}\n" + textwrap.dedent(""" + test_ec_txt = f"{TOY_EC_TXT}\n" + textwrap.dedent(""" exts_defaultclass = "DummyExtension" exts_list = [ ("bar", "0.0", { @@ -1406,11 +1384,6 @@ def test_toy_extension_patches_postinstallcmds(self): def test_toy_extension_sources(self): """Test install toy that includes extensions with 'sources' spec (as single-item list).""" - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') bar_sources_specs = [ @@ -1422,7 +1395,7 @@ def test_toy_extension_sources(self): # test use of single-element list in 'sources' with just the filename test_ec_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', @@ -1436,7 +1409,7 @@ def test_toy_extension_sources(self): # copy bar-0.0.tar.gz to /bar-0.0-local.tar.gz, to be used below test_source_path = os.path.join(self.test_prefix, 'sources') - toy_ext_sources = os.path.join(topdir, 'sandbox', 'sources', 'toy', 'extensions') + toy_ext_sources = os.path.join(TEST_DIR, 'sandbox', 'sources', 'toy', 'extensions') bar_source = os.path.join(toy_ext_sources, 'bar-0.0.tar.gz') copy_file(bar_source, os.path.join(test_source_path, 'bar-0.0-local.tar.gz')) @@ -1450,7 +1423,7 @@ def test_toy_extension_sources(self): bar_sources_spec = bar_sources_spec.replace('bar-%(version)s.tar.gz', 'bar-0.0-local.tar.gz') test_ec_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', @@ -1466,7 +1439,7 @@ def test_toy_extension_sources(self): # check that checksums are picked up and verified test_ec_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', @@ -1490,7 +1463,7 @@ def test_toy_extension_sources(self): # test again with correct checksum for bar-0.0.tar.gz, but faulty checksum for patch file test_ec_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', @@ -1514,7 +1487,7 @@ def test_toy_extension_sources(self): # test again with correct checksums test_ec_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', @@ -1534,13 +1507,9 @@ def test_toy_extension_sources(self): def test_toy_extension_extract_cmd(self): """Test for custom extract_cmd specified for an extension.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', @@ -1563,10 +1532,6 @@ def test_toy_extension_extract_cmd(self): def test_toy_extension_sources_git_config(self): """Test install toy that includes extensions with 'sources' spec including 'git_config'.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - # Tar-ball which should be created via 'git_config', and one file ext_tgz = 'exts-git.tar.gz' ext_tarball = os.path.join(self.test_sourcepath, 't', 'toy', ext_tgz) @@ -1578,7 +1543,7 @@ def test_toy_extension_sources_git_config(self): test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, 'prebuildopts = "echo \\\"%s\\\" > %s && ",' % (ext_code, ext_cfile), 'exts_defaultclass = "DummyExtension"', 'exts_list = [', @@ -1741,8 +1706,7 @@ def test_toy_module_fulltxt(self): def test_external_dependencies(self): """Test specifying external (build) dependencies.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - ectxt = read_file(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb')) + ectxt = read_file(os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb')) toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb') # just specify some of the test modules we ship, doesn't matter where they come from @@ -1804,19 +1768,18 @@ def test_external_dependencies(self): def test_module_only(self): """Test use of --module-only.""" - ec_files_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(ec_files_path, 't', 'toy', 'toy-0.0-deps.eb') + ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') # only consider provided test modules - self.reset_modulepath([os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules')]) + self.reset_modulepath([os.path.join(TEST_DIR, 'modules')]) # sanity check fails without --force if software is not installed yet common_args = [ ec_file, '--debug', '--unittest-file=%s' % self.logfile, - '--robot=%s' % ec_files_path, + '--robot=%s' % TEST_ECS_DIR, '--module-syntax=Tcl', ] args = common_args + ['--module-only'] @@ -1931,15 +1894,12 @@ def test_module_only_extensions(self): Sanity check should catch problems with extensions, extensions can be skipped using --skip-exts. """ - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_mod += '.lua' test_ec = os.path.join(self.test_prefix, 'test.ec') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\n' + '\n'.join([ "sanity_check_commands = ['barbar', 'toy']", "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", @@ -2003,15 +1963,12 @@ def test_toy_exts_parallel(self): """ Test parallel installation of extensions (--parallel-extensions-install) """ - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_mod += '.lua' test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\n' + '\n'.join([ "exts_defaultclass = 'DummyExtension'", "exts_list = [", @@ -2065,7 +2022,7 @@ def test_toy_exts_parallel(self): # check behaviour when using Toy_Extension easyblock that doesn't implement required_deps method; # framework should fall back to installing extensions sequentially - toy_ext_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py') + toy_ext_eb = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py') copy_file(toy_ext_eb, self.test_prefix) toy_ext_eb = os.path.join(self.test_prefix, 'toy_extension.py') toy_ext_eb_txt = read_file(toy_ext_eb) @@ -2093,8 +2050,7 @@ def test_toy_exts_parallel(self): def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" - ec_files_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - ec_file = os.path.join(ec_files_path, 't', 'toy', 'toy-0.0-deps.eb') + ec_file = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-deps.eb') toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0-deps') toy_mod_dir, toy_mod_fn = os.path.split(toy_mod) @@ -2102,7 +2058,7 @@ def test_backup_modules(self): ec_file, '--debug', '--unittest-file=%s' % self.logfile, - '--robot=%s' % ec_files_path, + '--robot=%s' % TEST_ECS_DIR, '--force', '--disable-cleanup-tmpdir' ] @@ -2387,13 +2343,9 @@ def test_reproducibility(self): def test_reproducibility_ext_easyblocks(self): """Test toy build produces expected reproducibility files also when extensions are used""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec_file) - ec1 = os.path.join(self.test_prefix, 'toy1.eb') ec1_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, "exts_defaultclass = 'DummyExtension'", "exts_list = [('barbar', '1.2', {'start_dir': 'src'})]", "", @@ -2428,13 +2380,9 @@ def test_reproducibility_ext_easyblocks(self): def test_toy_toy(self): """Test building two easyconfigs in a single go, with one depending on the other.""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec_file) - ec1 = os.path.join(self.test_prefix, 'toy1.eb') ec1_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, "versionsuffix = '-one'", ]) write_file(ec1, ec1_txt) @@ -2443,7 +2391,7 @@ def test_toy_toy(self): # name ('toy2' instead of 'toy') ec2 = os.path.join(self.test_prefix, 'toy2.eb') ec2_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, "name = 'toy2'", "easyblock = 'EB_toy'", "sources = ['toy/toy-0.0.tar.gz']", @@ -2494,13 +2442,10 @@ def test_toy_sanity_check_commands(self): self.setup_hierarchical_modules() - test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_txt = read_file(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb')) - out_file = os.path.join(self.test_prefix, 'out.txt') toy_ec_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, "toolchain = {'name': 'foss', 'version': '2018a'}", # specially construct (sort of senseless) sanity check commands, # that will fail if the corresponding modules are not loaded @@ -2527,7 +2472,7 @@ def test_toy_sanity_check_commands(self): '--debug', '--unittest-file=%s' % self.logfile, '--force', - '--robot=%s' % test_easyconfigs, + '--robot=%s' % TEST_ECS_DIR, '--module-naming-scheme=HierarchicalMNS', ] with self.mocked_stdout_stderr(): @@ -2549,12 +2494,8 @@ def test_toy_sanity_check_commands(self): def test_sanity_check_paths_lib64(self): """Test whether fallback in sanity check for lib64/ equivalents of library files works.""" - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') - ec_file = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') - ectxt = read_file(ec_file) - # modify test easyconfig: move lib/libtoy.a to lib64/libtoy.a - ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", ectxt) + ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", TOY_EC_TXT) postinstallcmd = ' && '.join([ # remove lib64 symlink (if it's there) "rm -f %(installdir)s/lib64", @@ -2594,8 +2535,7 @@ def test_sanity_check_paths_lib64(self): self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True) # also check other way around (lib64 -> lib) - ectxt = read_file(ec_file) - ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', 'lib64/libtoy.a'],", ectxt) + ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', 'lib64/libtoy.a'],", TOY_EC_TXT) write_file(test_ec, ectxt) # sanity check fails if lib64 fallback in sanity check is disabled, since lib64/libtoy.a is not there @@ -2624,8 +2564,7 @@ def test_sanity_check_paths_lib64(self): self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink'], raise_error=True) # check whether fallback works for files that's more than 1 subdir deep - ectxt = read_file(ec_file) - ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', 'lib/test/libtoy.a'],", ectxt) + ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', 'lib/test/libtoy.a'],", TOY_EC_TXT) postinstallcmd = "mkdir -p %(installdir)s/lib64/test && " postinstallcmd += "mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/test/libtoy.a" ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt) @@ -2641,21 +2580,17 @@ def test_toy_build_enhanced_sanity_check(self): if 'easybuild.easyblocks.toy' in sys.modules: del sys.modules['easybuild.easyblocks.toy'] - test_dir = os.path.join(os.path.abspath(os.path.dirname(__file__))) - toy_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') # get rid of custom sanity check paths in test easyconfig regex = re.compile(r'^sanity_check_paths\s*=\s*{[^}]+}', re.M) - test_ec_txt = regex.sub('', toy_ec_txt) + test_ec_txt = regex.sub('', TOY_EC_TXT) write_file(test_ec, test_ec_txt) self.assertNotIn('sanity_check_', test_ec_txt) # create custom easyblock for toy that has a custom sanity_check_step - toy_easyblock = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + toy_easyblock = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') toy_easyblock_txt = read_file(toy_easyblock) @@ -2805,22 +2740,18 @@ def test_toy_build_enhanced_sanity_check_templated_multi_dep(self): if 'easybuild.easyblocks.toy' in sys.modules: del sys.modules['easybuild.easyblocks.toy'] - test_dir = os.path.join(os.path.abspath(os.path.dirname(__file__))) - toy_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') # get rid of custom sanity check paths in test easyconfig regex = re.compile(r'^sanity_check_paths\s*=\s*{[^}]+}', re.M) - test_ec_txt = regex.sub('', toy_ec_txt) + test_ec_txt = regex.sub('', TOY_EC_TXT) self.assertNotIn('sanity_check_', test_ec_txt) test_ec_txt += "\nmulti_deps = {'Python': ['3.7.2', '2.7.15']}" write_file(test_ec, test_ec_txt) # create custom easyblock for toy that has a custom sanity_check_step - toy_easyblock = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + toy_easyblock = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') toy_easyblock_txt = read_file(toy_easyblock) @@ -2893,9 +2824,8 @@ def sanity_check_step(self): def test_toy_dumped_easyconfig(self): """ Test dumping of file in eb_filerepo in both .eb format """ filename = 'toy-0.0' - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') paths = [ - os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', '%s.eb' % filename), + os.path.join(TEST_ECS_DIR, 't', 'toy', '%s.eb' % filename), ] for path in paths: @@ -2945,8 +2875,7 @@ def test_toy_filter_env_vars(self): def test_toy_iter(self): """Test toy build that involves iterating over buildopts.""" - topdir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-iter.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy', 'toy-0.0-iter.eb') expected_buildopts = ['', '-O2; mv %(name)s toy_O2_$EBVERSIONGCC', '-O1; mv %(name)s toy_O1_$EBVERSIONGCC'] @@ -2966,7 +2895,7 @@ def test_toy_rpath(self): # find_eb_script function used to find rpath_args.py requires that location where easybuild/scripts # resides is listed in sys.path via absolute path; # this is only needed to make this test pass when it's being called from that same location... - top_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + top_path = os.path.dirname(os.path.dirname(TEST_DIR)) sys.path.insert(0, top_path) def grab_gcc_rpath_wrapper_args(): @@ -3040,8 +2969,7 @@ def grab_gcc_rpath_wrapper_args(): # test use of rpath toolchain option with SYSTEM and gompi 2018b toolchains for toolchain in ['', '-gompi-2018a']: - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', f'toy-0.0{toolchain}.eb')) + toy_ec_txt = read_file(os.path.join(TEST_ECS_DIR, 't', 'toy', f'toy-0.0{toolchain}.eb')) toy_ec_txt += "\ntoolchainopts = {'rpath': False}\n" # overwrites existing toolchainopts toy_ec = os.path.join(self.test_prefix, 'toy.eb') write_file(toy_ec, toy_ec_txt) @@ -3049,8 +2977,7 @@ def grab_gcc_rpath_wrapper_args(): self._test_toy_build(ec_file=toy_ec, extra_args=['--rpath'], raise_error=True) # test check_readelf_rpath easyconfig parameter - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_txt = read_file(os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')) + toy_ec_txt = TOY_EC_TXT toy_ec_txt += "\ncheck_readelf_rpath = False\n" toy_ec = os.path.join(self.test_prefix, 'toy.eb') write_file(toy_ec, toy_ec_txt) @@ -3060,8 +2987,7 @@ def grab_gcc_rpath_wrapper_args(): def test_toy_filter_rpath_sanity_libs(self): """Test use of --filter-rpath-sanity-libs.""" - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy-app', 'toy-app-0.0.eb') + toy_ec = os.path.join(TEST_ECS_DIR, 't', 'toy-app', 'toy-app-0.0.eb') # This should just build succesfully rpath_args = ['--rpath', '--strict-rpath-sanity-check'] @@ -3145,16 +3071,12 @@ def test_toy_filter_rpath_sanity_libs(self): def test_toy_cuda_sanity_check(self): """Test the CUDA sanity check""" update_build_option('trace', True) - # Define the toy_ec file we want to use - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_bin = '%(installdir)s/bin/toy' py_site_pkgs = '%(installdir)s/lib/python3.9/site-packages' shlib_ext = get_shared_lib_ext() toy_ec_cuda = os.path.join(self.test_prefix, 'toy-0.0-cuda.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT toy_ec_txt += '\n' + '\n'.join([ "dependencies = [('CUDA', '5.5.22', '', SYSTEM)]", "postinstallcmds += [", @@ -3316,7 +3238,7 @@ def assert_cuda_report(missing_cc, additional_cc, missing_ptx, log, stdout=None, # 8.0 device code # This should succeed (since the default for --cuda-sanity-check-error-on-failed-checks is False) # as to not break backwards compatibility - write_file(cuobjdump_file, cuobjdump_txt_shebang), + write_file(cuobjdump_file, cuobjdump_txt_shebang) write_file(cuobjdump_file, cuobjdump_txt_sm80, append=True) adjust_permissions(cuobjdump_file, stat.S_IXUSR, add=True) # Make sure our mock cuobjdump is executable args = ['--cuda-compute-capabilities=8.0'] @@ -3434,7 +3356,7 @@ def assert_cuda_report(missing_cc, additional_cc, missing_ptx, log, stdout=None, # This is expected to succeed: the individual file which _would_ cause the sanity check to fail is # now on the ignore list toy_whitelist_ec = os.path.join(self.test_prefix, 'toy-0.0-cuda-whitelist.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT toy_ec_txt += '\n' + '\n'.join([ "dependencies = [('CUDA', '5.5.22', '', SYSTEM)]", "cuda_sanity_ignore_files = ['bin/toy']", @@ -3458,7 +3380,7 @@ def assert_cuda_report(missing_cc, additional_cc, missing_ptx, log, stdout=None, # ordering (i.e. 9.0a > 9.0). It should pass, since device code is present for both CCs and PTX # code is present for the highest CC, and there is no additiona device code present # This also tests a case with multiple compute capabilities. - write_file(cuobjdump_file, cuobjdump_txt_shebang), + write_file(cuobjdump_file, cuobjdump_txt_shebang) write_file(cuobjdump_file, cuobjdump_txt_sm90, append=True) write_file(cuobjdump_file, cuobjdump_txt_sm90a, append=True) write_file(cuobjdump_file, cuobjdump_txt_sm90a_ptx, append=True) @@ -3536,15 +3458,11 @@ def assert_cuda_report(missing_cc, additional_cc, missing_ptx, log, stdout=None, def test_toy_modaltsoftname(self): """Build two dependent toys as in test_toy_toy but using modaltsoftname""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec_file) - - self.assertFalse(re.search('^modaltsoftname', toy_ec_txt, re.M)) + self.assertFalse(re.search('^modaltsoftname', TOY_EC_TXT, re.M)) ec1 = os.path.join(self.test_prefix, 'toy-0.0-one.eb') ec1_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, "versionsuffix = '-one'", "modaltsoftname = 'yot'" ]) @@ -3552,7 +3470,7 @@ def test_toy_modaltsoftname(self): ec2 = os.path.join(self.test_prefix, 'toy-0.0-two.eb') ec2_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, "versionsuffix = '-two'", "dependencies = [('toy', '0.0', '-one')]", "modaltsoftname = 'toytwo'", @@ -3595,11 +3513,8 @@ def test_toy_modaltsoftname(self): def test_toy_build_trace(self): """Test use of --trace""" - topdir = os.path.dirname(os.path.abspath(__file__)) - toy_ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, read_file(toy_ec_file) + '\nsanity_check_commands = ["toy"]') + write_file(test_ec, TOY_EC_TXT + '\nsanity_check_commands = ["toy"]') self.mock_stderr(True) self.mock_stdout(True) @@ -3641,9 +3556,8 @@ def test_toy_build_trace(self): def test_toy_build_hooks(self): """Test use of --hooks.""" - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + '\n'.join([ + test_ec_txt = TOY_EC_TXT + '\n'.join([ "exts_list = [('bar', '0.0'), ('toy', '0.0')]", "exts_defaultclass = 'DummyExtension'", ]) @@ -3802,9 +3716,7 @@ def post_build_and_install_loop_hook(ecs): def test_toy_multi_deps(self): """Test installation of toy easyconfig that uses multi_deps.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -4024,13 +3936,10 @@ def check_toy_load(depends_on=False): def test_fix_shebang(self): """Test use of fix_python_shebang_for & co.""" - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_txt = read_file(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) - test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ - toy_ec_txt, + TOY_EC_TXT, "postinstallcmds = [" # copy of bin/toy to use in fix_python_shebang_for and fix_perl_shebang_for " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.python',", @@ -4198,9 +4107,6 @@ def check_shebangs(): def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" - toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') tc_regex = re.compile('^toolchain = .*', re.M) @@ -4211,7 +4117,7 @@ def test_toy_system_toolchain_alias(self): ] for tc in test_tcs: - test_ec_txt = tc_regex.sub(tc, toy_ec_txt) + test_ec_txt = tc_regex.sub(tc, TOY_EC_TXT) write_file(test_ec, test_ec_txt) with self.mocked_stdout_stderr(): @@ -4418,11 +4324,8 @@ def __exit__(self, type, value, traceback): signal.alarm(0) # add extra sleep command to ensure session takes long enough - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec_txt = read_file(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) - test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, toy_ec_txt + '\npostinstallcmds = ["sleep 10"]') + write_file(test_ec, TOY_EC_TXT + '\npostinstallcmds = ["sleep 10"]') extra_args = ['--locks-dir=%s' % locks_dir, '--wait-on-lock-limit=3', '--wait-on-lock-interval=3'] @@ -4460,16 +4363,12 @@ def test_toy_build_unicode_description(self): """Test installation of easyconfig file that has non-ASCII characters in description.""" # cfr. https://github.com/easybuilders/easybuild-framework/issues/3284 - test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') - toy_ec_txt = read_file(toy_ec) - # the tilde character included here is a Unicode tilde character, not a regular ASCII tilde (~) descr = "This description includes a unicode tilde character: ∼, for your entertainment." self.assertNotIn('~', descr) regex = re.compile(r'^description\s*=.*', re.M) - test_ec_txt = regex.sub(r'description = "%s"' % descr, toy_ec_txt) + test_ec_txt = regex.sub(r'description = "%s"' % descr, TOY_EC_TXT) test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -4517,13 +4416,8 @@ def test_toy_build_lib64_lib_symlink(self): def test_toy_build_lib_lib64_symlink(self): """Check whether lib64 symlink to lib subdirectory is created.""" - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - - test_ec_txt = read_file(toy_ec) - test_ec = os.path.join(self.test_prefix, 'test.eb') - write_file(test_ec, test_ec_txt) + write_file(test_ec, TOY_EC_TXT) # by default, lib -> lib64 symlink is created (--lib-lib64-symlink is enabled by default) with self.mocked_stdout_stderr(): @@ -4561,8 +4455,7 @@ def test_toy_build_lib_lib64_symlink(self): def test_toy_build_sanity_check_linked_libs(self): """Test sanity checks for banned/requires libraries.""" - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - libtoy_ec = os.path.join(test_ecs, 'l', 'libtoy', 'libtoy-0.0.eb') + libtoy_ec = os.path.join(TEST_ECS_DIR, 'l', 'libtoy', 'libtoy-0.0.eb') libtoy_modfile_path = os.path.join(self.test_installpath, 'modules', 'all', 'libtoy', '0.0') if get_module_syntax() == 'Lua': @@ -4650,9 +4543,7 @@ def test_toy_build_sanity_check_linked_libs(self): def test_toy_mod_files(self): """Check detection of .mod files""" - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -4698,10 +4589,7 @@ def test_toy_post_install_patches(self): """ Test use of post-install patches """ - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "\npostinstallpatches = ['toy-0.0_fix-README.patch']" test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -4720,10 +4608,7 @@ def test_toy_unavailable_os_dep(self): Existence of OS dependencies is checking during the parsing of the easyconfig. Test here that this problem is caught and a test report generated (#4102). """ - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "\nosdependencies = [('package-does-not-exist')]" test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -4751,10 +4636,7 @@ def test_toy_post_install_messages(self): """ Test use of post-install messages """ - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "\npostinstallmsgs = ['This is post install message 1', 'This is post install message 2']" test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -4780,10 +4662,7 @@ def test_toy_build_info_msg(self): """ Test use of build info message """ - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\nbuild_info_msg = "Are you sure you want to install this toy software?"' test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -4804,13 +4683,11 @@ def test_toy_failing_test_step(self): """ Test behaviour when test step fails, using toy easyconfig. """ - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') toy_mod_path = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') if get_module_syntax() == 'Lua': toy_mod_path += '.lua' - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\nruntest = "false"' test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -4827,7 +4704,7 @@ def test_toy_failing_test_step(self): remove_file(toy_mod_path) # ignoring test failure should also work if an EasyBuildError is raises from test step - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += '\nruntest = "RAISE_ERROR"' write_file(test_ec, test_ec_txt) @@ -4840,6 +4717,25 @@ def test_toy_failing_test_step(self): raise_error=True, verbose=True) self.assertExists(toy_mod_path) + def test_system_ec_with_gcccore_build_dep(self): + """ + Test building a SYSTEM level software that has a module from GCCcore as build dependency""" + test_ec_txt = TOY_EC_TXT + # Using a software with existing (test) module and no dependencies itself for simplicity + test_ec_txt += "\nbuilddependencies = [('ncurses', '6.4', '', ('GCCcore', '12.3.0'))]" + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + common_args = ["--rebuild", test_ec] + for mns in ['EasyBuildMNS', 'HierarchicalMNS']: + with self.subTest(mns=mns): + args = common_args + [f'--module-naming-scheme={mns}'] + modulepath = os.path.join(TEST_DIR, 'modules') + if mns == 'HierarchicalMNS': + modulepath = os.path.join(modulepath, 'HierarchicalMNS') + os.environ['MODULEPATH'] = modulepath + outtxt = self.run_eb_main_capture_output(args, do_build=True, verbose=True, raise_error=True) + self.assertRegex(outtxt, r'\[SUCCESS\] .*toy/0.0') + def test_eb_crash(self): """ Test behaviour when EasyBuild crashes, for example due to a buggy hook @@ -4851,12 +4747,10 @@ def pre_configure_hook(self, *args, **kwargs): """) write_file(hooks_file, hooks_file_txt) - topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - toy_eb = os.path.join(topdir, 'test', 'framework', 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') - toy_ec = os.path.join(topdir, 'test', 'framework', 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_eb = os.path.join(TEST_DIR, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') args = [ - toy_ec, + TOY_EC, f'--hooks={hooks_file}', '--force', f'--installpath={self.test_prefix}', @@ -4867,7 +4761,7 @@ def pre_configure_hook(self, *args, **kwargs): cleanup() try: main_with_hooks(args=args) - self.assertFalse("This should never be reached, main function should have crashed!") + self.fail("This should never be reached, main function should have crashed!") except NameError as err: self.assertEqual(str(err), "name 'no_such_thing' is not defined") @@ -4879,11 +4773,8 @@ def test_eb_error(self): """ Test whether main function as run by 'eb' command print error messages to stderr. """ - topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - toy_ec = os.path.join(topdir, 'test', 'framework', 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "\ndependencies = [('nosuchdep', '1.0')]" write_file(test_ec, test_ec_txt) @@ -4914,10 +4805,7 @@ def test_toy_python(self): write_file(fake_python_mod, '#%Module') self.modtool.use(fake_mods_path) - test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - - test_ec_txt = read_file(toy_ec) + test_ec_txt = TOY_EC_TXT test_ec_txt += "\npostinstallcmds.append('mkdir -p %(installdir)s/lib/python3.6/site-packages')" test_ec_txt += "\npostinstallcmds.append('touch %(installdir)s/lib/python3.6/site-packages/foo.py')" @@ -4966,13 +4854,10 @@ def test_toy_multiple_ecs_module(self): """ Verify whether module file is correct when multiple easyconfigs are being installed. """ - test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') - # modify 'toy' easyconfig so toy-headers subdirectory is created, # which is taken into account by EB_toy easyblock for $CPATH test_toy_ec = os.path.join(self.test_prefix, 'test-toy.eb') - toy_ec_txt = read_file(toy_ec) + toy_ec_txt = TOY_EC_TXT toy_ec_txt += "\npostinstallcmds += ['mkdir %(installdir)s/toy-headers']" toy_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/toy-headers/toy.h']" write_file(test_toy_ec, toy_ec_txt) @@ -4980,7 +4865,7 @@ def test_toy_multiple_ecs_module(self): # modify 'toy-app' easyconfig so toy-headers subdirectory is created, # which is consider by EB_toy easyblock for $CPATH, # but should *not* be actually used because software name is not 'toy' - toy_app_ec = os.path.join(test_ecs, 't', 'toy-app', 'toy-app-0.0.eb') + toy_app_ec = os.path.join(TEST_ECS_DIR, 't', 'toy-app', 'toy-app-0.0.eb') test_toy_app_ec = os.path.join(self.test_prefix, 'test-toy-app.eb') toy_ec_txt = read_file(toy_app_ec) toy_ec_txt += "\npostinstallcmds += ['mkdir %(installdir)s/toy-headers']"