diff --git a/compass/clean.py b/compass/clean.py index 77f8ecdb87..5f6d946d84 100644 --- a/compass/clean.py +++ b/compass/clean.py @@ -7,7 +7,7 @@ from compass import provenance -def clean_cases(tests=None, numbers=None, work_dir=None): +def clean_cases(tests=None, numbers=None, work_dir=None, suite_name='custom'): """ Set up one or more test cases @@ -21,6 +21,10 @@ def clean_cases(tests=None, numbers=None, work_dir=None): work_dir : str, optional A directory that will serve as the base for creating case directories + + suite_name : str, optional + The name of the test suite if tests are being set up through a test + suite or ``'custom'`` if not """ if tests is None and numbers is None: @@ -28,6 +32,7 @@ def clean_cases(tests=None, numbers=None, work_dir=None): if work_dir is None: work_dir = os.getcwd() + work_dir = os.path.abspath(work_dir) mpas_cores = get_mpas_cores() all_test_cases = dict() @@ -65,6 +70,14 @@ def clean_cases(tests=None, numbers=None, work_dir=None): except OSError: pass + # delete the pickle file for the test suite (if any) + pickle_file = os.path.join(work_dir, '{}.pickle'.format(suite_name)) + + try: + os.remove(pickle_file) + except OSError: + pass + def main(): parser = argparse.ArgumentParser( diff --git a/compass/setup.py b/compass/setup.py index d562db2bac..537a7ec622 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -11,7 +11,8 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, - work_dir=None, baseline_dir=None, mpas_model_path=None): + work_dir=None, baseline_dir=None, mpas_model_path=None, + suite_name='custom'): """ Set up one or more test cases @@ -41,6 +42,10 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, The relative or absolute path to the root of a branch where the MPAS model has been built + suite_name : str, optional + The name of the test suite if tests are being set up through a test + suite or ``'custom'`` if not + Returns ------- test_cases : dict of compass.TestCase @@ -58,6 +63,7 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, if work_dir is None: work_dir = os.getcwd() + work_dir = os.path.abspath(work_dir) mpas_cores = get_mpas_cores() @@ -97,6 +103,26 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None, setup_case(path, test_case, config_file, machine, work_dir, baseline_dir, mpas_model_path) + test_suite = {'name': suite_name, + 'test_cases': test_cases, + 'work_dir': work_dir} + + # pickle the test or step dictionary for use at runtime + pickle_file = os.path.join(test_suite['work_dir'], + '{}.pickle'.format(suite_name)) + with open(pickle_file, 'wb') as handle: + pickle.dump(test_suite, handle, protocol=pickle.HIGHEST_PROTOCOL) + + if 'LOAD_COMPASS_ENV' in os.environ: + script_filename = os.environ['LOAD_COMPASS_ENV'] + # make a symlink to the script for loading the compass conda env. + symlink(script_filename, os.path.join(work_dir, 'load_compass_env.sh')) + + max_cores, max_of_min_cores = _get_required_cores(test_cases) + + print('target cores: {}'.format(max_cores)) + print('minimum cores: {}'.format(max_of_min_cores)) + return test_cases @@ -274,6 +300,10 @@ def main(): help="The path to the build of the MPAS model for the " "core.", metavar="PATH") + parser.add_argument("--suite_name", dest="suite_name", default="custom", + help="The name to use for the 'custom' test suite" + "containing all setup test cases.", + metavar="SUITE") args = parser.parse_args(sys.argv[2:]) if args.test is None: @@ -283,4 +313,18 @@ def main(): setup_cases(tests=tests, numbers=args.case_num, config_file=args.config_file, machine=args.machine, work_dir=args.work_dir, baseline_dir=args.baseline_dir, - mpas_model_path=args.mpas_model) + mpas_model_path=args.mpas_model, suite_name=args.suite_name) + + +def _get_required_cores(test_cases): + """ Get the maximum number of target cores and the max of min cores """ + + max_cores = 0 + max_of_min_cores = 0 + for test_case in test_cases.values(): + for step_name in test_case.steps_to_run: + step = test_case.steps[step_name] + max_cores = max(max_cores, step.cores) + max_of_min_cores = max(max_of_min_cores, step.min_cores) + + return max_cores, max_of_min_cores diff --git a/compass/suite.py b/compass/suite.py index 0e9c0ec441..a93b9d4c55 100644 --- a/compass/suite.py +++ b/compass/suite.py @@ -1,11 +1,8 @@ import argparse import sys -import os from importlib import resources -import pickle from compass.setup import setup_cases -from compass.io import symlink from compass.clean import clean_cases @@ -43,12 +40,6 @@ def setup_suite(mpas_core, suite_name, config_file=None, machine=None, The relative or absolute path to the root of a branch where the MPAS model has been built """ - if machine is None and 'COMPASS_MACHINE' in os.environ: - machine = os.environ['COMPASS_MACHINE'] - - if config_file is None and machine is None: - raise ValueError('At least one of config_file and machine is needed.') - text = resources.read_text('compass.{}.suites'.format(mpas_core), '{}.txt'.format(suite_name)) tests = list() @@ -58,33 +49,9 @@ def setup_suite(mpas_core, suite_name, config_file=None, machine=None, and not test.startswith('#')): tests.append(test) - if work_dir is None: - work_dir = os.getcwd() - work_dir = os.path.abspath(work_dir) - - test_cases = setup_cases(tests, config_file=config_file, machine=machine, - work_dir=work_dir, baseline_dir=baseline_dir, - mpas_model_path=mpas_model_path) - - test_suite = {'name': suite_name, - 'test_cases': test_cases, - 'work_dir': work_dir} - - # pickle the test or step dictionary for use at runtime - pickle_file = os.path.join(test_suite['work_dir'], - '{}.pickle'.format(suite_name)) - with open(pickle_file, 'wb') as handle: - pickle.dump(test_suite, handle, protocol=pickle.HIGHEST_PROTOCOL) - - if 'LOAD_COMPASS_ENV' in os.environ: - script_filename = os.environ['LOAD_COMPASS_ENV'] - # make a symlink to the script for loading the compass conda env. - symlink(script_filename, os.path.join(work_dir, 'load_compass_env.sh')) - - max_cores, max_of_min_cores = _get_required_cores(test_cases) - - print('target cores: {}'.format(max_cores)) - print('minimum cores: {}'.format(max_of_min_cores)) + setup_cases(tests, config_file=config_file, machine=machine, + work_dir=work_dir, baseline_dir=baseline_dir, + mpas_model_path=mpas_model_path, suite_name=suite_name) def clean_suite(mpas_core, suite_name, work_dir=None): @@ -111,19 +78,7 @@ def clean_suite(mpas_core, suite_name, work_dir=None): tests = [test.strip() for test in text.split('\n') if len(test.strip()) > 0 and not test.startswith('#')] - if work_dir is None: - work_dir = os.getcwd() - work_dir = os.path.abspath(work_dir) - - clean_cases(tests=tests, work_dir=work_dir) - - # delete the pickle file - pickle_file = os.path.join(work_dir, '{}.pickle'.format(suite_name)) - - try: - os.remove(pickle_file) - except OSError: - pass + clean_cases(tests=tests, work_dir=work_dir, suite_name=suite_name) def main(): @@ -173,16 +128,3 @@ def main(): config_file=args.config_file, machine=args.machine, work_dir=args.work_dir, baseline_dir=args.baseline_dir, mpas_model_path=args.mpas_model) - - -def _get_required_cores(test_cases): - """ Get the maximum number of target cores and the max of min cores """ - - max_cores = 0 - max_of_min_cores = 0 - for test_case in test_cases.values(): - for step in test_case.steps.values(): - max_cores = max(max_cores, step.cores) - max_of_min_cores = max(max_of_min_cores, step.min_cores) - - return max_cores, max_of_min_cores diff --git a/docs/developers_guide/command_line.rst b/docs/developers_guide/command_line.rst index c36434d4bc..f6620ab183 100644 --- a/docs/developers_guide/command_line.rst +++ b/docs/developers_guide/command_line.rst @@ -95,7 +95,7 @@ The command-line options are: .. code-block:: none compass setup [-h] [-t PATH] [-n NUM [NUM ...]] [-f FILE] [-m MACH] - [-w PATH] [-b PATH] [-p PATH] + [-w PATH] [-b PATH] [-p PATH] [--suite_name SUITE] The ``-h`` or ``--help`` options will display the help message describing the command-line options. @@ -142,8 +142,18 @@ previous run. Many test cases validate variables to make sure they are identical between runs, compare timers to see how much performance has changed, or both. See :ref:`dev_validation`. -See :ref:`dev_setup` for more about the underlying framework. +The test cases will be included in a "custom" test suite in the order they are +named or numbered. You can give this suite a name with ``--suite_name`` or +leave it with the default name ``custom``. You can run this test suite with +``compass run [suite_name]`` as with the predefined test suites (see +:ref:`dev_compass_suite`). + +Test cases within the custom suite are run in the order they are supplied to +``compass setup``, so keep this in mind when providing the list. Any test +cases that depend on the output of other test cases must run afther their +dependencies. +See :ref:`dev_setup` for more about the underlying framework. .. _dev_compass_clean: @@ -228,9 +238,10 @@ Whereas other ``compass`` commands are typically run in the local clone of the compass repo, ``compass run`` needs to be run in the appropriate work directory. If you are running a test suite, you may need to provide the name of the test suite if more than one suite has been set up in the same work -directory (with or without the ``.pickle`` suffix that exists on the suite's -file in the working directory). If you are in the work directory for a test -case or step, you do not need to provide any arguments. +directory. You can provide either just the suite name or +``.pickle`` (the latter is convenient for tab completion). If you +are in the work directory for a test case or step, you do not need to provide +any arguments. If you want to explicitly select which steps in a test case you want to run, you have two options. You can either edit the ``steps_to_run`` config options diff --git a/docs/users_guide/quick_start.rst b/docs/users_guide/quick_start.rst index de09a471ed..aad211b476 100644 --- a/docs/users_guide/quick_start.rst +++ b/docs/users_guide/quick_start.rst @@ -79,8 +79,8 @@ Each time you want to work with compass, you will need to run: .. _setup_overview: -Setting up a test case ----------------------- +Setting up test cases +--------------------- Before you set up a test case with ``compass``, you will need to build the MPAS component you wish to test with. Since the instructions for building @@ -213,6 +213,11 @@ in the repository. In order to run a bit-for-bit test with a previous test case, use ``-b $PREVIOUS_WORKDIR`` to specify a "baseline". +When you set up one or more test cases, they will also be included in a custom +test suite, which is called ``custom`` by default. (You can give it another +name with the ``--suite_name`` flag.) You can run all the test cases in +sequence with one command as described in :ref:`suite_overview` or run them +one at a time as follows. Running a test case -------------------