From 4aac9636a3b6f26f9ab45404a7dff4b30322834f Mon Sep 17 00:00:00 2001 From: tldahlgren Date: Mon, 19 Jul 2021 17:54:06 -0700 Subject: [PATCH 1/7] Snapshot of very preliminary work --- outputs/testing/cleanup.out | 3 ++ outputs/testing/repo-add.out | 2 ++ tutorial_testing.rst | 60 ++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 outputs/testing/cleanup.out create mode 100644 outputs/testing/repo-add.out create mode 100644 tutorial_testing.rst diff --git a/outputs/testing/cleanup.out b/outputs/testing/cleanup.out new file mode 100644 index 0000000000..0f79de32ad --- /dev/null +++ b/outputs/testing/cleanup.out @@ -0,0 +1,3 @@ +$ spack repo remove spack-tests +==> Removed repository /home/spack/spack/var/spack/repos/spack-tests with namespace 'spack-tests'. +$ rm -rf $SPACK_ROOT/var/spack/repos/spack-tests diff --git a/outputs/testing/repo-add.out b/outputs/testing/repo-add.out new file mode 100644 index 0000000000..17fbffc1e9 --- /dev/null +++ b/outputs/testing/repo-add.out @@ -0,0 +1,2 @@ +$ spack repo add $SPACK_ROOT/var/spack/repos/spack-tests/ +==> Added repo with namespace 'tutorial'. diff --git a/tutorial_testing.rst b/tutorial_testing.rst new file mode 100644 index 0000000000..d24d68a856 --- /dev/null +++ b/tutorial_testing.rst @@ -0,0 +1,60 @@ +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. include:: common/setup.rst + +.. _testing-tutorial: + +========================= +Package Testing Tutorial +========================= + +This tutorial walks you through the steps for adding and running Spack +package tests during and after the software installation process. A package +that *appears* to install successfully may not actually be installed +correctly or continue to work indefinitely. There are a number of possible +reasons. For example, the installation process may not have fully installed +the software. The installed software may not work. Or the software may +work right after it is installed but, due to system changes, it stops +working days, weeks, or months later. So Spack provides features for +`checking installed software +`_ + +Recall from `Package Creation +`_ +that Spack packages are installation scripts, or recipes, for building +software. As such, they are well suited for also encapsulating recipes +for testing the installed software. + +Tests can be performed at two points in the life of an installed +package: build-time and stand-alone. **Build-time** tests run as +part of the package installation process. **Stand-alone** tests run +at any point after the software is installed. + +--------------- +Getting started +--------------- + +In order to avoid modifying your Spack instance with changes from this +tutorial, let's add a package repository called ``spack-tests``: + +.. literalinclude:: outputs/testing/repo-add.out + :language: console + +Doing this ensures changes we make here do not adversely affect other +parts of the tutorial. You can find out more about repositories at +`Package Repositories `_. + +----------- +Cleaning up +----------- + +Before leaving, let's ensure what we have done does not interfere with your +Spack instance or future sections of the tutorial. Undo the work by entering +the following commands: + +.. literalinclude:: outputs/testing/cleanup.out + :language: console + From ca2094f9799fed3e1e77c997df1ab4f744bade13 Mon Sep 17 00:00:00 2001 From: tldahlgren Date: Mon, 19 Jul 2021 17:59:18 -0700 Subject: [PATCH 2/7] Add the new module to the index --- index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/index.rst b/index.rst index 5aa62acc44..f4b55aed92 100644 --- a/index.rst +++ b/index.rst @@ -91,4 +91,5 @@ Full contents: tutorial_spack_scripting tutorial_modules tutorial_buildsystems + tutorial_testing tutorial_advanced_packaging From 2ebd574547e71ac5d56ebcd510f504b15288534a Mon Sep 17 00:00:00 2001 From: tldahlgren Date: Wed, 11 Aug 2021 18:46:13 -0700 Subject: [PATCH 3/7] Snapshot --- outputs/testing/1.package.py | 39 +++++++++++ outputs/testing/cleanup.out | 6 +- outputs/testing/repo-add.out | 8 ++- tutorial_testing.rst | 132 +++++++++++++++++++++++++++++------ 4 files changed, 161 insertions(+), 24 deletions(-) create mode 100644 outputs/testing/1.package.py diff --git a/outputs/testing/1.package.py b/outputs/testing/1.package.py new file mode 100644 index 0000000000..18018d226e --- /dev/null +++ b/outputs/testing/1.package.py @@ -0,0 +1,39 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack import * + + +class Mpileaks(AutotoolsPackage): + """Tool to detect and report MPI objects like MPI_Requests and + MPI_Datatypes.""" + + homepage = "https://github.com/LLNL/mpileaks" + url = "https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" + + version('1.0', sha256='2e34cc4505556d1c1f085758e26f2f8eea0972db9382f051b2dcfb1d7d9e1825') + + variant('stackstart', values=int, default=0, + description='Specify the number of stack frames to truncate') + + depends_on('mpi') + depends_on('adept-utils') + depends_on('callpath') + + def configure_args(self): + stackstart = int(self.spec.variants['stackstart'].value) + + args = [ + '--with-adept-utils={0}'.format(self.spec['adept-utils'].prefix), + '--with-callpath={0}'.format(self.spec['callpath'].prefix), + ] + + if stackstart: + args.extend([ + '--with-stack-start-c={0}'.format(stackstart), + '--with-stack-start-fortran={0}'.format(stackstart) + ]) + + return args diff --git a/outputs/testing/cleanup.out b/outputs/testing/cleanup.out index 0f79de32ad..4fcbdc11d9 100644 --- a/outputs/testing/cleanup.out +++ b/outputs/testing/cleanup.out @@ -1,3 +1,5 @@ +$ spack uninstall -ay mpileaks +==> Successfully uninstalled mpileaks@1.0%gcc@7.5.0 arch=linux-ubuntu18.04-x86_64/dx5qx6s $ spack repo remove spack-tests -==> Removed repository /home/spack/spack/var/spack/repos/spack-tests with namespace 'spack-tests'. -$ rm -rf $SPACK_ROOT/var/spack/repos/spack-tests +==> Removed repository /home/spack/spack/spack-tests +$ rm -rf $SPACK_ROOT/spack-tests diff --git a/outputs/testing/repo-add.out b/outputs/testing/repo-add.out index 17fbffc1e9..0c0336e1a3 100644 --- a/outputs/testing/repo-add.out +++ b/outputs/testing/repo-add.out @@ -1,2 +1,6 @@ -$ spack repo add $SPACK_ROOT/var/spack/repos/spack-tests/ -==> Added repo with namespace 'tutorial'. +$ spack repo create spack-tests +==> Created repo with namespace 'spack-tests'. +==> To register it with spack, run this command: + spack repo add /home/spack/spack/spack-tests +$ spack repo add $SPACK_ROOT/spack-tests +==> Added repo with namespace 'spack-tests'. diff --git a/tutorial_testing.rst b/tutorial_testing.rst index d24d68a856..82925df6a6 100644 --- a/tutorial_testing.rst +++ b/tutorial_testing.rst @@ -12,49 +12,141 @@ Package Testing Tutorial ========================= This tutorial walks you through the steps for adding and running Spack -package tests during and after the software installation process. A package -that *appears* to install successfully may not actually be installed -correctly or continue to work indefinitely. There are a number of possible -reasons. For example, the installation process may not have fully installed -the software. The installed software may not work. Or the software may -work right after it is installed but, due to system changes, it stops -working days, weeks, or months later. So Spack provides features for -`checking installed software -`_ +package tests during and after the software installation process. + +A package that *appears* to install successfully may not actually be +installed correctly or continue to work indefinitely. There are a number +of possible reasons. For example, the installation process may not have +fully installed the software. The installed software may not work. Or the +software may work right after it is installed but, due to system changes, +it stops working days, weeks, or months later. So Spack provides features +for `checking installed software +`_. Recall from `Package Creation `_ that Spack packages are installation scripts, or recipes, for building software. As such, they are well suited for also encapsulating recipes -for testing the installed software. +for testing. -Tests can be performed at two points in the life of an installed -package: build-time and stand-alone. **Build-time** tests run as -part of the package installation process. **Stand-alone** tests run -at any point after the software is installed. +Tests can be performed at two phases in the life of an installed +package: build-time and stand-alone. `**Build-time** tests +`_ +run as part of the package installation process. `**Stand-alone** tests +`_ +run at any point after the software is installed. We will cover +adding both to a simple package. --------------- Getting started --------------- +First confirm you have three environment variables set as follows: + +* ``SPACK_ROOT``: consisting of the path to your Spack installation; +* ``PATH``: including ``$SPACK_ROOT/bin`` (so calls to the ``spack`` + command work); and +* ``EDITOR``: containing the path of your preferred text editor (so + Spack can run it when we modify the package). + +The first two variables are automatically set by ``setup-env.sh`` so, +if they aren't, run the following command: + +.. code-block:: console + + $ . ~/spack/share/spack/setup-env.sh + +You'll also need to create the ``EDITOR`` environment variable if it is +not set. + In order to avoid modifying your Spack instance with changes from this -tutorial, let's add a package repository called ``spack-tests``: +tutorial, let's add a **package repository** called ``spack-tests`` +just for this tutorial by entering the following command: .. literalinclude:: outputs/testing/repo-add.out :language: console + emphasize-lines: 1 Doing this ensures changes we make here do not adversely affect other parts of the tutorial. You can find out more about repositories at -`Package Repositories `_. +`Package Repositories +`_. + +------------------------- +Creating the package file +------------------------- + +Let's start with a fairly complete ``TBD`` example from the +`Package Creation Tutorial +`_. +It is an `AutotoolsPackage +`_ +so there are standard features we will consider leveraging (later) +in testing. + +Spack will create a suitable template when you run ``spack create`` +with the software's repository URL by entering: + +.. literalinclude:: outputs/testing/TBD-create.out + :language: console + emphasize-lines: 1 + +You should now be in your text editor of choice, with the ``package.py`` +file open for editing. Let's replace **all** of the templated contents +with the following: + +.. literalinclude:: outputs/testing/1.package.py + :caption: TBD/package.py (from outputs/testing/1.package.py) + :language: python + +and save the results. + +----------------------- +Adding build-time tests +----------------------- + +Let's first install the package before proceeding to add build-time +tests. We'll do this by entering the ``spack install`` command: + +.. literalinclude:: outputs/testing/TBD-install-1.out + :language: console + emphasize-lines: 1 + + +------------------- +More information +------------------- + +This tutorial only scratches the surface of adding package tests. For +more information, take a look at the Spack resources below. + +^^^^^^^^^^^^^^^^^^^^^ +Package test features +^^^^^^^^^^^^^^^^^^^^^ + +* `Build-time tests + `_ +* `Stand-alone (or smoke) tests + `_ + +^^^^^^^^^^^^^^ +Spack commands +^^^^^^^^^^^^^^ + +* `spack test + `_ + +* `spack repo + `_ ----------- Cleaning up ----------- -Before leaving, let's ensure what we have done does not interfere with your -Spack instance or future sections of the tutorial. Undo the work by entering -the following commands: +Before leaving, let's ensure what we have done does not interfere with +your Spack instance or future sections of the tutorial. Undo the work +by entering the following commands: .. literalinclude:: outputs/testing/cleanup.out :language: console - + :emphasize-lines: 1,3,5 From 4d83dcad8f4d3590e9d7610b5279e09184ad4856 Mon Sep 17 00:00:00 2001 From: tldahlgren Date: Thu, 12 Aug 2021 17:57:33 -0700 Subject: [PATCH 4/7] Elaborate on reference URLs --- tutorial_testing.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tutorial_testing.rst b/tutorial_testing.rst index 82925df6a6..2bdfe909ba 100644 --- a/tutorial_testing.rst +++ b/tutorial_testing.rst @@ -125,19 +125,20 @@ Package test features ^^^^^^^^^^^^^^^^^^^^^ * `Build-time tests - `_ + `_: sanity and installation phase checks +* `Built-in installation phase tests + `_: standard build-time test checks (when available) * `Stand-alone (or smoke) tests - `_ + `_: re-using, customizing, and inheriting checks ^^^^^^^^^^^^^^ Spack commands ^^^^^^^^^^^^^^ * `spack test - `_ - + `_: stand-alone (or smoke test) availability and execution * `spack repo - `_ + `_: tutorial sandbox ----------- Cleaning up From f0565f9e1b212981329c758f5ca4c3bb9b45ed40 Mon Sep 17 00:00:00 2001 From: tldahlgren Date: Thu, 12 Aug 2021 18:02:02 -0700 Subject: [PATCH 5/7] Fix emphasize lines syntax --- tutorial_testing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tutorial_testing.rst b/tutorial_testing.rst index 2bdfe909ba..067b47e709 100644 --- a/tutorial_testing.rst +++ b/tutorial_testing.rst @@ -65,7 +65,7 @@ just for this tutorial by entering the following command: .. literalinclude:: outputs/testing/repo-add.out :language: console - emphasize-lines: 1 + :emphasize-lines: 1 Doing this ensures changes we make here do not adversely affect other parts of the tutorial. You can find out more about repositories at @@ -89,7 +89,7 @@ with the software's repository URL by entering: .. literalinclude:: outputs/testing/TBD-create.out :language: console - emphasize-lines: 1 + :emphasize-lines: 1 You should now be in your text editor of choice, with the ``package.py`` file open for editing. Let's replace **all** of the templated contents @@ -110,7 +110,7 @@ tests. We'll do this by entering the ``spack install`` command: .. literalinclude:: outputs/testing/TBD-install-1.out :language: console - emphasize-lines: 1 + :emphasize-lines: 1 ------------------- From 403f23b14b65a44090573ecdc348c6e7359a56e8 Mon Sep 17 00:00:00 2001 From: tldahlgren Date: Wed, 4 May 2022 13:49:59 -0700 Subject: [PATCH 6/7] Fix copyright years --- outputs/testing/1.package.py | 2 +- tutorial_testing.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/outputs/testing/1.package.py b/outputs/testing/1.package.py index 18018d226e..1aa45cfd8f 100644 --- a/outputs/testing/1.package.py +++ b/outputs/testing/1.package.py @@ -1,4 +1,4 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) diff --git a/tutorial_testing.rst b/tutorial_testing.rst index 067b47e709..3308d945c8 100644 --- a/tutorial_testing.rst +++ b/tutorial_testing.rst @@ -1,4 +1,4 @@ -.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +.. Copyright 2013-2022 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details. SPDX-License-Identifier: (Apache-2.0 OR MIT) From 279c789a5b37379fd74a989e1ea67b24f2104ef6 Mon Sep 17 00:00:00 2001 From: tldahlgren Date: Wed, 4 May 2022 16:26:58 -0700 Subject: [PATCH 7/7] Revised introduction --- tutorial_testing.rst | 51 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/tutorial_testing.rst b/tutorial_testing.rst index 3308d945c8..3050442d44 100644 --- a/tutorial_testing.rst +++ b/tutorial_testing.rst @@ -11,31 +11,32 @@ Package Testing Tutorial ========================= -This tutorial walks you through the steps for adding and running Spack -package tests during and after the software installation process. - -A package that *appears* to install successfully may not actually be -installed correctly or continue to work indefinitely. There are a number -of possible reasons. For example, the installation process may not have -fully installed the software. The installed software may not work. Or the -software may work right after it is installed but, due to system changes, -it stops working days, weeks, or months later. So Spack provides features -for `checking installed software -`_. - -Recall from `Package Creation -`_ -that Spack packages are installation scripts, or recipes, for building -software. As such, they are well suited for also encapsulating recipes -for testing. - -Tests can be performed at two phases in the life of an installed -package: build-time and stand-alone. `**Build-time** tests -`_ -run as part of the package installation process. `**Stand-alone** tests -`_ -run at any point after the software is installed. We will cover -adding both to a simple package. +Once you have a recipe in a Spack package that can successfully build +your software (see `Package Creation +`_), +it's time to consider how people who use your package can gain confidence +that the software works. The package already encapsulates the installation +process, so it can do the same for testing. + +Just because `spack install` completes without reporting errors does +not necessarily mean the software installed correctly or will continue +to work indefinitely. How can that be true? There are a number of possible +reasons, including: + +* the installation process may not have fully installed the software; +* the installed software may not work; or +* the software may work right after it is installed but, due to system + changes, stop working weeks or months later. + +Spack provides several features for `checking installed software +`_: + +* sanity checks of installed files and or directories; +* post-installation phase checks; and +* stand-alone (or smoke) tests. + +This tutorial walks you through the steps for adding these types of +tests to your package and running them. --------------- Getting started