diff --git a/.gitmodules b/.gitmodules index 3d736105..6d879449 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "scripts/radiuss-spack-configs"] path = scripts/radiuss-spack-configs url = https://github.com/LLNL/radiuss-spack-configs +[submodule "scripts/uberenv"] + path = scripts/uberenv + url = https://github.com/LLNL/uberenv.git diff --git a/scripts/uberenv/project.json b/.uberenv_config.json similarity index 70% rename from scripts/uberenv/project.json rename to .uberenv_config.json index 1335d596..1a0136d9 100644 --- a/scripts/uberenv/project.json +++ b/.uberenv_config.json @@ -6,5 +6,6 @@ "spack_url": "https://github.com/davidbeckingsale/spack", "spack_branch": "feature/allow-untested-cuda-versions", "spack_commit": "f96e256bee1948aa030916aae0c1b2645230fb9f", -"spack_activate" : {} -} +"spack_activate" : {}, +"spack_configs_path": "scripts/radiuss-spack-configs", +"spack_packages_path": "scripts/spack_packages"} diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/spack_packages/chai/package.py similarity index 100% rename from scripts/uberenv/packages/chai/package.py rename to scripts/spack_packages/chai/package.py diff --git a/scripts/uberenv/packages/raja/package.py b/scripts/spack_packages/raja/package.py similarity index 100% rename from scripts/uberenv/packages/raja/package.py rename to scripts/spack_packages/raja/package.py diff --git a/scripts/uberenv/packages/umpire/camp_target_umpire_3.0.0.patch b/scripts/spack_packages/umpire/camp_target_umpire_3.0.0.patch similarity index 100% rename from scripts/uberenv/packages/umpire/camp_target_umpire_3.0.0.patch rename to scripts/spack_packages/umpire/camp_target_umpire_3.0.0.patch diff --git a/scripts/uberenv/packages/umpire/package.py b/scripts/spack_packages/umpire/package.py similarity index 100% rename from scripts/uberenv/packages/umpire/package.py rename to scripts/spack_packages/umpire/package.py diff --git a/scripts/uberenv b/scripts/uberenv new file mode 160000 index 00000000..105e384f --- /dev/null +++ b/scripts/uberenv @@ -0,0 +1 @@ +Subproject commit 105e384f585e2391c42b2def93124a6580319c1c diff --git a/scripts/uberenv/LICENSE b/scripts/uberenv/LICENSE deleted file mode 100644 index fcd00312..00000000 --- a/scripts/uberenv/LICENSE +++ /dev/null @@ -1,64 +0,0 @@ -Copyright (c) 2014-2018, Lawrence Livermore National Security, LLC. - -Produced at the Lawrence Livermore National Laboratory - -LLNL-CODE-666778 - -All rights reserved. - -This file is part of Conduit. - -For details, see: http://software.llnl.gov/conduit/. - -Please also read conduit/LICENSE - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the disclaimer below. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the disclaimer (as noted below) in the - documentation and/or other materials provided with the distribution. - -* Neither the name of the LLNS/LLNL nor the names of its contributors may - be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -Additional BSD Notice - - 1. This notice is required to be provided under our contract with the U.S. - Department of Energy (DOE). This work was produced at Lawrence - Livermore National Laboratory under Contract No. DE-AC52-07NA27344 with - the DOE. - - 2. Neither the United States Government nor Lawrence Livermore National - Security, LLC nor any of their employees, makes any warranty, express - or implied, or assumes any liability or responsibility for the - accuracy, completeness, or usefulness of any information, apparatus, - product, or process disclosed, or represents that its use would not - infringe privately-owned rights. - - 3. Also, reference herein to any specific commercial products, process, - or services by trade name, trademark, manufacturer or otherwise does - not necessarily constitute or imply its endorsement, recommendation, - or favoring by the United States Government or Lawrence Livermore - National Security, LLC. The views and opinions of authors expressed - herein do not necessarily state or reflect those of the United - States Government or Lawrence Livermore National Security, LLC, and - shall not be used for advertising or product endorsement purposes. - diff --git a/scripts/uberenv/Makefile b/scripts/uberenv/Makefile deleted file mode 100644 index 2760762d..00000000 --- a/scripts/uberenv/Makefile +++ /dev/null @@ -1,6 +0,0 @@ - -default: - sphinx-build -E -a -b html docs/sphinx/ _docs_html - -clean: - rm -rf _docs_html \ No newline at end of file diff --git a/scripts/uberenv/README.md b/scripts/uberenv/README.md deleted file mode 100644 index 82d68201..00000000 --- a/scripts/uberenv/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# uberenv -Automates using Spack (https://www.spack.io/) to build and deploy software. - -Uberenv is a short python script that helps automate using Spack to build -third-party dependencies for development and to deploy Spack packages. - -Uberenv was released as part of the Conduit (https://github.com/LLNL/conduit/). It is included in-source in several projects, this repo is used to hold the latest reference version. - -For more details, see Uberenv's documention: - -https://uberenv.readthedocs.io - -You can also find details about how it is used in Conduit's documentation: - -https://llnl-conduit.readthedocs.io/en/latest/building.html#building-conduit-and-third-party-dependencies - -Conduit's source repo also serves as an example for uberenv and spack configuration files, etc: - -https://github.com/LLNL/conduit/tree/master/scripts/uberenv diff --git a/scripts/uberenv/docs/sphinx/conf.py b/scripts/uberenv/docs/sphinx/conf.py deleted file mode 100644 index a8475c7b..00000000 --- a/scripts/uberenv/docs/sphinx/conf.py +++ /dev/null @@ -1,324 +0,0 @@ -# -*- coding: utf-8 -*- -# -############################################################################### -# Copyright (c) 2015-2019, Lawrence Livermore National Security, LLC. -# -# Produced at the Lawrence Livermore National Laboratory -# -# LLNL-CODE-666778 -# -# All rights reserved. -# -# This file is part of Conduit. -# -# For details, see: http://software.llnl.gov/conduit/. -# -# Please also read conduit/LICENSE -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the disclaimer below. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the disclaimer (as noted below) in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the LLNS/LLNL nor the names of its contributors may -# be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -############################################################################### -# -# Uberenv documentation build configuration file, created by -# sphinx-quickstart on Thu Oct 16 11:23:46 2014. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax' -] - -# try to add the breathe extension -try: - import breathe - extensions.append('breathe') -except: - pass - -# Add any paths that contain templates here, relative to this directory. -# templates_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Uberenv' -copyright = u'Copyright (c) 2015-2019, LLNS' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = 'current' -# The full version, including alpha/beta/rc tags. -release = 'current' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'sphinx_rtd_theme' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { 'logo_only' : True } - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'Uberenvdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'Uberenv.tex', u'Uberenv Documentation', - u'LLNS', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'uberenv', u'Uberenv Documentation', - [u'LLNS'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'Uberenv', u'Uberenv Documentation', - u'LLNS', 'Uberenv', 'Automates using spack to build and deploy software.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - - -# try to use the read the docs theme -try: - import sphinx_rtd_theme - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -except: - pass diff --git a/scripts/uberenv/docs/sphinx/index.rst b/scripts/uberenv/docs/sphinx/index.rst deleted file mode 100644 index 457ec596..00000000 --- a/scripts/uberenv/docs/sphinx/index.rst +++ /dev/null @@ -1,194 +0,0 @@ -.. ############################################################################ -.. # Copyright (c) 2014-2018, Lawrence Livermore National Security, LLC. -.. # -.. # Produced at the Lawrence Livermore National Laboratory -.. # -.. # LLNL-CODE-666778 -.. # -.. # All rights reserved. -.. # -.. # This file is part of Conduit. -.. # -.. # For details, see: http://software.llnl.gov/conduit/. -.. # -.. # Please also read conduit/LICENSE -.. # -.. # Redistribution and use in source and binary forms, with or without -.. # modification, are permitted provided that the following conditions are met: -.. # -.. # * Redistributions of source code must retain the above copyright notice, -.. # this list of conditions and the disclaimer below. -.. # -.. # * Redistributions in binary form must reproduce the above copyright notice, -.. # this list of conditions and the disclaimer (as noted below) in the -.. # documentation and/or other materials provided with the distribution. -.. # -.. # * Neither the name of the LLNS/LLNL nor the names of its contributors may -.. # be used to endorse or promote products derived from this software without -.. # specific prior written permission. -.. # -.. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -.. # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -.. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.. # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -.. # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -.. # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.. # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.. # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.. # POSSIBILITY OF SUCH DAMAGE. -.. # -.. ############################################################################ - -.. _building_with_uberenv: - -Uberenv -~~~~~~~~~~~~~~~ - -**Uberenv** automates using `Spack `_ to build and deploy software. - -Many projects leverage `Spack `_ to help build the software dependencies needed to develop and deploy their projects on HPC systems. Uberenv is a python script that helps automate using Spack to build -third-party dependencies for development and to deploy Spack packages. - -Uberenv was released as part of Conduit (https://github.com/LLNL/conduit/). It is included in-source in several projects. The -https://github.com/llnl/uberenv/ repo is used to hold the latest reference version of Uberenv. - - -uberenv.py -~~~~~~~~~~~~~~~~~~~~~ - -``uberenv.py`` is a single file python script that automates fetching Spack, building and installing third party dependencies, and can optionally install packages as well. To automate the full install process, ``uberenv.py`` uses a target Spack package along with extra settings such as Spack compiler and external third party package details for common HPC platforms. - -``uberenv.py`` is included directly in a project's source code repo in the folder: ``scripts/uberenv/`` -This folder is also used to store extra Spack and Uberenv configuration files unique to the target project. ``uberenv.py`` uses a ``project.json`` file to specify project details, including the target Spack package name and which Spack repo is used. Conduit's source repo serves as an example for Uberenv and Spack configuration files, etc: - -https://github.com/LLNL/conduit/tree/master/scripts/uberenv - - -``uberenv.py`` is developed by LLNL in support of the `Ascent `_, Axom, and `Conduit `_ projects. - - -Command Line Options -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Build configuration -------------------- - -``uberenv.py`` has a few options that allow you to control how dependencies are built: - - ======================= ============================================== ================================================ - Option Description Default - ======================= ============================================== ================================================ - ``--prefix`` Destination directory ``uberenv_libs`` - ``--spec`` Spack spec linux: **%gcc** - osx: **%clang** - ``--spack-config-dir`` Folder with Spack settings files linux: (empty) - osx: ``scripts/uberenv/spack_configs/darwin/`` - ``-k`` Ignore SSL Errors **False** - ``--install`` Fully install target, not just dependencies **False** - ``--run_tests`` Invoke tests during build and against install **False** - ``--project-json`` File for project specific settings ``project.json`` - ======================= ============================================== ================================================ - -The ``-k`` option exists for sites where SSL certificate interception undermines fetching -from github and https hosted source tarballs. When enabled, ``uberenv.py`` clones Spack using: - -.. code:: bash - - git -c http.sslVerify=false clone https://github.com/llnl/spack.git - -And passes ``-k`` to any Spack commands that may fetch via https. - - -Default invocation on Linux: - -.. code:: bash - - python scripts/uberenv/uberenv.py --prefix uberenv_libs \ - --spec %gcc - -Default invocation on OSX: - -.. code:: bash - - python scripts/uberenv/uberenv.py --prefix uberenv_libs \ - --spec %clang \ - --spack-config-dir scripts/uberenv/spack_configs/darwin/ - - -Use the ``--install`` option to install the target package (not just its development dependencies): - -.. code:: bash - - python scripts/uberenv/uberenv.py --install - - -If the target Spack package supports Spack's testing hooks, you can run tests during the build process to validate the build and install, using the ``--run_tests`` option: - -.. code:: bash - - python scripts/uberenv/uberenv.py --install \ - --run_tests - -For details on Spack's spec syntax, see the `Spack Specs & dependencies `_ documentation. - - -Uberenv looks for configuration yaml files under ``scripts/uberenv/spack_config/{platform}`` or you can use the **--spack-config-dir** option to specify a directory with compiler and packages yaml files to use with Spack. See the `Spack Compiler Configuration `_ -and `Spack System Packages -`_ -documentation for details. - -.. note:: - The bootstrapping process ignores ``~/.spack/compilers.yaml`` to avoid conflicts - and surprises from a user's specific Spack settings on HPC platforms. - -When run, ``uberenv.py`` checkouts a specific version of Spack from github as ``spack`` in the -destination directory. It then uses Spack to build and install the target packages' dependencies into -``spack/opt/spack/``. Finally, the target package generates a host-config file ``{hostname}.cmake``, which is -copied to destination directory. This file specifies the compiler settings and paths to all of the dependencies. - - -Project configuration ---------------------- - -Part of the configuration can also be addressed using a json file. By default, it is named ``project.json`` and some settings can be overridden on command line: - - ==================== ========================== ================================================ ======================================= - Setting Option Description Default - ==================== ========================== ================================================ ======================================= - package_name ``--package-name`` Spack package name **None** - package_version **None** Spack package version **None** - package_final_phase ``--package-final-phase`` Controls after which phase Spack should stop **None** - package_source_dir ``--package-source-dir`` Controls the source directory Spack should use **None** - spack_url **None** Url where to download Spack ``https://github.com/spack/spack.git`` - spack_commit **None** Spack commit to checkout **None** - spack_activate **None** Spack packages to activate **None** - ==================== ========================== ================================================ ======================================= - - -Optimization ------------- - -``uberenv.py`` also features options to optimize the installation - - ==================== ============================================== ================================================ - Option Description Default - ==================== ============================================== ================================================ - ``--mirror`` Location of a Spack mirror **None** - ``--create-mirror`` Creates a Spack mirror at specified location **None** - ``--upstream`` Location of a Spack upstream **None** - ==================== ============================================== ================================================ - - -Project Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A few notes on using ``uberenv.py`` in a new project: - -* For an example of how to craft a ``project.json`` file a target project, see: `Conduit's project.json file `_ - -* ``uberenv.py`` hot copies ``packages`` to the cloned Spack install, this allows you to easily version control any Spack package overrides necessary - - diff --git a/scripts/uberenv/gen_spack_env_script.py b/scripts/uberenv/gen_spack_env_script.py deleted file mode 100644 index a1e6ba5d..00000000 --- a/scripts/uberenv/gen_spack_env_script.py +++ /dev/null @@ -1,128 +0,0 @@ -############################################################################### -# Copyright (c) 2015-2019, Lawrence Livermore National Security, LLC. -# -# Produced at the Lawrence Livermore National Laboratory -# -# LLNL-CODE-716457 -# -# All rights reserved. -# -# This file is part of Ascent. -# -# For details, see: http://ascent.readthedocs.io/. -# -# Please also read ascent/LICENSE -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the disclaimer below. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the disclaimer (as noted below) in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the LLNS/LLNL nor the names of its contributors may -# be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -############################################################################### -import os -import sys -import subprocess - -from os.path import join as pjoin - -# if you have bad luck with spack load, this -# script is for you! -# -# Looks for subdir: spack or uberenv_libs/spack -# queries spack for given package names and -# creates a bash script that adds those to your path -# -# -# usage: -# python gen_spack_env_script.py [spack_pkg_1 spack_pkg_2 ...] -# - -def sexe(cmd,ret_output=False,echo = True): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {}]".format(cmd)) - if ret_output: - p = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - res = p.communicate()[0] - res = res.decode('utf8') - return p.returncode,res - else: - return subprocess.call(cmd,shell=True) - - -def spack_exe(spath=None): - if spath is None: - to_try = [pjoin("uberenv_libs","spack"), "spack"] - for p in to_try: - abs_p = os.path.abspath(p) - print("[looking for spack directory at: {}]".format(abs_p)) - if os.path.isdir(abs_p): - print("[FOUND spack directory at: {}]".format(abs_p)) - return os.path.abspath(pjoin(abs_p,"bin","spack")) - print("[ERROR: failed to find spack directory!]") - sys.exit(-1) - else: - spack_exe = os.path.abspath(spath,"bin","spack") - if not os.path.isfile(spack_exec): - print("[ERROR: failed to find spack directory at spath={}]").format(spath) - sys.exit(-1) - return spack_exe - -def find_pkg(pkg_name): - r,rout = sexe(spack_exe() + " find -p " + pkg_name,ret_output = True) - print(rout) - for l in rout.split("\n"): - print(l) - lstrip = l.strip() - if not lstrip == "" and \ - not lstrip.startswith("==>") and \ - not lstrip.startswith("--"): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - -def path_cmd(pkg): - return('export PATH={}:$PATH\n'.format((pjoin(pkg["path"],"bin")))) - -def write_env_script(pkgs): - ofile = open("s_env.sh","w") - for p in pkgs: - print("[found {} at {}]".format(p["name"],p["path"])) - ofile.write("# {}\n".format(p["name"])) - ofile.write(path_cmd(p)) - print("[created {}]".format(os.path.abspath("s_env.sh"))) - -def main(): - pkgs = [find_pkg(pkg) for pkg in sys.argv[1:]] - if len(pkgs) > 0: - write_env_script(pkgs) - else: - print("usage: python gen_spack_env_script.py spack_pkg_1 spack_pkg_2 ...") - -if __name__ == "__main__": - main() diff --git a/scripts/uberenv/spack_configs b/scripts/uberenv/spack_configs deleted file mode 120000 index 17d3bf7a..00000000 --- a/scripts/uberenv/spack_configs +++ /dev/null @@ -1 +0,0 @@ -../radiuss-spack-configs \ No newline at end of file diff --git a/scripts/uberenv/uberenv.py b/scripts/uberenv/uberenv.py deleted file mode 100755 index 7761be2e..00000000 --- a/scripts/uberenv/uberenv.py +++ /dev/null @@ -1,800 +0,0 @@ -#!/bin/sh -"exec" "python" "-u" "-B" "$0" "$@" -############################################################################### -# Copyright (c) 2014-2020, Lawrence Livermore National Security, LLC. -# -# Produced at the Lawrence Livermore National Laboratory -# -# LLNL-CODE-666778 -# -# All rights reserved. -# -# This file is part of Conduit. -# -# For details, see https://lc.llnl.gov/conduit/. -# -# Please also read conduit/LICENSE -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the disclaimer below. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the disclaimer (as noted below) in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the LLNS/LLNL nor the names of its contributors may -# be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -############################################################################### - -""" - file: uberenv.py - - description: automates using spack to install a project. - -""" - -import os -import sys -import subprocess -import shutil -import socket -import platform -import json -import datetime -import glob -import re - -from optparse import OptionParser - -from os import environ as env -from os.path import join as pjoin - - -def sexe(cmd,ret_output=False,echo=False): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {}]".format(cmd)) - if ret_output: - p = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - out = p.communicate()[0] - out = out.decode('utf8') - return p.returncode,out - else: - return subprocess.call(cmd,shell=True) - - -def parse_args(): - "Parses args from command line" - parser = OptionParser() - parser.add_option("--install", - action="store_true", - dest="install", - default=False, - help="Install `package_name`, not just its dependencies.") - - # where to install - parser.add_option("--prefix", - dest="prefix", - default="uberenv_libs", - help="destination directory") - - # what compiler to use - parser.add_option("--spec", - dest="spec", - default=None, - help="spack compiler spec") - - # optional location of spack mirror - parser.add_option("--mirror", - dest="mirror", - default=None, - help="spack mirror directory") - - # flag to create mirror - parser.add_option("--create-mirror", - action="store_true", - dest="create_mirror", - default=False, - help="Create spack mirror") - - # optional location of spack upstream - parser.add_option("--upstream", - dest="upstream", - default=None, - help="add an external spack instance as upstream") - - # this option allows a user to explicitly to select a - # group of spack settings files (compilers.yaml , packages.yaml) - parser.add_option("--spack-config-dir", - dest="spack_config_dir", - default=None, - help="dir with spack settings files (compilers.yaml, packages.yaml, etc)") - - # overrides package_name - parser.add_option("--package-name", - dest="package_name", - default=None, - help="override the default package name") - - # controls after which package phase spack should stop - parser.add_option("--package-final-phase", - dest="package_final_phase", - default=None, - help="override the default phase after which spack should stop") - - # controls source_dir spack should use to build the package - parser.add_option("--package-source-dir", - dest="package_source_dir", - default=None, - help="override the default source dir spack should use") - - # a file that holds settings for a specific project - # using uberenv.py - parser.add_option("--project-json", - dest="project_json", - default=pjoin(uberenv_script_dir(),"project.json"), - help="uberenv project settings json file") - - # flag to use insecure curl + git - parser.add_option("-k", - action="store_true", - dest="ignore_ssl_errors", - default=False, - help="Ignore SSL Errors") - - # option to force a spack pull - parser.add_option("--pull", - action="store_true", - dest="spack_pull", - default=False, - help="Pull if spack repo already exists") - - # option to force for clean of packages specified to - # be cleaned in the project.json - parser.add_option("--clean", - action="store_true", - dest="spack_clean", - default=False, - help="Force uninstall of packages specified in project.json") - - # option to tell spack to run tests - parser.add_option("--run_tests", - action="store_true", - dest="run_tests", - default=False, - help="Invoke build tests during spack install") - - # option to init osx sdk env flags - parser.add_option("--macos-sdk-env-setup", - action="store_true", - dest="macos_sdk_env_setup", - default=False, - help="Set several env vars to select OSX SDK settings." - "This was necessary for older versions of macOS " - " but can cause issues with macOS versions >= 10.13. " - " so it is disabled by default.") - - - ############### - # parse args - ############### - opts, extras = parser.parse_args() - # we want a dict b/c the values could - # be passed without using optparse - opts = vars(opts) - if not opts["spack_config_dir"] is None: - opts["spack_config_dir"] = os.path.abspath(opts["spack_config_dir"]) - if not os.path.isdir(opts["spack_config_dir"]): - print("[ERROR: invalid spack config dir: {} ]".format(opts["spack_config_dir"])) - sys.exit(-1) - # if rel path is given for the mirror, we need to evaluate here -- before any - # chdirs to avoid confusion related to what it is relative to. - # (it should be relative to where uberenv is run from, so it matches what you expect - # from shell completion, etc) - if not opts["mirror"] is None: - if not opts["mirror"].startswith("http") and not os.path.isabs(opts["mirror"]): - opts["mirror"] = os.path.abspath(opts["mirror"]) - return opts, extras - - -def uberenv_script_dir(): - # returns the directory of the uberenv.py script - return os.path.dirname(os.path.abspath(__file__)) - -def load_json_file(json_file): - # reads json file - return json.load(open(json_file)) - -def is_darwin(): - return "darwin" in platform.system().lower() - -def is_windows(): - return "windows" in platform.system().lower() - -class UberEnv(): - """ Base class for package manager """ - - def __init__(self, opts, extra_opts): - self.opts = opts - self.extra_opts = extra_opts - - # load project settings - self.project_opts = load_json_file(opts["project_json"]) - print("[uberenv project settings: {}]".format(str(self.project_opts))) - print("[uberenv options: {}]".format(str(self.opts))) - - def setup_paths_and_dirs(self): - self.uberenv_path = os.path.dirname(os.path.realpath(__file__)) - - def set_from_args_or_json(self,setting): - try: - setting_value = self.project_opts[setting] - except (KeyError): - print("ERROR: {} must at least be defined in project.json".format(setting)) - raise - else: - if self.opts[setting]: - setting_value = self.opts[setting] - return setting_value - - def set_from_json(self,setting): - try: - setting_value = self.project_opts[setting] - except (KeyError): - print("ERROR: {} must at least be defined in project.json".format(setting)) - raise - return setting_value - - def detect_platform(self): - # find supported sets of compilers.yaml, packages,yaml - res = None - if is_darwin(): - res = "darwin" - elif "SYS_TYPE" in os.environ.keys(): - sys_type = os.environ["SYS_TYPE"].lower() - res = sys_type - return res - - -class SpackEnv(UberEnv): - """ Helper to clone spack and install libraries on MacOS an Linux """ - - def __init__(self, opts, extra_opts): - UberEnv.__init__(self,opts,extra_opts) - - self.pkg_name = self.set_from_args_or_json("package_name") - self.pkg_version = self.set_from_json("package_version") - self.pkg_final_phase = self.set_from_args_or_json("package_final_phase") - self.pkg_src_dir = self.set_from_args_or_json("package_source_dir") - - self.spec_hash = "" - self.use_install = False - - # Some additional setup for macos - if is_darwin(): - if opts["macos_sdk_env_setup"]: - # setup osx deployment target and sdk settings - setup_osx_sdk_env_vars() - else: - print("[skipping MACOSX env var setup]") - - # setup default spec - if opts["spec"] is None: - if is_darwin(): - opts["spec"] = "%clang" - else: - opts["spec"] = "%gcc" - self.opts["spec"] = "@{}{}".format(self.pkg_version,opts["spec"]) - elif not opts["spec"].startswith("@"): - self.opts["spec"] = "@{}{}".format(self.pkg_version,opts["spec"]) - else: - self.opts["spec"] = "{}".format(opts["spec"]) - - print("[spack spec: {}]".format(self.opts["spec"])) - - def setup_paths_and_dirs(self): - # get the current working path, and the glob used to identify the - # package files we want to hot-copy to spack - - UberEnv.setup_paths_and_dirs(self) - - self.pkgs = pjoin(self.uberenv_path, "packages","*") - - # setup destination paths - self.dest_dir = os.path.abspath(self.opts["prefix"]) - self.dest_spack = pjoin(self.dest_dir,"spack") - print("[installing to: {0}]".format(self.dest_dir)) - - # print a warning if the dest path already exists - if not os.path.isdir(self.dest_dir): - os.mkdir(self.dest_dir) - else: - print("[info: destination '{}' already exists]".format(self.dest_dir)) - - if os.path.isdir(self.dest_spack): - print("[info: destination '{}' already exists]".format(self.dest_spack)) - - self.pkg_src_dir = os.path.join(self.uberenv_path,self.pkg_src_dir) - if not os.path.isdir(self.pkg_src_dir): - print("[ERROR: package_source_dir '{}' does not exist]".format(self.pkg_src_dir)) - sys.exit(-1) - - - def find_spack_pkg_path_from_hash(self, pkg_name, pkg_hash): - res, out = sexe("spack/bin/spack find -p /{}".format(pkg_hash), ret_output = True) - for l in out.split("\n"): - if l.startswith(pkg_name): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - - def find_spack_pkg_path(self, pkg_name, spec = ""): - res, out = sexe("spack/bin/spack find -p " + pkg_name + spec,ret_output = True) - for l in out.split("\n"): - # TODO: at least print a warning when several choices exist. This will - # pick the first in the list. - if l.startswith(pkg_name): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - - # Extract the first line of the full spec - def read_spack_full_spec(self,pkg_name,spec): - res, out = sexe("spack/bin/spack spec " + pkg_name + " " + spec, ret_output=True) - for l in out.split("\n"): - if l.startswith(pkg_name) and l.count("@") > 0 and l.count("arch=") > 0: - return l.strip() - - def clone_repo(self): - if not os.path.isdir(self.dest_spack): - - # compose clone command for the dest path, spack url and branch - print("[info: cloning spack develop branch from github]") - - os.chdir(self.dest_dir) - - clone_opts = ("-c http.sslVerify=false " - if self.opts["ignore_ssl_errors"] else "") - - spack_url = self.project_opts.get("spack_url", "https://github.com/spack/spack.git") - spack_branch = self.project_opts.get("spack_branch", "develop") - - clone_cmd = "git {0} clone --single-branch --depth=1 -b {1} {2}".format(clone_opts, spack_branch,spack_url) - sexe(clone_cmd, echo=True) - - if "spack_commit" in self.project_opts: - # optionally, check out a specific commit - os.chdir(pjoin(self.dest_dir,"spack")) - sha1 = self.project_opts["spack_commit"] - res, current_sha1 = sexe("git log -1 --pretty=%H", ret_output=True) - if sha1 != current_sha1: - print("[info: using spack commit {}]".format(sha1)) - sexe("git stash", echo=True) - sexe("git fetch --depth=1 origin {0}".format(sha1),echo=True) - sexe("git checkout {0}".format(sha1),echo=True) - - if self.opts["spack_pull"]: - # do a pull to make sure we have the latest - os.chdir(pjoin(self.dest_dir,"spack")) - sexe("git stash", echo=True) - sexe("git pull", echo=True) - - def config_dir(self): - """ path to compilers.yaml, which we will use for spack's compiler setup""" - spack_config_dir = self.opts["spack_config_dir"] - if spack_config_dir is None: - uberenv_plat = self.detect_platform() - if not uberenv_plat is None: - spack_config_dir = os.path.abspath(pjoin(self.uberenv_path,"spack_configs",uberenv_plat)) - return spack_config_dir - - - def disable_spack_config_scopes(self,spack_dir): - # disables all config scopes except "defaults", which we will - # force our settings into - spack_lib_config = pjoin(spack_dir,"lib","spack","spack","config.py") - print("[disabling config scope (except defaults) in: {}]".format(spack_lib_config)) - cfg_script = open(spack_lib_config).read() - for cfg_scope_stmt in ["('system', os.path.join(spack.paths.system_etc_path, 'spack')),", - "('site', os.path.join(spack.paths.etc_path, 'spack')),", - "('user', spack.paths.user_config_path)"]: - cfg_script = cfg_script.replace(cfg_scope_stmt, - "#DISABLED BY UBERENV: " + cfg_scope_stmt) - open(spack_lib_config,"w").write(cfg_script) - - - def patch(self): - - cfg_dir = self.config_dir() - spack_dir = self.dest_spack - - # force spack to use only "defaults" config scope - self.disable_spack_config_scopes(spack_dir) - spack_etc_defaults_dir = pjoin(spack_dir,"etc","spack","defaults") - - # copy in "defaults" config.yaml - config_yaml = os.path.abspath(pjoin(self.uberenv_path,"spack_configs","config.yaml")) - sexe("cp {} {}/".format(config_yaml, spack_etc_defaults_dir ), echo=True) - - # copy in other settings per platform - if not cfg_dir is None: - print("[copying uberenv compiler and packages settings from {0}]".format(cfg_dir)) - - config_yaml = pjoin(cfg_dir,"config.yaml") - compilers_yaml = pjoin(cfg_dir,"compilers.yaml") - packages_yaml = pjoin(cfg_dir,"packages.yaml") - - if os.path.isfile(config_yaml): - sexe("cp {} {}/".format(config_yaml , spack_etc_defaults_dir ), echo=True) - - if os.path.isfile(compilers_yaml): - sexe("cp {} {}/".format(compilers_yaml, spack_etc_defaults_dir ), echo=True) - - if os.path.isfile(packages_yaml): - sexe("cp {} {}/".format(packages_yaml, spack_etc_defaults_dir ), echo=True) - else: - # let spack try to auto find compilers - sexe("spack/bin/spack compiler find", echo=True) - - # hot-copy our packages into spack - if self.pkgs: - dest_spack_pkgs = pjoin(spack_dir,"var","spack","repos","builtin","packages") - print("[copying patched packages from {0}]".format(self.pkgs)) - sexe("cp -Rf {} {}".format(self.pkgs,dest_spack_pkgs)) - - - def clean_build(self): - # clean out any temporary spack build stages - cln_cmd = "spack/bin/spack clean " - res = sexe(cln_cmd, echo=True) - - # clean out any spack cached stuff - cln_cmd = "spack/bin/spack clean --all" - res = sexe(cln_cmd, echo=True) - - # check if we need to force uninstall of selected packages - if self.opts["spack_clean"]: - if self.project_opts.has_key("spack_clean_packages"): - for cln_pkg in self.project_opts["spack_clean_packages"]: - if not self.find_spack_pkg_path(cln_pkg) is None: - unist_cmd = "spack/bin/spack uninstall -f -y --all --dependents " + cln_pkg - res = sexe(unist_cmd, echo=True) - - def show_info(self): - # prints install status and 32 characters hash - options="--install-status --very-long" - spec_cmd = "spack/bin/spack spec {0} {1}{2}".format(options,self.pkg_name,self.opts["spec"]) - - res, out = sexe(spec_cmd, ret_output=True, echo=True) - print(out) - - #Check if spec is already installed - for line in out.split("\n"): - # Example of matching line: ("status" "hash" "package"...) - # [+] hf3cubkgl74ryc3qwen73kl4yfh2ijgd serac@develop%clang@10.0.0-apple~debug~devtools~glvis arch=darwin-mojave-x86_64 - if re.match(r"^(\[\+\]| - ) [a-z0-9]{32} " + re.escape(self.pkg_name), line): - self.spec_hash = line.split(" ")[1] - # if spec already installed - if line.startswith("[+]"): - pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name,self.spec_hash) - install_path = pkg_path["path"] - # testing that the path exists is mandatory until Spack team fixes - # https://github.com/spack/spack/issues/16329 - if os.path.isdir(install_path): - print("[Warning: {} {} has already been installed in {}]".format(self.pkg_name, self.opts["spec"],install_path)) - print("[Warning: Uberenv will proceed using this directory]".format(self.pkg_name)) - self.use_install = True - - return res - - def install(self): - # use the uberenv package to trigger the right builds - # and build an host-config.cmake file - - if not self.use_install: - install_cmd = "spack/bin/spack " - if self.opts["ignore_ssl_errors"]: - install_cmd += "-k " - if not self.opts["install"]: - install_cmd += "dev-build --quiet -d {} -u {} ".format(self.pkg_src_dir,self.pkg_final_phase) - else: - install_cmd += "install " - if self.opts["run_tests"]: - install_cmd += "--test=root " - install_cmd += self.pkg_name + self.opts["spec"] - res = sexe(install_cmd, echo=True) - - if res != 0: - print("[ERROR: failure of spack install/dev-build]") - return res - - full_spec = self.read_spack_full_spec(self.pkg_name,self.opts["spec"]) - if "spack_activate" in self.project_opts: - print("[activating dependent packages]") - # get the full spack spec for our project - pkg_names = self.project_opts["spack_activate"].keys() - for pkg_name in pkg_names: - pkg_spec_requirements = self.project_opts["spack_activate"][pkg_name] - activate=True - for req in pkg_spec_requirements: - if req not in full_spec: - activate=False - break - if activate: - activate_cmd = "spack/bin/spack activate " + pkg_name - sexe(activate_cmd, echo=True) - # note: this assumes package extends python when +python - # this may fail general cases - if self.opts["install"] and "+python" in full_spec: - activate_cmd = "spack/bin/spack activate /" + self.spec_hash - sexe(activate_cmd, echo=True) - # if user opt'd for an install, we want to symlink the final - # install to an easy place: - if self.opts["install"] or self.use_install: - pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name, self.spec_hash) - if self.pkg_name != pkg_path["name"]: - print("[ERROR: Could not find install of {}]".format(self.pkg_name)) - return -1 - else: - # Symlink host-config file - hc_glob = glob.glob(pjoin(pkg_path["path"],"*.cmake")) - if len(hc_glob) > 0: - hc_path = hc_glob[0] - hc_fname = os.path.split(hc_path)[1] - if os.path.islink(hc_fname): - os.unlink(hc_fname) - elif os.path.isfile(hc_fname): - sexe("rm -f {}".format(hc_fname)) - print("[symlinking host config file to {}]".format(pjoin(self.dest_dir,hc_fname))) - os.symlink(hc_path,hc_fname) - - # Symlink install directory - if self.opts["install"]: - pkg_lnk_dir = "{}-install".format(self.pkg_name) - if os.path.islink(pkg_lnk_dir): - os.unlink(pkg_lnk_dir) - print("") - print("[symlinking install to {}]".format(pjoin(self.dest_dir,pkg_lnk_dir))) - os.symlink(pkg_path["path"],os.path.abspath(pkg_lnk_dir)) - print("") - print("[install complete!]") - # otherwise we are in the "only dependencies" case and the host-config - # file has to be copied from the do-be-deleted spack-build dir. - else: - pattern = "*{}.cmake".format(self.pkg_name) - build_dir = pjoin(self.pkg_src_dir,"spack-build") - hc_glob = glob.glob(pjoin(build_dir,pattern)) - if len(hc_glob) > 0: - hc_path = hc_glob[0] - hc_fname = os.path.split(hc_path)[1] - if os.path.islink(hc_fname): - os.unlink(hc_fname) - print("[copying host config file to {}]".format(pjoin(self.dest_dir,hc_fname))) - sexe("cp {} {}".format(hc_path,hc_fname)) - print("[removing project build directory {}]".format(pjoin(build_dir))) - sexe("rm -rf {}".format(build_dir)) - - def get_mirror_path(self): - mirror_path = self.opts["mirror"] - if not mirror_path: - print("[--create-mirror requires a mirror directory]") - sys.exit(-1) - return mirror_path - - def create_mirror(self): - """ - Creates a spack mirror for pkg_name at mirror_path. - """ - - mirror_path = self.get_mirror_path() - - mirror_cmd = "spack/bin/spack " - if self.opts["ignore_ssl_errors"]: - mirror_cmd += "-k " - mirror_cmd += "mirror create -d {} --dependencies {}{}".format(mirror_path, - self.pkg_name, - self.opts["spec"]) - return sexe(mirror_cmd, echo=True) - - def find_spack_mirror(self, mirror_name): - """ - Returns the path of a defaults scoped spack mirror with the - given name, or None if no mirror exists. - """ - res, out = sexe("spack/bin/spack mirror list", ret_output=True) - mirror_path = None - for mirror in out.split('\n'): - if mirror: - parts = mirror.split() - if parts[0] == mirror_name: - mirror_path = parts[1] - return mirror_path - - def use_mirror(self): - """ - Configures spack to use mirror at a given path. - """ - mirror_name = self.pkg_name - mirror_path = self.get_mirror_path() - existing_mirror_path = self.find_spack_mirror(mirror_name) - - if existing_mirror_path and mirror_path != existing_mirror_path: - # Existing mirror has different URL, error out - print("[removing existing spack mirror `{}` @ {}]".format(mirror_name, - existing_mirror_path)) - # - # Note: In this case, spack says it removes the mirror, but we still - # get errors when we try to add a new one, sounds like a bug - # - sexe("spack/bin/spack mirror remove --scope=defaults {} ".format(mirror_name), - echo=True) - existing_mirror_path = None - if not existing_mirror_path: - # Add if not already there - sexe("spack/bin/spack mirror add --scope=defaults {} {}".format( - mirror_name, mirror_path), echo=True) - print("[using mirror {}]".format(mirror_path)) - - def find_spack_upstream(self, upstream_name): - """ - Returns the path of a defaults scoped spack upstream with the - given name, or None if no upstream exists. - """ - upstream_path = None - - res, out = sexe('spack/bin/spack config get upstreams', ret_output=True) - if (not out) and ("upstreams:" in out): - out = out.replace(' ', '') - out = out.replace('install_tree:', '') - out = out.replace(':', '') - out = out.splitlines() - out = out[1:] - upstreams = dict(zip(out[::2], out[1::2])) - - for name in upstreams.keys(): - if name == upstream_name: - upstream_path = upstreams[name] - - return upstream_path - - def use_spack_upstream(self): - """ - Configures spack to use upstream at a given path. - """ - upstream_path = self.opts["upstream"] - if not upstream_path: - print("[--create-upstream requires a upstream directory]") - sys.exit(-1) - upstream_path = os.path.abspath(upstream_path) - upstream_name = self.pkg_name - existing_upstream_path = self.find_spack_upstream(upstream_name) - if (not existing_upstream_path) or (upstream_path != os.path.abspath(existing_upstream_path)): - # Existing upstream has different URL, error out - print("[removing existing spack upstream configuration file]") - sexe("rm spack/etc/spack/defaults/upstreams.yaml") - with open('spack/etc/spack/defaults/upstreams.yaml','w+') as upstreams_cfg_file: - upstreams_cfg_file.write("upstreams:\n") - upstreams_cfg_file.write(" {}:\n".format(upstream_name)) - upstreams_cfg_file.write(" install_tree: {}\n".format(upstream_path)) - - -def find_osx_sdks(): - """ - Finds installed osx sdks, returns dict mapping version to file system path - """ - res = {} - sdks = glob.glob("/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk") - for sdk in sdks: - sdk_base = os.path.split(sdk)[1] - ver = sdk_base[len("MacOSX"):sdk_base.rfind(".")] - res[ver] = sdk - return res - -def setup_osx_sdk_env_vars(): - """ - Finds installed osx sdks, returns dict mapping version to file system path - """ - # find current osx version (10.11.6) - dep_tgt = platform.mac_ver()[0] - # sdk file names use short version (ex: 10.11) - dep_tgt_short = dep_tgt[:dep_tgt.rfind(".")] - # find installed sdks, ideally we want the sdk that matches the current os - sdk_root = None - sdks = find_osx_sdks() - if dep_tgt_short in sdks.keys(): - # matches our osx, use this one - sdk_root = sdks[dep_tgt_short] - elif len(sdks) > 0: - # for now, choose first one: - dep_tgt = sdks.keys()[0] - sdk_root = sdks[dep_tgt] - else: - # no valid sdks, error out - print("[ERROR: Could not find OSX SDK @ /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/]") - sys.exit(-1) - - env["MACOSX_DEPLOYMENT_TARGET"] = dep_tgt - env["SDKROOT"] = sdk_root - print("[setting MACOSX_DEPLOYMENT_TARGET to {}]".format(env["MACOSX_DEPLOYMENT_TARGET"])) - print("[setting SDKROOT to {}]".format(env[ "SDKROOT"])) - - - -def main(): - """ - Clones and runs a package manager to setup third_party libs. - Also creates a host-config.cmake file that can be used by our project. - """ - - # parse args from command line - opts, extra_opts = parse_args() - - # Initialize the environment - env = SpackEnv(opts, extra_opts) - - # Setup the necessary paths and directories - env.setup_paths_and_dirs() - - # Clone the package manager - env.clone_repo() - - os.chdir(env.dest_dir) - - # Patch the package manager, as necessary - env.patch() - - # Clean the build - env.clean_build() - - # Show the spec for what will be built - env.show_info() - - - ########################################################## - # we now have an instance of spack configured how we - # need it to build our tpls at this point there are two - # possible next steps: - # - # *) create a mirror of the packages - # OR - # *) build - # - ########################################################## - if opts["create_mirror"]: - return env.create_mirror() - else: - if not opts["mirror"] is None: - env.use_mirror() - - if not opts["upstream"] is None: - env.use_spack_upstream() - - res = env.install() - - return res - -if __name__ == "__main__": - sys.exit(main()) - -