CASM-4349 - add remote build node capabilities. (#75)

* CASMCMS-8724 - move ims-utils functionality into ims-python-helper.

* CASMCMS-8821 - Add remote build customize job options.

* CASMCMS-8818 - ssh key injection into recipe build jobs.

* CASMCMS-8897 - changes for aarch64 remote build node.

* CASMCMS-8895 - allow multiple concurrent remote customize jobs.

* Cleanup.

CHANGELOG.md

@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- CASMCMS-8821 - add support for remote build jobs.
+- CASMCMS-8818 - ssh key injection into jobs.
+- CASMCMS-8897 - changes for aarch64 remote build.
+- CASMCMS-8895 - allow multiple concurrent remote customize jobs.
 
 ## [2.12.0] - 2024-02-22
 ### Dependencies

Dockerfile

@@ -1,7 +1,7 @@
 #
 # MIT License
 #
-# (C) Copyright 2018-2023 Hewlett Packard Enterprise Development LP
+# (C) Copyright 2018-2024 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -41,6 +41,9 @@ RUN apk add --upgrade --no-cache apk-tools \
             gcc \
             python3-dev \
             libc-dev \
+            podman \
+            openssh \
+            bash \
         && apk -U upgrade --no-cache \
         &&  rm -rf \
            /var/cache/apk/* \
@@ -62,3 +65,4 @@ RUN --mount=type=secret,id=netrc,target=/root/.netrc \
 
 COPY scripts/* /scripts/
 COPY config/* /config/
+COPY Dockerfile.remote /Dockerfile.remote

Dockerfile.remote

@@ -0,0 +1,45 @@
+#
+# MIT License
+#
+# (C) Copyright 2018-2024 Hewlett Packard Enterprise Development LP
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+
+# Dockerfile for IMS remote customization jobs
+
+# Start with the ims-sshd image as it will have most of the tools needed
+FROM registry.local/$IMS_SSHD_IMAGE as base
+
+COPY /scripts/remote_customize_entrypoint.sh /entrypoint.sh
+COPY /etc/cray/ca/certificate_authority.crt /etc/cray/ca/certificate_authority.crt
+COPY /etc/admin-client-auth /etc/admin-client-auth
+COPY /mnt/image/image.sqsh /data/
+COPY /config/sshd_config /etc/cray/ims/sshd_config
+COPY /root/.ssh/id_ecdsa.pub /etc/cray/ims/authorized_keys
+
+# Copy in env vars needed for the remote job run
+ENV OAUTH_CONFIG_DIR=$OAUTH_CONFIG_DIR
+ENV BUILD_ARCH=$BUILD_ARCH
+ENV IMS_JOB_ID=$IMS_JOB_ID
+ENV IMAGE_ROOT_PARENT=$IMAGE_ROOT_PARENT
+ENV SSH_JAIL=$SSH_JAIL
+ENV JOB_ENABLE_DKMS=$JOB_ENABLE_DKMS
+
+ENTRYPOINT ["/entrypoint.sh"]

config/sshd_config

@@ -7,4 +7,3 @@ AuthorizedKeysFile    /etc/cray/ims/authorized_keys
 PasswordAuthentication no
 ChallengeResponseAuthentication no
 Subsystem sftp internal-sftp
-

scripts/build_ca_rpm.py

@@ -24,68 +24,10 @@
 #
 # Author: Eric Cozzi
 
-import errno
-import os
-import shutil
-import subprocess
-
-RPM_NAME = "cray_ca_cert"
-RPM_VERSION = "1.0.1"
-
-# There is nothing arch specific in here so build accordingly
-RPM_ARCHITECTURE = "noarch"
-
-ETC_CRAY_CA_DIR = "etc/cray/ca"
-CERTIFICATE_AUTHORITY_NAME = "certificate_authority.crt"
-ETC_CRAY_CA_CERT_FILE = os.path.join("/", ETC_CRAY_CA_DIR, CERTIFICATE_AUTHORITY_NAME)
-
-SOURCE_ARCHIVE_ROOT = os.path.expanduser(os.path.join("~", "{}-{}".format(RPM_NAME, RPM_VERSION)))
-SOURCE_ARCHIVE_ETC_CRAY_CA_DIR = os.path.join(SOURCE_ARCHIVE_ROOT, ETC_CRAY_CA_DIR)
-SOURCE_ARCHIVE_ETC_CRAY_CA_CERT_FILE = os.path.join(SOURCE_ARCHIVE_ETC_CRAY_CA_DIR, CERTIFICATE_AUTHORITY_NAME)
-SOURCE_TAR_FILE = "{}-{}.tar.gz".format(RPM_NAME, RPM_VERSION)
-
-RPM_BUILD_ROOT = os.path.expanduser("~/rpmbuild/")
-SPECFILE_NAME = "cray_ca_cert.spec"
-SPECFILE_SOURCE_FILE = os.path.join("/mnt/specfile/", SPECFILE_NAME)
-
+from ims_python_helper.build_ca_rpm import build_ca_rpm
 
 def main():
-    os.chdir(os.path.expanduser("~"))
-
-    # Create SOURCE archive root and sub directories
-    try:
-        os.makedirs(SOURCE_ARCHIVE_ETC_CRAY_CA_DIR)
-    except OSError as exc:  # Python >2.5
-        if exc.errno == errno.EEXIST and os.path.isdir(SOURCE_ARCHIVE_ETC_CRAY_CA_DIR):
-            pass
-        else:
-            raise
-
-    # Copy CA Certificate into SOURCE directory
-    shutil.copyfile(ETC_CRAY_CA_CERT_FILE, SOURCE_ARCHIVE_ETC_CRAY_CA_CERT_FILE)
-    os.chmod(SOURCE_ARCHIVE_ETC_CRAY_CA_CERT_FILE, 0o644)
-
-    # Archive SOURCE archive using tar
-    subprocess.check_call(["tar", "-zcvf", SOURCE_TAR_FILE, "{}-{}".format(RPM_NAME, RPM_VERSION)])
-
-    # Make RPMBUILD directories
-    for rpmbuild_directory in ["{}{}".format(RPM_BUILD_ROOT, subdir) for subdir in
-                               ("SOURCES", "RPMS", "SRPMS", "SPECS", "BUILD", "BUILDROOT")]:
-        try:
-            os.makedirs(os.path.expanduser(rpmbuild_directory))
-        except OSError as exc:  # Python >2.5
-            if exc.errno == errno.EEXIST and os.path.isdir(rpmbuild_directory):
-                pass
-            else:
-                raise
-
-    # Copy source archive and spec file into RPMBUILD directories
-    shutil.copyfile(SOURCE_TAR_FILE, os.path.join(RPM_BUILD_ROOT, "SOURCES", SOURCE_TAR_FILE))
-    shutil.copyfile(SPECFILE_SOURCE_FILE, os.path.join(RPM_BUILD_ROOT, "SPECS", SPECFILE_NAME))
-    subprocess.check_call(["rpmbuild", "-bb", "--target", RPM_ARCHITECTURE, os.path.join(RPM_BUILD_ROOT, "SPECS", SPECFILE_NAME)])
-    shutil.copyfile(os.path.join(RPM_BUILD_ROOT, "RPMS", RPM_ARCHITECTURE, f"cray_ca_cert-{RPM_VERSION}-1.{RPM_ARCHITECTURE}.rpm"),
-                    f"/mnt/ca-rpm/cray_ca_cert-{RPM_VERSION}-1.{RPM_ARCHITECTURE}.rpm")
-
+    build_ca_rpm()
 
 if __name__ == "__main__":
     main()

scripts/buildenv-sidecar.sh

@@ -2,7 +2,7 @@
 #
 # MIT License
 #
-# (C) Copyright 2018-2022 Hewlett Packard Enterprise Development LP
+# (C) Copyright 2018-2022, 2024 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -92,8 +92,8 @@ setup_user_shell() {
     # Set the permissions on the folder holding the keys
     chmod 600 /etc/cray/ims
 
-    # Change signal location if user if jailed
-    if [ "$SSH_JAIL" = "True" ]
+    # Change signal location if user if jailed and not running on remote
+    if [ "$SSH_JAIL" = "True" -a "$REMOTE_BUILD_NODE" = ""]
     then
         SIGNAL_FILE_COMPLETE=$IMAGE_ROOT_PARENT/image-root/tmp/complete
         SIGNAL_FILE_FAILED=$IMAGE_ROOT_PARENT/image-root/tmp/failed

scripts/fetch-recipe.sh

@@ -2,7 +2,7 @@
 #
 # MIT License
 #
-# (C) Copyright 2019-2023 Hewlett Packard Enterprise Development LP
+# (C) Copyright 2019-2024 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -27,5 +27,5 @@
 #     /scripts/fetch-recipe.sh /mnt/image/recipe http://example.com/path/to/recipe.tgz
 
 source /scripts/helper.sh
-python3 /scripts/fetch.py --recipe "$@"
+python3 /scripts/fetch.py --recipe True "$@"
 fail_if_error "Downloading recipe"