From 8edd58643b07ecd2ffe2d3e42eca99dcaf927f96 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Mon, 2 Mar 2020 14:50:57 +0100 Subject: [PATCH] tools: add 'artifacts' target --- Makefile | 6 + pipeline/resources/buildspec-build.yaml | 4 +- shared/makefiles/cfn-nocode.mk | 3 + shared/makefiles/cfn-python3.mk | 3 + shared/makefiles/empty.mk | 3 + tools/artifacts | 81 +++++++++ tools/clean | 2 +- tools/pipeline | 226 ------------------------ 8 files changed, 99 insertions(+), 229 deletions(-) create mode 100755 tools/artifacts delete mode 100755 tools/pipeline diff --git a/Makefile b/Makefile index 20b4e77..63a44ca 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,12 @@ ci-%: @${MAKE} build-$* @${MAKE} tests-unit-$* +# Artifacts services +artifacts: $(foreach service,${SERVICES_ENVONLY}, all-${service}) +artifacts-%: + @echo "[*] $(ccblue)artifacts $*$(ccend)" + @${MAKE} -C $* artifacts + # Build services build: $(foreach service,${SERVICES}, build-${service}) build-%: diff --git a/pipeline/resources/buildspec-build.yaml b/pipeline/resources/buildspec-build.yaml index cc1f10d..5615e1b 100644 --- a/pipeline/resources/buildspec-build.yaml +++ b/pipeline/resources/buildspec-build.yaml @@ -18,13 +18,13 @@ phases: # Tests for all services must work before moving to the next step - | for SERVICE in $(tools/services --changed-since $COMMIT_ID); do - make ci-$SERVICE package-$SERVICE || exit 1 + make ci-$SERVICE package-$SERVICE artifacts-$SERVICE || exit 1 done # Upload template artifacts to 'templates/$SERVICE.yaml' # This will trigger the pipelines per service - | for SERVICE in $(tools/services --changed-since $COMMIT_ID); do - tools/pipeline upload $SERVICE || exit 1 + aws s3 cp $SERVICE/artifacts.zip s3://$S3_BUCKET/templates/$SERVICE.zip || exit 1 done # Update the parameter - aws ssm put-parameter --name $COMMIT_PARAMETER --type String --value $CODEBUILD_RESOLVED_SOURCE_VERSION --overwrite \ No newline at end of file diff --git a/shared/makefiles/cfn-nocode.mk b/shared/makefiles/cfn-nocode.mk index e46ef8c..880afde 100644 --- a/shared/makefiles/cfn-nocode.mk +++ b/shared/makefiles/cfn-nocode.mk @@ -3,6 +3,9 @@ export ENVIRONMENT ?= dev export ROOT ?= $(shell dirname ${CURDIR}) export SERVICE ?= $(shell basename ${CURDIR}) +artifacts: + @${ROOT}/tools/artifacts cloudformation ${SERVICE} + build: @${ROOT}/tools/build resources ${SERVICE} @${ROOT}/tools/build openapi ${SERVICE} diff --git a/shared/makefiles/cfn-python3.mk b/shared/makefiles/cfn-python3.mk index 727d43b..0937551 100644 --- a/shared/makefiles/cfn-python3.mk +++ b/shared/makefiles/cfn-python3.mk @@ -3,6 +3,9 @@ export ENVIRONMENT ?= dev export ROOT ?= $(shell dirname ${CURDIR}) export SERVICE ?= $(shell basename ${CURDIR}) +artifacts: + @${ROOT}/tools/artifacts cloudformation ${SERVICE} + build: @${ROOT}/tools/build resources ${SERVICE} @${ROOT}/tools/build openapi ${SERVICE} diff --git a/shared/makefiles/empty.mk b/shared/makefiles/empty.mk index baec8d3..1ae2253 100644 --- a/shared/makefiles/empty.mk +++ b/shared/makefiles/empty.mk @@ -3,6 +3,9 @@ export ENVIRONMENT ?= dev export ROOT ?= $(shell dirname ${CURDIR}) export SERVICE ?= $(shell basename ${CURDIR}) +artifacts: + $(error "Target $@ is not implemented.") + build: $(error "Target $@ is not implemented.") .PHONY: build diff --git a/tools/artifacts b/tools/artifacts new file mode 100755 index 0000000..3d64c60 --- /dev/null +++ b/tools/artifacts @@ -0,0 +1,81 @@ +#!/bin/bash + +set -e + +ROOT=${ROOT:-$(pwd)} +TYPE=$1 +SERVICE=$2 + +service_dir=$ROOT/$SERVICE +build_dir=$service_dir/build + +display_usage () { + echo "Usage: $0 TYPE SERVICE" +} + +# Check if there are at least 2 arguments +if [ $# -lt 2 ]; then + display_usage + exit 1 +fi + +# Check if the service exists +if [ ! -f $service_dir/metadata.yaml ]; then + echo "Service $SERVICE does not exist" + exit 1 +fi + +# Check for quiet mode +if [ ! -z $QUIET ]; then + export OUTPUT_FILE=$(mktemp) + exec 5>&1 6>&2 1>$OUTPUT_FILE 2>&1 +fi + +cleanup () { + CODE=$? + if [ ! -z $QUIET ]; then + if [ ! $CODE -eq 0 ]; then + cat $OUTPUT_FILE >&5 + fi + rm $OUTPUT_FILE + fi +} +trap cleanup EXIT + +artifacts_cloudformation () { + if [ ! -d $build_dir ]; then + echo "$build_dir does not exist." + exit 1 + fi + + template_file=$build_dir/template.out + if [ ! -f $template_file ]; then + echo "$template_file does not exist." + exit 1 + fi + + # Create a temporary folder + tmpdir=$(mktemp -d) + + # Copy artifacts + cp -pv $template_file $tmpdir/template.yaml + for artifact_file in $build_dir/artifacts/*; do + cp -pv $artifact_file $tmpdir/$(basename $artifact_file) + done + + # Create zip file + pushd $tmpdir + zip $build_dir/artifacts.zip * + popd + + # Delete the temporary folder + rm -r $tmpdir +} + +type artifacts_$TYPE | grep -q "function" &>/dev/null || { + echo "Unsupported type: $TYPE" + echo + display_usage + exit 1 +} +artifacts_$TYPE \ No newline at end of file diff --git a/tools/clean b/tools/clean index b5f37c8..0e6ed29 100755 --- a/tools/clean +++ b/tools/clean @@ -50,7 +50,7 @@ clean () { echo "Removing $build_dir" rm -r $build_dir else - echo "$build_dir does not exit. Skipping" + echo "$build_dir does not exist. Skipping" fi } diff --git a/tools/pipeline b/tools/pipeline deleted file mode 100755 index a86a5b0..0000000 --- a/tools/pipeline +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python3.8 - - -import argparse -import os -import shutil -import subprocess -import sys -import tempfile -import traceback -from typing import List -import zipfile -import yaml - - -ROOT = os.environ.get("ROOT", os.getcwd()) - - -# If any file in these folders are changed, rebuild all services -REBUILD_FOLDERS = [ - "shared" -] - -# If any file in these service folders are changed, ignore it. -# If the folder does not have a `metadata.yaml` file, the folder is ignored -# anyway. -IGNORE_FOLDERS = [] - - -def get_args(): - """ - Parse command line arguments - """ - - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers() - - parser_changed = subparsers.add_parser("changed") - parser_changed.add_argument("commit_id") - parser_changed.set_defaults(command="changed") - - parser_services = subparsers.add_parser("services") - parser_services.add_argument("--env-only", default=False, action="store_true") - parser_services.set_defaults(command="services") - - parser_upload = subparsers.add_parser("upload") - parser_upload.add_argument("service") - parser_upload.set_defaults(command="upload") - - return parser.parse_args() - - -def get_metadata(service_name: str) -> List[str]: - """ - Get metadata for a service - """ - - metafile = os.path.join(ROOT, service_name, "metadata.yaml") - if not os.path.isfile(metafile): - raise ValueError("Metadata file not found for {}".format(service_name)) - - with open(metafile) as fp: - metadata = yaml.load(fp, Loader=yaml.SafeLoader) - - return metadata - - -def get_services(env_only: bool=False) -> List[str]: - """ - Gather the list of services - """ - - # Gather all services - services = { - # Mapping name: reverse dependencies - name: {"deps": [], "rdeps": [], "metadata": get_metadata(name)} - for name in os.listdir(ROOT) - if os.path.isfile(os.path.join(ROOT, name, "metadata.yaml")) - and name not in IGNORE_FOLDERS - } - - if env_only: - for name in list(services.keys()): - if not services[name]["metadata"].get("flags", {}).get("environment", True): - del services[name] - - to_scan = [] - - # Parse dependencies - for name in services.keys(): - if len(services[name]["metadata"].get("dependencies", [])) == 0: - to_scan.append(name) - for dep in services[name]["metadata"].get("dependencies", []): - if name == dep: - continue - # This service depends on all the services - elif dep == "*": - for dname in services.keys(): - # Only inject if this is not the same service - if dname != name: - services[name]["deps"].append(dname) - services[dname]["rdeps"].append(name) - elif dep not in services: - raise ValueError("Dependency {} of {} not found".format(dep, name)) - else: - services[name]["deps"].append(dep) - # Inject as a reverse dependency - services[dep]["rdeps"].append(name) - - # Scan all services until there are not more services to scan. This looks - # for services with no dependencies that were unprocessed. - # The initial `to_scan` list is seeded by all services without any - # dependency. Whenever this scans a dependency, it goes through the list of - # reverse dependencies (services depending on this one) and remove itself - # from the list of dependencies of that service. Whenever the list of - # dependencies for a service is empty, this means that all dependencies are - # satisfied by values in the `retval` list. - # At the end, all services should be resolved. If it's not possible to - # remove all dependencies of a service, this means that there is a circular - # dependency. - retval = [] - while len(to_scan) > 0: - name = to_scan.pop(0) - # If the service is already processed, something has gone wrong. - if name in retval: - raise ValueError("Service {} already present in services: {}".format(name, retval)) - retval.append(name) - for rname in services[name]["rdeps"]: - services[rname]["deps"].remove(name) - if len(services[rname]["deps"]) == 0: - to_scan.append(rname) - - # If there is a discrepancy, this means that there is a circular - # dependency. - if len(retval) != len(services): - for key, value in services.items(): - print(key, value) - raise ValueError("Potential circular dependency found: mismatch between number of services: {} vs {}".format(len(retval), len(services))) - - return retval - - -def get_changed(commit_id: str="0") -> List[str]: - """ - Gather the services that have changed - """ - - services = get_services() - - # '0' is a special value, usually for first runs, that trigger a complete - # (re)build. - if commit_id == "0": - return services - - process = subprocess.run(["git", "diff", commit_id, "--name-only"], capture_output=True, text=True) - process.check_returncode() - - changed = [] - for filename in process.stdout.split("\n"): - basename = filename.split("/", 1)[0] - - # If one file from the special folders has changed, all services need - # to be rebuilt. - if basename in REBUILD_FOLDERS: - return services - - if basename in services: - changed.append(basename) - - # Only return each service once - return list(set(changed)) - - -def upload_template(service: str): - """ - Upload templates to S3 - """ - - # Import here to prevent issues during first execution - import boto3 - - template_file = os.path.join(service, "build", "template.out") - if not os.path.isfile(template_file): - raise ValueError("File '{}' does not exist".format(template_file)) - - artifacts_dir = os.path.join(service, "build", "artifacts") - artifact_files = os.listdir(artifacts_dir) - - # Create zip archive - tmpdir = tempfile.mkdtemp() - try: - archive = os.path.join(tmpdir, "{}.zip".format(service)) - - with zipfile.ZipFile(archive, mode="w") as zip: - zip.write(template_file, arcname="template.yaml") - for artifact_file in artifact_files: - zip.write( - os.path.join(artifacts_dir, artifact_file), - arcname=artifact_file - ) - - s3 = boto3.resource("s3") - bucket = s3.Bucket(os.environ.get("S3_BUCKET", "{}-src".format(os.environ.get("USER", "")))) - - bucket.upload_file(archive, "templates/{}.zip".format(service)) - finally: - shutil.rmtree(tmpdir) - - -if __name__ == "__main__": - args = get_args() - - try: - if args.command == "changed": - for service in get_changed(args.commit_id): - print(service) - - if args.command == "services": - for service in get_services(args.env_only): - print(service) - - if args.command == "upload": - upload_template(args.service) - except Exception as exc: - traceback.print_exc() - sys.exit(1) \ No newline at end of file