From 9f056f1ff7e6fc3d133659f45d5f32bd6d98dd08 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Thu, 24 Oct 2024 17:45:28 +0200 Subject: [PATCH] Generate child pipeline --- .gitlab-ci.yml | 52 ++++++++++++++++++++++------- .gitlab/trigger_template.yml | 57 ++++++++++++++++++++++++++++++++ tasks/__init__.py | 4 +-- tasks/gatekeeper.py | 64 ------------------------------------ tasks/gitlab.py | 21 ++++++++++++ 5 files changed, 121 insertions(+), 77 deletions(-) create mode 100644 .gitlab/trigger_template.yml delete mode 100644 tasks/gatekeeper.py create mode 100644 tasks/gitlab.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e478b7bbf..013e897cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -158,6 +158,10 @@ get_agent_version: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi + - touch $CI_WORKING_DIR/built_$IMAGE.txt + artifacts: + paths: + - $CI_WORKING_DIR/built_$IMAGE.txt .build_ddregistry: @@ -188,19 +192,27 @@ get_agent_version: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi + - touch $CI_WORKING_DIR/built_$IMAGE.txt + artifacts: + paths: + - $CI_WORKING_DIR/built_$IMAGE.txt build_ci_image: stage: build image: "${CI_IMAGE}" tags: ["arch:amd64"] - script: |- - docker buildx build \ + script: + - docker buildx build \ --file ci/Dockerfile \ --platform linux/amd64 \ --label target=build \ --tag registry.ddbuild.io/${CI_IMAGE_REPO}:$IMAGE_VERSION \ --push \ . + - touch $CI_WORKING_DIR/built_$CI_IMAGE.txt + artifacts: + paths: + - $CI_WORKING_DIR/built_$CI_IMAGE.txt rules: - changes: paths: @@ -481,6 +493,10 @@ build_circleci_runner: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi + - touch $CI_WORKING_DIR/built_$IMAGE.txt + artifacts: + paths: + - $CI_WORKING_DIR/built_$IMAGE.txt build_gitlab_agent_deploy: extends: [.build, .x64] @@ -515,6 +531,25 @@ trigger_build_kernels: - local: .gitlab/kernel_version_testing.yml strategy: depend +generate_tests: + stage: test + tags: ["arch:amd64"] + image: $SETUP_IMAGE_NAME + script: + - inv -e gitlab.generate-tests -t $ECR_TEST_ONLY + artifacts: + paths: + - $CI_PROJECT_DIR/datadog-agent-trigger-gitlab-ci.yml + +trigger_datadog_agent_tests: + stage: test + needs: [generate_tests] + trigger: + include: + - artifact: datadog-agent-trigger-gitlab-ci.yml + job: generate_tests + allow_failure: true + trigger_tests: stage: test rules: @@ -582,6 +617,10 @@ push_to_datadog_agent: - docker rmi $SRC_IMAGE - If ($CI_JOB_STATUS -ne "success") { Write-Host "Build failed."; exit 0 } - If ($CI_PIPELINE_SOURCE -ne "schedule") { Write-Host "Image $SRC_IMAGE is available." } else { Write-Host "Scheduled pipeline, image push skipped." } + - New-Item -Path $CI_WORKING_DIR -Name "built_$IMAGE.txt" -ItemType "file" + artifacts: + paths: + - $CI_WORKING_DIR/built_$IMAGE.txt build_windows_1809_x64: extends: .winbuild @@ -764,15 +803,6 @@ release_circleci_runner_gcr: project: DataDog/public-images strategy: depend -gatekeeper: - stage: monitoring - tags: ["arch:amd64"] - image: $SETUP_IMAGE_NAME - dependencies: [] - script: - - echo "Gatekeeper job to ensure that the pipeline is running correctly" - - pip install invoke requests - - inv gatekeeper.control notify-images-available: extends: .slack-notifier-base diff --git a/.gitlab/trigger_template.yml b/.gitlab/trigger_template.yml new file mode 100644 index 000000000..86e92c3eb --- /dev/null +++ b/.gitlab/trigger_template.yml @@ -0,0 +1,57 @@ +--- +stages: + - test + +trigger_tests: + stage: test + rules: + - when: manual + allow_failure: true + variables: + RUN_KITCHEN_TESTS: "false" + BUCKET_BRANCH: "dev" + DATADOG_AGENT_BUILDIMAGES: "" + DATADOG_AGENT_BUILDIMAGES_SUFFIX: "" + DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "" + DATADOG_AGENT_WINBUILDIMAGES: "" + DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "" + DATADOG_AGENT_ARMBUILDIMAGES: "" + DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "" + DATADOG_AGENT_SYSPROBE_BUILDIMAGES: "" + CI_IMAGE_AGENT_DEPLOY: "" + CI_IMAGE_AGENT_DEPLOY_SUFFIX: "" + CI_IMAGE_BTFGEN: "" + CI_IMAGE_BTFGEN_SUFFIX: "" + CI_IMAGE_DEB_X64: "" + CI_IMAGE_DEB_X64_SUFFIX: "" + CI_IMAGE_DEB_ARM64: "" + CI_IMAGE_DEB_ARM64_SUFFIX: "" + CI_IMAGE_DEB_ARMHF: "" + CI_IMAGE_DEB_ARMHF_SUFFIX: "" + CI_IMAGE_DD_AGENT_TESTING: "" + CI_IMAGE_DD_AGENT_TESTING_SUFFIX: "" + CI_IMAGE_DOCKER_X64: "" + CI_IMAGE_DOCKER_X64_SUFFIX: "" + CI_IMAGE_DOCKER_ARM64: "" + CI_IMAGE_DOCKER_ARM64_SUFFIX: "" + CI_IMAGE_GLIBC_2_17_X64: "" + CI_IMAGE_GLIBC_2_17_X64_SUFFIX: "" + CI_IMAGE_GLIBC_2_23_ARM64: "" + CI_IMAGE_GLIBC_2_23_ARM64_SUFFIX: "" + CI_IMAGE_SYSTEM_PROBE_X64: "" + CI_IMAGE_SYSTEM_PROBE_X64_SUFFIX: "" + CI_IMAGE_SYSTEM_PROBE_ARM64: "" + CI_IMAGE_SYSTEM_PROBE_ARM64_SUFFIX: "" + CI_IMAGE_RPM_X64: "" + CI_IMAGE_RPM_X64_SUFFIX: "" + CI_IMAGE_RPM_ARM64: "" + CI_IMAGE_RPM_ARM64_SUFFIX: "" + CI_IMAGE_RPM_ARMHF: "" + CI_IMAGE_RPM_ARMHF_SUFFIX: "" + CI_IMAGE_WIN_1809_X64: "" + CI_IMAGE_WIN_1809_X64_SUFFIX: "" + CI_IMAGE_WIN_LTSC2022_X64: "" + CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: "" + trigger: + project: DataDog/datadog-agent + strategy: depend diff --git a/tasks/__init__.py b/tasks/__init__.py index 733fdf6b4..d1481b7af 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -5,7 +5,7 @@ from invoke import Collection -from tasks import agent +from tasks import agent, gitlab from tasks.datadog_agent import update_datadog_agent_buildimages from tasks.update_go import update_go @@ -15,7 +15,7 @@ # add single tasks to the root ns.add_collection(agent) -ns.add_collection(gatekeeper) +ns.add_collection(gitlab) ns.add_task(update_go) ns.add_task(update_datadog_agent_buildimages) ns.configure( diff --git a/tasks/gatekeeper.py b/tasks/gatekeeper.py deleted file mode 100644 index 73ab1abe9..000000000 --- a/tasks/gatekeeper.py +++ /dev/null @@ -1,64 +0,0 @@ -import yaml -import re -import os -import requests -import fnmatch -from invoke import task, Exit -from collections import UserList - -@task -def control(ctx): - gl = _load_gitlab_config() - build_jobs = [job for job in gl.keys() if job.startswith("build")] - jobs = get_jobs(os.environ["CI_PIPELINE_ID"]) - for job in expected_jobs(ctx, gl, build_jobs): - job_info = next((j for j in jobs if j["name"] == job), None) - if job_info and job_info["status"] != "success": - raise Exit(f"Job {job} failed and is required", 1) - print("All required jobs passed") - -class ReferenceTag(yaml.YAMLObject): - """ - Custom yaml tag to handle references in gitlab-ci configuration - """ - yaml_tag = "!reference" - - def __init__(self, references): - self.references = references - - @classmethod - def from_yaml(cls, loader, node): - return UserList(loader.construct_sequence(node)) - - @classmethod - def to_yaml(cls, dumper, data): - return dumper.represent_sequence(cls.yaml_tag, data.data, flow_style=True) - -def expected_jobs(ctx, gl, build_jobs): - modified_files = _get_modified_files(ctx, "main") - for job in build_jobs: - change_paths = [] - for rule in gl[job]["rules"]: - if "changes" in rule: - change_paths += rule["changes"]["paths"] - # print(f"in job {job} with changes {change_paths}") - for path in change_paths: - for file in modified_files: - if re.match(fnmatch.translate(path), file): - yield job - break - -def _get_modified_files(ctx, base): - last_main_commit = ctx.run(f"git merge-base HEAD origin/{base}", hide=True).stdout - modified_files = ctx.run(f"git diff --name-only --no-renames {last_main_commit}", hide=True).stdout.splitlines() - return modified_files - -def _load_gitlab_config(): - yaml.SafeLoader.add_constructor(ReferenceTag.yaml_tag, ReferenceTag.from_yaml) - with open(".gitlab-ci.yml") as f: - return yaml.safe_load(f) - -def get_jobs(pipeline): - headers = {"PRIVATE-TOKEN": os.environ["GITLAB_TOKEN"]} - url = f"https://gitlab.ddbuild.io/api/v4/projects/291/pipelines/{pipeline}/jobs?per_page=100" - return requests.get(url, headers=headers).json() \ No newline at end of file diff --git a/tasks/gitlab.py b/tasks/gitlab.py new file mode 100644 index 000000000..c4a270ef7 --- /dev/null +++ b/tasks/gitlab.py @@ -0,0 +1,21 @@ +import yaml +import os +from invoke import task + +@task +def generate_test(_): + with open(".gitlab/trigger_template.yml") as f: + trigger_template = yaml.safe_load(f) + version = f"v{os.environ['CI_PIPELINE_ID']}-{os.environ['CI_COMMIT_SHORT_SHA']}" + + for file in os.listdir(os.environ["CI_PROJECT_DIR"]): + if file.startswith("built"): + print(f"Adding {file} to the trigger template") + image = file.removeprefix("built_").removesuffix(".txt").replace("_", "").replace("-", "").casefold() + for variable, value in trigger_template["variables"].items(): + if variable.replace("_", "").replace("-", "").casefold()["key"] == image: + variable["value"] = version + break + print(f"triggering with {trigger_template}") + with open("datadog-agent-trigger-gitlab-ci.yml", "w") as f: + yaml.dump(trigger_template, f)