From 7954207e56e2c840c723512e85d2ee1bc5c85933 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Fri, 19 Apr 2024 19:14:12 +0200 Subject: [PATCH 01/11] add rules:changes and gatekeeper job --- .gitlab-ci.yml | 205 +++++++++++++++++++++++++++++++++++++++++++++- tasks/__init__.py | 7 +- 2 files changed, 207 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9009e7827..27b70115f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,6 +79,32 @@ default: - unknown_failure - api_failure +# Change path for "dynamic" generation of the pipeline +.on_config_files: &on_config_files + changes: + paths: + - .awsconfig + - .curlrc + - .wgetrc + - .gitlab-ci.yml + - tasks/** + compare_to: 'main' + +.on_python_setup: &on_python_setup + changes: + paths: + - python-packages-versions.txt + - requirements-py2.txt + - requirements.txt + - requirements/constraints.txt + - setup_python.sh + compare_to: 'main' + +.manual: &manual + - when: manual + allow_failure: true + +# Jobs definitions get_agent_version: stage: setup image: $SETUP_IMAGE_NAME @@ -200,17 +226,38 @@ build_ci_image: paths: - "ci/Dockerfile" compare_to: 'main' - - when: manual - allow_failure: true + - <<: *manual build_deb_x64: extends: [.build, .x64] + rules: + - <<: *on_config_files + - <<: *on_python_setup + - changes: + paths: + - entrypoint.sh + - rvm/** + - deb-x64/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: deb-x64/Dockerfile IMAGE: deb_x64 build_rpm_x64: extends: [.build, .x64] + rules: + - <<: *on_config_files + - <<: *on_python_setup + - changes: + paths: + - patches/rpm-4.15.1-fix-rpmbuild-segfault.patch + - rvm/** + - entrypoint.sh + - rpm-headers/** + - rpm-x64/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: rpm-x64/Dockerfile IMAGE: rpm_x64 @@ -218,6 +265,16 @@ build_rpm_x64: build_deb_arm64: extends: [.build, .arm] + rules: + - <<: *on_config_files + - <<: *on_python_setup + - changes: + paths: + - entrypoint.sh + - rmv/** + - deb-arm/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: deb-arm/Dockerfile IMAGE: deb_arm64 @@ -225,6 +282,16 @@ build_deb_arm64: build_deb_armhf: extends: [.build, .arm] + rules: + - <<: *on_config_files + - <<: *on_python_setup + - changes: + paths: + - entrypoint.sh + - rvm/** + - deb-arm/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: deb-arm/Dockerfile IMAGE: deb_armhf @@ -233,6 +300,17 @@ build_deb_armhf: build_rpm_arm64: extends: [.build, .arm] + rules: + - <<: *on_config_files + - <<: *on_python_setup + - changes: + paths: + - patches/rpm-4.15.1-fix-rpmbuild-segfault.patch + - rvm/** + - entrypoint.sh + - rpm-arm64/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: rpm-arm64/Dockerfile IMAGE: rpm_arm64 @@ -240,6 +318,17 @@ build_rpm_arm64: build_rpm_armhf: extends: [.build, .arm] + rules: + - <<: *on_config_files + - <<: *on_python_setup + - changes: + paths: + - patches/rpm-4.15.1-fix-rpmbuild-segfault.patch + - rvm/** + - entrypoint.sh + - rpm-armhf/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: rpm-armhf/Dockerfile IMAGE: rpm_armhf @@ -248,18 +337,43 @@ build_rpm_armhf: build_system-probe_x64: extends: [.build, .x64] + rules: + - <<: *on_config_files + - <<: *on_python_setup + - changes: + paths: + - system-probe_x64/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: system-probe_x64/Dockerfile IMAGE: system-probe_x64 build_system-probe_arm64: extends: [.build, .arm] + rules: + - <<: *on_config_files + - <<: *on_python_setup + - changes: + paths: + - system-probe_arm64/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: system-probe_arm64/Dockerfile IMAGE: system-probe_arm64 build_dd_agent_testing: extends: [.build, .x64] + rules: + - <<: *on_config_files + - changes: + paths: + - requirements.txt + - requirements/constraints.txt + - dd-agent-testing/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: dd-agent-testing/Dockerfile IMAGE: dd-agent-testing @@ -278,24 +392,59 @@ build_linux_glibc_2_23_arm64: build_docker_arm64: extends: [.build, .arm] + rules: + - <<: *on_config_files + - changes: + paths: + - requirements.txt + - requirements/constraints.txt + - docker-arm64/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: docker-arm64/Dockerfile IMAGE: docker_arm64 build_docker_x64: extends: [.build, .x64] + rules: + - <<: *on_config_files + - changes: + paths: + - requirements.txt + - requirements/constraints.txt + - docker-x64/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: docker-x64/Dockerfile IMAGE: docker_x64 build_btf_gen: extends: [.build, .x64] + rules: + - <<: *on_config_files + - changes: + paths: + - requirements/btf_gen.txt + - requirements/constraints.txt + - btf-gen/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: btf-gen/Dockerfile IMAGE: btf-gen build_kernel_version_testing_x64: extends: [.build, .x64] + rules: + - <<: *on_config_files + - changes: + paths: + - entrypoint-sysprobe.sh + - kernel-version-testing/** + compare_to: 'main' + - <<: *manual variables: CUSTOM_BUILD_ARGS: --build-arg KERNEL_EXTRA_CONFIG_VERSION=$EXTRA_KCONFIG_VERSION DOCKERFILE: kernel-version-testing/kernel-version-testing_x64/Dockerfile @@ -303,6 +452,14 @@ build_kernel_version_testing_x64: build_kernel_version_testing_arm64: extends: [.build, .arm] + rules: + - <<: *on_config_files + - changes: + paths: + - entrypoint-sysprobe.sh + - kernel-version-testing/** + compare_to: 'main' + - <<: *manual variables: CUSTOM_BUILD_ARGS: --build-arg KERNEL_EXTRA_CONFIG_VERSION=$EXTRA_KCONFIG_VERSION DOCKERFILE: kernel-version-testing/kernel-version-testing_arm64/Dockerfile @@ -313,6 +470,15 @@ build_circleci_runner: stage: build rules: - if: $CI_COMMIT_TAG == null + - <<: *on_config_files + - changes: + paths: + - requirements.txt + - requirements/circleci.txt + - requirements/constraints.txt + - circleci/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: circleci/Dockerfile IMAGE: circleci-runner @@ -338,6 +504,17 @@ build_circleci_runner: build_gitlab_agent_deploy: extends: [.build, .x64] + rules: + - <<: *on_config_files + - changes: + paths: + - requirements.txt + - requirements/agent-deploy.txt + - requirements/constraints.txt + - rvm/** + - agent-deploy/** + compare_to: 'main' + - <<: *manual variables: DOCKERFILE: agent-deploy/Dockerfile IMAGE: gitlab_agent_deploy @@ -428,6 +605,18 @@ push_to_datadog_agent: build_windows_1809_x64: extends: .winbuild + rules: + - <<: *on_config_files + - changes: + paths: + - go.env + - requirements.txt + - requirements-py2.txt + - requirements/constraints.txt + - python-packages-versions.txt + - windows/** + compare_to: 'main' + - <<: *manual tags: ["runner:windows-docker", "windowsversion:1809"] variables: DOCKERFILE: windows/Dockerfile @@ -436,6 +625,18 @@ build_windows_1809_x64: build_windows_ltsc2022_x64: extends: .winbuild + rules: + - <<: *on_config_files + - changes: + paths: + - go.env + - requirements.txt + - requirements-py2.txt + - requirements/constraints.txt + - python-packages-versions.txt + - windows/** + compare_to: 'main' + - <<: *manual tags: ["runner:windows-docker", "windowsversion:2022"] variables: DOCKERFILE: windows/Dockerfile diff --git a/tasks/__init__.py b/tasks/__init__.py index 2bd81fca7..733fdf6b4 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -5,9 +5,9 @@ from invoke import Collection -from . import agent -from .datadog_agent import update_datadog_agent_buildimages -from .update_go import update_go +from tasks import agent +from tasks.datadog_agent import update_datadog_agent_buildimages +from tasks.update_go import update_go # the root namespace ns = Collection() @@ -15,6 +15,7 @@ # add single tasks to the root ns.add_collection(agent) +ns.add_collection(gatekeeper) ns.add_task(update_go) ns.add_task(update_datadog_agent_buildimages) ns.configure( From d2ec5a27c53b556b2025045819e851e63e394e6f Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Fri, 19 Apr 2024 19:20:57 +0200 Subject: [PATCH 02/11] add a new script --- tasks/gatekeeper.py | 65 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 tasks/gatekeeper.py diff --git a/tasks/gatekeeper.py b/tasks/gatekeeper.py new file mode 100644 index 000000000..4c856c1f7 --- /dev/null +++ b/tasks/gatekeeper.py @@ -0,0 +1,65 @@ +import yaml +import re +import os +import requests +import fnmatch +from invoke import task, Exit +from collections import UserList + +@task +def control(ctx): + gl = _load_gitlab_config() + build_jobs = [job for job in gl.keys() if job.startswith("build")] + jobs = get_jobs(os.environ["CI_PIPELINE_ID"]) + for job in expected_jobs(ctx, gl, build_jobs): + print(f"Checking job {job}") + job_info = next((j for j in jobs if j["name"] == job), None) + if job_info and job_info["status"] != "success": + Exit(f"Job {job} failed and is required", 1) + print("All required jobs passed") + +class ReferenceTag(yaml.YAMLObject): + """ + Custom yaml tag to handle references in gitlab-ci configuration + """ + yaml_tag = "!reference" + + def __init__(self, references): + self.references = references + + @classmethod + def from_yaml(cls, loader, node): + return UserList(loader.construct_sequence(node)) + + @classmethod + def to_yaml(cls, dumper, data): + return dumper.represent_sequence(cls.yaml_tag, data.data, flow_style=True) + +def expected_jobs(ctx, gl, build_jobs): + modified_files = _get_modified_files(ctx, "main") + print("modified", modified_files) + for job in build_jobs: + change_paths = [] + for rule in gl[job]["rules"]: + if "changes" in rule: + change_paths += rule["changes"]["paths"] + # print(f"in job {job} with changes {change_paths}") + for path in change_paths: + for file in modified_files: + if re.match(fnmatch.translate(path), file): + yield job + +def _get_modified_files(ctx, base): + last_main_commit = ctx.run(f"git merge-base HEAD origin/{base}", hide=True).stdout + modified_files = ctx.run(f"git diff --name-only --no-renames {last_main_commit}", hide=True).stdout.splitlines() + return modified_files + +def _load_gitlab_config(): + yaml.SafeLoader.add_constructor(ReferenceTag.yaml_tag, ReferenceTag.from_yaml) + with open(".gitlab-ci.yml") as f: + return yaml.safe_load(f) + +def get_jobs(pipeline): + headers = {"PRIVATE-TOKEN": os.environ["GITLAB_TOKEN"]} + url = f"https://gitlab.ddbuild.io/api/v4/projects/291/pipelines/{pipeline}/jobs?per_page=100" + return requests.get(url, headers=headers).json() \ No newline at end of file From 2d12fd3ad2f0bae72bd212b26b0f06312f6e488b Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Sat, 20 Apr 2024 08:59:39 +0200 Subject: [PATCH 03/11] fix yaml and activate job --- .gitlab-ci.yml | 96 +++++++++++++++++++++++++-------------------- tasks/gatekeeper.py | 5 +-- 2 files changed, 55 insertions(+), 46 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27b70115f..a7828064b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ include: stages: - setup - build - - availability_message + - monitoring - test - dev_envs - devcontainer @@ -226,20 +226,20 @@ build_ci_image: paths: - "ci/Dockerfile" compare_to: 'main' - - <<: *manual + - !reference [.manual] build_deb_x64: extends: [.build, .x64] rules: - - <<: *on_config_files - - <<: *on_python_setup + - !reference [.on_config_files] + - !reference [.on_python_setup] - changes: paths: - entrypoint.sh - rvm/** - deb-x64/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: deb-x64/Dockerfile IMAGE: deb_x64 @@ -247,8 +247,8 @@ build_deb_x64: build_rpm_x64: extends: [.build, .x64] rules: - - <<: *on_config_files - - <<: *on_python_setup + - !reference [.on_config_files] + - !reference [.on_python_setup] - changes: paths: - patches/rpm-4.15.1-fix-rpmbuild-segfault.patch @@ -257,7 +257,7 @@ build_rpm_x64: - rpm-headers/** - rpm-x64/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: rpm-x64/Dockerfile IMAGE: rpm_x64 @@ -266,15 +266,15 @@ build_rpm_x64: build_deb_arm64: extends: [.build, .arm] rules: - - <<: *on_config_files - - <<: *on_python_setup + - !reference [.on_config_files] + - !reference [.on_python_setup] - changes: paths: - entrypoint.sh - rmv/** - deb-arm/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: deb-arm/Dockerfile IMAGE: deb_arm64 @@ -283,15 +283,15 @@ build_deb_arm64: build_deb_armhf: extends: [.build, .arm] rules: - - <<: *on_config_files - - <<: *on_python_setup + - !reference [.on_config_files] + - !reference [.on_python_setup] - changes: paths: - entrypoint.sh - rvm/** - deb-arm/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: deb-arm/Dockerfile IMAGE: deb_armhf @@ -301,8 +301,8 @@ build_deb_armhf: build_rpm_arm64: extends: [.build, .arm] rules: - - <<: *on_config_files - - <<: *on_python_setup + - !reference [.on_config_files] + - !reference [.on_python_setup] - changes: paths: - patches/rpm-4.15.1-fix-rpmbuild-segfault.patch @@ -310,7 +310,7 @@ build_rpm_arm64: - entrypoint.sh - rpm-arm64/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: rpm-arm64/Dockerfile IMAGE: rpm_arm64 @@ -319,8 +319,8 @@ build_rpm_arm64: build_rpm_armhf: extends: [.build, .arm] rules: - - <<: *on_config_files - - <<: *on_python_setup + - !reference [.on_config_files] + - !reference [.on_python_setup] - changes: paths: - patches/rpm-4.15.1-fix-rpmbuild-segfault.patch @@ -328,7 +328,7 @@ build_rpm_armhf: - entrypoint.sh - rpm-armhf/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: rpm-armhf/Dockerfile IMAGE: rpm_armhf @@ -338,13 +338,13 @@ build_rpm_armhf: build_system-probe_x64: extends: [.build, .x64] rules: - - <<: *on_config_files - - <<: *on_python_setup + - !reference [.on_config_files] + - !reference [.on_python_setup] - changes: paths: - system-probe_x64/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: system-probe_x64/Dockerfile IMAGE: system-probe_x64 @@ -366,14 +366,14 @@ build_system-probe_arm64: build_dd_agent_testing: extends: [.build, .x64] rules: - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - requirements.txt - requirements/constraints.txt - dd-agent-testing/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: dd-agent-testing/Dockerfile IMAGE: dd-agent-testing @@ -393,14 +393,14 @@ build_linux_glibc_2_23_arm64: build_docker_arm64: extends: [.build, .arm] rules: - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - requirements.txt - requirements/constraints.txt - docker-arm64/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: docker-arm64/Dockerfile IMAGE: docker_arm64 @@ -408,14 +408,14 @@ build_docker_arm64: build_docker_x64: extends: [.build, .x64] rules: - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - requirements.txt - requirements/constraints.txt - docker-x64/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: docker-x64/Dockerfile IMAGE: docker_x64 @@ -423,14 +423,14 @@ build_docker_x64: build_btf_gen: extends: [.build, .x64] rules: - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - requirements/btf_gen.txt - requirements/constraints.txt - btf-gen/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: btf-gen/Dockerfile IMAGE: btf-gen @@ -438,13 +438,13 @@ build_btf_gen: build_kernel_version_testing_x64: extends: [.build, .x64] rules: - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - entrypoint-sysprobe.sh - kernel-version-testing/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: CUSTOM_BUILD_ARGS: --build-arg KERNEL_EXTRA_CONFIG_VERSION=$EXTRA_KCONFIG_VERSION DOCKERFILE: kernel-version-testing/kernel-version-testing_x64/Dockerfile @@ -453,13 +453,13 @@ build_kernel_version_testing_x64: build_kernel_version_testing_arm64: extends: [.build, .arm] rules: - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - entrypoint-sysprobe.sh - kernel-version-testing/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: CUSTOM_BUILD_ARGS: --build-arg KERNEL_EXTRA_CONFIG_VERSION=$EXTRA_KCONFIG_VERSION DOCKERFILE: kernel-version-testing/kernel-version-testing_arm64/Dockerfile @@ -470,7 +470,7 @@ build_circleci_runner: stage: build rules: - if: $CI_COMMIT_TAG == null - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - requirements.txt @@ -478,7 +478,7 @@ build_circleci_runner: - requirements/constraints.txt - circleci/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: circleci/Dockerfile IMAGE: circleci-runner @@ -505,7 +505,7 @@ build_circleci_runner: build_gitlab_agent_deploy: extends: [.build, .x64] rules: - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - requirements.txt @@ -514,7 +514,7 @@ build_gitlab_agent_deploy: - rvm/** - agent-deploy/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: agent-deploy/Dockerfile IMAGE: gitlab_agent_deploy @@ -626,7 +626,7 @@ build_windows_1809_x64: build_windows_ltsc2022_x64: extends: .winbuild rules: - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - go.env @@ -636,7 +636,7 @@ build_windows_ltsc2022_x64: - python-packages-versions.txt - windows/** compare_to: 'main' - - <<: *manual + - !reference [.manual] tags: ["runner:windows-docker", "windowsversion:2022"] variables: DOCKERFILE: windows/Dockerfile @@ -784,9 +784,19 @@ release_circleci_runner_gcr: project: DataDog/public-images strategy: depend +gatekeeper: + stage: monitoring + tags: ["arch:amd64"] + image: $SETUP_IMAGE_NAME + dependencies: [] + script: + - echo "Gatekeeper job to ensure that the pipeline is running correctly" + - pip install invoke requests + - inv gatekeeper.control + notify-images-available: extends: .slack-notifier-base - stage: availability_message # Created a new stage to depend upon all image builds without explicitely name them in 'needs' + stage: monitoring # Created a new stage to depend upon all image builds without explicitely name them in 'needs' rules: - if: $CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "main" when: never @@ -804,7 +814,7 @@ notify-images-available: notify-images-failure: extends: .slack-notifier-base - stage: availability_message + stage: monitoring rules: - if: $CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "main" when: never diff --git a/tasks/gatekeeper.py b/tasks/gatekeeper.py index 4c856c1f7..73ab1abe9 100644 --- a/tasks/gatekeeper.py +++ b/tasks/gatekeeper.py @@ -12,10 +12,9 @@ def control(ctx): build_jobs = [job for job in gl.keys() if job.startswith("build")] jobs = get_jobs(os.environ["CI_PIPELINE_ID"]) for job in expected_jobs(ctx, gl, build_jobs): - print(f"Checking job {job}") job_info = next((j for j in jobs if j["name"] == job), None) if job_info and job_info["status"] != "success": - Exit(f"Job {job} failed and is required", 1) + raise Exit(f"Job {job} failed and is required", 1) print("All required jobs passed") class ReferenceTag(yaml.YAMLObject): @@ -37,7 +36,6 @@ def to_yaml(cls, dumper, data): def expected_jobs(ctx, gl, build_jobs): modified_files = _get_modified_files(ctx, "main") - print("modified", modified_files) for job in build_jobs: change_paths = [] for rule in gl[job]["rules"]: @@ -48,6 +46,7 @@ def expected_jobs(ctx, gl, build_jobs): for file in modified_files: if re.match(fnmatch.translate(path), file): yield job + break def _get_modified_files(ctx, base): last_main_commit = ctx.run(f"git merge-base HEAD origin/{base}", hide=True).stdout From 386e5ede5f3e4423a59fb61c260f28288c087ca4 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Sun, 21 Apr 2024 22:45:41 +0200 Subject: [PATCH 04/11] save image version as variable --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a7828064b..ec079322c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -218,7 +218,7 @@ build_ci_image: --file ci/Dockerfile \ --platform linux/amd64 \ --label target=build \ - --tag registry.ddbuild.io/${CI_IMAGE_REPO}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} \ + --tag registry.ddbuild.io/${CI_IMAGE_REPO}:$IMAGE_VERSION \ --push \ . rules: @@ -543,14 +543,14 @@ trigger_tests: variables: RUN_KITCHEN_TESTS: "false" BUCKET_BRANCH: "dev" - DATADOG_AGENT_BUILDIMAGES: "v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" + DATADOG_AGENT_BUILDIMAGES: "$IMAGE_VERSION" DATADOG_AGENT_BUILDIMAGES_SUFFIX: "${ECR_TEST_ONLY}" DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "${ECR_TEST_ONLY}" - DATADOG_AGENT_WINBUILDIMAGES: "v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" + DATADOG_AGENT_WINBUILDIMAGES: "$IMAGE_VERSION" DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "${ECR_TEST_ONLY}" - DATADOG_AGENT_ARMBUILDIMAGES: "v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" + DATADOG_AGENT_ARMBUILDIMAGES: "$IMAGE_VERSION" DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "${ECR_TEST_ONLY}" - DATADOG_AGENT_SYSPROBE_BUILDIMAGES: "v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" + DATADOG_AGENT_SYSPROBE_BUILDIMAGES: "$IMAGE_VERSION" trigger: project: DataDog/datadog-agent strategy: depend From dcad5409eb227720cdebf69b59e3713bc7402d9c Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Thu, 24 Oct 2024 16:29:38 +0200 Subject: [PATCH 05/11] Fix configuration --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ec079322c..62b632625 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,7 +80,7 @@ default: - api_failure # Change path for "dynamic" generation of the pipeline -.on_config_files: &on_config_files +.on_config_files: changes: paths: - .awsconfig @@ -90,7 +90,7 @@ default: - tasks/** compare_to: 'main' -.on_python_setup: &on_python_setup +.on_python_setup: changes: paths: - python-packages-versions.txt @@ -100,7 +100,7 @@ default: - setup_python.sh compare_to: 'main' -.manual: &manual +.manual: - when: manual allow_failure: true @@ -352,13 +352,13 @@ build_system-probe_x64: build_system-probe_arm64: extends: [.build, .arm] rules: - - <<: *on_config_files - - <<: *on_python_setup + - !reference [.on_config_files] + - !reference [.on_python_setup] - changes: paths: - system-probe_arm64/** compare_to: 'main' - - <<: *manual + - !reference [.manual] variables: DOCKERFILE: system-probe_arm64/Dockerfile IMAGE: system-probe_arm64 @@ -606,7 +606,7 @@ push_to_datadog_agent: build_windows_1809_x64: extends: .winbuild rules: - - <<: *on_config_files + - !reference [.on_config_files] - changes: paths: - go.env @@ -616,7 +616,7 @@ build_windows_1809_x64: - python-packages-versions.txt - windows/** compare_to: 'main' - - <<: *manual + - !reference [.manual] tags: ["runner:windows-docker", "windowsversion:1809"] variables: DOCKERFILE: windows/Dockerfile From 5b45bd2a03574e9bdc2814e11bc3387e16b17011 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Thu, 24 Oct 2024 17:45:28 +0200 Subject: [PATCH 06/11] Generate child pipeline --- .gitlab-ci.yml | 52 ++++++++++++++++++++++------- .gitlab/trigger_template.yml | 57 ++++++++++++++++++++++++++++++++ tasks/__init__.py | 4 +-- tasks/gatekeeper.py | 64 ------------------------------------ tasks/gitlab.py | 21 ++++++++++++ 5 files changed, 121 insertions(+), 77 deletions(-) create mode 100644 .gitlab/trigger_template.yml delete mode 100644 tasks/gatekeeper.py create mode 100644 tasks/gitlab.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62b632625..a32187c17 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -168,6 +168,10 @@ get_agent_version: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi + - touch $CI_WORKING_DIR/built_$IMAGE.txt + artifacts: + paths: + - $CI_WORKING_DIR/built_$IMAGE.txt .build_ddregistry: @@ -208,19 +212,27 @@ get_agent_version: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi + - touch $CI_WORKING_DIR/built_$IMAGE.txt + artifacts: + paths: + - $CI_WORKING_DIR/built_$IMAGE.txt build_ci_image: stage: build image: "${CI_IMAGE}" tags: ["arch:amd64"] - script: |- - docker buildx build \ + script: + - docker buildx build \ --file ci/Dockerfile \ --platform linux/amd64 \ --label target=build \ --tag registry.ddbuild.io/${CI_IMAGE_REPO}:$IMAGE_VERSION \ --push \ . + - touch $CI_WORKING_DIR/built_$CI_IMAGE.txt + artifacts: + paths: + - $CI_WORKING_DIR/built_$CI_IMAGE.txt rules: - changes: paths: @@ -501,6 +513,10 @@ build_circleci_runner: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi + - touch $CI_WORKING_DIR/built_$IMAGE.txt + artifacts: + paths: + - $CI_WORKING_DIR/built_$IMAGE.txt build_gitlab_agent_deploy: extends: [.build, .x64] @@ -535,6 +551,25 @@ trigger_build_kernels: - local: .gitlab/kernel_version_testing.yml strategy: depend +generate_tests: + stage: test + tags: ["arch:amd64"] + image: $SETUP_IMAGE_NAME + script: + - inv -e gitlab.generate-tests -t $ECR_TEST_ONLY + artifacts: + paths: + - $CI_PROJECT_DIR/datadog-agent-trigger-gitlab-ci.yml + +trigger_datadog_agent_tests: + stage: test + needs: [generate_tests] + trigger: + include: + - artifact: datadog-agent-trigger-gitlab-ci.yml + job: generate_tests + allow_failure: true + trigger_tests: stage: test rules: @@ -602,6 +637,10 @@ push_to_datadog_agent: - docker rmi $SRC_IMAGE - If ($CI_JOB_STATUS -ne "success") { Write-Host "Build failed."; exit 0 } - If ($CI_PIPELINE_SOURCE -ne "schedule") { Write-Host "Image $SRC_IMAGE is available." } else { Write-Host "Scheduled pipeline, image push skipped." } + - New-Item -Path $CI_WORKING_DIR -Name "built_$IMAGE.txt" -ItemType "file" + artifacts: + paths: + - $CI_WORKING_DIR/built_$IMAGE.txt build_windows_1809_x64: extends: .winbuild @@ -784,15 +823,6 @@ release_circleci_runner_gcr: project: DataDog/public-images strategy: depend -gatekeeper: - stage: monitoring - tags: ["arch:amd64"] - image: $SETUP_IMAGE_NAME - dependencies: [] - script: - - echo "Gatekeeper job to ensure that the pipeline is running correctly" - - pip install invoke requests - - inv gatekeeper.control notify-images-available: extends: .slack-notifier-base diff --git a/.gitlab/trigger_template.yml b/.gitlab/trigger_template.yml new file mode 100644 index 000000000..86e92c3eb --- /dev/null +++ b/.gitlab/trigger_template.yml @@ -0,0 +1,57 @@ +--- +stages: + - test + +trigger_tests: + stage: test + rules: + - when: manual + allow_failure: true + variables: + RUN_KITCHEN_TESTS: "false" + BUCKET_BRANCH: "dev" + DATADOG_AGENT_BUILDIMAGES: "" + DATADOG_AGENT_BUILDIMAGES_SUFFIX: "" + DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "" + DATADOG_AGENT_WINBUILDIMAGES: "" + DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "" + DATADOG_AGENT_ARMBUILDIMAGES: "" + DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "" + DATADOG_AGENT_SYSPROBE_BUILDIMAGES: "" + CI_IMAGE_AGENT_DEPLOY: "" + CI_IMAGE_AGENT_DEPLOY_SUFFIX: "" + CI_IMAGE_BTFGEN: "" + CI_IMAGE_BTFGEN_SUFFIX: "" + CI_IMAGE_DEB_X64: "" + CI_IMAGE_DEB_X64_SUFFIX: "" + CI_IMAGE_DEB_ARM64: "" + CI_IMAGE_DEB_ARM64_SUFFIX: "" + CI_IMAGE_DEB_ARMHF: "" + CI_IMAGE_DEB_ARMHF_SUFFIX: "" + CI_IMAGE_DD_AGENT_TESTING: "" + CI_IMAGE_DD_AGENT_TESTING_SUFFIX: "" + CI_IMAGE_DOCKER_X64: "" + CI_IMAGE_DOCKER_X64_SUFFIX: "" + CI_IMAGE_DOCKER_ARM64: "" + CI_IMAGE_DOCKER_ARM64_SUFFIX: "" + CI_IMAGE_GLIBC_2_17_X64: "" + CI_IMAGE_GLIBC_2_17_X64_SUFFIX: "" + CI_IMAGE_GLIBC_2_23_ARM64: "" + CI_IMAGE_GLIBC_2_23_ARM64_SUFFIX: "" + CI_IMAGE_SYSTEM_PROBE_X64: "" + CI_IMAGE_SYSTEM_PROBE_X64_SUFFIX: "" + CI_IMAGE_SYSTEM_PROBE_ARM64: "" + CI_IMAGE_SYSTEM_PROBE_ARM64_SUFFIX: "" + CI_IMAGE_RPM_X64: "" + CI_IMAGE_RPM_X64_SUFFIX: "" + CI_IMAGE_RPM_ARM64: "" + CI_IMAGE_RPM_ARM64_SUFFIX: "" + CI_IMAGE_RPM_ARMHF: "" + CI_IMAGE_RPM_ARMHF_SUFFIX: "" + CI_IMAGE_WIN_1809_X64: "" + CI_IMAGE_WIN_1809_X64_SUFFIX: "" + CI_IMAGE_WIN_LTSC2022_X64: "" + CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: "" + trigger: + project: DataDog/datadog-agent + strategy: depend diff --git a/tasks/__init__.py b/tasks/__init__.py index 733fdf6b4..d1481b7af 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -5,7 +5,7 @@ from invoke import Collection -from tasks import agent +from tasks import agent, gitlab from tasks.datadog_agent import update_datadog_agent_buildimages from tasks.update_go import update_go @@ -15,7 +15,7 @@ # add single tasks to the root ns.add_collection(agent) -ns.add_collection(gatekeeper) +ns.add_collection(gitlab) ns.add_task(update_go) ns.add_task(update_datadog_agent_buildimages) ns.configure( diff --git a/tasks/gatekeeper.py b/tasks/gatekeeper.py deleted file mode 100644 index 73ab1abe9..000000000 --- a/tasks/gatekeeper.py +++ /dev/null @@ -1,64 +0,0 @@ -import yaml -import re -import os -import requests -import fnmatch -from invoke import task, Exit -from collections import UserList - -@task -def control(ctx): - gl = _load_gitlab_config() - build_jobs = [job for job in gl.keys() if job.startswith("build")] - jobs = get_jobs(os.environ["CI_PIPELINE_ID"]) - for job in expected_jobs(ctx, gl, build_jobs): - job_info = next((j for j in jobs if j["name"] == job), None) - if job_info and job_info["status"] != "success": - raise Exit(f"Job {job} failed and is required", 1) - print("All required jobs passed") - -class ReferenceTag(yaml.YAMLObject): - """ - Custom yaml tag to handle references in gitlab-ci configuration - """ - yaml_tag = "!reference" - - def __init__(self, references): - self.references = references - - @classmethod - def from_yaml(cls, loader, node): - return UserList(loader.construct_sequence(node)) - - @classmethod - def to_yaml(cls, dumper, data): - return dumper.represent_sequence(cls.yaml_tag, data.data, flow_style=True) - -def expected_jobs(ctx, gl, build_jobs): - modified_files = _get_modified_files(ctx, "main") - for job in build_jobs: - change_paths = [] - for rule in gl[job]["rules"]: - if "changes" in rule: - change_paths += rule["changes"]["paths"] - # print(f"in job {job} with changes {change_paths}") - for path in change_paths: - for file in modified_files: - if re.match(fnmatch.translate(path), file): - yield job - break - -def _get_modified_files(ctx, base): - last_main_commit = ctx.run(f"git merge-base HEAD origin/{base}", hide=True).stdout - modified_files = ctx.run(f"git diff --name-only --no-renames {last_main_commit}", hide=True).stdout.splitlines() - return modified_files - -def _load_gitlab_config(): - yaml.SafeLoader.add_constructor(ReferenceTag.yaml_tag, ReferenceTag.from_yaml) - with open(".gitlab-ci.yml") as f: - return yaml.safe_load(f) - -def get_jobs(pipeline): - headers = {"PRIVATE-TOKEN": os.environ["GITLAB_TOKEN"]} - url = f"https://gitlab.ddbuild.io/api/v4/projects/291/pipelines/{pipeline}/jobs?per_page=100" - return requests.get(url, headers=headers).json() \ No newline at end of file diff --git a/tasks/gitlab.py b/tasks/gitlab.py new file mode 100644 index 000000000..c4a270ef7 --- /dev/null +++ b/tasks/gitlab.py @@ -0,0 +1,21 @@ +import yaml +import os +from invoke import task + +@task +def generate_test(_): + with open(".gitlab/trigger_template.yml") as f: + trigger_template = yaml.safe_load(f) + version = f"v{os.environ['CI_PIPELINE_ID']}-{os.environ['CI_COMMIT_SHORT_SHA']}" + + for file in os.listdir(os.environ["CI_PROJECT_DIR"]): + if file.startswith("built"): + print(f"Adding {file} to the trigger template") + image = file.removeprefix("built_").removesuffix(".txt").replace("_", "").replace("-", "").casefold() + for variable, value in trigger_template["variables"].items(): + if variable.replace("_", "").replace("-", "").casefold()["key"] == image: + variable["value"] = version + break + print(f"triggering with {trigger_template}") + with open("datadog-agent-trigger-gitlab-ci.yml", "w") as f: + yaml.dump(trigger_template, f) From 3cf05675b5c4c1aaec12caab51728700b463dd61 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Fri, 25 Oct 2024 12:12:52 +0200 Subject: [PATCH 07/11] fix(generate_test): Improve the trigger test generation - Fix the task name typo - Aligned the variable names to the IMAGE name used in gitlab configuration - Add tests for the parts of the generate task --- .gitignore | 1 + .gitlab-ci.yml | 16 +-- .gitlab/trigger_template.yml | 26 ++-- tasks/gitlab.py | 57 ++++++-- tasks/unit_tests/gitlab_tests.py | 134 ++++++++++++++++++ .../test_data/all_images/built_btf-gen.txt | 0 .../all_images/built_dd-agent-testing.txt | 0 .../test_data/all_images/built_deb_arm64.txt | 0 .../test_data/all_images/built_deb_armhf.txt | 0 .../test_data/all_images/built_deb_x64.txt | 0 .../all_images/built_docker_arm64.txt | 0 .../test_data/all_images/built_docker_x64.txt | 0 .../all_images/built_gitlab_agent_deploy.txt | 0 .../all_images/built_linux-glibc-2-17-x64.txt | 0 .../built_linux-glibc-2-23-arm64.txt | 0 .../test_data/all_images/built_rpm_arm64.txt | 0 .../test_data/all_images/built_rpm_armhf.txt | 0 .../test_data/all_images/built_rpm_x64.txt | 0 .../all_images/built_system-probe_arm64.txt | 0 .../all_images/built_system-probe_x64.txt | 0 .../all_images/built_windows_1809_x64.txt | 0 .../all_images/built_windows_ltsc2022_x64.txt | 0 .../test_data/one_image/built_btf-gen.txt | 0 23 files changed, 199 insertions(+), 35 deletions(-) create mode 100644 tasks/unit_tests/gitlab_tests.py create mode 100644 tasks/unit_tests/test_data/all_images/built_btf-gen.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_dd-agent-testing.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_deb_arm64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_deb_armhf.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_deb_x64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_docker_arm64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_docker_x64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_gitlab_agent_deploy.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_linux-glibc-2-17-x64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_linux-glibc-2-23-arm64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_rpm_arm64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_rpm_armhf.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_rpm_x64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_system-probe_arm64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_system-probe_x64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_windows_1809_x64.txt create mode 100644 tasks/unit_tests/test_data/all_images/built_windows_ltsc2022_x64.txt create mode 100644 tasks/unit_tests/test_data/one_image/built_btf-gen.txt diff --git a/.gitignore b/.gitignore index 73e8f8a1a..20027c1ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vscode/settings.json **/__pycache__ venv +datadog-agent-trigger-gitlab-ci.yml \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a32187c17..d9f5af8f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -168,7 +168,7 @@ get_agent_version: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi - - touch $CI_WORKING_DIR/built_$IMAGE.txt + - touch built_$IMAGE.txt artifacts: paths: - $CI_WORKING_DIR/built_$IMAGE.txt @@ -212,7 +212,7 @@ get_agent_version: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi - - touch $CI_WORKING_DIR/built_$IMAGE.txt + - touch built_$IMAGE.txt artifacts: paths: - $CI_WORKING_DIR/built_$IMAGE.txt @@ -229,7 +229,7 @@ build_ci_image: --tag registry.ddbuild.io/${CI_IMAGE_REPO}:$IMAGE_VERSION \ --push \ . - - touch $CI_WORKING_DIR/built_$CI_IMAGE.txt + - touch built_$CI_IMAGE.txt artifacts: paths: - $CI_WORKING_DIR/built_$CI_IMAGE.txt @@ -513,10 +513,6 @@ build_circleci_runner: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi - - touch $CI_WORKING_DIR/built_$IMAGE.txt - artifacts: - paths: - - $CI_WORKING_DIR/built_$IMAGE.txt build_gitlab_agent_deploy: extends: [.build, .x64] @@ -556,7 +552,7 @@ generate_tests: tags: ["arch:amd64"] image: $SETUP_IMAGE_NAME script: - - inv -e gitlab.generate-tests -t $ECR_TEST_ONLY + - inv -e gitlab.generate-test -t $ECR_TEST_ONLY artifacts: paths: - $CI_PROJECT_DIR/datadog-agent-trigger-gitlab-ci.yml @@ -637,10 +633,10 @@ push_to_datadog_agent: - docker rmi $SRC_IMAGE - If ($CI_JOB_STATUS -ne "success") { Write-Host "Build failed."; exit 0 } - If ($CI_PIPELINE_SOURCE -ne "schedule") { Write-Host "Image $SRC_IMAGE is available." } else { Write-Host "Scheduled pipeline, image push skipped." } - - New-Item -Path $CI_WORKING_DIR -Name "built_$IMAGE.txt" -ItemType "file" + - New-Item -Name "built_$IMAGE.txt" -ItemType "file" artifacts: paths: - - $CI_WORKING_DIR/built_$IMAGE.txt + - built_$IMAGE.txt build_windows_1809_x64: extends: .winbuild diff --git a/.gitlab/trigger_template.yml b/.gitlab/trigger_template.yml index 86e92c3eb..89a50f8b6 100644 --- a/.gitlab/trigger_template.yml +++ b/.gitlab/trigger_template.yml @@ -2,7 +2,7 @@ stages: - test -trigger_tests: +trigger_datadog_agent: stage: test rules: - when: manual @@ -18,10 +18,8 @@ trigger_tests: DATADOG_AGENT_ARMBUILDIMAGES: "" DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "" DATADOG_AGENT_SYSPROBE_BUILDIMAGES: "" - CI_IMAGE_AGENT_DEPLOY: "" - CI_IMAGE_AGENT_DEPLOY_SUFFIX: "" - CI_IMAGE_BTFGEN: "" - CI_IMAGE_BTFGEN_SUFFIX: "" + CI_IMAGE_BTF_GEN: "" + CI_IMAGE_BTF_GEN_SUFFIX: "" CI_IMAGE_DEB_X64: "" CI_IMAGE_DEB_X64_SUFFIX: "" CI_IMAGE_DEB_ARM64: "" @@ -34,10 +32,12 @@ trigger_tests: CI_IMAGE_DOCKER_X64_SUFFIX: "" CI_IMAGE_DOCKER_ARM64: "" CI_IMAGE_DOCKER_ARM64_SUFFIX: "" - CI_IMAGE_GLIBC_2_17_X64: "" - CI_IMAGE_GLIBC_2_17_X64_SUFFIX: "" - CI_IMAGE_GLIBC_2_23_ARM64: "" - CI_IMAGE_GLIBC_2_23_ARM64_SUFFIX: "" + CI_IMAGE_GITLAB_AGENT_DEPLOY: "" + CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX: "" + CI_IMAGE_LINUX_GLIBC_2_17_X64: "" + CI_IMAGE_LINUX_GLIBC_2_17_X64_SUFFIX: "" + CI_IMAGE_LINUX_GLIBC_2_23_ARM64: "" + CI_IMAGE_LINUX_GLIBC_2_23_ARM64_SUFFIX: "" CI_IMAGE_SYSTEM_PROBE_X64: "" CI_IMAGE_SYSTEM_PROBE_X64_SUFFIX: "" CI_IMAGE_SYSTEM_PROBE_ARM64: "" @@ -48,10 +48,10 @@ trigger_tests: CI_IMAGE_RPM_ARM64_SUFFIX: "" CI_IMAGE_RPM_ARMHF: "" CI_IMAGE_RPM_ARMHF_SUFFIX: "" - CI_IMAGE_WIN_1809_X64: "" - CI_IMAGE_WIN_1809_X64_SUFFIX: "" - CI_IMAGE_WIN_LTSC2022_X64: "" - CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: "" + CI_IMAGE_WINDOWS_1809_X64: "" + CI_IMAGE_WINDOWS_1809_X64_SUFFIX: "" + CI_IMAGE_WINDOWS_LTSC2022_X64: "" + CI_IMAGE_WINDOWS_LTSC2022_X64_SUFFIX: "" trigger: project: DataDog/datadog-agent strategy: depend diff --git a/tasks/gitlab.py b/tasks/gitlab.py index c4a270ef7..a958c1a79 100644 --- a/tasks/gitlab.py +++ b/tasks/gitlab.py @@ -2,20 +2,53 @@ import os from invoke import task + @task -def generate_test(_): +def generate_test(_, test=""): + """ + Generate a gitlab yml configuration to trigger datadog-agent pipeline with only the modified images updated + """ with open(".gitlab/trigger_template.yml") as f: trigger_template = yaml.safe_load(f) - version = f"v{os.environ['CI_PIPELINE_ID']}-{os.environ['CI_COMMIT_SHORT_SHA']}" - - for file in os.listdir(os.environ["CI_PROJECT_DIR"]): - if file.startswith("built"): - print(f"Adding {file} to the trigger template") - image = file.removeprefix("built_").removesuffix(".txt").replace("_", "").replace("-", "").casefold() - for variable, value in trigger_template["variables"].items(): - if variable.replace("_", "").replace("-", "").casefold()["key"] == image: - variable["value"] = version - break - print(f"triggering with {trigger_template}") + built_images = list_built_images() + trigger_template["trigger_datadog_agent"]["variables"] = update_variables( + trigger_template["trigger_datadog_agent"]["variables"], built_images, test + ) + print(f"Generated test with images: {built_images}") + print(f"Gitlab configuration: {trigger_template}") with open("datadog-agent-trigger-gitlab-ci.yml", "w") as f: yaml.dump(trigger_template, f) + + +def list_built_images(): + """ + Reshape the artefacts generated by build stage to list what image has been built + """ + return [ + file.removeprefix("built_").removesuffix(".txt").replace("-", "_").casefold() + for file in os.listdir(os.environ["CI_PROJECT_DIR"]) + if file.startswith("built_") + ] + + +def update_variables(variables, images, test): + """ + Update the variables to be used by the trigger_datadog_agent job. We always update the former variables + """ + version = f"v{os.environ['CI_PIPELINE_ID']}-{os.environ['CI_COMMIT_SHORT_SHA']}" + for image in images: + variable = f"CI_IMAGE_{image.upper()}" + if variable in variables: + # print(f"Updating {variable} to {version}") + variables[variable] = version + if test: + variables[f"{variable}_SUFFIX"] = test + else: + print(f"WARNING variable {variable} not found in the template") + # Update the former images as well + for variable in variables: + if variable.startswith("DATADOG_AGENT_") and not "SUFFIX" in variable: + variables[variable] = version + if test: + variables[f"{variable}_SUFFIX"] = test + return variables diff --git a/tasks/unit_tests/gitlab_tests.py b/tasks/unit_tests/gitlab_tests.py new file mode 100644 index 000000000..97c99fdbf --- /dev/null +++ b/tasks/unit_tests/gitlab_tests.py @@ -0,0 +1,134 @@ +import unittest +import os +import yaml + +from unittest.mock import patch + +import tasks.gitlab as gitlab + + +class TestListBuildImages(unittest.TestCase): + @patch.dict( + os.environ, + {"CI_PROJECT_DIR": "tasks/unit_tests/test_data/one_image"}, + clear=True, + ) + def test_one_image(self): + self.assertListEqual(gitlab.list_built_images(), ["btf_gen"]) + + @patch.dict( + os.environ, + {"CI_PROJECT_DIR": "tasks/unit_tests/test_data/all_images"}, + clear=True, + ) + def test_all_images(self): + self.assertListEqual( + gitlab.list_built_images(), + [ + "docker_x64", + "deb_armhf", + "docker_arm64", + "windows_ltsc2022_x64", + "windows_1809_x64", + "rpm_armhf", + "gitlab_agent_deploy", + "system_probe_arm64", + "btf_gen", + "deb_arm64", + "linux_glibc_2_17_x64", + "deb_x64", + "system_probe_x64", + "linux_glibc_2_23_arm64", + "rpm_arm64", + "dd_agent_testing", + "rpm_x64", + ], + ) + + @patch.dict( + os.environ, {"CI_PROJECT_DIR": "tasks/unit_tests/test_data"}, clear=True + ) + def test_no_update(self): + self.assertListEqual(gitlab.list_built_images(), []) + + +class TestUpdateVariables(unittest.TestCase): + def setUp(self): + with open(".gitlab/trigger_template.yml") as f: + trigger_template = yaml.safe_load(f) + self.variables = trigger_template["trigger_datadog_agent"]["variables"] + + @patch.dict( + os.environ, + {"CI_PIPELINE_ID": "42", "CI_COMMIT_SHORT_SHA": "5h0rtc4t"}, + clear=True, + ) + def test_one_update(self): + images = ["btf_gen"] + test = "" + new_variables = gitlab.update_variables(self.variables, images, test) + self.assertEqual(new_variables["RUN_KITCHEN_TESTS"], "false") + self.assertEqual(new_variables["BUCKET_BRANCH"], "dev") + self.assertEqual( + sum([1 for v in new_variables.values() if v == "v42-5h0rtc4t"]), 5 + ) # 4 for former variables + 1 for btf_gen + self.assertEqual(sum([1 for v in new_variables.values() if v == ""]), 37) + self.assertEqual( + sum([1 for v in new_variables.values() if v == "_test_only"]), 0 + ) + + @patch.dict( + os.environ, + {"CI_PIPELINE_ID": "1789", "CI_COMMIT_SHORT_SHA": "fr4nc3"}, + clear=True, + ) + def test_all_updates_with_test(self): + images = [ + "docker_x64", + "deb_armhf", + "docker_arm64", + "windows_ltsc2022_x64", + "windows_1809_x64", + "rpm_armhf", + "gitlab_agent_deploy", + "system_probe_arm64", + "btf_gen", + "deb_arm64", + "linux_glibc_2_17_x64", + "deb_x64", + "system_probe_x64", + "linux_glibc_2_23_arm64", + "rpm_arm64", + "dd_agent_testing", + "rpm_x64", + ] + test = "_test_only" + new_variables = gitlab.update_variables(self.variables, images, test) + self.assertEqual(new_variables["RUN_KITCHEN_TESTS"], "false") + self.assertEqual(new_variables["BUCKET_BRANCH"], "dev") + self.assertEqual( + sum([1 for v in new_variables.values() if v == "v1789-fr4nc3"]), 21 + ) + self.assertEqual( + sum([1 for v in new_variables.values() if v == "_test_only"]), 21 + ) + self.assertEqual(sum([1 for v in new_variables.values() if v == ""]), 0) + + @patch.dict( + os.environ, + {"CI_PIPELINE_ID": "1000000", "CI_COMMIT_SHORT_SHA": "m1n10n5"}, + clear=True, + ) + def test_updates_no_images(self): + images = ["circleci_runner"] + test = "_test_only" + new_variables = gitlab.update_variables(self.variables, images, test) + self.assertEqual(new_variables["RUN_KITCHEN_TESTS"], "false") + self.assertEqual(new_variables["BUCKET_BRANCH"], "dev") + self.assertEqual( + sum([1 for v in new_variables.values() if v == "v1000000-m1n10n5"]), 4 + ) # only BUILDIMAGES + self.assertEqual( + sum([1 for v in new_variables.values() if v == "_test_only"]), 4 + ) + self.assertEqual(sum([1 for v in new_variables.values() if v == ""]), 34) diff --git a/tasks/unit_tests/test_data/all_images/built_btf-gen.txt b/tasks/unit_tests/test_data/all_images/built_btf-gen.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_dd-agent-testing.txt b/tasks/unit_tests/test_data/all_images/built_dd-agent-testing.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_deb_arm64.txt b/tasks/unit_tests/test_data/all_images/built_deb_arm64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_deb_armhf.txt b/tasks/unit_tests/test_data/all_images/built_deb_armhf.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_deb_x64.txt b/tasks/unit_tests/test_data/all_images/built_deb_x64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_docker_arm64.txt b/tasks/unit_tests/test_data/all_images/built_docker_arm64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_docker_x64.txt b/tasks/unit_tests/test_data/all_images/built_docker_x64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_gitlab_agent_deploy.txt b/tasks/unit_tests/test_data/all_images/built_gitlab_agent_deploy.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_linux-glibc-2-17-x64.txt b/tasks/unit_tests/test_data/all_images/built_linux-glibc-2-17-x64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_linux-glibc-2-23-arm64.txt b/tasks/unit_tests/test_data/all_images/built_linux-glibc-2-23-arm64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_rpm_arm64.txt b/tasks/unit_tests/test_data/all_images/built_rpm_arm64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_rpm_armhf.txt b/tasks/unit_tests/test_data/all_images/built_rpm_armhf.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_rpm_x64.txt b/tasks/unit_tests/test_data/all_images/built_rpm_x64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_system-probe_arm64.txt b/tasks/unit_tests/test_data/all_images/built_system-probe_arm64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_system-probe_x64.txt b/tasks/unit_tests/test_data/all_images/built_system-probe_x64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_windows_1809_x64.txt b/tasks/unit_tests/test_data/all_images/built_windows_1809_x64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/all_images/built_windows_ltsc2022_x64.txt b/tasks/unit_tests/test_data/all_images/built_windows_ltsc2022_x64.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tasks/unit_tests/test_data/one_image/built_btf-gen.txt b/tasks/unit_tests/test_data/one_image/built_btf-gen.txt new file mode 100644 index 000000000..e69de29bb From ac78bde385e13a0b5360e18c3dc6c6eac1c29b9c Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Fri, 25 Oct 2024 18:33:54 +0200 Subject: [PATCH 08/11] Use the correct CI variable --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d9f5af8f4..45adf859c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -168,10 +168,10 @@ get_agent_version: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi - - touch built_$IMAGE.txt + - touch $CI_PROJECT_DIR/built_$IMAGE.txt artifacts: paths: - - $CI_WORKING_DIR/built_$IMAGE.txt + - $CI_PROJECT_DIR/built_$IMAGE.txt .build_ddregistry: @@ -212,10 +212,10 @@ get_agent_version: else echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m" fi - - touch built_$IMAGE.txt + - touch $CI_PROJECT_DIR/built_$IMAGE.txt artifacts: paths: - - $CI_WORKING_DIR/built_$IMAGE.txt + - $CI_PROJECT_DIR/built_$IMAGE.txt build_ci_image: stage: build @@ -229,10 +229,10 @@ build_ci_image: --tag registry.ddbuild.io/${CI_IMAGE_REPO}:$IMAGE_VERSION \ --push \ . - - touch built_$CI_IMAGE.txt + - touch $CI_PROJECT_DIR/built_$CI_IMAGE.txt artifacts: paths: - - $CI_WORKING_DIR/built_$CI_IMAGE.txt + - $CI_PROJECT_DIR/built_$CI_IMAGE.txt rules: - changes: paths: From 5e8d2ed28c559120d6044ab5933f63d97d53ff57 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Thu, 31 Oct 2024 09:50:01 +0100 Subject: [PATCH 09/11] remove redundant test job --- .gitignore | 2 +- .gitlab-ci.yml | 20 -------------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 20027c1ea..430e0251d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ .vscode/settings.json **/__pycache__ venv -datadog-agent-trigger-gitlab-ci.yml \ No newline at end of file +datadog-agent-trigger-gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 45adf859c..c8a604d58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -566,26 +566,6 @@ trigger_datadog_agent_tests: job: generate_tests allow_failure: true -trigger_tests: - stage: test - rules: - - when: manual - allow_failure: true - variables: - RUN_KITCHEN_TESTS: "false" - BUCKET_BRANCH: "dev" - DATADOG_AGENT_BUILDIMAGES: "$IMAGE_VERSION" - DATADOG_AGENT_BUILDIMAGES_SUFFIX: "${ECR_TEST_ONLY}" - DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "${ECR_TEST_ONLY}" - DATADOG_AGENT_WINBUILDIMAGES: "$IMAGE_VERSION" - DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "${ECR_TEST_ONLY}" - DATADOG_AGENT_ARMBUILDIMAGES: "$IMAGE_VERSION" - DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "${ECR_TEST_ONLY}" - DATADOG_AGENT_SYSPROBE_BUILDIMAGES: "$IMAGE_VERSION" - trigger: - project: DataDog/datadog-agent - strategy: depend - push_to_datadog_agent: stage: test rules: From 9e691f9317f58111d625ed5cb363b57d6c080070 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Thu, 31 Oct 2024 09:54:21 +0100 Subject: [PATCH 10/11] Ignore pyenv file --- .gitignore | 1 + .gitlab-ci.yml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 430e0251d..e52bfbd8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vscode/settings.json **/__pycache__ venv +.python-version datadog-agent-trigger-gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c8a604d58..61e0e76cf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ include: stages: - setup - build - - monitoring + - notify_slack - test - dev_envs - devcontainer @@ -802,7 +802,7 @@ release_circleci_runner_gcr: notify-images-available: extends: .slack-notifier-base - stage: monitoring # Created a new stage to depend upon all image builds without explicitely name them in 'needs' + stage: notify_slack # Created a new stage to depend upon all image builds without explicitely name them in 'needs' rules: - if: $CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "main" when: never @@ -820,7 +820,7 @@ notify-images-available: notify-images-failure: extends: .slack-notifier-base - stage: monitoring + stage: notify_slack rules: - if: $CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "main" when: never From a2e9215e2ea6358da88f3db3bddcf3abd64a7d31 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Tue, 5 Nov 2024 10:26:57 +0100 Subject: [PATCH 11/11] Update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5fa1d425c..65fa9901e 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ ![Docker Image Version](https://img.shields.io/docker/v/datadog/agent-buildimages-deb_x64) ![GitHub License](https://img.shields.io/github/license/datadog/datadog-agent-buildimages) +update readme This repo contains the Dockerfiles of the images used to build the rpm and deb packages for the Datadog [Agent][agent].