Generate child pipeline

.gitlab-ci.yml

@@ -158,6 +158,10 @@ get_agent_version:
       else
         echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m"
       fi
+    - touch $CI_WORKING_DIR/built_$IMAGE.txt
+  artifacts:
+    paths:
+      - $CI_WORKING_DIR/built_$IMAGE.txt
 
 
 .build_ddregistry:
@@ -188,19 +192,27 @@ get_agent_version:
       else
         echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m"
       fi
+    - touch $CI_WORKING_DIR/built_$IMAGE.txt
+  artifacts:
+    paths:
+      - $CI_WORKING_DIR/built_$IMAGE.txt
 
 build_ci_image:
   stage: build
   image: "${CI_IMAGE}"
   tags: ["arch:amd64"]
-  script: |-
-    docker buildx build \
+  script:
+    - docker buildx build \
       --file ci/Dockerfile \
       --platform linux/amd64 \
       --label target=build \
       --tag registry.ddbuild.io/${CI_IMAGE_REPO}:$IMAGE_VERSION \
       --push \
       .
+    - touch $CI_WORKING_DIR/built_$CI_IMAGE.txt
+  artifacts:
+    paths:
+      - $CI_WORKING_DIR/built_$CI_IMAGE.txt
   rules:
     - changes:
         paths:
@@ -481,6 +493,10 @@ build_circleci_runner:
       else
         echo -e "\033[0;33m\033[1mScheduled pipeline, image push skipped.\033[0m"
       fi
+    - touch $CI_WORKING_DIR/built_$IMAGE.txt
+  artifacts:
+    paths:
+      - $CI_WORKING_DIR/built_$IMAGE.txt
 
 build_gitlab_agent_deploy:
   extends: [.build, .x64]
@@ -515,6 +531,25 @@ trigger_build_kernels:
       - local: .gitlab/kernel_version_testing.yml
     strategy: depend
 
+generate_tests:
+  stage: test
+  tags: ["arch:amd64"]
+  image: $SETUP_IMAGE_NAME
+  script:
+    - inv -e gitlab.generate-tests -t $ECR_TEST_ONLY
+  artifacts:
+    paths:
+      - $CI_PROJECT_DIR/datadog-agent-trigger-gitlab-ci.yml
+
+trigger_datadog_agent_tests:
+  stage: test
+  needs: [generate_tests]
+  trigger:
+    include:
+      - artifact: datadog-agent-trigger-gitlab-ci.yml
+        job: generate_tests
+  allow_failure: true
+
 trigger_tests:
   stage: test
   rules:
@@ -582,6 +617,10 @@ push_to_datadog_agent:
     - docker rmi $SRC_IMAGE
     - If ($CI_JOB_STATUS -ne "success") { Write-Host "Build failed."; exit 0 }
     - If ($CI_PIPELINE_SOURCE -ne "schedule") { Write-Host "Image $SRC_IMAGE is available." } else { Write-Host "Scheduled pipeline, image push skipped." }
+    - New-Item -Path $CI_WORKING_DIR -Name "built_$IMAGE.txt" -ItemType "file"
+  artifacts:
+    paths:
+      - $CI_WORKING_DIR/built_$IMAGE.txt
 
 build_windows_1809_x64:
   extends: .winbuild
@@ -764,15 +803,6 @@ release_circleci_runner_gcr:
     project: DataDog/public-images
     strategy: depend
 
-gatekeeper:
-  stage: monitoring
-  tags: ["arch:amd64"]
-  image: $SETUP_IMAGE_NAME
-  dependencies: []
-  script:
-    - echo "Gatekeeper job to ensure that the pipeline is running correctly"
-    - pip install invoke requests
-    - inv gatekeeper.control
 
 notify-images-available:
   extends: .slack-notifier-base

.gitlab/trigger_template.yml

@@ -0,0 +1,57 @@
+---
+stages:
+  - test
+
+trigger_tests:
+  stage: test
+  rules:
+    - when: manual
+      allow_failure: true
+  variables:
+    RUN_KITCHEN_TESTS: "false"
+    BUCKET_BRANCH: "dev"
+    DATADOG_AGENT_BUILDIMAGES: ""
+    DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
+    DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
+    DATADOG_AGENT_WINBUILDIMAGES: ""
+    DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
+    DATADOG_AGENT_ARMBUILDIMAGES: ""
+    DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
+    DATADOG_AGENT_SYSPROBE_BUILDIMAGES: ""
+    CI_IMAGE_AGENT_DEPLOY: ""
+    CI_IMAGE_AGENT_DEPLOY_SUFFIX: ""
+    CI_IMAGE_BTFGEN: ""
+    CI_IMAGE_BTFGEN_SUFFIX: ""
+    CI_IMAGE_DEB_X64: ""
+    CI_IMAGE_DEB_X64_SUFFIX: ""
+    CI_IMAGE_DEB_ARM64: ""
+    CI_IMAGE_DEB_ARM64_SUFFIX: ""
+    CI_IMAGE_DEB_ARMHF: ""
+    CI_IMAGE_DEB_ARMHF_SUFFIX: ""
+    CI_IMAGE_DD_AGENT_TESTING: ""
+    CI_IMAGE_DD_AGENT_TESTING_SUFFIX: ""
+    CI_IMAGE_DOCKER_X64: ""
+    CI_IMAGE_DOCKER_X64_SUFFIX: ""
+    CI_IMAGE_DOCKER_ARM64: ""
+    CI_IMAGE_DOCKER_ARM64_SUFFIX: ""
+    CI_IMAGE_GLIBC_2_17_X64: ""
+    CI_IMAGE_GLIBC_2_17_X64_SUFFIX: ""
+    CI_IMAGE_GLIBC_2_23_ARM64: ""
+    CI_IMAGE_GLIBC_2_23_ARM64_SUFFIX: ""
+    CI_IMAGE_SYSTEM_PROBE_X64: ""
+    CI_IMAGE_SYSTEM_PROBE_X64_SUFFIX: ""
+    CI_IMAGE_SYSTEM_PROBE_ARM64: ""
+    CI_IMAGE_SYSTEM_PROBE_ARM64_SUFFIX: ""
+    CI_IMAGE_RPM_X64: ""
+    CI_IMAGE_RPM_X64_SUFFIX: ""
+    CI_IMAGE_RPM_ARM64: ""
+    CI_IMAGE_RPM_ARM64_SUFFIX: ""
+    CI_IMAGE_RPM_ARMHF: ""
+    CI_IMAGE_RPM_ARMHF_SUFFIX: ""
+    CI_IMAGE_WIN_1809_X64: ""
+    CI_IMAGE_WIN_1809_X64_SUFFIX: ""
+    CI_IMAGE_WIN_LTSC2022_X64: ""
+    CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: ""
+  trigger:
+    project: DataDog/datadog-agent
+    strategy: depend

tasks/__init__.py

@@ -5,7 +5,7 @@
 
 from invoke import Collection
 
-from tasks import agent
+from tasks import agent, gitlab
 from tasks.datadog_agent import update_datadog_agent_buildimages
 from tasks.update_go import update_go
 
@@ -15,7 +15,7 @@
 # add single tasks to the root
 
 ns.add_collection(agent)
-ns.add_collection(gatekeeper)
+ns.add_collection(gitlab)
 ns.add_task(update_go)
 ns.add_task(update_datadog_agent_buildimages)
 ns.configure(

tasks/gitlab.py

@@ -0,0 +1,21 @@
+import yaml
+import os
+from invoke import task
+
+@task
+def generate_test(_):
+    with open(".gitlab/trigger_template.yml") as f:
+        trigger_template = yaml.safe_load(f)
+    version = f"v{os.environ['CI_PIPELINE_ID']}-{os.environ['CI_COMMIT_SHORT_SHA']}"    
+
+    for file in os.listdir(os.environ["CI_PROJECT_DIR"]):
+        if file.startswith("built"):
+            print(f"Adding {file} to the trigger template")
+            image = file.removeprefix("built_").removesuffix(".txt").replace("_", "").replace("-", "").casefold()
+            for variable, value in trigger_template["variables"].items():
+                if variable.replace("_", "").replace("-", "").casefold()["key"] == image:
+                    variable["value"] = version
+                    break
+    print(f"triggering with {trigger_template}")
+    with open("datadog-agent-trigger-gitlab-ci.yml", "w") as f:
+        yaml.dump(trigger_template, f)