Build the builder image on native systems

.github/workflows/collector-builder.yml

@@ -11,7 +11,7 @@ on:
     outputs:
       collector-builder-tag:
         description: The builder tag used by the build
-        value: ${{ jobs.build-builder-image.outputs.collector-builder-tag || 'master' }}
+        value: ${{ jobs.builder-needs-rebuilding.outputs.collector-builder-tag || 'master' }}
 
 env:
   COLLECTOR_TAG: ${{ inputs.collector-tag }}
@@ -23,7 +23,9 @@ jobs:
     name: Determine if builder image needs to be built
     runs-on: ubuntu-24.04
     outputs:
-      build-image: ${{ steps.changed.outputs.builder-changed }}
+      build-image: ${{ steps.check-builder.outputs.build-image }}
+      build-multiarch: ${{ steps.build-multiarch.outputs.build-multiarch }}
+      collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }}
 
     steps:
       - uses: actions/checkout@v4
@@ -38,30 +40,90 @@ jobs:
               - builder/Dockerfile
               - .github/workflows/collector-builder.yml
 
+      - name: Define builder tag
+        id: builder-tag
+        run: |
+          COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}"
+          if [[ "${{ github.event_name }}" == 'pull_request' || \
+                "${{ github.ref_type }}" == 'tag' || \
+                "${{ github.ref_name }}" =~ ^release- ]]; then
+            COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}"
+          fi
+
+          echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV"
+          echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT"
+
+      - name: Check if the builder image should be built
+        id: check-builder
+        run: |
+          function build_builder() {
+            echo "build-image=$1" >> "$GITHUB_OUTPUT"
+            exit 0
+          }
+
+          if [[ "${{ steps.changed.outputs.builder-changed }}" == "true" ]]; then
+            build_builder "true"
+          fi
+
+          if [[ "${GITHUB_EVENT_NAME}" == "schedule" ]]; then
+            build_builder "true"
+          fi
+
+          if [[ "${GITHUB_EVENT_NAME}" == "push" ]]; then
+            if [[ "${GITHUB_REF_TYPE}" == "tag" || "${GITHUB_REF_NAME}" == release-* ]]; then
+              build_builder "true"
+            fi
+
+            build_builder "false"
+          fi
+
+          # If we got here, we are on a PR event
+          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'build-builder-image') }}" == "true" ]]; then
+            build_builder "true"
+          fi
+
+          build_builder "false"
+
+      - name: Specify if multiarch image should be built
+        id: build-multiarch
+        run: |
+          function run_multiarch() {
+            echo "build-multiarch=$1" >> "$GITHUB_OUTPUT"
+            exit 0
+          }
+
+          if [[ "${{ steps.check-builder.outputs.build-image }}" == "false" ]]; then
+            run_multiarch "false"
+          fi
+
+          if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+            # PRs only run multiarch on demand
+            if [[ "${{ contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') }}" == "true" ]]; then
+              run_multiarch "true"
+            fi
+            run_multiarch "false"
+          fi
+
+          # Events that are not on PRs always run multiarch
+          run_multiarch "true"
+
   build-builder-image:
     name: Build the builder image
     runs-on: ubuntu-24.04
     # Multiarch builds sometimes take for eeeeeeeeeever
     timeout-minutes: 480
     needs:
     - builder-needs-rebuilding
-    if: |
-      needs.builder-needs-rebuilding.outputs.build-image == 'true' ||
-      (github.event_name == 'push' && (
-        github.ref_type == 'tag' || startsWith(github.ref_name, 'release-')
-      )) ||
-      contains(github.event.pull_request.labels.*.name, 'build-builder-image') ||
-      github.event_name == 'schedule'
-    outputs:
-      collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }}
+    if: needs.builder-needs-rebuilding.outputs.build-image == 'true'
     strategy:
       fail-fast: false
       matrix:
-        arch: [amd64, ppc64le, s390x, arm64]
+        arch: [amd64]
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
       BUILD_TYPE: ci
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
 
     steps:
       - uses: actions/checkout@v4
@@ -74,13 +136,58 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Create ansible vars
+        run: |
+          cat << EOF > ${{ github.workspace }}/ansible/secrets.yml
+          ---
+          stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
+          stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
+          rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
+          rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
+          collector_git_ref: ${{ github.ref }}
+          collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}
+          clone_repository: false
+          EOF
+
+      - name: Build images
+        timeout-minutes: 480
+        run: |
+          ansible-galaxy install -r ansible/requirements.yml
+          ansible-playbook \
+            --connection local \
+            -i localhost, \
+            --limit localhost \
+            -e arch='${{ matrix.arch }}' \
+            -e @'${{ github.workspace }}/ansible/secrets.yml' \
+            ansible/ci-build-builder.yml
+
+  build-builder-image-remote-vm:
+    name: Build the builder image on a native VM
+    runs-on: ubuntu-24.04
+    timeout-minutes: 480
+    needs:
+    -  builder-needs-rebuilding
+    if: needs.builder-needs-rebuilding.outputs.build-multiarch == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [ppc64le, s390x, arm64]
+
+    env:
+      PLATFORM: linux/${{ matrix.arch }}
+      BUILD_TYPE: ci
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
+
+    steps:
+      - uses: actions/checkout@v4
+
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 
       - uses: 'google-github-actions/auth@v2'
         with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_CI_VM_SVC_ACCT }}'
 
       - uses: 'google-github-actions/setup-gcloud@v2'
 
@@ -95,63 +202,33 @@ jobs:
           ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_API_KEY }}
           redhat-username: ${{ secrets.REDHAT_USERNAME }}
           redhat-password: ${{ secrets.REDHAT_PASSWORD }}
-          vm-type: all
+          # TODO: this only works because RHEL VMs for power, Z and arm only
+          # have a single VM each, we might need to define builder VMs
+          # explicitly in ansible/group_vars/all.yml
+          vm-type: rhel-${{ matrix.arch }}
           job-tag: builder
 
       - name: Create Build VMs
-        if: |
-          matrix.arch == 's390x' &&
-          (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds'))
         run: |
           make -C "${{ github.workspace }}/ansible" create-build-vms
 
-      - name: Define builder tag
-        id: builder-tag
-        run: |
-          COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}"
-          if [[ "${{ github.event_name }}" == 'pull_request' || \
-                "${{ github.ref_type }}" == 'tag' || \
-                "${{ github.ref_name }}" =~ ^release- ]]; then
-            COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}"
-          fi
-
-          echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV"
-          echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT"
-
       - name: Create ansible vars
         run: |
-          {
-            echo "---"
-            echo "stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}"
-            echo "stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}"
-            echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}"
-            echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
-            echo "collector_git_ref: ${{ github.ref }}"
-            echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}"
-          } > ${{ github.workspace }}/ansible/secrets.yml
+          cat << EOF > ${{ github.workspace }}/ansible/secrets.yml
+          ---
+          stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}
+          stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}
+          rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}
+          rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}
+          collector_git_ref: ${{ github.ref }}
+          collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}
+          clone_repository: true
+          EOF
 
       - name: Build images
-        if: |
-          (github.event_name != 'pull_request' && matrix.arch != 's390x') ||
-          matrix.arch == 'amd64' ||
-          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x')
         timeout-minutes: 480
         run: |
           ansible-galaxy install -r ansible/requirements.yml
-          ansible-playbook \
-            --connection local \
-            -i localhost, \
-            --limit localhost \
-            -e arch='${{ matrix.arch }}' \
-            -e @'${{ github.workspace }}/ansible/secrets.yml' \
-            ansible/ci-build-builder.yml
-
-      - name: Build s390x images
-        if: |
-          (github.event_name != 'pull_request' && matrix.arch == 's390x') ||
-          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
-        timeout-minutes: 480
-        run: |
           ansible-playbook \
             -i ansible/ci \
             -e build_hosts='job_id_${{ env.JOB_ID }}' \
@@ -160,21 +237,20 @@ jobs:
             ansible/ci-build-builder.yml
 
       - name: Destroy VMs
-        if: always() && matrix.arch == 's390x'
+        if: always()
         run: |
           make -C ansible destroy-vms
 
   create-multiarch-manifest:
     needs:
+    - builder-needs-rebuilding
     - build-builder-image
+    - build-builder-image-remote-vm
     name: Create Multiarch manifest
     runs-on: ubuntu-24.04
-    if: |
-      github.event_name != 'pull_request' ||
-      (needs.build-builder-image.outputs.collector-builder-tag != 'cache' &&
-       contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds'))
+    if : needs.builder-needs-rebuilding.outputs.build-multiarch == 'true'
     env:
-      COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }}
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
       ARCHS: amd64 ppc64le s390x arm64
 
     steps:
@@ -208,15 +284,13 @@ jobs:
 
   retag-x86-image:
     needs:
+    - builder-needs-rebuilding
     - build-builder-image
     name: Retag x86 builder image
     runs-on: ubuntu-24.04
-    if: |
-      github.event_name == 'pull_request' &&
-      needs.build-builder-image.outputs.collector-builder-tag != 'cache' &&
-      !contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
+    if : needs.builder-needs-rebuilding.outputs.build-multiarch == 'false'
     env:
-      COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }}
+      COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}
     steps:
       - name: Pull image to retag
         run: |

ansible/ci-build-builder.yml

@@ -22,58 +22,38 @@
         version: "{{ local_branch }}"
         refspec: "+{{ collector_git_ref | replace('refs/', '') }}:{{ local_branch }}"
         recursive: true
-      when: arch == "s390x"
+      when: clone_repository
 
     - name: Build the collector builder image
       community.general.make:
         chdir: "{{ ansible_env.GITHUB_WORKSPACE | default(collector_root) }}"
         target: builder
-
-    - name: Retag collector builder image to arch specific
-      community.docker.docker_image:
-        name: "quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}"
-        repository: "quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}-{{ arch }}"
-        source: local
-
-    - name: Untag collector builder image to prevent issues with multiarch
-      community.docker.docker_image:
-        name: "quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}"
-        state: absent
-
-    - name: Login to quay.io
-      community.docker.docker_login:
-        registry_url: quay.io
+      environment:
+        BUILDAH_FORMAT: 'docker'
+
+    - name: Retag and push to stackrox-io
+      ansible.builtin.include_role:
+        name: push-image
+      vars:
+        original_image: "quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}"
+        target_image: "quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}-{{ arch }}"
+        untag_original: true
         username: "{{ stackrox_io_username }}"
         password: "{{ stackrox_io_password }}"
 
-    - name: Push builder image to quay.io/stackrox-io
-      community.docker.docker_image:
-        name: "quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}-{{ arch }}"
-        push: true
-        source: local
-
-    - name: Login to quay.io
-      community.docker.docker_login:
-        registry_url: quay.io
+    - name: Retag and push to rhacs-eng
+      ansible.builtin.include_role:
+        name: push-image
+      vars:
+        original_image: "quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}-{{ arch }}"
+        target_image: "quay.io/rhacs-eng/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}-{{ arch }}"
+        untag_original: false
         username: "{{ rhacs_eng_username }}"
         password: "{{ rhacs_eng_password }}"
 
-    - name: Push builder image to quay.io/rhacs-eng
-      community.docker.docker_image:
-        name: "quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}-{{ arch }}"
-        repository: "quay.io/rhacs-eng/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}-{{ arch }}"
-        push: true
-        source: local
-
     - name: Print builder images pushed
-      debug:
+      ansible.builtin.debug:
         msg:
           - "Pushed the following builder images:"
           - "  quay.io/stackrox-io/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}-{{ arch }}"
           - "  quay.io/rhacs-eng/collector-builder:{{ ansible_env.COLLECTOR_BUILDER_TAG }}-{{ arch }}"
-
-    - name: Logout of quay.io
-      community.docker.docker_login:
-        registry_url: quay.io
-        state: absent
-      when: true

ansible/ci-create-build-vms.yml

@@ -8,9 +8,11 @@
       include_role:
         name: create-all-vms
       vars:
-        vm_list:
-          # s390x
-          rhel-s390x: "{{ virtual_machines['rhel-s390x'] }}"
+        vm_list: "{{ virtual_machines }}"
+  post_tasks:
+    # We have all the VMs created now, so refresh the inventory - this allows
+    # us to use the provisioning role on the VMs we just created
+    - meta: refresh_inventory
 
 - name: Provision Build VMs
   hosts: "job_id_{{ job_id }}"