From 93d59dfe435011887aae0a01d7f2aae57c66176b Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Wed, 10 Jul 2024 20:58:14 +0000
Subject: [PATCH 01/43] feat: attempt at abstracting buildx config in bake file

---
 .gitlab-ci.yml             | 32 +++++++++-----------------------
 containers/debian/base.hcl | 29 +++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 23 deletions(-)
 create mode 100644 containers/debian/base.hcl

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 870020c6..f876d585 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -276,29 +276,15 @@ base:
     - attempts=0
     - nocache=""
     - while !
-      docker buildx build --push ${BUILD_OPTIONS} ${nocache}
-                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
-                   ${EXPORT_TAG:+
-                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
-                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
-                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
-                   }
-                   --file containers/debian/base.Dockerfile
-                   --platform ${PLATFORM}
-                   --build-arg BASE_IMAGE=${BASE_IMAGE}
-                   --build-arg BUILD_IMAGE=${BUILD_IMAGE}
-                   --build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
-                   --build-arg SPACK_VERSION=${SPACK_VERSION}
-                   --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
-                   --build-arg SPACK_CHERRYPICKS_FILES="${SPACK_CHERRYPICKS_FILES}"
-                   --build-arg KEY4HEPSPACK_ORGREPO=${KEY4HEPSPACK_ORGREPO}
-                   --build-arg KEY4HEPSPACK_VERSION=${KEY4HEPSPACK_VERSION}
-                   --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
-                   --build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
-                   --build-arg jobs=${JOBS}
-                   --provenance false
-                   containers/debian
-                   2>&1 | tee build.log
+      docker buildx bake --print ${BUILD_OPTIONS} ${nocache}
+        ${EXPORT_TAG:+
+          ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
+          ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
+          ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
+        }
+        --build-arg jobs=${JOBS}
+        --file containers/debian/base.hcl
+        2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
           nocache="--no-cache" ;
diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
new file mode 100644
index 00000000..18d7bd10
--- /dev/null
+++ b/containers/debian/base.hcl
@@ -0,0 +1,29 @@
+variable "BASE_IMAGE" {
+  default = "${BASE_IMAGE}"
+}
+
+variable "BUILD_IMAGE" {
+  default = "${BUILD_IMAGE}"
+}
+
+variable "INTERNAL_TAG" {
+  default = "${INTERNAL_TAG}"
+}
+
+variable "EXPORT_TAG" {
+  default = "${EXPORT_TAG}"
+}
+
+target "default" {
+  context = "containers/debian"
+  dockerfile = "containers/debian/base.Dockerfile"
+  platforms = [
+    "linux/amd64"
+  ]
+  BASE_IMAGE = "${BASE_IMAGE}"
+  BUILD_IMAGE = "${BUILD_IMAGE}"
+  tags = [
+    "${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}",
+    ""
+  ]
+}

From 77d75215cb62cda4675dd8d05390458a36f1e883 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Wed, 10 Jul 2024 23:57:20 +0000
Subject: [PATCH 02/43] fix: no --tag for bake command

---
 .gitlab-ci.yml             | 10 +---------
 containers/debian/base.hcl |  5 +----
 2 files changed, 2 insertions(+), 13 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f876d585..8fdaab17 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -276,15 +276,7 @@ base:
     - attempts=0
     - nocache=""
     - while !
-      docker buildx bake --print ${BUILD_OPTIONS} ${nocache}
-        ${EXPORT_TAG:+
-          ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
-          ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
-          ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
-        }
-        --build-arg jobs=${JOBS}
-        --file containers/debian/base.hcl
-        2>&1 | tee build.log
+      docker buildx bake ${BUILD_OPTIONS} ${nocache} --push --build-arg jobs=${JOBS} --file containers/debian/base.hcl 2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
           nocache="--no-cache" ;
diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 18d7bd10..de433e5b 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -17,13 +17,10 @@ variable "EXPORT_TAG" {
 target "default" {
   context = "containers/debian"
   dockerfile = "containers/debian/base.Dockerfile"
-  platforms = [
-    "linux/amd64"
-  ]
+  platforms = [ "linux/amd64" ]
   BASE_IMAGE = "${BASE_IMAGE}"
   BUILD_IMAGE = "${BUILD_IMAGE}"
   tags = [
     "${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}",
-    ""
   ]
 }

From 1b47f9ad927aab248228bf1e276280a9b014319e Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 00:06:10 +0000
Subject: [PATCH 03/43] fix: null default, print only for now

---
 .gitlab-ci.yml             |  2 +-
 containers/debian/base.hcl | 16 +++++++++-------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8fdaab17..a4c8da58 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -276,7 +276,7 @@ base:
     - attempts=0
     - nocache=""
     - while !
-      docker buildx bake ${BUILD_OPTIONS} ${nocache} --push --build-arg jobs=${JOBS} --file containers/debian/base.hcl 2>&1 | tee build.log
+      docker buildx bake ${BUILD_OPTIONS} ${nocache} --print --build-arg jobs=${JOBS} --file containers/debian/base.hcl 2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
           nocache="--no-cache" ;
diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index de433e5b..674cb74b 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -1,26 +1,28 @@
 variable "BASE_IMAGE" {
-  default = "${BASE_IMAGE}"
+  default = null
 }
 
 variable "BUILD_IMAGE" {
-  default = "${BUILD_IMAGE}"
+  default = null
 }
 
 variable "INTERNAL_TAG" {
-  default = "${INTERNAL_TAG}"
+  default = null
 }
 
 variable "EXPORT_TAG" {
-  default = "${EXPORT_TAG}"
+  default = null
 }
 
 target "default" {
   context = "containers/debian"
   dockerfile = "containers/debian/base.Dockerfile"
   platforms = [ "linux/amd64" ]
-  BASE_IMAGE = "${BASE_IMAGE}"
-  BUILD_IMAGE = "${BUILD_IMAGE}"
+  args = {
+    BASE_IMAGE = "${BASE_IMAGE}"
+    BUILD_IMAGE = "${BUILD_IMAGE}"
+  }
   tags = [
-    "${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}",
+    "${BUILD_IMAGE}:${INTERNAL_TAG}",
   ]
 }

From 946aa43ccbc94bac057511d07a509dbec5eea863 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 00:33:36 +0000
Subject: [PATCH 04/43] feat: activate build with push to registry

---
 .gitlab-ci.yml             |  2 +-
 containers/debian/base.hcl | 16 +++++++++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a4c8da58..8fdaab17 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -276,7 +276,7 @@ base:
     - attempts=0
     - nocache=""
     - while !
-      docker buildx bake ${BUILD_OPTIONS} ${nocache} --print --build-arg jobs=${JOBS} --file containers/debian/base.hcl 2>&1 | tee build.log
+      docker buildx bake ${BUILD_OPTIONS} ${nocache} --push --build-arg jobs=${JOBS} --file containers/debian/base.hcl 2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
           nocache="--no-cache" ;
diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 674cb74b..4fba5cc7 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -14,7 +14,21 @@ variable "EXPORT_TAG" {
   default = null
 }
 
+variable "CI_REGISTRY" {
+  default = null
+}
+
+variable "CI_PROJECT_PATH" {
+  default = null
+}
+
 target "default" {
+  matrix = {
+    registry = [
+      "${CI_REGISTRY}/${CI_PROJECT_PATH}"
+    ]
+  }
+  name = "${replace(registry,"/","-")}"
   context = "containers/debian"
   dockerfile = "containers/debian/base.Dockerfile"
   platforms = [ "linux/amd64" ]
@@ -23,6 +37,6 @@ target "default" {
     BUILD_IMAGE = "${BUILD_IMAGE}"
   }
   tags = [
-    "${BUILD_IMAGE}:${INTERNAL_TAG}",
+    "{registry}/${BUILD_IMAGE}:${INTERNAL_TAG}",
   ]
 }

From 3e282b27182ad9ad1135635b892bfcccd034919d Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 00:35:23 +0000
Subject: [PATCH 05/43] fix: typo

---
 containers/debian/base.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 4fba5cc7..655ea115 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -37,6 +37,6 @@ target "default" {
     BUILD_IMAGE = "${BUILD_IMAGE}"
   }
   tags = [
-    "{registry}/${BUILD_IMAGE}:${INTERNAL_TAG}",
+    "${registry}/${BUILD_IMAGE}:${INTERNAL_TAG}",
   ]
 }

From 9b936a77b8c995ea745a228b4b5de5e97e135a32 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 01:18:04 +0000
Subject: [PATCH 06/43] feat: better matrix var handling

---
 .gitlab-ci.yml             |  6 +++++-
 containers/debian/base.hcl | 31 +++----------------------------
 containers/variables.hcl   | 23 +++++++++++++++++++++++
 3 files changed, 31 insertions(+), 29 deletions(-)
 create mode 100644 containers/variables.hcl

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8fdaab17..d3a5a4d0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -276,7 +276,11 @@ base:
     - attempts=0
     - nocache=""
     - while !
-      docker buildx bake ${BUILD_OPTIONS} ${nocache} --push --build-arg jobs=${JOBS} --file containers/debian/base.hcl 2>&1 | tee build.log
+      docker buildx bake ${BUILD_OPTIONS} ${nocache} --push
+        --build-arg jobs=${JOBS}
+        --file containers/variables.hcl
+        --file containers/debian/base.hcl
+        2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
           nocache="--no-cache" ;
diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 655ea115..f905cb1b 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -1,32 +1,6 @@
-variable "BASE_IMAGE" {
-  default = null
-}
-
-variable "BUILD_IMAGE" {
-  default = null
-}
-
-variable "INTERNAL_TAG" {
-  default = null
-}
-
-variable "EXPORT_TAG" {
-  default = null
-}
-
-variable "CI_REGISTRY" {
-  default = null
-}
-
-variable "CI_PROJECT_PATH" {
-  default = null
-}
-
 target "default" {
   matrix = {
-    registry = [
-      "${CI_REGISTRY}/${CI_PROJECT_PATH}"
-    ]
+    registry = registries
   }
   name = "${replace(registry,"/","-")}"
   context = "containers/debian"
@@ -37,6 +11,7 @@ target "default" {
     BUILD_IMAGE = "${BUILD_IMAGE}"
   }
   tags = [
-    "${registry}/${BUILD_IMAGE}:${INTERNAL_TAG}",
+    "${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}",
+    "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}",
   ]
 }
diff --git a/containers/variables.hcl b/containers/variables.hcl
new file mode 100644
index 00000000..6cdfe856
--- /dev/null
+++ b/containers/variables.hcl
@@ -0,0 +1,23 @@
+variable "BASE_IMAGE" { default = null }
+variable "BUILD_IMAGE" { default = null }
+
+variable "INTERNAL_TAG" { default = null }
+variable "EXPORT_TAG" { default = null }
+
+variable "CI_PUSH" { default = null }
+variable "CI_REGISTRY" { default = null }
+variable "CI_PROJECT_PATH" { default = null }
+
+variable "DH_PUSH" { default = null }
+variable "DH_REGISTRY" { default = null }
+variable "DH_REGISTRY_USER" { default = null }
+
+variable "GH_PUSH" { default = null }
+variable "GH_REGISTRY" { default = null }
+variable "GH_REGISTRY_USER" { default = null }
+
+registries = compact([
+  CI_PUSH != null ? "${CI_REGISTRY}/${CI_PROJECT_PATH}" : null,
+  DH_PUSH != null ? "${DH_REGISTRY}/${DH_REGISTRY_USER}" : null,
+  GH_PUSH != null ? "${GH_REGISTRY}/${GH_REGISTRY_USER}" : null,
+])
\ No newline at end of file

From 04a26f37e955310cb5747568f54a64bf997070f0 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 02:26:29 +0000
Subject: [PATCH 07/43] fix: full regex_replace of disallowed chars

---
 containers/debian/base.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index f905cb1b..ea23387b 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -2,7 +2,7 @@ target "default" {
   matrix = {
     registry = registries
   }
-  name = "${replace(registry,"/","-")}"
+  name = "${regex_replace(registry,"[^a-zA-Z0-9_-]","-")}"
   context = "containers/debian"
   dockerfile = "containers/debian/base.Dockerfile"
   platforms = [ "linux/amd64" ]

From 25063694913cd1c59ea8e848c5e6648b8fa6a311 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 04:32:24 +0000
Subject: [PATCH 08/43] fix: dockerfile relative to context

---
 containers/debian/base.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index ea23387b..3e7390e0 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -4,7 +4,7 @@ target "default" {
   }
   name = "${regex_replace(registry,"[^a-zA-Z0-9_-]","-")}"
   context = "containers/debian"
-  dockerfile = "containers/debian/base.Dockerfile"
+  dockerfile = "base.Dockerfile"
   platforms = [ "linux/amd64" ]
   args = {
     BASE_IMAGE = "${BASE_IMAGE}"

From 5ddac8e8826d826c04b2ebf6fa65033f0d30b159 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 16:08:36 +0000
Subject: [PATCH 09/43] feat: dev.hcl

---
 .gitlab-ci.yml         | 83 +++---------------------------------------
 containers/jug/dev.hcl | 73 +++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+), 78 deletions(-)
 create mode 100644 containers/jug/dev.hcl

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d3a5a4d0..7da7fc38 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -347,9 +347,6 @@ eic:
       if [ "$CI_COMMIT_BRANCH" == "master" ]; then
         PUSH_NIGHTLY_WITH_DATE="1"
       fi
-    - if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY= ; fi
-    - if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT= ; fi
-    - apk add envsubst git
     - source spack.sh ; 
       source key4hep-spack.sh ;
       source eic-spack.sh ;
@@ -358,81 +355,11 @@ eic:
     - attempts=0
     - nocache=""
     - while !
-      docker buildx build --push ${BUILD_OPTIONS} ${nocache}
-                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE}
-                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE}
-                   ${EXPORT_TAG:+
-                     ${IF_BUILD_DEFAULT+
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}}
-                     }
-                     ${IF_BUILD_DEFAULT-
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                     }
-                   }
-                   ${IF_BUILD_NIGHTLY+
-                     ${NIGHTLY:+
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
-                       ${PUSH_NIGHTLY_WITH_DATE:+
-                         ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                         ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                       }
-                     }
-                   }
-                   ${EXPORT_TAG:+
-                     ${IF_BUILD_DEFAULT+
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}}
-                     }
-                     ${IF_BUILD_DEFAULT-
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
-                     }
-                   }
-                   ${IF_BUILD_NIGHTLY+
-                     ${NIGHTLY:+
-                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}}
-                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}}
-                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}}
-                     }
-                       ${PUSH_NIGHTLY_WITH_DATE:+
-                         ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                         ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                       }
-                   }
-                   --file containers/jug/dev.Dockerfile
-                   --platform ${PLATFORM}
-                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
-                   --build-arg BUILDER_IMAGE=${BUILDER_IMAGE}
-                   --build-arg RUNTIME_IMAGE=${RUNTIME_IMAGE}
-                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
-                   --build-arg JUG_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD)
-                   ${IF_BUILD_DEFAULT+
-                     ${EDM4EIC_VERSION:+--build-arg EDM4EIC_VERSION=${EDM4EIC_VERSION}}
-                     ${EICRECON_VERSION:+--build-arg EICRECON_VERSION=${EICRECON_VERSION}}
-                     ${EPIC_VERSION:+--build-arg EPIC_VERSION=${EPIC_VERSION}}
-                     ${JUGGLER_VERSION:+--build-arg JUGGLER_VERSION=${JUGGLER_VERSION}}
-                   }
-                   ${IF_BUILD_NIGHTLY+
-                     --build-arg EDM4EIC_VERSION=${EDM4EIC_VERSION:-main}
-                     --build-arg EICRECON_VERSION=${EICRECON_VERSION:-main}
-                     --build-arg EPIC_VERSION=${EPIC_VERSION:-main}
-                     --build-arg JUGGLER_VERSION=${JUGGLER_VERSION:-main}
-                   }
-                   --build-arg ENV=${ENV}
-                   --build-arg jobs=${JOBS}
-                   --build-context spack-environment=spack-environment
-                   --secret id=mirrors,src=mirrors.yaml
-                   --provenance false
-                   containers/jug
-                   2>&1 | tee build.log
+      docker buildx bake --print ${BUILD_OPTIONS} ${nocache} --push
+        --file containers/variables.hcl
+        --file containers/debian/dev.hcl
+        --provenance false
+        2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
           nocache="--no-cache" ;
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
new file mode 100644
index 00000000..60de817f
--- /dev/null
+++ b/containers/jug/dev.hcl
@@ -0,0 +1,73 @@
+variable "jobs" { default = null }
+variable "ENV" { default = null }
+
+variable "DOCKER_REGISTRY" { default = null }
+variable "BUILDER_IMAGE" { default = null }
+variable "RUNTIME_IMAGE" { default = null }
+
+variable "SPACK_ORGREPO" { default = null }
+variable "SPACK_VERSION" { default = null }
+variable "SPACK_CHERRYPICKS" { default = null }
+variable "SPACK_CHERRYPICKS_FILES" { default = null }
+variable "KEY4HEPSPACK_ORGREPO" { default = null }
+variable "KEY4HEPSPACK_VERSION" { default = null }
+variable "EICSPACK_ORGREPO" { default = null }
+variable "EICSPACK_VERSION" { default = null }
+variable "S3_ACCESS_KEY" { default = null }
+variable "S3_SECRET_KEY" { default = null }
+
+variable "NIGHTLY" { default = null }
+variable "BUILD_TYPE" { default = null }
+variable "NIGHTLY_TAG" { default = null }
+
+variable "EDM4EIC_VERSION" { default = null }
+variable "EICRECON_VERSION" { default = null }
+variable "EPIC_VERSION" { default = null }
+variable "JUGGLER_VERSION" { default = null }
+
+variable "CI_COMMIT_SHA" { default = null }
+
+image_name = ${BUILD_TYPE} == "default" ? "${BUILD_IMAGE}${ENV}" : "${BUILD_IMAGE}${ENV}-${BUILD_TYPE}"
+
+target "default" {
+  matrix = {
+    registry = registries,
+    image_names = [
+      "${image_name}", 
+      "replace(${image_tag},"eic","jug")${ENV}"
+    ]
+  }
+  name = "${regex_replace(join("-",[registry,image_name]),"[^a-zA-Z0-9_-]","-")}"
+  context = "containers/jug"
+  contexts = {
+    spack-environment = "spack-environment"
+  }
+  dockerfile = "dev.Dockerfile"
+  platforms = [ "linux/amd64" ]
+  secret = [
+    "id=mirrors,src=mirrors.yaml",
+  ]
+  args = {
+    DOCKER_REGISTRY = "${DOCKER_REGISTRY}"
+    BUILDER_IMAGE = "${BUILDER_IMAGE}"
+    RUNTIME_IMAGE = "${RUNTIME_IMAGE}"
+    SPACK_ORGREPO = "${SPACK_ORGREPO}"
+    SPACK_VERSION = "${SPACK_VERSION}"
+    SPACK_CHERRYPICKS = "${SPACK_CHERRYPICKS}"
+    SPACK_CHERRYPICKS_FILES = "${SPACK_CHERRYPICKS_FILES}"
+    KEY4HEPSPACK_ORGREPO = "${KEY4HEPSPACK_ORGREPO}"
+    KEY4HEPSPACK_VERSION = "${KEY4HEPSPACK_VERSION}"
+    EICSPACK_ORGREPO = "${EICSPACK_ORGREPO}"
+    EICSPACK_VERSION = "${EICSPACK_VERSION}"
+    S3_ACCESS_KEY = "${S3_ACCESS_KEY}"
+    S3_SECRET_KEY = "${S3_SECRET_KEY}"
+    EDM4EIC_VERSION = ${BUILD_TYPE} == "default" ? "${EDM4EIC_VERSION}" : "main"
+    EICRECON_VERSION = ${BUILD_TYPE} == "default" ? "${EICRECON_VERSION}" : "main"
+    EPIC_VERSION = ${BUILD_TYPE} == "default" ? "${EPIC_VERSION}" : "main"
+    JUGGLER_VERSION = ${BUILD_TYPE} == "default" ? "${JUGGLER_VERSION}" : "main"
+  }
+  tags = compact([
+    "${CI_REGISTRY}/${CI_PROJECT_PATH}/${image_name}:${INTERNAL_TAG}",
+    EXPORT_TAG != null ? "${registry}/${image_name}:${EXPORT_TAG}" : null,
+  ])
+}
\ No newline at end of file

From c16d6976fb0a8a5843526b60928db7fecc1b7707 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 16:17:26 +0000
Subject: [PATCH 10/43] fix: apk add envsubst

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7da7fc38..dc856bb3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -347,6 +347,7 @@ eic:
       if [ "$CI_COMMIT_BRANCH" == "master" ]; then
         PUSH_NIGHTLY_WITH_DATE="1"
       fi
+    - apk add envsubst
     - source spack.sh ; 
       source key4hep-spack.sh ;
       source eic-spack.sh ;

From 2eaa9a36e0e43b74fc7d288cbcd1aea28b438d56 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 12:10:06 -0500
Subject: [PATCH 11/43] fix: correct use of variables

---
 containers/jug/dev.hcl | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 60de817f..938621e2 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -27,17 +27,17 @@ variable "JUGGLER_VERSION" { default = null }
 
 variable "CI_COMMIT_SHA" { default = null }
 
-image_name = ${BUILD_TYPE} == "default" ? "${BUILD_IMAGE}${ENV}" : "${BUILD_IMAGE}${ENV}-${BUILD_TYPE}"
+image_name = BUILD_TYPE == "default" ? "${BUILD_IMAGE}${ENV}" : "${BUILD_IMAGE}${ENV}-${BUILD_TYPE}"
 
 target "default" {
   matrix = {
     registry = registries,
     image_names = [
-      "${image_name}", 
-      "replace(${image_tag},"eic","jug")${ENV}"
+      image_name,
+      replace(image_name,"eic","jug")
     ]
   }
-  name = "${regex_replace(join("-",[registry,image_name]),"[^a-zA-Z0-9_-]","-")}"
+  name = "${regex_replace(join("-",[registry,image_names]),"[^a-zA-Z0-9_-]","-")}"
   context = "containers/jug"
   contexts = {
     spack-environment = "spack-environment"
@@ -61,13 +61,13 @@ target "default" {
     EICSPACK_VERSION = "${EICSPACK_VERSION}"
     S3_ACCESS_KEY = "${S3_ACCESS_KEY}"
     S3_SECRET_KEY = "${S3_SECRET_KEY}"
-    EDM4EIC_VERSION = ${BUILD_TYPE} == "default" ? "${EDM4EIC_VERSION}" : "main"
-    EICRECON_VERSION = ${BUILD_TYPE} == "default" ? "${EICRECON_VERSION}" : "main"
-    EPIC_VERSION = ${BUILD_TYPE} == "default" ? "${EPIC_VERSION}" : "main"
-    JUGGLER_VERSION = ${BUILD_TYPE} == "default" ? "${JUGGLER_VERSION}" : "main"
+    EDM4EIC_VERSION = BUILD_TYPE == "default" ? "${EDM4EIC_VERSION}" : "main"
+    EICRECON_VERSION = BUILD_TYPE == "default" ? "${EICRECON_VERSION}" : "main"
+    EPIC_VERSION = BUILD_TYPE == "default" ? "${EPIC_VERSION}" : "main"
+    JUGGLER_VERSION = BUILD_TYPE == "default" ? "${JUGGLER_VERSION}" : "main"
   }
   tags = compact([
     "${CI_REGISTRY}/${CI_PROJECT_PATH}/${image_name}:${INTERNAL_TAG}",
     EXPORT_TAG != null ? "${registry}/${image_name}:${EXPORT_TAG}" : null,
   ])
-}
\ No newline at end of file
+}

From 4e05235a46ac8f775c2920123d2446c8223af021 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 17:15:41 +0000
Subject: [PATCH 12/43] fix: base provenance false, respect EXPORT_TAG unset

---
 .gitlab-ci.yml             | 1 +
 containers/debian/base.hcl | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dc856bb3..63cefa74 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -280,6 +280,7 @@ base:
         --build-arg jobs=${JOBS}
         --file containers/variables.hcl
         --file containers/debian/base.hcl
+        --provenance false
         2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 3e7390e0..3b2e351f 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -10,8 +10,8 @@ target "default" {
     BASE_IMAGE = "${BASE_IMAGE}"
     BUILD_IMAGE = "${BUILD_IMAGE}"
   }
-  tags = [
+  tags = compact([
     "${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}",
-    "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}",
-  ]
+    EXPORT_TAG != null ? "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}" : null,
+  ])
 }

From 7903f20b5e98eb4270da9a21f71b7fada2900dc3 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 17:19:12 +0000
Subject: [PATCH 13/43] feat: bake --print before bake --push

---
 .gitlab-ci.yml | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 63cefa74..15cf204d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -270,13 +270,17 @@ base:
   needs:
     - version
   script:
-    - source spack.sh ; 
-      source key4hep-spack.sh ;
-      source eic-spack.sh ;
     - attempts=0
     - nocache=""
+    - docker buildx bake ${BUILD_OPTIONS} ${nocache}
+        --print
+        --build-arg jobs=${JOBS}
+        --file containers/variables.hcl
+        --file containers/debian/base.hcl
+        --provenance false
     - while !
-      docker buildx bake ${BUILD_OPTIONS} ${nocache} --push
+      docker buildx bake ${BUILD_OPTIONS} ${nocache}
+        --push
         --build-arg jobs=${JOBS}
         --file containers/variables.hcl
         --file containers/debian/base.hcl
@@ -356,8 +360,14 @@ eic:
       cat mirrors.yaml.in | envsubst > mirrors.yaml
     - attempts=0
     - nocache=""
+    - docker buildx bake ${BUILD_OPTIONS} ${nocache}
+        --print
+        --file containers/variables.hcl
+        --file containers/debian/dev.hcl
+        --provenance false
     - while !
-      docker buildx bake --print ${BUILD_OPTIONS} ${nocache} --push
+      docker buildx bake ${BUILD_OPTIONS} ${nocache}
+        --push
         --file containers/variables.hcl
         --file containers/debian/dev.hcl
         --provenance false

From 32dcbbc8ebfee5c037c71ca2a2db692d7e9342fe Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 12:28:50 -0500
Subject: [PATCH 14/43] feat: use bake-action in .github/workflows

---
 .github/workflows/build-push.yml | 125 +++----------------------------
 1 file changed, 12 insertions(+), 113 deletions(-)

diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml
index 96d034bb..218613f2 100644
--- a/.github/workflows/build-push.yml
+++ b/.github/workflows/build-push.yml
@@ -93,20 +93,14 @@ jobs:
           registry: ${{ env.GH_REGISTRY }}
           username: ${{ secrets.GHCR_REGISTRY_USER }}
           password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v5
+      - name: Buildx Bake
+        uses: docker/bake-action@v5
         with:
-          file: containers/debian/base.Dockerfile
-          context: containers/debian
-          platforms: ${{ matrix.PLATFORM }}
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            BASE_IMAGE=${{ matrix.BASE_IMAGE }}
-            BUILD_IMAGE=${{ matrix.BUILD_IMAGE }}
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
+          provenance: false
+          files: |
+            containers/debian/base.hcl
+            ${{ steps.meta.outputs.bake-file }}
 
   dev:
     runs-on: ubuntu-latest
@@ -194,106 +188,11 @@ jobs:
           registry: ${{ env.GH_REGISTRY }}
           username: ${{ secrets.GHCR_REGISTRY_USER }}
           password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v5
+      - name: Buildx Bake
+        uses: docker/bake-action@v5
         with:
-          file: containers/jug/dev.Dockerfile
-          context: containers/jug
-          build-contexts: |
-            spack-environment=spack-environment
-          secret-files: |
-            mirrors=mirrors.yaml
-          platforms: ${{ matrix.PLATFORM }}
           push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/
-            BUILDER_IMAGE=${{ matrix.BUILDER_IMAGE }}
-            RUNTIME_IMAGE=${{ matrix.RUNTIME_IMAGE }}
-            BUILD_IMAGE=eic_${{ matrix.BUILD_IMAGE }}
-            INTERNAL_TAG=${{ env.INTERNAL_TAG }}
-            SPACK_ORGREPO=${{ steps.spack.outputs.orgrepo }}
-            SPACK_VERSION=${{ steps.spack.outputs.version }}
-            SPACK_CHERRYPICKS=${{ steps.spack.outputs.cherrypicks }}
-            SPACK_CHERRYPICKS_FILES=${{ steps.spack.outputs.cherrypicks_files }}
-            KEY4HEPSPACK_ORGREPO=${{ steps.eic-spack.outputs.orgrepo }}
-            KEY4HEPSPACK_VERSION=${{ steps.eic-spack.outputs.version }}
-            EICSPACK_ORGREPO=${{ steps.eic-spack.outputs.orgrepo }}
-            EICSPACK_VERSION=${{ steps.eic-spack.outputs.version }}
-            KEY4HEPSPACK_ORGREPO=${{ steps.key4hep-spack.outputs.orgrepo }}
-            KEY4HEPSPACK_VERSION=${{ steps.key4hep-spack.outputs.version }}
-            S3_ACCESS_KEY=${{ secrets.S3_ACCESS_KEY }}
-            S3_SECRET_KEY=${{ secrets.S3_SECRET_KEY }}
-            jobs=${{ env.JOBS }}
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
-
-  xl:
-    runs-on: ubuntu-latest
-    needs: dev
-    strategy:
-      matrix:
-        include:
-        - BASE_IMAGE: dev
-          BUILD_IMAGE: xl
-          PLATFORM: linux/amd64
-    steps:
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@v1.3.1
-        with:
-          android: true
-          dotnet: true
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: linux/amd64
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            name=${{ env.DH_REGISTRY }}/${{ env.DH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.DH_PUSH != 0 }}
-            name=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.GH_PUSH != 0 }}
-          tags: |
-            ${{ env.INTERNAL_TAG }}
-            type=schedule,pattern={{date 'YYYY-MM-DD'}}
-            type=ref,prefix=unstable-pr-,event=pr
-            type=match,pattern=^v(\d+\.\d+\.\d+-.*)$,group=1
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: ${{ env.DH_PUSH == '1' }}
-        with:
-          registry: ${{ env.DH_REGISTRY }}
-          username: ${{ env.DH_REGISTRY_USER }}
-          password: ${{ secrets.DH_EICWEB_TOKEN }}
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        if: ${{ env.GH_PUSH == '1' }}
-        with:
-          registry: ${{ env.GH_REGISTRY }}
-          username: ${{ secrets.GHCR_REGISTRY_USER }}
-          password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          file: containers/jug/xl.Dockerfile
-          context: containers/jug
-          build-contexts: |
-            detectors=.
-          platforms: ${{ matrix.PLATFORM }}
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/
-            BASE_IMAGE=eic_${{ matrix.BASE_IMAGE }}
-            BUILD_IMAGE=eic_${{ matrix.BUILD_IMAGE }}
-            INTERNAL_TAG=${{ env.INTERNAL_TAG }}
-            jobs=${{ env.JOBS }}
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
+          provenance: false
+          files: |
+            containers/jug/dev.hcl
+            ${{ steps.meta.outputs.bake-file }}

From 4f82c96263abe365584467a2e71a93ead722a787 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 12:32:15 -0500
Subject: [PATCH 15/43] fix: store regex in variable printable

---
 containers/debian/base.hcl | 2 +-
 containers/jug/dev.hcl     | 2 +-
 containers/variables.hcl   | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 3b2e351f..75d6f0b9 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -2,7 +2,7 @@ target "default" {
   matrix = {
     registry = registries
   }
-  name = "${regex_replace(registry,"[^a-zA-Z0-9_-]","-")}"
+  name = "${regex_replace(registry,"[^${printable}]","-")}"
   context = "containers/debian"
   dockerfile = "base.Dockerfile"
   platforms = [ "linux/amd64" ]
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 938621e2..f7e074c9 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -37,7 +37,7 @@ target "default" {
       replace(image_name,"eic","jug")
     ]
   }
-  name = "${regex_replace(join("-",[registry,image_names]),"[^a-zA-Z0-9_-]","-")}"
+  name = "${regex_replace(join("-",[registry,image_names]),"[^${printable}]","-")}"
   context = "containers/jug"
   contexts = {
     spack-environment = "spack-environment"
diff --git a/containers/variables.hcl b/containers/variables.hcl
index 6cdfe856..47a10047 100644
--- a/containers/variables.hcl
+++ b/containers/variables.hcl
@@ -1,3 +1,5 @@
+variable "printable" { default = "a-zA-Z0-9_-" }
+
 variable "BASE_IMAGE" { default = null }
 variable "BUILD_IMAGE" { default = null }
 
@@ -20,4 +22,4 @@ registries = compact([
   CI_PUSH != null ? "${CI_REGISTRY}/${CI_PROJECT_PATH}" : null,
   DH_PUSH != null ? "${DH_REGISTRY}/${DH_REGISTRY_USER}" : null,
   GH_PUSH != null ? "${GH_REGISTRY}/${GH_REGISTRY_USER}" : null,
-])
\ No newline at end of file
+])

From 50fd3a7a65a35259c9f953b063779dd9b8ff193a Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 12:57:34 -0500
Subject: [PATCH 16/43] fix: no matrix target, just expand registries in tags

---
 containers/debian/base.hcl | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 75d6f0b9..2198defa 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -1,8 +1,4 @@
 target "default" {
-  matrix = {
-    registry = registries
-  }
-  name = "${regex_replace(registry,"[^${printable}]","-")}"
   context = "containers/debian"
   dockerfile = "base.Dockerfile"
   platforms = [ "linux/amd64" ]
@@ -10,8 +6,8 @@ target "default" {
     BASE_IMAGE = "${BASE_IMAGE}"
     BUILD_IMAGE = "${BUILD_IMAGE}"
   }
-  tags = compact([
+  tags = compact(flatten([
     "${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}",
-    EXPORT_TAG != null ? "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}" : null,
-  ])
+    EXPORT_TAG != null ? [ for registry in registries: "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}" ] : null,
+  ]))
 }

From 54c0c8e6b3b9830801c1bb31d5f1ca59e1d5e6a9 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 13:50:27 -0500
Subject: [PATCH 17/43] fix: no matrix target, just expand registries in tags
 (in dev)

---
 containers/jug/dev.hcl | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index f7e074c9..d7c09419 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -28,16 +28,12 @@ variable "JUGGLER_VERSION" { default = null }
 variable "CI_COMMIT_SHA" { default = null }
 
 image_name = BUILD_TYPE == "default" ? "${BUILD_IMAGE}${ENV}" : "${BUILD_IMAGE}${ENV}-${BUILD_TYPE}"
+image_names = [
+  image_name,
+  replace(image_name,"eic","jug")
+]
 
 target "default" {
-  matrix = {
-    registry = registries,
-    image_names = [
-      image_name,
-      replace(image_name,"eic","jug")
-    ]
-  }
-  name = "${regex_replace(join("-",[registry,image_names]),"[^${printable}]","-")}"
   context = "containers/jug"
   contexts = {
     spack-environment = "spack-environment"
@@ -66,8 +62,11 @@ target "default" {
     EPIC_VERSION = BUILD_TYPE == "default" ? "${EPIC_VERSION}" : "main"
     JUGGLER_VERSION = BUILD_TYPE == "default" ? "${JUGGLER_VERSION}" : "main"
   }
-  tags = compact([
-    "${CI_REGISTRY}/${CI_PROJECT_PATH}/${image_name}:${INTERNAL_TAG}",
-    EXPORT_TAG != null ? "${registry}/${image_name}:${EXPORT_TAG}" : null,
-  ])
+  tags = compact(flatten([
+    [ for image_name in image_names: "${CI_REGISTRY}/${CI_PROJECT_PATH}/${image_name}:${INTERNAL_TAG}" ],
+    EXPORT_TAG != null && EXPORT_TAG != "" ? [
+      for registry_image_name in setproduct(registries, image_names):
+        format("%s:%s", join("/", registry_image_name), EXPORT_TAG )
+    ] : [ null ]
+  ]))
 }

From 5600754c75a282d42aabfcffddde09fc9857829a Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 13:51:57 -0500
Subject: [PATCH 18/43] fix: correct path to containers/jug/dev.hcl

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 15cf204d..4fce14c8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -363,7 +363,7 @@ eic:
     - docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --print
         --file containers/variables.hcl
-        --file containers/debian/dev.hcl
+        --file containers/jug/dev.hcl
         --provenance false
     - while !
       docker buildx bake ${BUILD_OPTIONS} ${nocache}

From a09aab9f5d608fdeb46f9cd348d3040dac96dde4 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 13:54:30 -0500
Subject: [PATCH 19/43] fix: treat empty DH_PUSH as undefined

---
 containers/variables.hcl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/containers/variables.hcl b/containers/variables.hcl
index 47a10047..fa3eda67 100644
--- a/containers/variables.hcl
+++ b/containers/variables.hcl
@@ -19,7 +19,7 @@ variable "GH_REGISTRY" { default = null }
 variable "GH_REGISTRY_USER" { default = null }
 
 registries = compact([
-  CI_PUSH != null ? "${CI_REGISTRY}/${CI_PROJECT_PATH}" : null,
-  DH_PUSH != null ? "${DH_REGISTRY}/${DH_REGISTRY_USER}" : null,
-  GH_PUSH != null ? "${GH_REGISTRY}/${GH_REGISTRY_USER}" : null,
+  CI_PUSH != null && CI_PUSH != "" ? "${CI_REGISTRY}/${CI_PROJECT_PATH}" : null,
+  DH_PUSH != null && DH_PUSH != "" ? "${DH_REGISTRY}/${DH_REGISTRY_USER}" : null,
+  GH_PUSH != null && GH_PUSH != "" ? "${GH_REGISTRY}/${GH_REGISTRY_USER}" : null,
 ])

From 68da46c6ad4bf0fb50bcb787a39e622d42e92abb Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 13:56:52 -0500
Subject: [PATCH 20/43] feat: move provenance to bake file

---
 .gitlab-ci.yml             | 4 ----
 containers/debian/base.hcl | 3 +++
 containers/jug/dev.hcl     | 3 +++
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4fce14c8..5ac4e698 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -277,14 +277,12 @@ base:
         --build-arg jobs=${JOBS}
         --file containers/variables.hcl
         --file containers/debian/base.hcl
-        --provenance false
     - while !
       docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --push
         --build-arg jobs=${JOBS}
         --file containers/variables.hcl
         --file containers/debian/base.hcl
-        --provenance false
         2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
@@ -364,13 +362,11 @@ eic:
         --print
         --file containers/variables.hcl
         --file containers/jug/dev.hcl
-        --provenance false
     - while !
       docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --push
         --file containers/variables.hcl
         --file containers/debian/dev.hcl
-        --provenance false
         2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then
diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 2198defa..23b4d498 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -1,4 +1,7 @@
 target "default" {
+  attest = [
+    "type=provenance,disabled=true"
+  ]
   context = "containers/debian"
   dockerfile = "base.Dockerfile"
   platforms = [ "linux/amd64" ]
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index d7c09419..f9c6ccb1 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -34,6 +34,9 @@ image_names = [
 ]
 
 target "default" {
+  attest = [
+    "type=provenance,disabled=true"
+  ]
   context = "containers/jug"
   contexts = {
     spack-environment = "spack-environment"

From 960ad610eb75106a99cc8c97b4e407f71abefcbb Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 14:48:09 -0500
Subject: [PATCH 21/43] fix: correct path to containers/jug/dev.hcl

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5ac4e698..c1ac74aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -366,7 +366,7 @@ eic:
       docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --push
         --file containers/variables.hcl
-        --file containers/debian/dev.hcl
+        --file containers/jug/dev.hcl
         2>&1 | tee build.log
       ; do
         if grep "unknown blob" build.log ; then

From 11943f543df9a63fb2ec84e1afea51f05997c9cb Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 14:50:12 -0500
Subject: [PATCH 22/43] fix: mv {BASE,BUILD}_IMAGE into endpoint hcl

---
 containers/debian/base.hcl | 6 ++++++
 containers/jug/dev.hcl     | 1 +
 containers/variables.hcl   | 3 ---
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 23b4d498..e022b911 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -1,3 +1,9 @@
+# Variables which are required to be defined
+variable "BUILD_IMAGE" { default = "debian_base" }
+
+# Variables whose defaults can be overridden on build
+variable "BASE_IMAGE" { default = null }
+
 target "default" {
   attest = [
     "type=provenance,disabled=true"
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index f9c6ccb1..5ca5b3e5 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -1,5 +1,6 @@
 variable "jobs" { default = null }
 variable "ENV" { default = null }
+variable "BUILD_IMAGE" { default = "eic_" }
 
 variable "DOCKER_REGISTRY" { default = null }
 variable "BUILDER_IMAGE" { default = null }
diff --git a/containers/variables.hcl b/containers/variables.hcl
index fa3eda67..886c55b1 100644
--- a/containers/variables.hcl
+++ b/containers/variables.hcl
@@ -1,8 +1,5 @@
 variable "printable" { default = "a-zA-Z0-9_-" }
 
-variable "BASE_IMAGE" { default = null }
-variable "BUILD_IMAGE" { default = null }
-
 variable "INTERNAL_TAG" { default = null }
 variable "EXPORT_TAG" { default = null }
 

From f460f6716e92c9cabd8cd662e3b151b9112c5086 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 14:51:01 -0500
Subject: [PATCH 23/43] fix: INTERNAL_TAG default to "testing"

---
 containers/variables.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/variables.hcl b/containers/variables.hcl
index 886c55b1..76decc74 100644
--- a/containers/variables.hcl
+++ b/containers/variables.hcl
@@ -1,6 +1,6 @@
 variable "printable" { default = "a-zA-Z0-9_-" }
 
-variable "INTERNAL_TAG" { default = null }
+variable "INTERNAL_TAG" { default = "testing" }
 variable "EXPORT_TAG" { default = null }
 
 variable "CI_PUSH" { default = null }

From 96dd09160d4f0a6e947d9ecd84f6dbb42f66884a Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 14:51:33 -0500
Subject: [PATCH 24/43] fix: docs

---
 containers/jug/dev.hcl   | 1 +
 containers/variables.hcl | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 5ca5b3e5..c2c00dcd 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -2,6 +2,7 @@ variable "jobs" { default = null }
 variable "ENV" { default = null }
 variable "BUILD_IMAGE" { default = "eic_" }
 
+# Variables whose defaults can be overridden on build
 variable "DOCKER_REGISTRY" { default = null }
 variable "BUILDER_IMAGE" { default = null }
 variable "RUNTIME_IMAGE" { default = null }
diff --git a/containers/variables.hcl b/containers/variables.hcl
index 76decc74..6cea0b7b 100644
--- a/containers/variables.hcl
+++ b/containers/variables.hcl
@@ -1,20 +1,27 @@
+# Helper to map onto printable target names
 variable "printable" { default = "a-zA-Z0-9_-" }
 
+# Internal tag (never pushed)
 variable "INTERNAL_TAG" { default = "testing" }
+# External tag (pushed to registries)
 variable "EXPORT_TAG" { default = null }
 
+# Local registry
 variable "CI_PUSH" { default = null }
 variable "CI_REGISTRY" { default = null }
 variable "CI_PROJECT_PATH" { default = null }
 
+# Docker Hub
 variable "DH_PUSH" { default = null }
 variable "DH_REGISTRY" { default = null }
 variable "DH_REGISTRY_USER" { default = null }
 
+# GitHub Container Registry
 variable "GH_PUSH" { default = null }
 variable "GH_REGISTRY" { default = null }
 variable "GH_REGISTRY_USER" { default = null }
 
+# List of enabled registries
 registries = compact([
   CI_PUSH != null && CI_PUSH != "" ? "${CI_REGISTRY}/${CI_PROJECT_PATH}" : null,
   DH_PUSH != null && DH_PUSH != "" ? "${DH_REGISTRY}/${DH_REGISTRY_USER}" : null,

From 0f2779120f1fc6efbc443c8b93325a7331bca8f0 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 14:52:29 -0500
Subject: [PATCH 25/43] fix: simplify syntax args from variable

---
 containers/debian/base.hcl |  4 ++--
 containers/jug/dev.hcl     | 35 ++++++++++++++++++-----------------
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index e022b911..4f1cd429 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -12,8 +12,8 @@ target "default" {
   dockerfile = "base.Dockerfile"
   platforms = [ "linux/amd64" ]
   args = {
-    BASE_IMAGE = "${BASE_IMAGE}"
-    BUILD_IMAGE = "${BUILD_IMAGE}"
+    BASE_IMAGE = BASE_IMAGE
+    BUILD_IMAGE = BUILD_IMAGE
   }
   tags = compact(flatten([
     "${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}",
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index c2c00dcd..8c91f56a 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -49,23 +49,24 @@ target "default" {
     "id=mirrors,src=mirrors.yaml",
   ]
   args = {
-    DOCKER_REGISTRY = "${DOCKER_REGISTRY}"
-    BUILDER_IMAGE = "${BUILDER_IMAGE}"
-    RUNTIME_IMAGE = "${RUNTIME_IMAGE}"
-    SPACK_ORGREPO = "${SPACK_ORGREPO}"
-    SPACK_VERSION = "${SPACK_VERSION}"
-    SPACK_CHERRYPICKS = "${SPACK_CHERRYPICKS}"
-    SPACK_CHERRYPICKS_FILES = "${SPACK_CHERRYPICKS_FILES}"
-    KEY4HEPSPACK_ORGREPO = "${KEY4HEPSPACK_ORGREPO}"
-    KEY4HEPSPACK_VERSION = "${KEY4HEPSPACK_VERSION}"
-    EICSPACK_ORGREPO = "${EICSPACK_ORGREPO}"
-    EICSPACK_VERSION = "${EICSPACK_VERSION}"
-    S3_ACCESS_KEY = "${S3_ACCESS_KEY}"
-    S3_SECRET_KEY = "${S3_SECRET_KEY}"
-    EDM4EIC_VERSION = BUILD_TYPE == "default" ? "${EDM4EIC_VERSION}" : "main"
-    EICRECON_VERSION = BUILD_TYPE == "default" ? "${EICRECON_VERSION}" : "main"
-    EPIC_VERSION = BUILD_TYPE == "default" ? "${EPIC_VERSION}" : "main"
-    JUGGLER_VERSION = BUILD_TYPE == "default" ? "${JUGGLER_VERSION}" : "main"
+    jobs = jobs
+    DOCKER_REGISTRY = DOCKER_REGISTRY
+    BUILDER_IMAGE = BUILDER_IMAGE
+    RUNTIME_IMAGE = RUNTIME_IMAGE
+    SPACK_ORGREPO = SPACK_ORGREPO
+    SPACK_VERSION = SPACK_VERSION
+    SPACK_CHERRYPICKS = SPACK_CHERRYPICKS
+    SPACK_CHERRYPICKS_FILES = SPACK_CHERRYPICKS_FILES
+    KEY4HEPSPACK_ORGREPO = KEY4HEPSPACK_ORGREPO
+    KEY4HEPSPACK_VERSION = KEY4HEPSPACK_VERSION
+    EICSPACK_ORGREPO = EICSPACK_ORGREPO
+    EICSPACK_VERSION = EICSPACK_VERSION
+    S3_ACCESS_KEY = S3_ACCESS_KEY
+    S3_SECRET_KEY = S3_SECRET_KEY
+    EDM4EIC_VERSION = BUILD_TYPE == "default" ? EDM4EIC_VERSION : "main"
+    EICRECON_VERSION = BUILD_TYPE == "default" ? EICRECON_VERSION : "main"
+    EPIC_VERSION = BUILD_TYPE == "default" ? EPIC_VERSION : "main"
+    JUGGLER_VERSION = BUILD_TYPE == "default" ? JUGGLER_VERSION : "main"
   }
   tags = compact(flatten([
     [ for image_name in image_names: "${CI_REGISTRY}/${CI_PROJECT_PATH}/${image_name}:${INTERNAL_TAG}" ],

From 64c84152dbfc96196d78503c677c59d304c4768d Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 14:59:44 -0500
Subject: [PATCH 26/43] fix: eic_dev:testing default settings

---
 containers/jug/dev.hcl | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 8c91f56a..e0df7869 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -1,6 +1,8 @@
-variable "jobs" { default = null }
-variable "ENV" { default = null }
+# Variables which are required to be defined
+variable "jobs" { default = 1 }
+variable "ENV" { default = "dev" }
 variable "BUILD_IMAGE" { default = "eic_" }
+variable "BUILD_TYPE" { default = "default" }
 
 # Variables whose defaults can be overridden on build
 variable "DOCKER_REGISTRY" { default = null }
@@ -19,7 +21,6 @@ variable "S3_ACCESS_KEY" { default = null }
 variable "S3_SECRET_KEY" { default = null }
 
 variable "NIGHTLY" { default = null }
-variable "BUILD_TYPE" { default = null }
 variable "NIGHTLY_TAG" { default = null }
 
 variable "EDM4EIC_VERSION" { default = null }

From 1a79ca8a0ff4a58a3c47daa2ba6deb1eaafc4048 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 15:00:19 -0500
Subject: [PATCH 27/43] fix: allow undefined CI_REGISTRY and CI_PROJECT_PATH

---
 containers/debian/base.hcl | 6 ++++--
 containers/jug/dev.hcl     | 5 ++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 4f1cd429..a51195e4 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -16,7 +16,9 @@ target "default" {
     BUILD_IMAGE = BUILD_IMAGE
   }
   tags = compact(flatten([
-    "${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}",
-    EXPORT_TAG != null ? [ for registry in registries: "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}" ] : null,
+    join("/", compact([ CI_REGISTRY, CI_PROJECT_PATH, "${BUILD_IMAGE}:${INTERNAL_TAG}"]) ),
+    EXPORT_TAG != null && EXPORT_TAG != "" ? [
+      for registry in registries: "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}"
+    ] : [ null ]
   ]))
 }
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index e0df7869..72a522a1 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -70,7 +70,10 @@ target "default" {
     JUGGLER_VERSION = BUILD_TYPE == "default" ? JUGGLER_VERSION : "main"
   }
   tags = compact(flatten([
-    [ for image_name in image_names: "${CI_REGISTRY}/${CI_PROJECT_PATH}/${image_name}:${INTERNAL_TAG}" ],
+    [
+      for image_name in image_names:
+        join("/", compact([ CI_REGISTRY, CI_PROJECT_PATH, "${image_name}:${INTERNAL_TAG}"]) )
+    ],
     EXPORT_TAG != null && EXPORT_TAG != "" ? [
       for registry_image_name in setproduct(registries, image_names):
         format("%s:%s", join("/", registry_image_name), EXPORT_TAG )

From 9167b576275b9d9ce61f6011e5cce0622303d548 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 11 Jul 2024 15:00:44 -0500
Subject: [PATCH 28/43] fix: update default spack and eic-spack versions

---
 containers/jug/dev.Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/containers/jug/dev.Dockerfile b/containers/jug/dev.Dockerfile
index 4b2c241b..b7aeac25 100644
--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
@@ -1,8 +1,8 @@
 #syntax=docker/dockerfile:1.8
 #check
 ARG DOCKER_REGISTRY="eicweb/"
-ARG BUILDER_IMAGE="debian_stable_base"
-ARG RUNTIME_IMAGE="debian_stable_base"
+ARG BUILDER_IMAGE="debian_base"
+ARG RUNTIME_IMAGE="debian_base"
 ARG INTERNAL_TAG="testing"
 
 ##
@@ -39,7 +39,7 @@ ARG TARGETPLATFORM
 
 ## Copy our default environment
 COPY --from=spack-environment . /opt/spack-environment/
-ARG ENV=dev
+ARG ENV=xl
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
 
 # Concretization (default environment)

From 0ea4b169dd793df3a192b4c7cad1003d058426d0 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 02:29:11 +0000
Subject: [PATCH 29/43] fix: set default DOCKER_REGISTRY to CI_REGISTRY

---
 containers/jug/dev.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 72a522a1..96259151 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -5,7 +5,7 @@ variable "BUILD_IMAGE" { default = "eic_" }
 variable "BUILD_TYPE" { default = "default" }
 
 # Variables whose defaults can be overridden on build
-variable "DOCKER_REGISTRY" { default = null }
+variable "DOCKER_REGISTRY" { default = join("/", [ CI_REGISTRY, CI_PROJECT_PATH ]) }
 variable "BUILDER_IMAGE" { default = null }
 variable "RUNTIME_IMAGE" { default = null }
 

From fa3cce873e1fc8e9894f51d5caf0839a1498f10a Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 02:38:15 +0000
Subject: [PATCH 30/43] fix: DOCKER_REGISTRY end with /

---
 containers/jug/dev.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 96259151..96c9080a 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -5,7 +5,7 @@ variable "BUILD_IMAGE" { default = "eic_" }
 variable "BUILD_TYPE" { default = "default" }
 
 # Variables whose defaults can be overridden on build
-variable "DOCKER_REGISTRY" { default = join("/", [ CI_REGISTRY, CI_PROJECT_PATH ]) }
+variable "DOCKER_REGISTRY" { default = join("/", [ CI_REGISTRY, CI_PROJECT_PATH, ""]) }
 variable "BUILDER_IMAGE" { default = null }
 variable "RUNTIME_IMAGE" { default = null }
 

From 849dc6e1465b01e0072d24faf6d35da2828a18bd Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 02:52:42 +0000
Subject: [PATCH 31/43] fix: dev.hcl args INTERNAL_TAG

---
 containers/jug/dev.hcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 96c9080a..5498f571 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -54,6 +54,7 @@ target "default" {
     DOCKER_REGISTRY = DOCKER_REGISTRY
     BUILDER_IMAGE = BUILDER_IMAGE
     RUNTIME_IMAGE = RUNTIME_IMAGE
+    INTERNAL_TAG = INTERNAL_TAGN
     SPACK_ORGREPO = SPACK_ORGREPO
     SPACK_VERSION = SPACK_VERSION
     SPACK_CHERRYPICKS = SPACK_CHERRYPICKS

From 2bf48bd95bd5800f3d9660a499d660e5448b0017 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 03:01:03 +0000
Subject: [PATCH 32/43] fix: typo

---
 containers/jug/dev.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 5498f571..797bd785 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -54,7 +54,7 @@ target "default" {
     DOCKER_REGISTRY = DOCKER_REGISTRY
     BUILDER_IMAGE = BUILDER_IMAGE
     RUNTIME_IMAGE = RUNTIME_IMAGE
-    INTERNAL_TAG = INTERNAL_TAGN
+    INTERNAL_TAG = INTERNAL_TAG
     SPACK_ORGREPO = SPACK_ORGREPO
     SPACK_VERSION = SPACK_VERSION
     SPACK_CHERRYPICKS = SPACK_CHERRYPICKS

From a7882f2a16233dde618cec58c7ec705bb55a04a0 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 15:18:41 -0500
Subject: [PATCH 33/43] fix: default ENV=xl now

---
 containers/jug/dev.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 797bd785..6aa3de23 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -1,6 +1,6 @@
 # Variables which are required to be defined
 variable "jobs" { default = 1 }
-variable "ENV" { default = "dev" }
+variable "ENV" { default = "xl" }
 variable "BUILD_IMAGE" { default = "eic_" }
 variable "BUILD_TYPE" { default = "default" }
 

From 0c3d78548f9b70445276a7701a1285559c939a2e Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 15:19:02 -0500
Subject: [PATCH 34/43] fix: pass ENV in args

---
 containers/jug/dev.hcl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 6aa3de23..956f935e 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -50,6 +50,7 @@ target "default" {
     "id=mirrors,src=mirrors.yaml",
   ]
   args = {
+    ENV = ENV
     jobs = jobs
     DOCKER_REGISTRY = DOCKER_REGISTRY
     BUILDER_IMAGE = BUILDER_IMAGE

From 589dbb616e51e2bad9c276767827ce48d757878f Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Mon, 23 Sep 2024 12:54:38 -0500
Subject: [PATCH 35/43] feat: use hcl lists for cherry-picks, allow undefined

---
 .gitlab-ci.yml                        |  8 ++++++--
 containers/jug/dev.hcl                | 24 ++++++++----------------
 eic-spack.sh => eic-spack.hcl         |  4 ++--
 key4hep-spack.sh => key4hep-spack.hcl |  4 ++--
 spack.hcl                             | 18 ++++++++++++++++++
 spack.sh                              | 21 ---------------------
 6 files changed, 36 insertions(+), 43 deletions(-)
 rename eic-spack.sh => eic-spack.hcl (56%)
 rename key4hep-spack.sh => key4hep-spack.hcl (55%)
 create mode 100644 spack.hcl
 delete mode 100644 spack.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c1ac74aa..7dc9a907 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -352,19 +352,23 @@ eic:
       fi
     - apk add envsubst
     - source spack.sh ; 
-      source key4hep-spack.sh ;
-      source eic-spack.sh ;
       export SPACK_VERSION ;
       cat mirrors.yaml.in | envsubst > mirrors.yaml
     - attempts=0
     - nocache=""
     - docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --print
+        --file spack.hcl
+        --file eic-spack.hcl
+        --file key4hep-spack.hcl
         --file containers/variables.hcl
         --file containers/jug/dev.hcl
     - while !
       docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --push
+        --file spack.hcl
+        --file eic-spack.hcl
+        --file key4hep-spack.hcl
         --file containers/variables.hcl
         --file containers/jug/dev.hcl
         2>&1 | tee build.log
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 956f935e..3b867e6a 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -9,14 +9,6 @@ variable "DOCKER_REGISTRY" { default = join("/", [ CI_REGISTRY, CI_PROJECT_PATH,
 variable "BUILDER_IMAGE" { default = null }
 variable "RUNTIME_IMAGE" { default = null }
 
-variable "SPACK_ORGREPO" { default = null }
-variable "SPACK_VERSION" { default = null }
-variable "SPACK_CHERRYPICKS" { default = null }
-variable "SPACK_CHERRYPICKS_FILES" { default = null }
-variable "KEY4HEPSPACK_ORGREPO" { default = null }
-variable "KEY4HEPSPACK_VERSION" { default = null }
-variable "EICSPACK_ORGREPO" { default = null }
-variable "EICSPACK_VERSION" { default = null }
 variable "S3_ACCESS_KEY" { default = null }
 variable "S3_SECRET_KEY" { default = null }
 
@@ -56,14 +48,14 @@ target "default" {
     BUILDER_IMAGE = BUILDER_IMAGE
     RUNTIME_IMAGE = RUNTIME_IMAGE
     INTERNAL_TAG = INTERNAL_TAG
-    SPACK_ORGREPO = SPACK_ORGREPO
-    SPACK_VERSION = SPACK_VERSION
-    SPACK_CHERRYPICKS = SPACK_CHERRYPICKS
-    SPACK_CHERRYPICKS_FILES = SPACK_CHERRYPICKS_FILES
-    KEY4HEPSPACK_ORGREPO = KEY4HEPSPACK_ORGREPO
-    KEY4HEPSPACK_VERSION = KEY4HEPSPACK_VERSION
-    EICSPACK_ORGREPO = EICSPACK_ORGREPO
-    EICSPACK_VERSION = EICSPACK_VERSION
+    SPACK_ORGREPO = try(SPACK_ORGREPO, null)
+    SPACK_VERSION = try(SPACK_VERSION, null)
+    SPACK_CHERRYPICKS = join("\n", try(SPACK_CHERRYPICKS, []))
+    SPACK_CHERRYPICKS_FILES = join("\n", try(SPACK_CHERRYPICKS_FILES, []))
+    KEY4HEPSPACK_ORGREPO = try(KEY4HEPSPACK_ORGREPO, null)
+    KEY4HEPSPACK_VERSION = try(KEY4HEPSPACK_VERSION, null)
+    EICSPACK_ORGREPO = try(EICSPACK_ORGREPO, null)
+    EICSPACK_VERSION = try(EICSPACK_VERSION, null)
     S3_ACCESS_KEY = S3_ACCESS_KEY
     S3_SECRET_KEY = S3_SECRET_KEY
     EDM4EIC_VERSION = BUILD_TYPE == "default" ? EDM4EIC_VERSION : "main"
diff --git a/eic-spack.sh b/eic-spack.hcl
similarity index 56%
rename from eic-spack.sh
rename to eic-spack.hcl
index f08987eb..7d1bf500 100644
--- a/eic-spack.sh
+++ b/eic-spack.hcl
@@ -1,6 +1,6 @@
 ## EIC spack organization and repository, e.g. eic/eic-spack
-EICSPACK_ORGREPO="eic/eic-spack"
+variable "EICSPACK_ORGREPO" { default = "eic/eic-spack" }
 
 ## EIC spack commit hash or github version, e.g. v0.19.7
 ## note: nightly builds could use a branch e.g. releases/v0.19
-EICSPACK_VERSION="accaafa04638287146be96b865b0ddf33f97df02"
+variable "EICSPACK_VERSION" { default = "accaafa04638287146be96b865b0ddf33f97df02" }
diff --git a/key4hep-spack.sh b/key4hep-spack.hcl
similarity index 55%
rename from key4hep-spack.sh
rename to key4hep-spack.hcl
index 90f2a1a3..da6434cd 100644
--- a/key4hep-spack.sh
+++ b/key4hep-spack.hcl
@@ -1,6 +1,6 @@
 ## Key4HEP spack organization and repository, e.g. key4hep/key4hep-spack
-KEY4HEPSPACK_ORGREPO="key4hep/key4hep-spack"
+variable "KEY4HEPSPACK_ORGREPO" { default = "key4hep/key4hep-spack" }
 
 ## Key4HEP spack commit hash or github version, e.g. v0.19.7
 ## note: nightly builds could use a branch e.g. releases/v0.19
-KEY4HEPSPACK_VERSION="ce50d74dba7c665157cbef63cd5c9cc4d7758f72"
+variable "KEY4HEPSPACK_VERSION" { default = "ce50d74dba7c665157cbef63cd5c9cc4d7758f72" }
diff --git a/spack.hcl b/spack.hcl
new file mode 100644
index 00000000..a094fa56
--- /dev/null
+++ b/spack.hcl
@@ -0,0 +1,18 @@
+## Spack organization and repository, e.g. spack/spack
+variable "SPACK_ORGREPO" { default = "spack/spack" }
+
+## Spack github version, e.g. v0.18.1 or commit hash
+## note: nightly builds will use e.g. releases/v0.19
+variable "SPACK_VERSION" { default = "v0.23.0" }
+
+## Space-separated list of spack cherry-picks
+variable "SPACK_CHERRYPICKS" {
+  default = [
+    "09f75ee426a2e05e0543570821582480ff823ba5", # setup-env.sh: if exe contains qemu, use /proc/$$/comm instead
+    "b90ac6441cfdf6425cb59551e7b0538899b69527", # celeritas: remove ancient versions and add CUDA package dependency
+  ]
+}
+## Optional hash table with comma-separated file list
+variable "SPACK_CHERRYPICKS_FILES" {
+  default = []
+}
diff --git a/spack.sh b/spack.sh
deleted file mode 100644
index f3adcb39..00000000
--- a/spack.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-## Spack organization and repository, e.g. spack/spack
-SPACK_ORGREPO="spack/spack"
-
-## Spack github version, e.g. v0.18.1 or commit hash
-## note: nightly builds will use e.g. releases/v0.19
-SPACK_VERSION="v0.23.0"
-
-## Space-separated list of spack cherry-picks
-read -r -d '' SPACK_CHERRYPICKS <<- \
---- || true
-09f75ee426a2e05e0543570821582480ff823ba5
-b90ac6441cfdf6425cb59551e7b0538899b69527
----
-## Optional hash table with comma-separated file list
-read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
---- || true
----
-## Ref: https://github.com/spack/spack/commit/[hash]
-## [hash]: [description]
-## 09f75ee426a2e05e0543570821582480ff823ba5: setup-env.sh: if exe contains qemu, use /proc/$$/comm instead
-## b90ac6441cfdf6425cb59551e7b0538899b69527: celeritas: remove ancient versions and add CUDA package dependency

From fc2db7ad06f1aa53aeafd9ad21787386a7448625 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 15:35:55 -0500
Subject: [PATCH 36/43] fix: consistently use debian_base as output of
 debian/base.Dockerfile

---
 containers/debian/base.Dockerfile | 2 +-
 containers/jug/dev.hcl            | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/containers/debian/base.Dockerfile b/containers/debian/base.Dockerfile
index b2632eb3..694c01c1 100644
--- a/containers/debian/base.Dockerfile
+++ b/containers/debian/base.Dockerfile
@@ -1,7 +1,7 @@
 #syntax=docker/dockerfile:1.8
 #check
 ARG BASE_IMAGE="amd64/debian:stable-slim"
-ARG BUILD_IMAGE="debian_stable_base"
+ARG BUILD_IMAGE="debian_base"
 
 # Minimal container based on Debian base systems for up-to-date packages. 
 FROM  ${BASE_IMAGE}
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 3b867e6a..143699dc 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -6,8 +6,8 @@ variable "BUILD_TYPE" { default = "default" }
 
 # Variables whose defaults can be overridden on build
 variable "DOCKER_REGISTRY" { default = join("/", [ CI_REGISTRY, CI_PROJECT_PATH, ""]) }
-variable "BUILDER_IMAGE" { default = null }
-variable "RUNTIME_IMAGE" { default = null }
+variable "BUILDER_IMAGE" { default = "debian_base" }
+variable "RUNTIME_IMAGE" { default = "debian_base" }
 
 variable "S3_ACCESS_KEY" { default = null }
 variable "S3_SECRET_KEY" { default = null }

From 93062b70c300ff525a21d18730ae3bde937e3ccb Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 15:36:27 -0500
Subject: [PATCH 37/43] fix: properly form DOCKER_REGISTRY when CI_REGISTRY
 defined/undefined

---
 containers/jug/dev.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index 143699dc..e48e1d48 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -5,7 +5,7 @@ variable "BUILD_IMAGE" { default = "eic_" }
 variable "BUILD_TYPE" { default = "default" }
 
 # Variables whose defaults can be overridden on build
-variable "DOCKER_REGISTRY" { default = join("/", [ CI_REGISTRY, CI_PROJECT_PATH, ""]) }
+variable "DOCKER_REGISTRY" { default = join("/", concat(compact([CI_REGISTRY, CI_PROJECT_PATH]), [""])) }
 variable "BUILDER_IMAGE" { default = "debian_base" }
 variable "RUNTIME_IMAGE" { default = "debian_base" }
 

From 8a7f17f3c0cf2105657e9adaeed164ffa826a4e3 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 15:58:36 -0500
Subject: [PATCH 38/43] feat: support docker/metadata-action for tags on GitHub

---
 containers/debian/base.hcl | 18 ++++++++++++------
 containers/jug/dev.hcl     | 26 ++++++++++++++++----------
 2 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index a51195e4..cb5b5195 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -4,7 +4,19 @@ variable "BUILD_IMAGE" { default = "debian_base" }
 # Variables whose defaults can be overridden on build
 variable "BASE_IMAGE" { default = null }
 
+# docker/metadata-action overrides the following target with tags
+# but we implement it for use outside docker/metadata-action
+target "docker-metadata-action" {
+  tags = compact(flatten([
+    join("/", compact([ CI_REGISTRY, CI_PROJECT_PATH, "${BUILD_IMAGE}:${INTERNAL_TAG}"]) ),
+    EXPORT_TAG != null && EXPORT_TAG != "" ? [
+      for registry in registries: "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}"
+    ] : [ null ]
+  ]))
+}
+
 target "default" {
+  inherits = ["docker-metadata-action"]
   attest = [
     "type=provenance,disabled=true"
   ]
@@ -15,10 +27,4 @@ target "default" {
     BASE_IMAGE = BASE_IMAGE
     BUILD_IMAGE = BUILD_IMAGE
   }
-  tags = compact(flatten([
-    join("/", compact([ CI_REGISTRY, CI_PROJECT_PATH, "${BUILD_IMAGE}:${INTERNAL_TAG}"]) ),
-    EXPORT_TAG != null && EXPORT_TAG != "" ? [
-      for registry in registries: "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}"
-    ] : [ null ]
-  ]))
 }
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index e48e1d48..fef63a50 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -28,7 +28,23 @@ image_names = [
   replace(image_name,"eic","jug")
 ]
 
+# docker/metadata-action overrides the following target with tags
+# but we implement it for use outside docker/metadata-action
+target "docker-metadata-action" {
+  tags = compact(flatten([
+    [
+      for image_name in image_names:
+        join("/", compact([ CI_REGISTRY, CI_PROJECT_PATH, "${image_name}:${INTERNAL_TAG}"]) )
+    ],
+    EXPORT_TAG != null && EXPORT_TAG != "" ? [
+      for registry_image_name in setproduct(registries, image_names):
+        format("%s:%s", join("/", registry_image_name), EXPORT_TAG )
+    ] : [ null ]
+  ]))
+}
+
 target "default" {
+  inherits = ["docker-metadata-action"]
   attest = [
     "type=provenance,disabled=true"
   ]
@@ -63,14 +79,4 @@ target "default" {
     EPIC_VERSION = BUILD_TYPE == "default" ? EPIC_VERSION : "main"
     JUGGLER_VERSION = BUILD_TYPE == "default" ? JUGGLER_VERSION : "main"
   }
-  tags = compact(flatten([
-    [
-      for image_name in image_names:
-        join("/", compact([ CI_REGISTRY, CI_PROJECT_PATH, "${image_name}:${INTERNAL_TAG}"]) )
-    ],
-    EXPORT_TAG != null && EXPORT_TAG != "" ? [
-      for registry_image_name in setproduct(registries, image_names):
-        format("%s:%s", join("/", registry_image_name), EXPORT_TAG )
-    ] : [ null ]
-  ]))
 }

From 0afb873ca031b6865603bd89d951b4966518a78e Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Fri, 12 Jul 2024 16:10:30 -0500
Subject: [PATCH 39/43] fix: JOBS -> jobs

---
 .github/workflows/build-push.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml
index 218613f2..376bbb60 100644
--- a/.github/workflows/build-push.yml
+++ b/.github/workflows/build-push.yml
@@ -45,7 +45,7 @@ env:
   GH_PUSH: 1
 
   ## Number of jobs to start during container builds
-  JOBS: 4
+  jobs: 4
 
   ## Internal tag used for the CI
   INTERNAL_TAG: pipeline-${{ github.run_id }}

From 7e5bafeef6eeda9f63aee6cff20771475ac51ec3 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Mon, 23 Sep 2024 12:59:33 -0500
Subject: [PATCH 40/43] fix: mv jobs into base.hcl

---
 .gitlab-ci.yml             | 2 --
 containers/debian/base.hcl | 2 ++
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7dc9a907..a0d82341 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -274,13 +274,11 @@ base:
     - nocache=""
     - docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --print
-        --build-arg jobs=${JOBS}
         --file containers/variables.hcl
         --file containers/debian/base.hcl
     - while !
       docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --push
-        --build-arg jobs=${JOBS}
         --file containers/variables.hcl
         --file containers/debian/base.hcl
         2>&1 | tee build.log
diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index cb5b5195..37fc6588 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -1,4 +1,5 @@
 # Variables which are required to be defined
+variable "jobs" { default = 1 }
 variable "BUILD_IMAGE" { default = "debian_base" }
 
 # Variables whose defaults can be overridden on build
@@ -24,6 +25,7 @@ target "default" {
   dockerfile = "base.Dockerfile"
   platforms = [ "linux/amd64" ]
   args = {
+    jobs = jobs
     BASE_IMAGE = BASE_IMAGE
     BUILD_IMAGE = BUILD_IMAGE
   }

From 46bf491ead28aa38503b1a643fbd32f33669808d Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Mon, 23 Sep 2024 13:13:09 -0500
Subject: [PATCH 41/43] fix: don't require spack.sh in mirrors.yaml.in

---
 .gitlab-ci.yml  | 4 +---
 mirrors.yaml.in | 4 ++--
 spack.hcl       | 1 +
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a0d82341..59f07762 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -349,9 +349,7 @@ eic:
         PUSH_NIGHTLY_WITH_DATE="1"
       fi
     - apk add envsubst
-    - source spack.sh ; 
-      export SPACK_VERSION ;
-      cat mirrors.yaml.in | envsubst > mirrors.yaml
+    - cat mirrors.yaml.in | envsubst > mirrors.yaml
     - attempts=0
     - nocache=""
     - docker buildx bake ${BUILD_OPTIONS} ${nocache}
diff --git a/mirrors.yaml.in b/mirrors.yaml.in
index 6c0fdc5c..45c9fd39 100644
--- a/mirrors.yaml.in
+++ b/mirrors.yaml.in
@@ -1,7 +1,7 @@
 mirrors:
   eicweb:
     autopush: true
-    url: oci://${CI_REGISTRY}/${CI_PROJECT_PATH}/spack-${SPACK_VERSION}
+    url: oci://${CI_REGISTRY}/${CI_PROJECT_PATH}/spack-0.23.0
     signed: false
     fetch:
       access_pair:
@@ -13,7 +13,7 @@ mirrors:
       - ${CI_REGISTRY_PASSWORD}
   ghcr:
     autopush: true
-    url: oci://ghcr.io/eic/spack-${SPACK_VERSION}
+    url: oci://ghcr.io/eic/spack-0.23.0
     signed: false
     fetch:
       access_pair:
diff --git a/spack.hcl b/spack.hcl
index a094fa56..26c13567 100644
--- a/spack.hcl
+++ b/spack.hcl
@@ -3,6 +3,7 @@ variable "SPACK_ORGREPO" { default = "spack/spack" }
 
 ## Spack github version, e.g. v0.18.1 or commit hash
 ## note: nightly builds will use e.g. releases/v0.19
+## note: update mirrors.yaml.in to match
 variable "SPACK_VERSION" { default = "v0.23.0" }
 
 ## Space-separated list of spack cherry-picks

From 6ce3d085da8bf174c18e3d424e51f8d2987c3917 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Mon, 23 Sep 2024 13:22:43 -0500
Subject: [PATCH 42/43] fix: mv eic/key4hep/spack.hcl to base.hcl

---
 .gitlab-ci.yml             | 12 ++++++------
 containers/debian/base.hcl | 13 +++++++++++++
 containers/jug/dev.hcl     | 13 -------------
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 59f07762..91adcae9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -274,11 +274,17 @@ base:
     - nocache=""
     - docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --print
+        --file spack.hcl
+        --file eic-spack.hcl
+        --file key4hep-spack.hcl
         --file containers/variables.hcl
         --file containers/debian/base.hcl
     - while !
       docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --push
+        --file spack.hcl
+        --file eic-spack.hcl
+        --file key4hep-spack.hcl
         --file containers/variables.hcl
         --file containers/debian/base.hcl
         2>&1 | tee build.log
@@ -354,17 +360,11 @@ eic:
     - nocache=""
     - docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --print
-        --file spack.hcl
-        --file eic-spack.hcl
-        --file key4hep-spack.hcl
         --file containers/variables.hcl
         --file containers/jug/dev.hcl
     - while !
       docker buildx bake ${BUILD_OPTIONS} ${nocache}
         --push
-        --file spack.hcl
-        --file eic-spack.hcl
-        --file key4hep-spack.hcl
         --file containers/variables.hcl
         --file containers/jug/dev.hcl
         2>&1 | tee build.log
diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl
index 37fc6588..26a3f43a 100644
--- a/containers/debian/base.hcl
+++ b/containers/debian/base.hcl
@@ -5,6 +5,9 @@ variable "BUILD_IMAGE" { default = "debian_base" }
 # Variables whose defaults can be overridden on build
 variable "BASE_IMAGE" { default = null }
 
+variable "S3_ACCESS_KEY" { default = null }
+variable "S3_SECRET_KEY" { default = null }
+
 # docker/metadata-action overrides the following target with tags
 # but we implement it for use outside docker/metadata-action
 target "docker-metadata-action" {
@@ -28,5 +31,15 @@ target "default" {
     jobs = jobs
     BASE_IMAGE = BASE_IMAGE
     BUILD_IMAGE = BUILD_IMAGE
+    SPACK_ORGREPO = try(SPACK_ORGREPO, null)
+    SPACK_VERSION = try(SPACK_VERSION, null)
+    SPACK_CHERRYPICKS = join("\n", try(SPACK_CHERRYPICKS, []))
+    SPACK_CHERRYPICKS_FILES = join("\n", try(SPACK_CHERRYPICKS_FILES, []))
+    KEY4HEPSPACK_ORGREPO = try(KEY4HEPSPACK_ORGREPO, null)
+    KEY4HEPSPACK_VERSION = try(KEY4HEPSPACK_VERSION, null)
+    EICSPACK_ORGREPO = try(EICSPACK_ORGREPO, null)
+    EICSPACK_VERSION = try(EICSPACK_VERSION, null)
+    S3_ACCESS_KEY = S3_ACCESS_KEY
+    S3_SECRET_KEY = S3_SECRET_KEY
   }
 }
diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index fef63a50..fc655b7d 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -9,9 +9,6 @@ variable "DOCKER_REGISTRY" { default = join("/", concat(compact([CI_REGISTRY, CI
 variable "BUILDER_IMAGE" { default = "debian_base" }
 variable "RUNTIME_IMAGE" { default = "debian_base" }
 
-variable "S3_ACCESS_KEY" { default = null }
-variable "S3_SECRET_KEY" { default = null }
-
 variable "NIGHTLY" { default = null }
 variable "NIGHTLY_TAG" { default = null }
 
@@ -64,16 +61,6 @@ target "default" {
     BUILDER_IMAGE = BUILDER_IMAGE
     RUNTIME_IMAGE = RUNTIME_IMAGE
     INTERNAL_TAG = INTERNAL_TAG
-    SPACK_ORGREPO = try(SPACK_ORGREPO, null)
-    SPACK_VERSION = try(SPACK_VERSION, null)
-    SPACK_CHERRYPICKS = join("\n", try(SPACK_CHERRYPICKS, []))
-    SPACK_CHERRYPICKS_FILES = join("\n", try(SPACK_CHERRYPICKS_FILES, []))
-    KEY4HEPSPACK_ORGREPO = try(KEY4HEPSPACK_ORGREPO, null)
-    KEY4HEPSPACK_VERSION = try(KEY4HEPSPACK_VERSION, null)
-    EICSPACK_ORGREPO = try(EICSPACK_ORGREPO, null)
-    EICSPACK_VERSION = try(EICSPACK_VERSION, null)
-    S3_ACCESS_KEY = S3_ACCESS_KEY
-    S3_SECRET_KEY = S3_SECRET_KEY
     EDM4EIC_VERSION = BUILD_TYPE == "default" ? EDM4EIC_VERSION : "main"
     EICRECON_VERSION = BUILD_TYPE == "default" ? EICRECON_VERSION : "main"
     EPIC_VERSION = BUILD_TYPE == "default" ? EPIC_VERSION : "main"

From 3c469610bb8a91ce9312964bb2cc229df7c99bd4 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Tue, 18 Mar 2025 11:56:57 -0500
Subject: [PATCH 43/43] fix: more robust handling of passed commit versions

---
 containers/jug/dev.hcl | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl
index fc655b7d..7b74628f 100644
--- a/containers/jug/dev.hcl
+++ b/containers/jug/dev.hcl
@@ -54,16 +54,23 @@ target "default" {
   secret = [
     "id=mirrors,src=mirrors.yaml",
   ]
-  args = {
+  args = merge({
     ENV = ENV
     jobs = jobs
     DOCKER_REGISTRY = DOCKER_REGISTRY
     BUILDER_IMAGE = BUILDER_IMAGE
     RUNTIME_IMAGE = RUNTIME_IMAGE
     INTERNAL_TAG = INTERNAL_TAG
-    EDM4EIC_VERSION = BUILD_TYPE == "default" ? EDM4EIC_VERSION : "main"
-    EICRECON_VERSION = BUILD_TYPE == "default" ? EICRECON_VERSION : "main"
-    EPIC_VERSION = BUILD_TYPE == "default" ? EPIC_VERSION : "main"
-    JUGGLER_VERSION = BUILD_TYPE == "default" ? JUGGLER_VERSION : "main"
-  }
+  },
+  BUILD_TYPE == "default" ? merge(
+    { EPIC_VERSION = try(EPIC_VERSION, null) },
+    ( EDM4EIC_VERSION != "" ? { EDM4EIC_VERSION = EDM4EIC_VERSION } : { } ),
+    ( EICRECON_VERSION != "" ? { EICRECON_VERSION = EICRECON_VERSION } : { } ),
+    ( JUGGLER_VERSION != "" ? { JUGGLER_VERSION = JUGGLER_VERSION } : { } ),
+  ) : {
+    EPIC_VERSION = "main"
+    EDM4EIC_VERSION = "main"
+    EICRECON_VERSION = "main"
+    JUGGLER_VERSION = "main"
+  })
 }