Add github action to push docker images with latest tag.

Signed-off-by: Revital Sur <[email protected]>
dolfim-ibm · Jul 1, 2024 · 73fb10a · 73fb10a
1 parent 31d073b
commit 73fb10a
Show file tree

Hide file tree

Showing 60 changed files with 425 additions and 75 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -48,7 +48,17 @@ jobs:
               uses: actions/checkout@v4
             - name: Test Code Transforms
               run: |
-                  make -C transforms/code DOCKER=docker venv test-src
+                  make USE_LATEST_TAG=1 -C transforms/code DOCKER=docker venv test-src
+            - name: Save images
+              if: ${{ github.event_name != 'pull_request' }}
+              run: make -C transforms -C kfp-kfp_ray_componenets docker-save-images
+            - name: Upload images
+              if: ${{ github.event_name != 'pull_request' }}
+              uses: actions/upload-artifact@v3
+              with:
+                name: images
+                path: artifacts/*
+                retention-days: 1
     test-language:
         runs-on: ubuntu-latest
         steps:
@@ -159,3 +169,35 @@ jobs:
                   header_text "Running ${transforms[$index]} workflow test"
                   make -C ${transforms[$index]} workflow-test
                   header_text "Run ${transforms[$index]} completed"
+    push_images:
+      name: Push images with latest tag
+      runs-on: ubuntu-latest
+      env:
+        DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+        DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
+      needs: [test-make, test-python-lib, test-ray-lib, test-kfp-v1]
+      if: ${{ github.event_name != 'pull_request' && github.repository == 'IBM/data-prep-kit' }}
+      steps:
+      - uses: actions/checkout@v3
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: images
+      - name: Load images
+        run: make -C transforms -C kfp-kfp_ray_componenets docker-load-image
+      - id: version
+        name: Infer version
+        run: |
+          publish_images='false'
+          if  [[ ${GITHUB_REF} == refs/heads/dev ]] ;
+          then
+              publish_images='true'
+          fi
+          echo ::set-output name=publish_images::$publish_images
+       - name: Publish images
+         if: ${{ steps.version.outputs.publish_images == 'true' }}
+         env:
+           USE_LATEST_TAG: 1
+#         run: make -C transforms -C kfp-kfp_ray_componenets publish
+          run: echo "make -C transforms -C kfp-kfp_ray_componenets publish"
+
diff --git a/.make.defaults b/.make.defaults
@@ -44,11 +44,14 @@ DOCKER_REGISTRY_KEY?=$(DPK_DOCKER_REGISTRY_KEY)
 DOCKER_REGISTRY_ENDPOINT?=$(DOCKER_HOSTNAME)/$(DOCKER_NAMESPACE)
 DOCKER_LOCAL_IMAGE=$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
 DOCKER_REMOTE_IMAGE=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
+DOCKER_REMOTE_IMAGE_LATEST=$(DOCKER_REGISTRY_ENDPOINT)/$(DOCKER_IMAGE_NAME):latest
 DOCKER_SPARK_BASE_IMAGE_NAME=data-prep-kit-spark-3.5.1
 DOCKER_SPARK_BASE_IMAGE=$(DOCKER_SPARK_BASE_IMAGE_NAME):$(DPK_LIB_VERSION)
 # Deprecated in favor of DOCKER_REMOTE_IMAGE
 DOCKER_IMAGE?=$(DOCKER_REMOTE_IMAGE)
 KIND_CLUSTER_NAME=dataprep
+ARTIFACTS_DIR?=$(REPOROOT)/artifacts
+USE_LATEST_TAG?=0
 
 DPK_PYTHON_LIB_DIR=$(REPOROOT)/data-processing-lib/python
 DPK_RAY_LIB_DIR=$(REPOROOT)/data-processing-lib/ray
@@ -179,6 +182,18 @@ __check_defined = \
 	-find . -name '*.egg-info' | xargs rm -rf
 	-find . -name '__pycache__' | xargs rm -rf
 	-rm -rf dist
+	-rm -rf $(REPOROOT)/artifacts > /dev/null 2>&1
+
+.PHONY: .defaults.docker-save-image
+.defaults.docker-save-image:
+ifeq ($(USE_LATEST_TAG),1)
+	   docker tag $(DOCKER_REMOTE_IMAGE) $(DOCKER_REMOTE_IMAGE_LATEST)
+endif
+	docker save -o ${ARTIFACTS_DIR}/${DOCKER_IMAGE_NAME}.tar $(DOCKER_REMOTE_IMAGE)
+
+.PHONY: .defaults.docker-load-image
+.defaults.docker-load-image:
+	docker load -i ${ARTIFACTS_DIR}/${DOCKER_IMAGE_NAME}.tar
 
 # We create both local and remote tags.  Local seems to be needed when using our spark
 # base image.  Remote seems to be needed by kfp.
@@ -392,7 +407,12 @@ __check_defined = \
 	@# Help: Publish the $(DOCKER_LOCAL_IMAGE) to $(DOCKER_HOSTNAME) container registry
 	-$(DOCKER) logout $(DOCKER_HOSTNAME)
 	$(DOCKER) login $(DOCKER_HOSTNAME) -u '$(DOCKER_REGISTRY_USER)' -p '$(DOCKER_REGISTRY_KEY)'
-	$(DOCKER) push  $(DOCKER_REMOTE_IMAGE)
+ifeq ($(USE_LATEST_TAG),1)
+	   $(DOCKER) push $(DOCKER_REMOTE_IMAGE_LATEST); 
+else  
+	   $(DOCKER) push $(DOCKER_REMOTE_IMAGE);
+endif
+
 
 # Create the local virtual environment, assuming python is already installed and available
 # We upgrade pip as that seems to be required by watson_nlp

diff --git a/kfp/kfp_ray_components/Makefile b/kfp/kfp_ray_components/Makefile
@@ -57,11 +57,18 @@ ifneq ($(KFPv2), 1)
 	@$(MAKE) .reconcile-requirements FILE=executeSubWorkflowComponent.yaml
 endif
 
-.PHONY: load-image
-load-image:
+.PHONY: kind-load-image
+kind-load-image:
 	@# Help: Load the image to the kind cluster created with make setup.
 	kind load docker-image $(DOCKER_REMOTE_IMAGE) --name=$(KIND_CLUSTER_NAME)
 
+.PHONY: docker-load-image
+docker-load-image:
+	@$(MAKE) .defaults.docker-load-image
+
+.PHONY: docker-save-image
+docker-save-image:
+	@$(MAKE) .defaults.docker-load-image
 
 .PHONY: publish
 publish:

diff --git a/transforms/.make.transforms b/transforms/.make.transforms
@@ -210,8 +210,8 @@ test-locals:: .transforms.test-locals
 	    echo Directory $(CHECK_DIR) is very large;		\
 	fi
 
-.PHONY: .transforms.load-image
-.transforms.load-image:
+.PHONY: .transforms.kind-load-image
+.transforms.kind-load-image:
 	@# Help: Load the transform image to the kind cluster created with make setup.
 	kind load docker-image $(DOCKER_REMOTE_IMAGE) --name=$(KIND_CLUSTER_NAME)
 

diff --git a/transforms/.make.workflows b/transforms/.make.workflows
@@ -48,8 +48,8 @@ FORCE:
 		cd ${REPOROOT} && make -C scripts/k8s-setup setup;  \
 	fi
 ifeq ($(USE_DEV_IMAGES), 1)
-	cd ${TRANSFORM_SRC} && $(MAKE) image && $(MAKE) load-image
-	cd ${REPOROOT}/kfp/kfp_ray_components && $(MAKE) image && $(MAKE) load-image
+	cd ${TRANSFORM_SRC} && $(MAKE) image && $(MAKE) kind-load-image
+	cd ${REPOROOT}/kfp/kfp_ray_components && $(MAKE) image && $(MAKE) kind-load-image
 endif
 	. ${WORKFLOW_VENV_ACTIVATE}  && ${PYTHON} -m workflow_support.pipeline_utils.pipelines_tests_utils -c "sanity-test" -p ${CURDIR}/${PIPELINE_FILE} -e ${KFP_ENDPOINT}
 

diff --git a/transforms/Makefile b/transforms/Makefile
@@ -44,7 +44,13 @@ test-src::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 
-load-image::
+kind-load-image::
+	@# Help: Recursively make $@ in all subdirs 
+	$(MAKE) RULE=$@ .recurse
+
+docker-load-image::
+
+docker-save-image::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 

diff --git a/transforms/code/Makefile b/transforms/code/Makefile
@@ -35,10 +35,17 @@ publish::
 	@# Help: Recursively make $@ in all subdirs 
 	@$(MAKE) RULE=$@ .recurse
 
-load-image::
+kind-load-image::
 	@# Help: Recursively make $@ in all subdirs
 	@$(MAKE) RULE=$@ .recurse
 
+docker-load-image::
+	@# Help: Recursively make $@ in all subdirs 
+	$(MAKE) RULE=$@ .recurse
+
+docker-save-image::
+	@# Help: Recursively make $@ in all subdirs 
+	$(MAKE) RULE=$@ .recurse
 
 set-versions:  
 	@# Help: Recursively $@ in all subdirs 

diff --git a/transforms/code/code2parquet/Makefile b/transforms/code/code2parquet/Makefile
@@ -37,7 +37,13 @@ test-src::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 
-load-image::
+kind-load-image::
+	@# Help: Recursively make $@ in all subdirs 
+	$(MAKE) RULE=$@ .recurse
+
+docker-load-image::
+
+docker-save-image::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 

diff --git a/transforms/code/code2parquet/kfp_ray/Makefile b/transforms/code/code2parquet/kfp_ray/Makefile
@@ -30,8 +30,11 @@ image::
 
 test-image::
 
-load-image::
+kind-load-image::
 
+docker-load-image::
+
+docker-save-image::
 
 .PHONY: workflow-build
 workflow-build: workflow-venv

diff --git a/transforms/code/code2parquet/python/Makefile b/transforms/code/code2parquet/python/Makefile
@@ -60,4 +60,8 @@ run-local-python-sample: .transforms.run-local-python-sample
 
 minio-start:	.minio-start
 
-load-image:: .transforms.load-image
+kind-load-image:: .transforms.kind-load-image
+
+docker-load-image: .defaults.docker-load-image
+
+docker-save-image: .defaults.docker-save-image
diff --git a/transforms/code/code2parquet/ray/Makefile b/transforms/code/code2parquet/ray/Makefile
@@ -55,4 +55,8 @@ run-s3-sample: .transforms.run-s3-ray-sample
 
 minio-start:	.minio-start
 
-load-image:: .transforms.load-image
+kind-load-image:: .transforms.kind-load-image
+
+docker-load-image: .defaults.docker-load-image
+
+docker-save-image: .defaults.docker-save-image
diff --git a/transforms/code/code_quality/Makefile b/transforms/code/code_quality/Makefile
@@ -41,7 +41,15 @@ test-src::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 
-load-image::
+kind-load-image::
+	@# Help: Recursively make $@ in all subdirs 
+	$(MAKE) RULE=$@ .recurse
+
+docker-load-image::
+	@# Help: Recursively make $@ in all subdirs
+	$(MAKE) RULE=$@ .recurse
+
+docker-save-image::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 
@@ -59,4 +67,4 @@ workflow-upload:
 
 .PHONY: workflow-build
 workflow-build:
-	$(MAKE) -C  kfp_ray workflow-build
+	$(MAKE) -C  kfp_ray workflow-build
diff --git a/transforms/code/code_quality/kfp_ray/Makefile b/transforms/code/code_quality/kfp_ray/Makefile
@@ -30,8 +30,11 @@ publish::
 
 image::
 
-load-image::
+kind-load-image::
 
+docker-load-image::
+
+docker-save-image::
 
 .PHONY: workflow-build
 workflow-build: workflow-venv

diff --git a/transforms/code/code_quality/python/Makefile b/transforms/code/code_quality/python/Makefile
@@ -48,4 +48,8 @@ run-local-sample: .transforms.run-local-sample
 
 run-local-python-sample: .transforms.run-local-python-sample
 
-load-image:: .transforms.load-image
+kind-load-image:: .transforms.kind-load-image
+
+docker-load-image: .defaults.docker-load-image
+
+docker-save-image: .defaults.docker-save-image
diff --git a/transforms/code/code_quality/ray/Makefile b/transforms/code/code_quality/ray/Makefile
@@ -43,4 +43,8 @@ run-s3-sample: .transforms.run-s3-ray-sample
 
 minio-start:	.minio-start
 
-load-image:: .transforms.load-image
+kind-load-image:: .transforms.kind-load-image
+
+docker-load-image: .defaults.docker-load-image
+
+docker-save-image: .defaults.docker-save-image
diff --git a/transforms/code/malware/Makefile b/transforms/code/malware/Makefile
@@ -41,7 +41,15 @@ test-src::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 
-load-image::
+kind-load-image::
+	@# Help: Recursively make $@ in all subdirs 
+	$(MAKE) RULE=$@ .recurse
+
+docker-load-image::
+	@# Help: Recursively make $@ in all subdirs
+	$(MAKE) RULE=$@ .recurse
+
+docker-save-image::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 
@@ -59,4 +67,4 @@ workflow-upload:
 
 .PHONY: workflow-build
 workflow-build:
-	$(MAKE) -C  kfp_ray workflow-build
+	$(MAKE) -C  kfp_ray workflow-build
diff --git a/transforms/code/malware/kfp_ray/Makefile b/transforms/code/malware/kfp_ray/Makefile
@@ -30,7 +30,11 @@ publish::
 
 image::
 
-load-image::
+kind-load-image::
+
+docker-load-image::
+
+docker-save-image::
 
 .PHONY: workflow-build
 workflow-build: workflow-venv

diff --git a/transforms/code/malware/python/Makefile b/transforms/code/malware/python/Makefile
@@ -59,4 +59,8 @@ run-local-python-sample: .transforms.run-local-python-sample
 
 minio-start:	.minio-start
 
-load-image:: .transforms.load-image
+kind-load-image:: .transforms.kind-load-image
+
+docker-load-image: .defaults.docker-load-image
+
+docker-save-image: .defaults.docker-save-image
diff --git a/transforms/code/malware/ray/Makefile b/transforms/code/malware/ray/Makefile
@@ -62,4 +62,8 @@ run-s3-ray-sample: #.transforms.run-s3-ray-sample
 
 minio-start:	.minio-start
 
-load-image:: .transforms.load-image
+kind-load-image:: .transforms.kind-load-image
+
+docker-load-image: .defaults.docker-load-image
+
+docker-save-image: .defaults.docker-save-image
diff --git a/transforms/code/proglang_select/Makefile b/transforms/code/proglang_select/Makefile
@@ -41,7 +41,15 @@ test-src::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 
-load-image::
+kind-load-image::
+	@# Help: Recursively make $@ in all subdirs 
+	$(MAKE) RULE=$@ .recurse
+
+docker-load-image::
+	@# Help: Recursively make $@ in all subdirs
+	$(MAKE) RULE=$@ .recurse
+
+docker-save-image::
 	@# Help: Recursively make $@ in all subdirs 
 	$(MAKE) RULE=$@ .recurse
 
@@ -59,4 +67,4 @@ workflow-upload:
 
 .PHONY: workflow-build
 workflow-build:
-	$(MAKE) -C  kfp_ray workflow-build
+	$(MAKE) -C  kfp_ray workflow-build
diff --git a/transforms/code/proglang_select/kfp_ray/Makefile b/transforms/code/proglang_select/kfp_ray/Makefile
@@ -30,7 +30,11 @@ publish::
 
 image::
 
-load-image::
+kind-load-image::
+
+docker-load-image::
+
+docker-save-image::
 
 .PHONY: workflow-build
 workflow-build: workflow-venv

diff --git a/transforms/code/proglang_select/python/Makefile b/transforms/code/proglang_select/python/Makefile
@@ -51,4 +51,8 @@ run-local-sample: .transforms.run-local-sample
 
 run-local-python-sample: .transforms.run-local-python-sample
 
-load-image:: .transforms.load-image
+kind-load-image:: .transforms.kind-load-image
+
+docker-load-image: .defaults.docker-load-image
+
+docker-save-image: .defaults.docker-save-image
diff --git a/transforms/code/proglang_select/ray/Makefile b/transforms/code/proglang_select/ray/Makefile
@@ -54,4 +54,8 @@ run-local-ray-sample: .transforms.run-local-ray-sample
 
 minio-start:	.minio-start
 
-load-image:: .transforms.load-image
+kind-load-image:: .transforms.kind-load-image
+
+docker-load-image: .defaults.docker-load-image
+
+docker-save-image: .defaults.docker-save-image