Handle latest tag.

Signed-off-by: Revital Sur <[email protected]>

.github/workflows/test.yml

@@ -170,34 +170,34 @@ jobs:
                   make -C ${transforms[$index]} workflow-test
                   header_text "Run ${transforms[$index]} completed"
     push_images:
-      name: Push images with latest tag
-      runs-on: ubuntu-latest
-      env:
-        DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
-        DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
-      needs: [test-make, test-python-lib, test-ray-lib, test-kfp-v1]
-      # if: ${{ github.event_name != 'pull_request' && github.repository == 'IBM/data-prep-kit' }}
-      steps:
-      - uses: actions/checkout@v3
-      - name: Download artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: images
-      - name: Load images
-        run: make -C transforms -C kfp-kfp_ray_componenets docker-load-image
-      - id: version
-        name: Infer version
-        run: |
-          publish_images='false'
-          if  [[ ${GITHUB_REF} == refs/heads/dev ]] ;
-          then
+        name: Push images with latest tag
+        runs-on: ubuntu-latest
+        env:
+           DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+           DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
+        needs: [test-make, test-python-lib, test-ray-lib, test-kfp-v1]
+        # if: ${{ github.event_name != 'pull_request' && github.repository == 'IBM/data-prep-kit' }}
+        steps:
+        - uses: actions/checkout@v3
+        - name: Download artifact
+          uses: actions/download-artifact@v3
+          with:
+            name: images
+        - name: Load images
+          run: make -C transforms -C kfp-kfp_ray_componenets docker-load-image
+        - id: version
+          name: Infer version
+          run: |
+            publish_images='false'
+            if  [[ ${GITHUB_REF} == refs/heads/dev ]] ;
+            then
               publish_images='true'
-          fi
-          echo ::set-output name=publish_images::$publish_images
-       - name: Publish images
-         if: ${{ steps.version.outputs.publish_images == 'true' }}
-         env:
-           USE_LATEST_TAG: 1
-#         run: make -C transforms -C kfp-kfp_ray_componenets publish
+            fi
+            echo ::set-output name=publish_images::$publish_images
+        - name: Publish images
+          if: ${{ steps.version.outputs.publish_images == 'true' }}
+          env:
+             USE_LATEST_TAG: 1
+#           run: make -C transforms -C kfp-kfp_ray_componenets publish
           run: echo "make -C transforms -C kfp-kfp_ray_componenets publish"
 

.make.versions

@@ -23,6 +23,13 @@ DPK_VERSION_SUFFIX=.dev0
 
 DPK_VERSION=$(DPK_MAJOR_VERSION).$(DPK_MINOR_VERSION).$(DPK_MICRO_VERSION)$(DPK_VERSION_SUFFIX)
 
+# publish docker images with latest tag
+ifeq ($(DPK_VERSION_SUFFIX), )
+	DOCKER_IMAGE_VERSION=$(DPK_VERSION)
+else
+	DOCKER_IMAGE_VERSION=latest
+endif
+
 # Data prep lab wheel version
 DPK_LIB_VERSION=$(DPK_VERSION)
 DPK_LIB_KFP_VERSION=$(DPK_VERSION)
@@ -50,7 +57,6 @@ NOOP_PYTHON_VERSION=$(DPK_VERSION)
 NOOP_RAY_VERSION=$(DPK_VERSION)
 NOOP_SPARK_VERSION=$(DPK_VERSION)
 
-PROFILER_VERSION=$(DPK_VERSION)
 PROFILER_RAY_VERSION=$(DPK_VERSION)
 
 RESIZE_PYTHON_VERSION=$(DPK_VERSION)
@@ -75,9 +81,6 @@ CODE2PARQUET_PYTHON_VERSION=$(DPK_VERSION)
 CODE2PARQUET_RAY_VERSION=$(DPK_VERSION)
 INGEST_TO_PARQUET_VERSION=$(DPK_VERSION)
 
-KFP_DOCKER_VERSION=$(DPK_VERSION)
-KFP_DOCKER_VERSION_v2=$(DPK_VERSION)
-
 ################## ################## ################## ################## ################## ##################
 # Begin versions that the repo depends on. 
 

kfp/kfp_ray_components/Makefile

@@ -12,10 +12,8 @@ DOCKER_IMG=$(DOCKER_LOCAL_IMAGE)
 
 ifeq ($(KFPv2), 1)
         DOCKER_IMAGE_NAME=kfp-data-processing_v2
-        DOCKER_IMAGE_VERSION=${KFP_DOCKER_VERSION_v2}
 else
         DOCKER_IMAGE_NAME=kfp-data-processing
-        DOCKER_IMAGE_VERSION=${KFP_DOCKER_VERSION}
 endif
 
 .PHONY: .lib-src-image
@@ -40,7 +38,7 @@ image: Dockerfile requirements.txt
 
 .PHONY: .reconcile-requirements
 .reconcile-requirements:
-	@sed -i.back "s/kfp-data-processing.*:[0-9].*/$(DOCKER_IMAGE_NAME):${DOCKER_IMAGE_VERSION}\"/" ${FILE}
+	@sed -i.back "s/kfp-data-processing.*/$(DOCKER_IMAGE_NAME):${DOCKER_IMAGE_VERSION}\"/" ${FILE}
 
 .PHONE: set-versions
 set-versions:

kfp/kfp_ray_components/createRayClusterComponent.yaml

@@ -11,7 +11,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.1.dev0"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

kfp/kfp_ray_components/deleteRayClusterComponent.yaml

@@ -8,7 +8,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.1.dev0"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

kfp/kfp_ray_components/executeRayJobComponent.yaml

@@ -12,7 +12,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.1.dev0"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

kfp/kfp_ray_components/executeRayJobComponent_multi_s3.yaml

@@ -13,7 +13,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.1.dev0"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

kfp/kfp_ray_components/executeSubWorkflowComponent.yaml

@@ -27,7 +27,7 @@ outputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.1.dev0"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists, and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

kfp/superworkflows/ray/kfp_v1/superworkflow_code_sample_wf.py

@@ -17,14 +17,14 @@
 run_fuzzy_dedup_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml")
 run_tokenization_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml")
 
-code_to_parquet_image = "quay.io/dataprep1/data-prep-kit/code2parquet-ray:0.2.1.dev0"
-proglang_select_image = "quay.io/dataprep1/data-prep-kit/proglang_select-ray:0.2.1.dev0"
-code_quality_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:0.2.1.dev0"
-malware_image = "quay.io/dataprep1/data-prep-kit/malware-ray:0.2.1.dev0"
-doc_id_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:0.2.1.dev0"
-ededup_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:0.2.1.dev0"
-fdedup_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:0.2.1.dev0"
-tokenizer_image = "quay.io/dataprep1/data-prep-kit/tokenization-ray:0.2.1.dev0"
+code_to_parquet_image = "quay.io/dataprep1/data-prep-kit/code2parquet-ray:latest"
+proglang_select_image = "quay.io/dataprep1/data-prep-kit/proglang_select-ray:latest"
+code_quality_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:latest"
+malware_image = "quay.io/dataprep1/data-prep-kit/malware-ray:latest"
+doc_id_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:latest"
+ededup_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:latest"
+fdedup_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:latest"
+tokenizer_image = "quay.io/dataprep1/data-prep-kit/tokenization-ray:latest"
 
 
 # Pipeline to invoke execution on remote resource

kfp/superworkflows/ray/kfp_v1/superworkflow_dedups_sample_wf.py

@@ -12,9 +12,9 @@
 run_exact_dedup_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml")
 run_fuzzy_dedup_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml")
 
-doc_id_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:0.2.1.dev0"
-ededup_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:0.2.1.dev0"
-fdedup_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:0.2.1.dev0"
+doc_id_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:latest"
+ededup_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:latest"
+fdedup_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:latest"
 
 # Pipeline to invoke execution on remote resource
 @dsl.pipeline(

scripts/transforms/update_workflow_tags.sh

@@ -1,15 +1,19 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-if [[ $# != 2 ]]; then
+set -x
+
+if [[ $# != 3 ]]; then
   cat << EOF
-	"Incorrect number of parameters provided. The required parameters are versions_file and pipeline_path. 
+	$1 $2 
+	"Incorrect number of parameters provided. The required parameters are versions_file, pipeline_path and docker_image_version. 
 EOF
   exit 1
 fi
 
 versions_file=$1
 pipeline_path=$2
+docker_image_version=$3
 
 # Modify the tasks tags as defined in the versions file
 while IFS= read -r line; do 
@@ -19,6 +23,5 @@ while IFS= read -r line; do
 	VERSION_NAME=$(echo $line |cut -d "=" -f 1)
 	DOCKER_IMAGE_NAME=$(echo $line |cut -d "=" -f 1 |sed "s/_VERSION//" |tr '[:upper:]' '[:lower:]')
 	DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE_NAME |sed "s/_ray$/\-ray/" | sed "s/_spark$/\-spark/" | sed "s/_parquet$/\-parquet/")
-	DOCKER_IMAGE_VERSION=$(eval echo ${!VERSION_NAME})
-	sed -i.back "s/data-prep-kit\/$DOCKER_IMAGE_NAME:.*/data-prep-kit\/$DOCKER_IMAGE_NAME:$DOCKER_IMAGE_VERSION\"/" $pipeline_path
+	sed -i.back "s/data-prep-kit\/$DOCKER_IMAGE_NAME:.*/data-prep-kit\/$DOCKER_IMAGE_NAME:$docker_image_version\"/" $pipeline_path
 done < $versions_file

tools/ingest2parquet/Makefile

@@ -5,8 +5,6 @@ include $(REPOROOT)/.make.defaults
 
 DOCKER_IMAGE_VERSION=${INGEST_TO_PARQUET_VERSION}
 DOCKER_IMAGE_NAME?=ingest2parquet
-#DOCKER_IMAGE=${DOCKER_REGISTRY_ENDPOINT}/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_VERSION)
-
 
 venv::	.defaults.ray-lib-src-venv
 

transforms/.make.transforms

@@ -323,9 +323,8 @@ minio-stop:
 # Requires version number of Python image as TRANSFORM_PYTHON_VERSION to be set when called
 # In addition, update Dockerfile with versions
 .transforms.set-versions: 
-	$(call check_defined,DOCKER_IMAGE_VERSION)
 	$(call check_defined, TRANSFORM_PYTHON_VERSION)
-	$(MAKE) TOML_VERSION=$(DOCKER_IMAGE_VERSION) .defaults.update-toml
+	$(MAKE) .defaults.update-toml
 	if [ -e pyproject.toml ]; then	\
 	    dash_name=$$(echo $(TRANSFORM_NAME) | sed -e 's/_/-/g');		\
 	    cat pyproject.toml | sed 	\

transforms/.make.workflows

@@ -1,7 +1,3 @@
-IGNORE := $(shell bash -c "sed -nr /^[^\#]*=/p ${REPOROOT}/.make.versions | sed 's/=/:=/' | sed 's/^/export /' > makeenv")
-
-include makeenv
-
 # Include the common rules.
 # Use "make help" to see them.
 include ${REPOROOT}/.make.defaults
@@ -17,10 +13,11 @@ KFP_ENDPOINT ?= "http://localhost:8080/"
 
 .PHONY: set-versions
 set-versions:
+	echo ${DOCKER_IMAGE_VERSION}
 	cd ${REPOROOT}/kfp/kfp_ray_components && $(MAKE) set-versions
 	@for file in $(PYTHON_WF); do \
 		cd ${REPOROOT}/kfp/kfp_ray_components && $(MAKE) .reconcile-requirements FILE=${CURDIR}/$$file; \
-		${REPOROOT}/scripts/transforms/update_workflow_tags.sh ${REPOROOT}/.make.versions ${CURDIR}/$$file; \
+		${REPOROOT}/scripts/transforms/update_workflow_tags.sh ${REPOROOT}/.make.versions ${CURDIR}/$$file ${DOCKER_IMAGE_VERSION}; \
 	done
 
 .PHONY: .workflows.compile-pipeline

transforms/code/code2parquet/kfp_ray/code2parquet_wf.py

@@ -21,11 +21,11 @@
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "code2parquet_transform_ray.py"
 
-task_image = "quay.io/dataprep1/data-prep-kit/code2parquet-ray:0.2.1.dev0"
+task_image = "quay.io/dataprep1/data-prep-kit/code2parquet-ray:latest"
 
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.1.dev0"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

transforms/code/code2parquet/python/Makefile

@@ -9,7 +9,6 @@ REPOROOT=../../../..
 
 TRANSFORM_RUNTIME=python
 TRANSFORM_NAME=code2parquet
-DOCKER_IMAGE_VERSION=${CODE2PARQUET_PYTHON_VERSION}
 
 include $(REPOROOT)/transforms/.make.transforms
 
@@ -35,7 +34,7 @@ setup:: .transforms.setup
 
 # distribution versions is the same as image version.
 set-versions:
-	$(MAKE) TOML_VERSION=$(DOCKER_IMAGE_VERSION) .defaults.update-toml
+	$(MAKE) TRANSFORM_PYTHON_VERSION=$(CODE2PARQUET_PYTHON_VERSION) TOML_VERSION=$(CODE2PARQUET_PYTHON_VERSION) .transforms.set-versions 
 
 build-dist:: set-versions .defaults.build-dist 
 

transforms/code/code2parquet/ray/Makefile

@@ -7,8 +7,6 @@ REPOROOT=../../../..
 include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=code2parquet
-# $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${CODE2PARQUET_RAY_VERSION}
 
 venv::	.transforms.ray-venv
 
@@ -34,7 +32,7 @@ setup:: .transforms.setup
 
 # set the version of python transform that this depends on.
 set-versions:
-	$(MAKE) TRANSFORM_PYTHON_VERSION=$(CODE2PARQUET_PYTHON_VERSION) .transforms.set-versions
+	$(MAKE) TRANSFORM_PYTHON_VERSION=$(CODE2PARQUET_PYTHON_VERSION) TOML_VERSION=$(CODE2PARQUET_RAY_VERSION) .transforms.set-versions
 
 build-dist:: set-versions .defaults.build-dist
 

transforms/code/code_quality/kfp_ray/code_quality_wf.py

@@ -21,10 +21,10 @@
 EXEC_SCRIPT_NAME: str = "code_quality_transform_ray.py"
 PREFIX: str = ""
 
-task_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:0.2.1.dev0"
+task_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:latest"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.1.dev0"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

transforms/code/code_quality/python/Makefile

@@ -5,8 +5,6 @@ REPOROOT=../../../..
 include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=code_quality
-# $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${CODE_QUALITY_PYTHON_VERSION}
 
 # Use default rule inherited from makefile.common
 clean:: .transforms.clean
@@ -34,7 +32,7 @@ setup:: .transforms.setup
 
 # distribution versions is the same as image version.
 set-versions:
-	$(MAKE) TRANSFORM_PYTHON_VERSION=${CODE_QUALITY_PYTHON_VERSION} TOML_VERSION=$(DOCKER_IMAGE_VERSION) .transforms.set-versions 
+	$(MAKE) TRANSFORM_PYTHON_VERSION=$(CODE_QUALITY_PYTHON_VERSION) TOML_VERSION=$(CODE_QUALITY_PYTHON_VERSION) .transforms.set-versions 
 
 build-dist:: set-versions .defaults.build-dist 
 

transforms/code/code_quality/ray/Makefile

@@ -5,8 +5,6 @@ REPOROOT=../../../..
 include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=code_quality
-# $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${CODE_QUALITY_RAY_VERSION}
 
 clean:: .transforms.clean
 
@@ -29,7 +27,7 @@ publish-image:: .transforms.publish-image-ray
 setup:: .transforms.setup
 
 set-versions:
-	$(MAKE) TRANSFORM_PYTHON_VERSION=$(CODE_QUALITY_PYTHON_VERSION) .transforms.set-versions
+	$(MAKE) TRANSFORM_PYTHON_VERSION=$(CODE_QUALITY_PYTHON_VERSION) TOML_VERSION=$(CODE_QUALITY_RAY_VERSION) .transforms.set-versions
 
 build-dist:: set-versions .defaults.build-dist 
 

transforms/code/malware/kfp_ray/malware_wf.py

@@ -21,10 +21,10 @@
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "malware_transform_ray.py"
 
-task_image = "quay.io/dataprep1/data-prep-kit/malware-ray:0.2.1.dev0"
+task_image = "quay.io/dataprep1/data-prep-kit/malware-ray:latest"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.1.dev0"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

transforms/code/malware/python/Makefile

@@ -7,8 +7,6 @@ REPOROOT=../../../..
 include $(REPOROOT)/transforms/.make.transforms
 
 TRANSFORM_NAME=malware
-# $(REPOROOT)/.make.versions file contains the versions
-DOCKER_IMAGE_VERSION=${MALWARE_PYTHON_VERSION}
 
 OS := $(shell uname -s)
 ifeq ($(OS),Darwin)
@@ -41,7 +39,7 @@ setup:: .transforms.setup
 
 # distribution versions is the same as image version.
 set-versions:
-	$(MAKE) TRANSFORM_PYTHON_VERSION=${MALWARE_PYTHON_VERSION} TOML_VERSION=$(DOCKER_IMAGE_VERSION) .transforms.set-versions 
+	$(MAKE) TRANSFORM_PYTHON_VERSION=$(MALWARE_PYTHON_VERSION) TOML_VERSION=$(MALWARE_PYTHON_VERSION) .transforms.set-versions 
 
 build-dist:: set-versions .defaults.build-dist