From cc9b33d9170bf99e7736ea21c1c5870b32150554 Mon Sep 17 00:00:00 2001
From: Alessio Pragliola <83355398+Al-Pragliola@users.noreply.github.com>
Date: Thu, 17 Oct 2024 19:41:19 +0200
Subject: [PATCH] fix(gha): prevent race condition on port-forward (#485)

* fix(gha): prevent race condition on port-forward

Signed-off-by: Alessio Pragliola <seth.pro@gmail.com>

* chore(gha): make it faster by waiting for db before mr

Signed-off-by: Alessio Pragliola <seth.pro@gmail.com>
Co-authored-by: Matteo Mortari <matteo.mortari@gmail.com>

---------

Signed-off-by: Alessio Pragliola <seth.pro@gmail.com>
Co-authored-by: Matteo Mortari <matteo.mortari@gmail.com>
---
 .github/workflows/python-tests.yml |  8 +++----
 scripts/deploy_on_kind.sh          | 13 ++++++++++-
 scripts/utils.sh                   | 35 ++++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 5 deletions(-)
 create mode 100644 scripts/utils.sh

diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml
index 90cc44829..556143a95 100644
--- a/.github/workflows/python-tests.yml
+++ b/.github/workflows/python-tests.yml
@@ -125,15 +125,15 @@ jobs:
         uses: helm/kind-action@v1.10.0
         with:
           node_image: "kindest/node:v1.27.11"
+          cluster_name: chart-testing-py-${{ matrix.python }}
       - name: Load Local Registry Test Image
-        if: matrix.session == 'tests'
         env:
-          IMG: "docker.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
+          IMG: "${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
         run: |
-          kind load docker-image -n chart-testing ${IMG}
+          kind load docker-image -n chart-testing-py-${{ matrix.python }} ${IMG}
       - name: Deploy Model Registry using manifests
         env:
-          IMG: "docker.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
+          IMG: "${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
         run: ./scripts/deploy_on_kind.sh
       - name: Nox test end-to-end
         working-directory: clients/python
diff --git a/scripts/deploy_on_kind.sh b/scripts/deploy_on_kind.sh
index 8cbc4bdd8..ecc3a5947 100755
--- a/scripts/deploy_on_kind.sh
+++ b/scripts/deploy_on_kind.sh
@@ -2,8 +2,11 @@
 
 set -e
 
+DIR="$(dirname "$0")"
 MR_NAMESPACE="${MR_NAMESPACE:-kubeflow}"
 
+source ./${DIR}/utils.sh
+
 # modularity to allow re-use this script against a remote k8s cluster
 if [[ -n "$LOCAL" ]]; then
     CLUSTER_NAME="${CLUSTER_NAME:-kind}"
@@ -33,6 +36,14 @@ else
 fi
 
 kubectl apply -k manifests/kustomize/overlays/db
-kubectl set image -n kubeflow deployment/model-registry-deployment rest-container="$IMG"
+kubectl patch deployment -n "$MR_NAMESPACE" model-registry-deployment \
+--patch '{"spec": {"template": {"spec": {"containers": [{"name": "rest-container", "image": "'$IMG'", "imagePullPolicy": "IfNotPresent"}]}}}}'
+
 kubectl wait --for=condition=available -n "$MR_NAMESPACE" deployment/model-registry-db --timeout=5m
+
+kubectl delete pod -n "$MR_NAMESPACE" --selector='component=model-registry-server'
+
+repeat_cmd_until "kubectl get pod -n "$MR_NAMESPACE" --selector='component=model-registry-server' \
+-o jsonpath=\"{.items[*].spec.containers[?(@.name=='rest-container')].image}\"" "= $IMG" 300 "kubectl describe pod -n $MR_NAMESPACE --selector='component=model-registry-server'"
+
 kubectl wait --for=condition=available -n "$MR_NAMESPACE" deployment/model-registry-deployment --timeout=5m
diff --git a/scripts/utils.sh b/scripts/utils.sh
new file mode 100644
index 000000000..904ca6593
--- /dev/null
+++ b/scripts/utils.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+repeat_cmd_until() {
+  local cmd=$1
+  local condition=$2
+  local max_wait_secs=$3
+  local debug_cmd=$4
+  local interval_secs=2
+  local start_time=$(date +%s)
+  local output
+
+  while true; do
+
+    current_time=$(date +%s)
+    if (( (current_time - start_time) > max_wait_secs )); then
+      echo "Waited for expression "$1" to satisfy condition "$2" for $max_wait_secs seconds without luck. Returning with error."
+      if [ -n "$debug_cmd" ]; then
+        echo "Running debug command: $debug_cmd"
+        eval $debug_cmd
+      fi
+      return 1
+    fi
+
+    output=$(eval $cmd)
+
+    if [ $output $condition ]; then
+      break
+    else
+      sleep $interval_secs
+    fi
+  done
+}