upgrade kubelet after kube components upgrade completes (#175)

Signed-off-by: Piyush Kumar <[email protected]>

go.mod

@@ -3,6 +3,7 @@ module github.com/kairos-io/kairos/provider-kubeadm
 go 1.23.1
 
 require (
+	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/kairos-io/kairos-sdk v0.5.0
 	github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5
 	github.com/mudler/yip v1.10.0
@@ -32,6 +33,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect

go.sum

@@ -75,6 +75,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -135,6 +137,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
+github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

scripts/kube-init.sh

@@ -15,6 +15,7 @@ proxy_no=$5
 KUBE_VIP_LOC="/etc/kubernetes/manifests/kube-vip.yaml"
 
 export PATH="$PATH:$root_path/usr/bin"
+export PATH="$PATH:$root_path/usr/local/bin"
 
 do_kubeadm_reset() {
   if [ -S /run/spectro/containerd/containerd.sock ]; then

scripts/kube-join.sh

@@ -16,6 +16,7 @@ proxy_https=$5
 proxy_no=$6
 
 export PATH="$PATH:$root_path/usr/bin"
+export PATH="$PATH:$root_path/usr/local/bin"
 
 KUBE_VIP_LOC="/etc/kubernetes/manifests/kube-vip.yaml"
 

scripts/kube-post-init.sh

@@ -11,6 +11,7 @@ root_path=$1
 
 export KUBECONFIG=/etc/kubernetes/admin.conf
 export PATH="$PATH:$root_path/usr/bin"
+export PATH="$PATH:$root_path/usr/local/bin"
 
 while true;
 do

scripts/kube-pre-init.sh

@@ -1,17 +1,33 @@
 #!/bin/bash
 
+exec   > >(tee -ia /var/log/kube-pre-init.log)
+exec  2> >(tee -ia /var/log/kube-pre-init.log >& 2)
+exec 19>> /var/log/kube-pre-init.log
+
+export BASH_XTRACEFD="19"
 set -x
 
 root_path=$1
 
 export PATH="$PATH:$root_path/usr/bin"
+export PATH="$PATH:$root_path/usr/local/bin"
 
 sysctl --system
 modprobe overlay
 modprobe br_netfilter
 systemctl daemon-reload
 
-systemctl enable kubelet && systemctl start kubelet
+if [ -f "$root_path"/opt/spectrocloud/kubeadm/bin/kubelet ]; then
+  cp "$root_path"/opt/spectrocloud/kubeadm/bin/kubelet "$root_path"/usr/local/bin/kubelet
+  systemctl daemon-reload
+  systemctl enable kubelet && systemctl restart kubelet
+  rm -rf "$root_path"/opt/spectrocloud/kubeadm/bin/kubelet
+fi
+
+if [ ! -f "$root_path"/usr/local/bin/kubelet ]; then
+  cp "$root_path"/opt/kubeadm/bin/kubelet "$root_path"/usr/local/bin/kubelet
+  systemctl enable kubelet && systemctl start kubelet
+fi
 
 if systemctl cat spectro-containerd >/dev/null 2<&1; then
   systemctl enable spectro-containerd && systemctl restart spectro-containerd

scripts/kube-reconfigure.sh

@@ -19,6 +19,7 @@ proxy_https=$6
 proxy_no=$7
 
 export PATH="$PATH:$root_path/usr/bin"
+export PATH="$PATH:$root_path/usr/local/bin"
 
 certs_sans_revision_path="$root_path/opt/kubeadm/.kubeadm_certs_sans_revision"
 

scripts/kube-reset.sh

@@ -8,6 +8,7 @@ if [ -f /etc/spectro/environment ]; then
 fi
 
 export PATH="$PATH:$STYLUS_ROOT/usr/bin"
+export PATH="$PATH:$STYLUS_ROOT/usr/local/bin"
 
 if [ -S /run/spectro/containerd/containerd.sock ]; then
     kubeadm reset -f --cri-socket unix:///run/spectro/containerd/containerd.sock --cleanup-tmp-dir

scripts/kube-upgrade.sh

@@ -15,6 +15,7 @@ proxy_https=$5
 proxy_no=$6
 
 export PATH="$PATH:$root_path/usr/bin"
+export PATH="$PATH:$root_path/usr/local/bin"
 
 if [ -n "$proxy_no" ]; then
   export NO_PROXY=$proxy_no
@@ -33,18 +34,38 @@ fi
 
 CURRENT_NODE_NAME=$(cat /etc/hostname)
 
+export KUBECONFIG=/etc/kubernetes/admin.conf
+
 get_current_upgrading_node_name() {
   kubectl get configmap upgrade-lock -n kube-system --kubeconfig /etc/kubernetes/admin.conf -o jsonpath="{['data']['node']}"
 }
 
-delete_lock_config_map(){
+delete_lock_config_map() {
   # Delete the configmap lock once the upgrade completes
   if [ "$NODE_ROLE" != "worker" ]
   then
     kubectl --kubeconfig /etc/kubernetes/admin.conf delete configmap upgrade-lock -n kube-system
   fi
 }
 
+upgrade_kubelet() {
+  echo "upgrading kubelet"
+  systemctl stop kubelet
+  cp "$root_path"/opt/kubeadm/bin/kubelet "$root_path"/usr/local/bin/kubelet
+  systemctl daemon-reload && systemctl restart kubelet
+  systemctl restart containerd
+  echo "kubelet upgraded"
+}
+
+apply_new_kubeadm_config() {
+  kubectl get cm kubeadm-config -n kube-system -o jsonpath="{['data']['ClusterConfiguration']}" --kubeconfig /etc/kubernetes/admin.conf > "$root_path"/opt/kubeadm/existing-cluster-config.yaml
+  kubeadm init phase upload-config kubeadm --config "$root_path"/opt/kubeadm/cluster-config.yaml
+}
+
+revert_kubeadm_config() {
+  kubeadm init phase upload-config kubeadm --config "$root_path"/opt/kubeadm/existing-cluster-config.yaml
+}
+
 run_upgrade() {
     echo "running upgrade process on $NODE_ROLE"
 
@@ -104,6 +125,7 @@ run_upgrade() {
 
             if [ "$master_api_version" = "$old_version" ]
             then
+                apply_new_kubeadm_config
                 upgrade_command="kubeadm upgrade apply -y $current_version"
                 if [ "$PROXY_CONFIGURED" = true ]; then
                   up=("kubeadm upgrade apply -y ${current_version}")
@@ -113,33 +135,25 @@ run_upgrade() {
         fi
         echo "upgrading node from $old_version to $current_version using command: $upgrade_command"
 
-        if [ "$PROXY_CONFIGURED" = true ]; then
-          if sudo -E bash -c "$upgrade_command"
-          then
-              # Update current client version in the version file
-              echo "$current_version" > "$root_path"/opt/sentinel_kubeadmversion
-              old_version=$current_version
-
-              delete_lock_config_map
-              echo "upgrade success"
-          else
-              echo "upgrade failed, retrying in 60 seconds"
-              sleep 60
-          fi
+        if sudo -E bash -c "$upgrade_command"
+        then
+            # Update current client version in the version file
+            echo "$current_version" > "$root_path"/opt/sentinel_kubeadmversion
+            old_version=$current_version
+
+            delete_lock_config_map
+            echo "upgrade success"
         else
-          if $upgrade_command
-          then
-              # Update current client version in the version file
-              echo "$current_version" > "$root_path"/opt/sentinel_kubeadmversion
-              old_version=$current_version
-
-              delete_lock_config_map
-              echo "upgrade success"
-          else
-              echo "upgrade failed, retrying in 60 seconds"
-              sleep 60
-          fi
+            echo "upgrade failed"
+            if echo "$upgrade_command" | grep -q "apply"; then
+              echo "reverting kubeadm config"
+              revert_kubeadm_config
+            fi
+            echo "retrying in 60 seconds"
+            sleep 60
         fi
     done
+    upgrade_kubelet
 }
+
 run_upgrade

stages/init.go

@@ -45,8 +45,8 @@ func GetInitYipStagesV1Beta3(clusterCtx *domain.ClusterContext, kubeadmConfig do
 		getKubeadmInitConfigStage(getInitNodeConfigurationBeta3(clusterCtx, kubeadmConfig.InitConfiguration, kubeadmConfig.ClusterConfiguration, kubeadmConfig.KubeletConfiguration), clusterCtx.RootPath),
 		getKubeadmInitStage(clusterCtx),
 		getKubeadmPostInitStage(clusterCtx.RootPath),
-		getKubeadmInitUpgradeStage(clusterCtx),
 		getKubeadmInitCreateClusterConfigStage(&kubeadmConfig.ClusterConfiguration, &kubeadmConfig.InitConfiguration, clusterCtx.RootPath),
+		getKubeadmInitUpgradeStage(clusterCtx),
 		getKubeadmInitCreateKubeletConfigStage(&kubeadmConfig.ClusterConfiguration, &kubeadmConfig.InitConfiguration, &kubeadmConfig.KubeletConfiguration, clusterCtx.RootPath),
 		getKubeadmInitReconfigureStage(clusterCtx),
 	}
@@ -66,8 +66,8 @@ func GetInitYipStagesV1Beta4(clusterCtx *domain.ClusterContext, kubeadmConfig do
 		getKubeadmInitConfigStage(getInitNodeConfigurationBeta4(clusterCtx, kubeadmConfig.InitConfiguration, kubeadmConfig.ClusterConfiguration, kubeadmConfig.KubeletConfiguration), clusterCtx.RootPath),
 		getKubeadmInitStage(clusterCtx),
 		getKubeadmPostInitStage(clusterCtx.RootPath),
-		getKubeadmInitUpgradeStage(clusterCtx),
 		getKubeadmInitCreateClusterConfigStage(&kubeadmConfig.ClusterConfiguration, &kubeadmConfig.InitConfiguration, clusterCtx.RootPath),
+		getKubeadmInitUpgradeStage(clusterCtx),
 		getKubeadmInitCreateKubeletConfigStage(&kubeadmConfig.ClusterConfiguration, &kubeadmConfig.InitConfiguration, &kubeadmConfig.KubeletConfiguration, clusterCtx.RootPath),
 		getKubeadmInitReconfigureStage(clusterCtx),
 	}