From 399f8f6227a9a423b0f078b26be7a0a29aa07345 Mon Sep 17 00:00:00 2001 From: Piyush Kumar Date: Fri, 21 Feb 2025 12:33:38 +0530 Subject: [PATCH] upgrade kubelet after kube components upgrade completes (#175) Signed-off-by: Piyush Kumar --- go.mod | 2 ++ go.sum | 4 +++ scripts/kube-init.sh | 1 + scripts/kube-join.sh | 1 + scripts/kube-post-init.sh | 1 + scripts/kube-pre-init.sh | 18 +++++++++- scripts/kube-reconfigure.sh | 1 + scripts/kube-reset.sh | 1 + scripts/kube-upgrade.sh | 66 ++++++++++++++++++++++--------------- stages/init.go | 4 +-- 10 files changed, 70 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 7507ac4..a9b5de6 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/kairos-io/kairos/provider-kubeadm go 1.23.1 require ( + github.com/coreos/go-systemd/v22 v22.5.0 github.com/kairos-io/kairos-sdk v0.5.0 github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5 github.com/mudler/yip v1.10.0 @@ -32,6 +33,7 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect diff --git a/go.sum b/go.sum index 4eecd7c..ac6fa1e 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -135,6 +137,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= diff --git a/scripts/kube-init.sh b/scripts/kube-init.sh index 3908017..e670f2d 100755 --- a/scripts/kube-init.sh +++ b/scripts/kube-init.sh @@ -15,6 +15,7 @@ proxy_no=$5 KUBE_VIP_LOC="/etc/kubernetes/manifests/kube-vip.yaml" export PATH="$PATH:$root_path/usr/bin" +export PATH="$PATH:$root_path/usr/local/bin" do_kubeadm_reset() { if [ -S /run/spectro/containerd/containerd.sock ]; then diff --git a/scripts/kube-join.sh b/scripts/kube-join.sh index 53d97c3..f5d36a2 100755 --- a/scripts/kube-join.sh +++ b/scripts/kube-join.sh @@ -16,6 +16,7 @@ proxy_https=$5 proxy_no=$6 export PATH="$PATH:$root_path/usr/bin" +export PATH="$PATH:$root_path/usr/local/bin" KUBE_VIP_LOC="/etc/kubernetes/manifests/kube-vip.yaml" diff --git a/scripts/kube-post-init.sh b/scripts/kube-post-init.sh index d15b09d..709f0a7 100755 --- a/scripts/kube-post-init.sh +++ b/scripts/kube-post-init.sh @@ -11,6 +11,7 @@ root_path=$1 export KUBECONFIG=/etc/kubernetes/admin.conf export PATH="$PATH:$root_path/usr/bin" +export PATH="$PATH:$root_path/usr/local/bin" while true; do diff --git a/scripts/kube-pre-init.sh b/scripts/kube-pre-init.sh index c7b5e44..0fbe0cb 100755 --- a/scripts/kube-pre-init.sh +++ b/scripts/kube-pre-init.sh @@ -1,17 +1,33 @@ #!/bin/bash +exec > >(tee -ia /var/log/kube-pre-init.log) +exec 2> >(tee -ia /var/log/kube-pre-init.log >& 2) +exec 19>> /var/log/kube-pre-init.log + +export BASH_XTRACEFD="19" set -x root_path=$1 export PATH="$PATH:$root_path/usr/bin" +export PATH="$PATH:$root_path/usr/local/bin" sysctl --system modprobe overlay modprobe br_netfilter systemctl daemon-reload -systemctl enable kubelet && systemctl start kubelet +if [ -f "$root_path"/opt/spectrocloud/kubeadm/bin/kubelet ]; then + cp "$root_path"/opt/spectrocloud/kubeadm/bin/kubelet "$root_path"/usr/local/bin/kubelet + systemctl daemon-reload + systemctl enable kubelet && systemctl restart kubelet + rm -rf "$root_path"/opt/spectrocloud/kubeadm/bin/kubelet +fi + +if [ ! -f "$root_path"/usr/local/bin/kubelet ]; then + cp "$root_path"/opt/kubeadm/bin/kubelet "$root_path"/usr/local/bin/kubelet + systemctl enable kubelet && systemctl start kubelet +fi if systemctl cat spectro-containerd >/dev/null 2<&1; then systemctl enable spectro-containerd && systemctl restart spectro-containerd diff --git a/scripts/kube-reconfigure.sh b/scripts/kube-reconfigure.sh index aedb998..669e828 100755 --- a/scripts/kube-reconfigure.sh +++ b/scripts/kube-reconfigure.sh @@ -19,6 +19,7 @@ proxy_https=$6 proxy_no=$7 export PATH="$PATH:$root_path/usr/bin" +export PATH="$PATH:$root_path/usr/local/bin" certs_sans_revision_path="$root_path/opt/kubeadm/.kubeadm_certs_sans_revision" diff --git a/scripts/kube-reset.sh b/scripts/kube-reset.sh index 1b5c911..55528e1 100755 --- a/scripts/kube-reset.sh +++ b/scripts/kube-reset.sh @@ -8,6 +8,7 @@ if [ -f /etc/spectro/environment ]; then fi export PATH="$PATH:$STYLUS_ROOT/usr/bin" +export PATH="$PATH:$STYLUS_ROOT/usr/local/bin" if [ -S /run/spectro/containerd/containerd.sock ]; then kubeadm reset -f --cri-socket unix:///run/spectro/containerd/containerd.sock --cleanup-tmp-dir diff --git a/scripts/kube-upgrade.sh b/scripts/kube-upgrade.sh index ee97d14..ba7c37c 100755 --- a/scripts/kube-upgrade.sh +++ b/scripts/kube-upgrade.sh @@ -15,6 +15,7 @@ proxy_https=$5 proxy_no=$6 export PATH="$PATH:$root_path/usr/bin" +export PATH="$PATH:$root_path/usr/local/bin" if [ -n "$proxy_no" ]; then export NO_PROXY=$proxy_no @@ -33,11 +34,13 @@ fi CURRENT_NODE_NAME=$(cat /etc/hostname) +export KUBECONFIG=/etc/kubernetes/admin.conf + get_current_upgrading_node_name() { kubectl get configmap upgrade-lock -n kube-system --kubeconfig /etc/kubernetes/admin.conf -o jsonpath="{['data']['node']}" } -delete_lock_config_map(){ +delete_lock_config_map() { # Delete the configmap lock once the upgrade completes if [ "$NODE_ROLE" != "worker" ] then @@ -45,6 +48,24 @@ delete_lock_config_map(){ fi } +upgrade_kubelet() { + echo "upgrading kubelet" + systemctl stop kubelet + cp "$root_path"/opt/kubeadm/bin/kubelet "$root_path"/usr/local/bin/kubelet + systemctl daemon-reload && systemctl restart kubelet + systemctl restart containerd + echo "kubelet upgraded" +} + +apply_new_kubeadm_config() { + kubectl get cm kubeadm-config -n kube-system -o jsonpath="{['data']['ClusterConfiguration']}" --kubeconfig /etc/kubernetes/admin.conf > "$root_path"/opt/kubeadm/existing-cluster-config.yaml + kubeadm init phase upload-config kubeadm --config "$root_path"/opt/kubeadm/cluster-config.yaml +} + +revert_kubeadm_config() { + kubeadm init phase upload-config kubeadm --config "$root_path"/opt/kubeadm/existing-cluster-config.yaml +} + run_upgrade() { echo "running upgrade process on $NODE_ROLE" @@ -104,6 +125,7 @@ run_upgrade() { if [ "$master_api_version" = "$old_version" ] then + apply_new_kubeadm_config upgrade_command="kubeadm upgrade apply -y $current_version" if [ "$PROXY_CONFIGURED" = true ]; then up=("kubeadm upgrade apply -y ${current_version}") @@ -113,33 +135,25 @@ run_upgrade() { fi echo "upgrading node from $old_version to $current_version using command: $upgrade_command" - if [ "$PROXY_CONFIGURED" = true ]; then - if sudo -E bash -c "$upgrade_command" - then - # Update current client version in the version file - echo "$current_version" > "$root_path"/opt/sentinel_kubeadmversion - old_version=$current_version - - delete_lock_config_map - echo "upgrade success" - else - echo "upgrade failed, retrying in 60 seconds" - sleep 60 - fi + if sudo -E bash -c "$upgrade_command" + then + # Update current client version in the version file + echo "$current_version" > "$root_path"/opt/sentinel_kubeadmversion + old_version=$current_version + + delete_lock_config_map + echo "upgrade success" else - if $upgrade_command - then - # Update current client version in the version file - echo "$current_version" > "$root_path"/opt/sentinel_kubeadmversion - old_version=$current_version - - delete_lock_config_map - echo "upgrade success" - else - echo "upgrade failed, retrying in 60 seconds" - sleep 60 - fi + echo "upgrade failed" + if echo "$upgrade_command" | grep -q "apply"; then + echo "reverting kubeadm config" + revert_kubeadm_config + fi + echo "retrying in 60 seconds" + sleep 60 fi done + upgrade_kubelet } + run_upgrade \ No newline at end of file diff --git a/stages/init.go b/stages/init.go index 3fecfa4..def9996 100644 --- a/stages/init.go +++ b/stages/init.go @@ -45,8 +45,8 @@ func GetInitYipStagesV1Beta3(clusterCtx *domain.ClusterContext, kubeadmConfig do getKubeadmInitConfigStage(getInitNodeConfigurationBeta3(clusterCtx, kubeadmConfig.InitConfiguration, kubeadmConfig.ClusterConfiguration, kubeadmConfig.KubeletConfiguration), clusterCtx.RootPath), getKubeadmInitStage(clusterCtx), getKubeadmPostInitStage(clusterCtx.RootPath), - getKubeadmInitUpgradeStage(clusterCtx), getKubeadmInitCreateClusterConfigStage(&kubeadmConfig.ClusterConfiguration, &kubeadmConfig.InitConfiguration, clusterCtx.RootPath), + getKubeadmInitUpgradeStage(clusterCtx), getKubeadmInitCreateKubeletConfigStage(&kubeadmConfig.ClusterConfiguration, &kubeadmConfig.InitConfiguration, &kubeadmConfig.KubeletConfiguration, clusterCtx.RootPath), getKubeadmInitReconfigureStage(clusterCtx), } @@ -66,8 +66,8 @@ func GetInitYipStagesV1Beta4(clusterCtx *domain.ClusterContext, kubeadmConfig do getKubeadmInitConfigStage(getInitNodeConfigurationBeta4(clusterCtx, kubeadmConfig.InitConfiguration, kubeadmConfig.ClusterConfiguration, kubeadmConfig.KubeletConfiguration), clusterCtx.RootPath), getKubeadmInitStage(clusterCtx), getKubeadmPostInitStage(clusterCtx.RootPath), - getKubeadmInitUpgradeStage(clusterCtx), getKubeadmInitCreateClusterConfigStage(&kubeadmConfig.ClusterConfiguration, &kubeadmConfig.InitConfiguration, clusterCtx.RootPath), + getKubeadmInitUpgradeStage(clusterCtx), getKubeadmInitCreateKubeletConfigStage(&kubeadmConfig.ClusterConfiguration, &kubeadmConfig.InitConfiguration, &kubeadmConfig.KubeletConfiguration, clusterCtx.RootPath), getKubeadmInitReconfigureStage(clusterCtx), }