Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: kairos-io/provider-kubeadm
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 002c33c8ada0d7a7884ac0194cf79847ce298633
Choose a base ref
..
head repository: kairos-io/provider-kubeadm
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 849a609c39b85c71d14c39306a040be478649be0
Choose a head ref
Showing with 303 additions and 225 deletions.
  1. +2 −0 domain/constants.go
  2. +8 −10 go.mod
  3. +31 −94 go.sum
  4. +4 −3 main.go
  5. +7 −1 scripts/import.sh
  6. +28 −13 scripts/kube-init.sh
  7. +27 −13 scripts/kube-join.sh
  8. +3 −0 scripts/kube-post-init.sh
  9. +26 −0 scripts/kube-pre-init.sh
  10. +16 −13 scripts/kube-reconfigure.sh
  11. +64 −0 scripts/kube-reset.sh
  12. +10 −7 scripts/kube-upgrade.sh
  13. +31 −25 stages/init.go
  14. +23 −18 stages/join.go
  15. +12 −27 stages/pre.go
  16. +1 −1 stages/proxy.go
  17. +10 −0 utils/misc.go
2 changes: 2 additions & 0 deletions domain/constants.go
Original file line number Diff line number Diff line change
@@ -2,4 +2,6 @@ package domain

const (
DefaultAPIAdvertiseAddress = "0.0.0.0"

ClusterRootPath = "cluster_root_path"
)
18 changes: 8 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
@@ -2,11 +2,13 @@ module github.com/kairos-io/kairos/provider-kubeadm

go 1.22.5

toolchain go1.22.6

require (
github.com/kairos-io/kairos-sdk v0.1.1
github.com/kairos-io/kairos-sdk v0.4.1
github.com/mudler/yip v1.9.4
github.com/sirupsen/logrus v1.9.3
k8s.io/apimachinery v0.31.0
k8s.io/apimachinery v0.31.1
k8s.io/cli-runtime v0.31.0
k8s.io/cluster-bootstrap v0.31.0
k8s.io/component-helpers v0.27.1
@@ -36,7 +38,6 @@ require (
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -55,23 +56,20 @@ require (
github.com/mudler/go-pluggable v0.0.0-20230126220627-7710299a0ae5 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo/v2 v2.19.0 // indirect
github.com/onsi/gomega v1.33.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.19.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/spf13/cobra v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/twpayne/go-vfs v1.7.2 // indirect
github.com/twpayne/go-vfs/v4 v4.3.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/sys v0.23.0 // indirect
golang.org/x/term v0.23.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.34.2 // indirect
125 changes: 31 additions & 94 deletions go.sum

Large diffs are not rendered by default.

7 changes: 4 additions & 3 deletions main.go
Original file line number Diff line number Diff line change
@@ -32,12 +32,13 @@ func clusterProvider(cluster clusterplugin.Cluster) yip.YipConfig {
_ = json.Unmarshal(userOptions, &kubeadmConfig)
}

clusterRootPath := utils.GetClusterRootPath(cluster)

preStage := []yip.Stage{
stages.GetPreKubeadmProxyStage(kubeadmConfig, cluster),
stages.GetPreKubeadmCommandStages(),
stages.GetPreKubeadmCommandStages(clusterRootPath),
stages.GetPreKubeadmSwapOffDisableStage(),
stages.GetPreKubeadmImportCoreK8sImageStage(),
stages.GetPreKubeadmStoreKubeadmVersionStage(),
stages.GetPreKubeadmImportCoreK8sImageStage(clusterRootPath),
}

if cluster.ImportLocalImages {
8 changes: 7 additions & 1 deletion scripts/import.sh
Original file line number Diff line number Diff line change
@@ -1,13 +1,19 @@
#!/bin/bash -x

CONTENT_PATH=$1
ROOT_PATH=$2

# find all tar files recursively
for tarfile in $(find $CONTENT_PATH -name "*.tar" -type f)
do
# try to import the tar file into containerd up to ten times
for i in {1..10}
do
/opt/bin/ctr -n k8s.io image import $tarfile --all-platforms
if [ "$ROOT_PATH" = "/" ]; then
/opt/bin/ctr -n k8s.io image import "$tarfile" --all-platforms
else
"$ROOT_PATH"/opt/spectro/bin/ctr -n k8s.io --address /run/spectro/containerd/containerd.sock image import "$tarfile" --all-platforms
fi
if [ $? -eq 0 ]; then
echo "Import successful: $tarfile (attempt $i)"
break
41 changes: 28 additions & 13 deletions scripts/kube-init.sh
Original file line number Diff line number Diff line change
@@ -7,34 +7,49 @@ exec 19>> /var/log/kube-init.log
export BASH_XTRACEFD="19"
set -ex

PROXY_CONFIGURED=$1
proxy_http=$2
proxy_https=$3
proxy_no=$4
root_path=$1
PROXY_CONFIGURED=$2
proxy_http=$3
proxy_https=$4
proxy_no=$5
KUBE_VIP_LOC="/etc/kubernetes/manifests/kube-vip.yaml"

export PATH="$PATH:$root_path/usr/bin"

do_kubeadm_reset() {
kubeadm reset -f
if [ -S /run/spectro/containerd/containerd.sock ]; then
kubeadm reset -f --cri-socket unix:///run/spectro/containerd/containerd.sock --cleanup-tmp-dir
else
kubeadm reset -f --cleanup-tmp-dir
fi

iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X && rm -rf /etc/kubernetes/etcd /etc/kubernetes/manifests /etc/kubernetes/pki
rm -rf /etc/cni/net.d
systemctl restart containerd
rm -rf "$root_path"/opt/spectro/cni/net.d
systemctl daemon-reload
if systemctl cat spectro-containerd >/dev/null 2<&1; then
systemctl restart spectro-containerd
fi

if systemctl cat containerd >/dev/null 2<&1; then
systemctl restart containerd
fi
}

backup_kube_vip_manifest_if_present() {
if [ -f "$KUBE_VIP_LOC" ]; then
cp $KUBE_VIP_LOC /opt/kubeadm/kube-vip.yaml
cp $KUBE_VIP_LOC "$root_path"/opt/kubeadm/kube-vip.yaml
fi
}

restore_kube_vip_manifest_after_reset() {
if [ -f "/opt/kubeadm/kube-vip.yaml" ]; then
mkdir -p /etc/kubernetes/manifests
cp /opt/kubeadm/kube-vip.yaml $KUBE_VIP_LOC
if [ -f "$root_path/opt/kubeadm/kube-vip.yaml" ]; then
mkdir -p "$root_path"/etc/kubernetes/manifests
cp "$root_path"/opt/kubeadm/kube-vip.yaml $KUBE_VIP_LOC
fi
}

if [ "$PROXY_CONFIGURED" = true ]; then
until HTTP_PROXY=$proxy_http http_proxy=$proxy_http HTTPS_PROXY=$proxy_https https_proxy=$proxy_https NO_PROXY=$proxy_no no_proxy=$proxy_no kubeadm init --config /opt/kubeadm/kubeadm.yaml --upload-certs --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
until HTTP_PROXY=$proxy_http http_proxy=$proxy_http HTTPS_PROXY=$proxy_https https_proxy=$proxy_https NO_PROXY=$proxy_no no_proxy=$proxy_no kubeadm init --config "$root_path"/opt/kubeadm/kubeadm.yaml --upload-certs --ignore-preflight-errors=NumCPU --ignore-preflight-errors=Mem --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
do
backup_kube_vip_manifest_if_present
echo "failed to apply kubeadm init, applying reset";
@@ -44,7 +59,7 @@ if [ "$PROXY_CONFIGURED" = true ]; then
restore_kube_vip_manifest_after_reset
done;
else
until kubeadm init --config /opt/kubeadm/kubeadm.yaml --upload-certs --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
until kubeadm init --config "$root_path"/opt/kubeadm/kubeadm.yaml --upload-certs --ignore-preflight-errors=NumCPU --ignore-preflight-errors=Mem --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
do
backup_kube_vip_manifest_if_present
echo "failed to apply kubeadm init, applying reset";
40 changes: 27 additions & 13 deletions scripts/kube-join.sh
Original file line number Diff line number Diff line change
@@ -9,35 +9,49 @@ set -ex

NODE_ROLE=$1

PROXY_CONFIGURED=$2
proxy_http=$3
proxy_https=$4
proxy_no=$5
root_path=$2
PROXY_CONFIGURED=$3
proxy_http=$4
proxy_https=$5
proxy_no=$6

export PATH="$PATH:$root_path/usr/bin"

KUBE_VIP_LOC="/etc/kubernetes/manifests/kube-vip.yaml"

do_kubeadm_reset() {
kubeadm reset -f
if [ -S /run/spectro/containerd/containerd.sock ]; then
kubeadm reset -f --cri-socket unix:///run/spectro/containerd/containerd.sock --cleanup-tmp-dir
else
kubeadm reset -f --cleanup-tmp-dir
fi
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X && rm -rf /etc/kubernetes/etcd /etc/kubernetes/manifests /etc/kubernetes/pki
rm -rf /etc/cni/net.d
systemctl restart containerd
rm -rf "$root_path"/opt/spectro/cni/net.d
systemctl daemon-reload
if systemctl cat spectro-containerd >/dev/null 2<&1; then
systemctl restart spectro-containerd
fi

if systemctl cat containerd >/dev/null 2<&1; then
systemctl restart containerd
fi
}

backup_kube_vip_manifest_if_present() {
if [ -f "$KUBE_VIP_LOC" ] && [ "$NODE_ROLE" != "worker" ]; then
cp $KUBE_VIP_LOC /opt/kubeadm/kube-vip.yaml
cp $KUBE_VIP_LOC "$root_path"/opt/kubeadm/kube-vip.yaml
fi
}

restore_kube_vip_manifest_after_reset() {
if [ -f "/opt/kubeadm/kube-vip.yaml" ] && [ "$NODE_ROLE" != "worker" ]; then
mkdir -p /etc/kubernetes/manifests
cp /opt/kubeadm/kube-vip.yaml $KUBE_VIP_LOC
if [ -f "$root_path/opt/kubeadm/kube-vip.yaml" ] && [ "$NODE_ROLE" != "worker" ]; then
mkdir -p "$root_path"/etc/kubernetes/manifests
cp "$root_path"/opt/kubeadm/kube-vip.yaml $KUBE_VIP_LOC
fi
}

if [ "$PROXY_CONFIGURED" = true ]; then
until HTTP_PROXY=$proxy_http http_proxy=$proxy_http HTTPS_PROXY=$proxy_https https_proxy=$proxy_https NO_PROXY=$proxy_no no_proxy=$proxy_no kubeadm join --config /opt/kubeadm/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
until HTTP_PROXY=$proxy_http http_proxy=$proxy_http HTTPS_PROXY=$proxy_https https_proxy=$proxy_https NO_PROXY=$proxy_no no_proxy=$proxy_no kubeadm join --config "$root_path"/opt/kubeadm/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
do
backup_kube_vip_manifest_if_present
echo "failed to apply kubeadm join, will retry in 10s";
@@ -48,7 +62,7 @@ if [ "$PROXY_CONFIGURED" = true ]; then
restore_kube_vip_manifest_after_reset
done;
else
until kubeadm join --config /opt/kubeadm/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
until kubeadm join --config "$root_path"/opt/kubeadm/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests -v=5 > /dev/null
do
backup_kube_vip_manifest_if_present
echo "failed to apply kubeadm join, will retry in 10s";
3 changes: 3 additions & 0 deletions scripts/kube-post-init.sh
Original file line number Diff line number Diff line change
@@ -7,7 +7,10 @@ exec 19>> /var/log/kube-post-init.log
export BASH_XTRACEFD="19"
set -x

root_path=$1

export KUBECONFIG=/etc/kubernetes/admin.conf
export PATH="$PATH:$root_path/usr/bin"

while true;
do
26 changes: 26 additions & 0 deletions scripts/kube-pre-init.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/bin/bash

set -x

export PATH="$PATH:$root_path/usr/bin"

root_path=$1

sysctl --system
modprobe overlay
modprobe br_netfilter
systemctl daemon-reload

systemctl enable kubelet && systemctl start kubelet

if systemctl cat spectro-containerd >/dev/null 2<&1; then
systemctl enable spectro-containerd && systemctl start spectro-containerd
fi

if systemctl cat containerd >/dev/null 2<&1; then
systemctl enable containerd && systemctl start containerd
fi

if [ ! -f "$root_path"/opt/sentinel_kubeadmversion ]; then
kubeadm version -o short > "$root_path"/opt/sentinel_kubeadmversion
fi
29 changes: 16 additions & 13 deletions scripts/kube-reconfigure.sh
Original file line number Diff line number Diff line change
@@ -10,14 +10,17 @@ info() {
echo "[INFO] " "$@"
}

certs_sans_revision_path="/opt/kubeadm/.kubeadm_certs_sans_revision"

node_role=$1
certs_sans_revision=$2
kubelet_envs=$3
proxy_http=$4
proxy_https=$5
proxy_no=$6
root_path=$4
proxy_http=$5
proxy_https=$6
proxy_no=$7

export PATH="$PATH:$root_path/usr/bin"

certs_sans_revision_path="$root_path/opt/kubeadm/.kubeadm_certs_sans_revision"

if [ -n "$proxy_no" ]; then
export NO_PROXY=$proxy_no
@@ -37,11 +40,11 @@ fi
export KUBECONFIG=/etc/kubernetes/admin.conf

regenerate_kube_components_manifests() {
sudo -E bash -c "kubeadm init phase control-plane apiserver --config /opt/kubeadm/cluster-config.yaml"
sudo -E bash -c "kubeadm init phase control-plane controller-manager --config /opt/kubeadm/cluster-config.yaml"
sudo -E bash -c "kubeadm init phase control-plane scheduler --config /opt/kubeadm/cluster-config.yaml"
sudo -E bash -c "kubeadm init phase control-plane apiserver --config $root_path/opt/kubeadm/cluster-config.yaml"
sudo -E bash -c "kubeadm init phase control-plane controller-manager --config $root_path/opt/kubeadm/cluster-config.yaml"
sudo -E bash -c "kubeadm init phase control-plane scheduler --config $root_path/opt/kubeadm/cluster-config.yaml"

kubeadm init phase upload-config kubeadm --config /opt/kubeadm/cluster-config.yaml
kubeadm init phase upload-config kubeadm --config "$root_path"/opt/kubeadm/cluster-config.yaml

info "regenerated kube components manifest"
}
@@ -62,13 +65,13 @@ regenerate_apiserver_certs_sans() {
rm /etc/kubernetes/pki/apiserver.{crt,key}
info "regenerated removed existing apiserver certs"

kubeadm init phase certs apiserver --config /opt/kubeadm/cluster-config.yaml
kubeadm init phase certs apiserver --config "$root_path"/opt/kubeadm/cluster-config.yaml
info "regenerated apiserver certs"

crictl pods 2>/dev/null | grep kube-apiserver | cut -d' ' -f1 | xargs -I %s sh -c '{ crictl stopp %s; crictl rmp %s; }' 2>/dev/null
info "deleted existing apiserver pod"

kubeadm init phase upload-config kubeadm --config /opt/kubeadm/cluster-config.yaml
kubeadm init phase upload-config kubeadm --config "$root_path"/opt/kubeadm/cluster-config.yaml

restart_kubelet
}
@@ -83,7 +86,7 @@ regenerate_kubelet_config() {
}

upload_kubelet_config() {
kubeadm init phase upload-config kubelet --config /opt/kubeadm/kubelet-config.yaml
kubeadm init phase upload-config kubelet --config "$root_path"/opt/kubeadm/kubelet-config.yaml
}

restart_kubelet() {
@@ -97,7 +100,7 @@ regenerate_etcd_manifests() {
sleep 60
continue
done
kubeadm init phase etcd local --config /opt/kubeadm/cluster-config.yaml
kubeadm init phase etcd local --config "$root_path"/opt/kubeadm/cluster-config.yaml
info "regenerated etcd manifest"
sleep 60
}
64 changes: 64 additions & 0 deletions scripts/kube-reset.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#!/bin/bash

set -x
trap 'echo -n $(date)' DEBUG

if [ -f /etc/spectro/environment ]; then
. /etc/spectro/environment
fi

export PATH="$PATH:$STYLUS_ROOT/usr/bin"

if [ -S /run/spectro/containerd/containerd.sock ]; then
kubeadm reset -f --cri-socket unix:///run/spectro/containerd/containerd.sock --cleanup-tmp-dir
else
kubeadm reset -f --cleanup-tmp-dir
fi

iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
rm -rf /etc/kubernetes/etcd
rm -rf /etc/kubernetes/manifests
rm -rf /etc/kubernetes/pki
rm -rf /etc/containerd/config.toml
systemctl stop kubelet
if systemctl cat spectro-containerd >/dev/null 2<&1; then
systemctl stop spectro-containerd
fi

if systemctl cat containerd >/dev/null 2<&1; then
systemctl stop containerd
fi

umount -l /var/lib/kubelet
rm -rf /var/lib/kubelet && rm -rf ${STYLUS_ROOT}/var/lib/kubelet
umount -l /var/lib/spectro/containerd
rm -rf /var/lib/spectro/containerd && rm -rf ${STYLUS_ROOT}/var/lib/spectro/containerd
umount -l /opt/spectro/bin
rm -rf /opt/spectro/bin && rm -rf ${STYLUS_ROOT}/opt/spectro/bin
umount -l /opt/spectro/cni/bin
rm -rf /opt/spectro/cni && rm -rf ${STYLUS_ROOT}/opt/spectro/cni
umount -l /etc/kubernetes
rm -rf /etc/kubernetes && rm -rf ${STYLUS_ROOT}/etc/kubernetes

rm -rf ${STYLUS_ROOT}/opt/kubeadm
rm -rf ${STYLUS_ROOT}/opt/*init
rm -rf ${STYLUS_ROOT}/opt/kube-images
rm -rf ${STYLUS_ROOT}/opt/sentinel_kubeadmversion

rm -rf /etc/systemd/system/etc-default-kubelet.mount
rm -rf /etc/systemd/system/etc-cni-net.d.mount
rm -rf /etc/systemd/system/opt-spectro-cni-bin.mount
rm -rf /etc/systemd/system/opt-spectro-bin.mount
rm -rf /etc/systemd/system/var-lib-spectro-containerd.mount
rm -rf /etc/systemd/system/var-lib-kubelet.mount

rm -rf /etc/systemd/system/spectro-kubelet.slice
rm -rf /etc/systemd/system/spectro-containerd.slice
rm -rf /etc/systemd/system/kubelet.service
rm -rf /etc/systemd/system/containerd.service 2> /dev/null
rm -rf /etc/systemd/system/spectro-containerd.service 2> /dev/null

rm -rf /var/log/kube*.log
rm -rf /var/log/apiserver
rm -rf /var/log/pods

17 changes: 10 additions & 7 deletions scripts/kube-upgrade.sh
Original file line number Diff line number Diff line change
@@ -8,10 +8,13 @@ set -x

NODE_ROLE=$1

PROXY_CONFIGURED=$2
proxy_http=$3
proxy_https=$4
proxy_no=$5
root_path=$2
PROXY_CONFIGURED=$3
proxy_http=$4
proxy_https=$5
proxy_no=$6

export PATH="$PATH:$root_path/usr/bin"

if [ -n "$proxy_no" ]; then
export NO_PROXY=$proxy_no
@@ -45,7 +48,7 @@ delete_lock_config_map(){
run_upgrade() {
echo "running upgrade process on $NODE_ROLE"

old_version=$(cat /opt/sentinel_kubeadmversion)
old_version=$(cat "$root_path"/opt/sentinel_kubeadmversion)
echo "found last deployed version $old_version"

current_version=$(kubeadm version -o short)
@@ -114,7 +117,7 @@ run_upgrade() {
if sudo -E bash -c "$upgrade_command"
then
# Update current client version in the version file
echo "$current_version" > /opt/sentinel_kubeadmversion
echo "$current_version" > "$root_path"/opt/sentinel_kubeadmversion
old_version=$current_version

delete_lock_config_map
@@ -127,7 +130,7 @@ run_upgrade() {
if $upgrade_command
then
# Update current client version in the version file
echo "$current_version" > /opt/sentinel_kubeadmversion
echo "$current_version" > "$root_path"/opt/sentinel_kubeadmversion
old_version=$current_version

delete_lock_config_map
56 changes: 31 additions & 25 deletions stages/init.go
Original file line number Diff line number Diff line change
@@ -29,30 +29,30 @@ func init() {
}

const (
configurationPath = "/opt/kubeadm"
configurationPath = "opt/kubeadm"
)

func GetInitYipStages(cluster clusterplugin.Cluster, initCfg kubeadmapiv3.InitConfiguration, clusterCfg kubeadmapiv3.ClusterConfiguration, kubeletCfg kubeletv1beta1.KubeletConfiguration) []yip.Stage {
utils.MutateClusterConfigDefaults(cluster, &clusterCfg)
utils.MutateKubeletDefaults(&clusterCfg, &kubeletCfg)

clusterRootPath := utils.GetClusterRootPath(cluster)
return []yip.Stage{
getKubeadmInitConfigStage(getInitNodeConfiguration(cluster, initCfg, clusterCfg, kubeletCfg)),
getKubeadmInitConfigStage(getInitNodeConfiguration(cluster, initCfg, clusterCfg, kubeletCfg), clusterRootPath),
getKubeadmInitStage(cluster, clusterCfg),
getKubeadmPostInitStage(),
getKubeadmPostInitStage(cluster),
getKubeadmInitUpgradeStage(cluster, clusterCfg),
getKubeadmInitCreateClusterConfigStage(clusterCfg, initCfg),
getKubeadmInitCreateKubeletConfigStage(kubeletCfg),
getKubeadmInitCreateClusterConfigStage(clusterCfg, initCfg, clusterRootPath),
getKubeadmInitCreateKubeletConfigStage(kubeletCfg, clusterRootPath),
getKubeadmInitReconfigureStage(cluster, clusterCfg, initCfg),
}
}

func getKubeadmInitConfigStage(kubeadmCfg string) yip.Stage {
func getKubeadmInitConfigStage(kubeadmCfg, rootPath string) yip.Stage {
return yip.Stage{
Name: "Generate Kubeadm Init Config File",
Files: []yip.File{
{
Path: filepath.Join(configurationPath, "kubeadm.yaml"),
Path: filepath.Join(rootPath, configurationPath, "kubeadm.yaml"),
Permissions: 0640,
Content: kubeadmCfg,
},
@@ -61,33 +61,37 @@ func getKubeadmInitConfigStage(kubeadmCfg string) yip.Stage {
}

func getKubeadmInitStage(cluster clusterplugin.Cluster, clusterCfg kubeadmapiv3.ClusterConfiguration) yip.Stage {
clusterRootPath := utils.GetClusterRootPath(cluster)

initStage := yip.Stage{
Name: "Run Kubeadm Init",
If: "[ ! -f /opt/kubeadm.init ]",
If: fmt.Sprintf("[ ! -f %s ]", filepath.Join(clusterRootPath, "opt/kubeadm.init")),
}

if utils.IsProxyConfigured(cluster.Env) {
proxy := cluster.Env
initStage.Commands = []string{
fmt.Sprintf("bash %s %t %s %s %s", filepath.Join(helperScriptPath, "kube-init.sh"), true, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
"touch /opt/kubeadm.init",
fmt.Sprintf("bash %s %s %t %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-init.sh"), clusterRootPath, true, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
fmt.Sprintf("touch %s", filepath.Join(clusterRootPath, "opt/kubeadm.init")),
}
} else {
initStage.Commands = []string{
fmt.Sprintf("bash %s", filepath.Join(helperScriptPath, "kube-init.sh")),
"touch /opt/kubeadm.init",
fmt.Sprintf("bash %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-init.sh"), clusterRootPath),
fmt.Sprintf("touch %s", filepath.Join(clusterRootPath, "opt/kubeadm.init")),
}
}
return initStage
}

func getKubeadmPostInitStage() yip.Stage {
func getKubeadmPostInitStage(cluster clusterplugin.Cluster) yip.Stage {
clusterRootPath := utils.GetClusterRootPath(cluster)

return yip.Stage{
Name: "Run Post Kubeadm Init",
If: "[ ! -f /opt/post-kubeadm.init ]",
If: fmt.Sprintf("[ ! -f %s ]", filepath.Join(clusterRootPath, "opt/post-kubeadm.init")),
Commands: []string{
fmt.Sprintf("bash %s", filepath.Join(helperScriptPath, "kube-post-init.sh")),
"touch /opt/post-kubeadm.init",
fmt.Sprintf("bash %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-post-init.sh"), clusterRootPath),
fmt.Sprintf("touch %s", filepath.Join(clusterRootPath, "opt/post-kubeadm.init")),
},
}
}
@@ -96,39 +100,40 @@ func getKubeadmInitUpgradeStage(cluster clusterplugin.Cluster, clusterCfg kubead
upgradeStage := yip.Stage{
Name: "Run Kubeadm Init Upgrade",
}
clusterRootPath := utils.GetClusterRootPath(cluster)

if utils.IsProxyConfigured(cluster.Env) {
proxy := cluster.Env
upgradeStage.Commands = []string{
fmt.Sprintf("bash %s %s %t %s %s %s", filepath.Join(helperScriptPath, "kube-upgrade.sh"), cluster.Role, true, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
fmt.Sprintf("bash %s %s %s %t %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-upgrade.sh"), cluster.Role, clusterRootPath, true, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
}
} else {
upgradeStage.Commands = []string{
fmt.Sprintf("bash %s %s", filepath.Join(helperScriptPath, "kube-upgrade.sh"), cluster.Role),
fmt.Sprintf("bash %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-upgrade.sh"), cluster.Role, clusterRootPath),
}
}
return upgradeStage
}

func getKubeadmInitCreateClusterConfigStage(clusterCfg kubeadmapiv3.ClusterConfiguration, initCfg kubeadmapiv3.InitConfiguration) yip.Stage {
func getKubeadmInitCreateClusterConfigStage(clusterCfg kubeadmapiv3.ClusterConfiguration, initCfg kubeadmapiv3.InitConfiguration, rootPath string) yip.Stage {
return yip.Stage{
Name: "Generate Cluster Config File",
Files: []yip.File{
{
Path: filepath.Join(configurationPath, "cluster-config.yaml"),
Path: filepath.Join(rootPath, configurationPath, "cluster-config.yaml"),
Permissions: 0640,
Content: getUpdatedInitClusterConfig(clusterCfg, initCfg),
},
},
}
}

func getKubeadmInitCreateKubeletConfigStage(kubeletCfg kubeletv1beta1.KubeletConfiguration) yip.Stage {
func getKubeadmInitCreateKubeletConfigStage(kubeletCfg kubeletv1beta1.KubeletConfiguration, rootPath string) yip.Stage {
return yip.Stage{
Name: "Generate Kubelet Config File",
Files: []yip.File{
{
Path: filepath.Join(configurationPath, "kubelet-config.yaml"),
Path: filepath.Join(rootPath, configurationPath, "kubelet-config.yaml"),
Permissions: 0640,
Content: getUpdatedKubeletConfig(kubeletCfg),
},
@@ -141,17 +146,18 @@ func getKubeadmInitReconfigureStage(cluster clusterplugin.Cluster, clusterCfg ku
Name: "Run Kubeadm Reconfiguration",
}

clusterRootPath := utils.GetClusterRootPath(cluster)
kubeletArgs := utils.RegenerateKubeletKubeadmArgsFile(&clusterCfg, &initCfg.NodeRegistration, string(cluster.Role))
sansRevision := utils.GetCertSansRevision(clusterCfg.APIServer.CertSANs)

if utils.IsProxyConfigured(cluster.Env) {
proxy := cluster.Env
reconfigureStage.Commands = []string{
fmt.Sprintf("bash %s %s %s %s %s %s %s", filepath.Join(helperScriptPath, "kube-reconfigure.sh"), cluster.Role, sansRevision, kubeletArgs, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
fmt.Sprintf("bash %s %s %s %s %s %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-reconfigure.sh"), cluster.Role, sansRevision, kubeletArgs, clusterRootPath, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
}
} else {
reconfigureStage.Commands = []string{
fmt.Sprintf("bash %s %s %s %s", filepath.Join(helperScriptPath, "kube-reconfigure.sh"), cluster.Role, sansRevision, kubeletArgs),
fmt.Sprintf("bash %s %s %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-reconfigure.sh"), cluster.Role, sansRevision, kubeletArgs, clusterRootPath),
}
}
return reconfigureStage
41 changes: 23 additions & 18 deletions stages/join.go
Original file line number Diff line number Diff line change
@@ -19,15 +19,15 @@ import (
func GetJoinYipStages(cluster clusterplugin.Cluster, clusterCfg kubeadmapiv3.ClusterConfiguration, initCfg kubeadmapiv3.InitConfiguration, joinCfg kubeadmapiv3.JoinConfiguration, kubeletCfg kubeletv1beta1.KubeletConfiguration) []yip.Stage {
utils.MutateClusterConfigDefaults(cluster, &clusterCfg)
utils.MutateKubeletDefaults(&clusterCfg, &kubeletCfg)

clusterRootPath := utils.GetClusterRootPath(cluster)
joinStg := []yip.Stage{
getKubeadmJoinConfigStage(getJoinNodeConfiguration(cluster, joinCfg)),
getKubeadmJoinConfigStage(getJoinNodeConfiguration(cluster, joinCfg), clusterRootPath),
getKubeadmJoinStage(cluster, clusterCfg),
getKubeadmJoinUpgradeStage(cluster, clusterCfg),
}

if cluster.Role != clusterplugin.RoleWorker {
joinStg = append(joinStg, getKubeadmJoinCreateClusterConfigStage(clusterCfg, initCfg, joinCfg), getKubeadmJoinCreateKubeletConfigStage(kubeletCfg))
joinStg = append(joinStg, getKubeadmJoinCreateClusterConfigStage(clusterCfg, initCfg, joinCfg, clusterRootPath), getKubeadmJoinCreateKubeletConfigStage(kubeletCfg, clusterRootPath))
}

return append(joinStg, getKubeadmJoinReconfigureStage(cluster, clusterCfg, joinCfg))
@@ -71,12 +71,12 @@ func getJoinNodeConfiguration(cluster clusterplugin.Cluster, joinCfg kubeadmapiv
return out.String()
}

func getKubeadmJoinConfigStage(kubeadmCfg string) yip.Stage {
func getKubeadmJoinConfigStage(kubeadmCfg, rootPath string) yip.Stage {
return yip.Stage{
Name: "Generate Kubeadm Join Config File",
Files: []yip.File{
{
Path: filepath.Join(configurationPath, "kubeadm.yaml"),
Path: filepath.Join(rootPath, configurationPath, "kubeadm.yaml"),
Permissions: 0640,
Content: kubeadmCfg,
},
@@ -85,21 +85,23 @@ func getKubeadmJoinConfigStage(kubeadmCfg string) yip.Stage {
}

func getKubeadmJoinStage(cluster clusterplugin.Cluster, clusterCfg kubeadmapiv3.ClusterConfiguration) yip.Stage {
clusterRootPath := utils.GetClusterRootPath(cluster)

joinStage := yip.Stage{
Name: "Run Kubeadm Join",
If: "[ ! -f /opt/kubeadm.join ]",
If: fmt.Sprintf("[ ! -f %s ]", filepath.Join(clusterRootPath, "opt/kubeadm.join")),
}

if utils.IsProxyConfigured(cluster.Env) {
proxy := cluster.Env
joinStage.Commands = []string{
fmt.Sprintf("bash %s %s %t %s %s %s", filepath.Join(helperScriptPath, "kube-join.sh"), cluster.Role, true, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
"touch /opt/kubeadm.join",
fmt.Sprintf("bash %s %s %s %t %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-join.sh"), cluster.Role, clusterRootPath, true, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
fmt.Sprintf("touch %s", filepath.Join(clusterRootPath, "opt/kubeadm.join")),
}
} else {
joinStage.Commands = []string{
fmt.Sprintf("bash %s %s", filepath.Join(helperScriptPath, "kube-join.sh"), cluster.Role),
"touch /opt/kubeadm.join",
fmt.Sprintf("bash %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-join.sh"), cluster.Role, clusterRootPath),
fmt.Sprintf("touch %s", filepath.Join(clusterRootPath, "opt/kubeadm.join")),
}
}
return joinStage
@@ -110,38 +112,40 @@ func getKubeadmJoinUpgradeStage(cluster clusterplugin.Cluster, clusterCfg kubead
Name: "Run Kubeadm Join Upgrade",
}

clusterRootPath := utils.GetClusterRootPath(cluster)

if utils.IsProxyConfigured(cluster.Env) {
proxy := cluster.Env
upgradeStage.Commands = []string{
fmt.Sprintf("bash %s %s %t %s %s %s", filepath.Join(helperScriptPath, "kube-upgrade.sh"), cluster.Role, true, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
fmt.Sprintf("bash %s %s %s %t %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-upgrade.sh"), cluster.Role, clusterRootPath, true, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
}
} else {
upgradeStage.Commands = []string{
fmt.Sprintf("bash %s %s", filepath.Join(helperScriptPath, "kube-upgrade.sh"), cluster.Role),
fmt.Sprintf("bash %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-upgrade.sh"), cluster.Role, clusterRootPath),
}
}
return upgradeStage
}

func getKubeadmJoinCreateClusterConfigStage(clusterCfg kubeadmapiv3.ClusterConfiguration, initCfg kubeadmapiv3.InitConfiguration, joinCfg kubeadmapiv3.JoinConfiguration) yip.Stage {
func getKubeadmJoinCreateClusterConfigStage(clusterCfg kubeadmapiv3.ClusterConfiguration, initCfg kubeadmapiv3.InitConfiguration, joinCfg kubeadmapiv3.JoinConfiguration, rootPath string) yip.Stage {
return yip.Stage{
Name: "Generate Cluster Config File",
Files: []yip.File{
{
Path: filepath.Join(configurationPath, "cluster-config.yaml"),
Path: filepath.Join(rootPath, configurationPath, "cluster-config.yaml"),
Permissions: 0640,
Content: getUpdatedJoinClusterConfig(clusterCfg, initCfg, joinCfg),
},
},
}
}

func getKubeadmJoinCreateKubeletConfigStage(kubeletCfg kubeletv1beta1.KubeletConfiguration) yip.Stage {
func getKubeadmJoinCreateKubeletConfigStage(kubeletCfg kubeletv1beta1.KubeletConfiguration, rootPath string) yip.Stage {
return yip.Stage{
Name: "Generate Kubelet Config File",
Files: []yip.File{
{
Path: filepath.Join(configurationPath, "kubelet-config.yaml"),
Path: filepath.Join(rootPath, configurationPath, "kubelet-config.yaml"),
Permissions: 0640,
Content: getUpdatedKubeletConfig(kubeletCfg),
},
@@ -154,17 +158,18 @@ func getKubeadmJoinReconfigureStage(cluster clusterplugin.Cluster, clusterCfg ku
Name: "Run Kubeadm Join Reconfiguration",
}

clusterRootPath := utils.GetClusterRootPath(cluster)
kubeletArgs := utils.RegenerateKubeletKubeadmArgsFile(&clusterCfg, &joinCfg.NodeRegistration, string(cluster.Role))
sansRevision := utils.GetCertSansRevision(clusterCfg.APIServer.CertSANs)

if utils.IsProxyConfigured(cluster.Env) {
proxy := cluster.Env
reconfigureStage.Commands = []string{
fmt.Sprintf("bash %s %s %s %s %s %s %s", filepath.Join(helperScriptPath, "kube-reconfigure.sh"), cluster.Role, sansRevision, kubeletArgs, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
fmt.Sprintf("bash %s %s %s %s %s %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-reconfigure.sh"), cluster.Role, sansRevision, kubeletArgs, clusterRootPath, proxy["HTTP_PROXY"], proxy["HTTPS_PROXY"], utils.GetNoProxyConfig(clusterCfg, cluster.Env)),
}
} else {
reconfigureStage.Commands = []string{
fmt.Sprintf("bash %s %s %s %s", filepath.Join(helperScriptPath, "kube-reconfigure.sh"), cluster.Role, sansRevision, kubeletArgs),
fmt.Sprintf("bash %s %s %s %s %s", filepath.Join(clusterRootPath, helperScriptPath, "kube-reconfigure.sh"), cluster.Role, sansRevision, kubeletArgs, clusterRootPath),
}
}
return reconfigureStage
39 changes: 12 additions & 27 deletions stages/pre.go
Original file line number Diff line number Diff line change
@@ -5,26 +5,19 @@ import (
"path/filepath"

"github.com/kairos-io/kairos-sdk/clusterplugin"
"github.com/kairos-io/kairos/provider-kubeadm/utils"
yip "github.com/mudler/yip/pkg/schema"
)

const (
helperScriptPath = "/opt/kubeadm/scripts"
helperScriptPath = "opt/kubeadm/scripts"
)

func GetPreKubeadmCommandStages() yip.Stage {
func GetPreKubeadmCommandStages(rootPath string) yip.Stage {
return yip.Stage{
Name: "Run Pre Kubeadm Commands",
Systemctl: yip.Systemctl{
Enable: []string{"kubelet"},
},
Commands: []string{
"sysctl --system",
"modprobe overlay",
"modprobe br_netfilter",
"systemctl daemon-reload",
"systemctl restart containerd",
"mkdir -p /etc/kubernetes/manifests",
fmt.Sprintf("/bin/bash %s %s", filepath.Join(rootPath, helperScriptPath, "kube-pre-init.sh"), rootPath),
},
}
}
@@ -40,36 +33,28 @@ func GetPreKubeadmSwapOffDisableStage() yip.Stage {
}

func GetPreKubeadmImportLocalImageStage(cluster clusterplugin.Cluster) yip.Stage {
clusterRootPath := utils.GetClusterRootPath(cluster)

if cluster.LocalImagesPath == "" {
cluster.LocalImagesPath = "/opt/content/images"
cluster.LocalImagesPath = filepath.Join(clusterRootPath, "opt/content/images")
}

return yip.Stage{
Name: "Run Import Local Images",
Commands: []string{
fmt.Sprintf("chmod +x %s", filepath.Join(helperScriptPath, "import.sh")),
fmt.Sprintf("/bin/sh %s %s > /var/log/import.log", filepath.Join(helperScriptPath, "import.sh"), cluster.LocalImagesPath),
fmt.Sprintf("chmod +x %s", filepath.Join(clusterRootPath, helperScriptPath, "import.sh")),
fmt.Sprintf("/bin/sh %s %s > /var/log/import.log", filepath.Join(clusterRootPath, helperScriptPath, "import.sh"), cluster.LocalImagesPath),
},
If: fmt.Sprintf("[ -d %s ]", cluster.LocalImagesPath),
}
}

func GetPreKubeadmImportCoreK8sImageStage() yip.Stage {
func GetPreKubeadmImportCoreK8sImageStage(rootPath string) yip.Stage {
return yip.Stage{
Name: "Run Load Kube Images",
Commands: []string{
fmt.Sprintf("chmod +x %s", filepath.Join(helperScriptPath, "import.sh")),
fmt.Sprintf("/bin/sh %s /opt/kube-images > /var/log/import-kube-images.log", filepath.Join(helperScriptPath, "import.sh")),
},
}
}

func GetPreKubeadmStoreKubeadmVersionStage() yip.Stage {
return yip.Stage{
If: "[ ! -f /opt/sentinel_kubeadmversion ]",
Name: "Create kubeadm sentinel version file",
Commands: []string{
"kubeadm version -o short > /opt/sentinel_kubeadmversion",
fmt.Sprintf("chmod +x %s", filepath.Join(rootPath, helperScriptPath, "import.sh")),
fmt.Sprintf("/bin/sh %s %s %s > /var/log/import-kube-images.log", filepath.Join(rootPath, helperScriptPath, "import.sh"), filepath.Join(rootPath, "opt/kube-images"), rootPath),
},
}
}
2 changes: 1 addition & 1 deletion stages/proxy.go
Original file line number Diff line number Diff line change
@@ -38,7 +38,7 @@ func kubeletProxyEnv(clusterCfg kubeadmapiv3.ClusterConfiguration, proxyMap map[
var proxy []string

httpProxy := proxyMap["HTTP_PROXY"]
httpsProxy := proxyMap["HTTP_PROXY"]
httpsProxy := proxyMap["HTTPS_PROXY"]
userNoProxy := proxyMap["NO_PROXY"]

if utils.IsProxyConfigured(proxyMap) {
10 changes: 10 additions & 0 deletions utils/misc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package utils

import (
"github.com/kairos-io/kairos-sdk/clusterplugin"
"github.com/kairos-io/kairos/provider-kubeadm/domain"
)

func GetClusterRootPath(cluster clusterplugin.Cluster) string {
return cluster.ProviderOptions[domain.ClusterRootPath]
}