feature: use keepalvied to add vip for virtual cluster

deploy/crds/kosmos.io_virtualclusters.yaml

@@ -202,6 +202,10 @@ spec:
               updateTime:
                 format: date-time
                 type: string
+              vipMap:
+                additionalProperties:
+                  type: string
+                type: object
             type: object
         required:
         - spec

deploy/k8s-in-k8s/manifest/keepalived/clusterrole.yaml

@@ -0,0 +1,8 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: node-reader
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list"]

deploy/k8s-in-k8s/manifest/keepalived/clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: node-reader-binding
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: node-reader
+  apiGroup: rbac.authorization.k8s.io

deploy/k8s-in-k8s/manifest/keepalived/keepalived-config.yaml

@@ -0,0 +1,44 @@
+apiVersion: v1
+data:
+  backup: |+
+    vrrp_instance VI_1 {
+        state BACKUP
+        interface __IFNAME__
+        virtual_router_id 51
+        priority 90
+        advert_int 1
+        authentication {
+            auth_type PASS
+            auth_pass XXXXXX
+        }
+        virtual_ipaddress {
+            {{ .Vip }}
+        }
+        track_interface {
+            __IFNAME__
+        }
+    }
+
+  master: |+
+    vrrp_instance VI_1 {
+        state MASTER
+        interface __IFNAME__
+        virtual_router_id 51
+        priority 100
+        advert_int 1
+        authentication {
+            auth_type PASS
+            auth_pass XXXXXX
+        }
+        virtual_ipaddress {
+            {{ .Vip }}
+        }
+        track_interface {
+            __IFNAME__
+        }
+    }
+
+kind: ConfigMap
+metadata:
+  name: keepalived-config
+  namespace: kube-system

deploy/k8s-in-k8s/manifest/keepalived/keepalived-sts.yaml

@@ -0,0 +1,71 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    virtualCluster-app: keepalived
+  name: keepalived
+  namespace: kube-system
+spec:
+  serviceName: "keepalived"
+  replicas: {{ .KeepalivedReplicas }}
+  selector:
+    matchLabels:
+      virtualCluster-app: keepalived
+  template:
+    metadata:
+      labels:
+        virtualCluster-app: keepalived
+    spec:
+      tolerations:
+        - key: "node-role.kubernetes.io/control-plane"
+          operator: "Exists"
+          effect: "NoSchedule"
+      hostNetwork: true
+      volumes:
+        - name: keepalived-conf
+          configMap:
+            name: keepalived-config
+        - name: shared
+          emptyDir: {}
+      nodeSelector:
+        kosmos.io/keepalived-node: "true"
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: virtualCluster-app
+                    operator: In
+                    values:
+                      - keepalived
+              topologyKey: "kubernetes.io/hostname"
+      initContainers:
+        - name: init-keepalived-config
+          image: {{ .ImageRepository }}/kubectl:v1.25.7
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          command:
+            - "sh"
+            - "-c"
+            - "ROLE=$(kubectl get no -L kosmos.io/keepalived-role | grep ${NODE_NAME} | awk '{print $6}'); NODEIP=$(kubectl get no -o wide | grep ${NODE_NAME} | awk '{print $6}'); IFNAME=$(ip -o addr show | grep $NODEIP | awk '{print $2}'); if [ $ROLE = 'master' ]; then cp /config/master /shared/keepalived.conf; else cp /config/backup /shared/keepalived.conf; fi; sed -i 's/__IFNAME__/'$IFNAME'/g' /shared/keepalived.conf;"
+          volumeMounts:
+            - name: keepalived-conf
+              mountPath: /config
+            - name: shared
+              mountPath: /shared/
+      containers:
+        - image: {{ .ImageRepository }}/keepalived:v1.25.7
+          volumeMounts:
+            - name: shared
+              mountPath: /usr/local/etc/keepalived/keepalived.conf
+              subPath: keepalived.conf
+          name: keepalived
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+          resources: {}
+  volumeClaimTemplates: []

deploy/virtual-cluster-components-manifest-cm.yaml

@@ -4,6 +4,7 @@ data:
     [
       {"name": "kube-proxy", "path": "/kosmos/manifest/kube-proxy/*.yaml"},
       {"name": "calico", "path": "/kosmos/manifest/calico/*.yaml"},
+      {"name": "keepalived", "path": "/kosmos/manifest/keepalived/*.yaml"},
     ]
   host-core-dns-components: |
     [

deploy/virtual-cluster-vip-pool-cm.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kosmos-vip-pool
+  namespace: kosmos-system
+data:
+  vip-config.yaml: |
+    # can be use for vc, the ip formate is 192.168.0.1 and 192.168.0.2-192.168.0.10
+    vipPool:
+      - 192.168.0.1-192.168.0.10

pkg/apis/kosmos/v1alpha1/virtualcluster_types.go

@@ -137,6 +137,8 @@ type VirtualClusterStatus struct {
 	Port int32 `json:"port,omitempty"`
 	// +optional
 	PortMap map[string]int32 `json:"portMap,omitempty"`
+	// +optional
+	VipMap map[string]string `json:"vipMap,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/kubenest/constants/constant.go

@@ -105,6 +105,17 @@ const (
 	ApiServerNetworkProxyAdminPortKey  = "apiserver-network-proxy-admin-port"
 	VirtualClusterPortNum              = 5
 
+	// vip
+	VipPoolConfigMapName        = "kosmos-vip-pool"
+	VipPoolKey                  = "vip-config.yaml"
+	VcVipStatusKey              = "vip-key"
+	VipKeepAlivedNodeLabelKey   = "kosmos.io/keepalived-node"
+	VipKeepAlivedNodeLabelValue = "true"
+	VipKeepAlivedNodeRoleKey    = "kosmos.io/keepalived-role"
+	VipKeepAlivedNodeRoleMaster = "master"
+	VipKeepalivedNodeRoleBackup = "backup"
+	VipKeepAlivedReplicas       = 3
+
 	ManifestComponentsConfigMap = "components-manifest-cm"
 
 	WaitAllPodsRunningTimeoutSeconds = 1800