cluster-template-rke2-dhcp.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: ${NAMESPACE}
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster 
metadata:
  namespace: ${NAMESPACE}
  name: ${CLUSTER_NAME} 
  labels:
${RANCHER_TURTLES_LABEL}
    ccm: external
    csi: external
    cni: external
spec:
  controlPlaneRef:
    apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
    kind: RKE2ControlPlane
    name: ${CLUSTER_NAME}-control-plane
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
    kind: HarvesterCluster
    name: ${CLUSTER_NAME}-hv
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: HarvesterCluster
metadata:
  name: ${CLUSTER_NAME}-hv
  namespace: ${NAMESPACE}
spec:
  targetNamespace: ${TARGET_HARVESTER_NAMESPACE}
  loadBalancerConfig:
    ipamType: dhcp
    listeners:
    - name: rke2-server
      port: 9345
      protocol: TCk
      backendPort: 9345
  server: ${HARVESTER_ENDPOINT}
  identitySecret: 
    namespace: ${NAMESPACE}
    name: hv-identity-secret
{UPDATE_CLOUD_PROVIDER:-  updateCloudProviderConfig: 
    cloudConfigCredentialsSecretKey: cloud-config
    cloudConfigCredentialsSecretName: cloud-config
    manifestsConfigMapKey: harvester-cloud-provider-deploy.yaml
    manifestsConfigMapName: harvester-csi-driver-addon
    manifestsConfigMapNamespace: ${NAMESPACE}
    }
---
apiVersion: v1
kind: Secret
metadata:
  namespace: ${NAMESPACE}
  name: hv-identity-secret
data: 
  kubeconfig: ${HARVESTER_KUBECONFIG_B64}
---
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
kind: RKE2ControlPlane
metadata:
  namespace: ${NAMESPACE}
  name: ${CLUSTER_NAME}-control-plane
spec:
  replicas: ${CONTROL_PLANE_MACHINE_COUNT}
  agentConfig:
    version: ${KUBERNETES_VERSION}+rke2r1
  serverConfig:
    cni: calico
    cloudProviderName: external
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
    kind: HarvesterMachineTemplate
    name: ${CLUSTER_NAME}-cp-machine
    namespace: ${NAMESPACE}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
kind: RKE2ConfigTemplate
metadata:
  namespace: ${NAMESPACE}
  name: ${CLUSTER_NAME}-worker
spec:
  template:
    spec:
      agentConfig:
        version: ${KUBERNETES_VERSION}+rke2r1
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
  namespace: ${NAMESPACE}
  name: ${CLUSTER_NAME}-workers
spec:
  clusterName: ${CLUSTER_NAME}
  replicas: ${WORKER_MACHINE_COUNT}
  selector:
    matchLabels:
      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
  template:
    spec:
      bootstrap:
        configRef:
          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
          kind: RKE2ConfigTemplate
          name: ${CLUSTER_NAME}-worker
          namespace: ${NAMESPACE}
      clusterName: ${CLUSTER_NAME}
      infrastructureRef:
        apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
        kind: HarvesterMachineTemplate
        name: ${CLUSTER_NAME}-wk-machine
        namespace: ${NAMESPACE}
      version: ${KUBERNETES_VERSION}+rke2r1
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: HarvesterMachineTemplate
metadata:
  namespace: ${NAMESPACE}
  name: ${CLUSTER_NAME}-wk-machine
spec:
  template: 
    spec:
      cpu: 1
      memory: 4Gi
      sshUser: ubuntu
      sshKeyPair: ${SSH_KEYPAIR}  
      networks:
      - ${VM_NETWORK}
      volumes:
      - volumeType: image 
        imageName: ${VM_IMAGE_NAME}
        volumeSize: ${VM_DISK_SIZE}
        bootOrder: 0
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: HarvesterMachineTemplate
metadata:
  namespace: ${NAMESPACE}
  name: ${CLUSTER_NAME}-cp-machine
spec:
  template: 
    spec:
      cpu: 2
      memory: 8Gi
      sshUser: ubuntu
      sshKeyPair: ${SSH_KEYPAIR}  
      networks:
      - ${VM_NETWORK}
      volumes:
      - volumeType: image 
        imageName: ${VM_IMAGE_NAME}
        volumeSize: ${VM_DISK_SIZE}
        bootOrder: 0
---
apiVersion: addons.cluster.x-k8s.io/v1beta1
kind: ClusterResourceSet
metadata:
  name: crs-harvester-ccm
  namespace: ${NAMESPACE}
spec:
  clusterSelector:
    matchLabels:
      ccm: external
  resources:
  - kind: ConfigMap
    name: cloud-controller-manager-addon
  strategy: ApplyOnce
---
apiVersion: addons.cluster.x-k8s.io/v1beta1
kind: ClusterResourceSet
metadata:
  name: crs-harvester-csi
  namespace: ${NAMESPACE}
spec:
  clusterSelector:
    matchLabels:
      csi: external
  resources:
  - kind: ConfigMap
    name: harvester-csi-driver-addon
  strategy: ApplyOnce
---
apiVersion: addons.cluster.x-k8s.io/v1beta1
kind: ClusterResourceSet
metadata:
  name: crs-calico-chart-config
  namespace: ${NAMESPACE}
spec:
  clusterSelector:
    matchLabels:
      cni: external
  resources:
  - kind: ConfigMap
    name: calico-helm-config
  strategy: ApplyOnce
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: cloud-controller-manager-addon
  namespace: ${NAMESPACE}
data:
  harvester-csi-deployment.yaml: |
    apiVersion: apps/v1
    kind: DaemonSet
    metadata:
      name: harvester-csi-plugin
      namespace: kube-system
    spec:
      selector:
        matchLabels:
          app: harvester-csi-plugin
      template:
        metadata:
          labels:
            app: harvester-csi-plugin
        spec:
          containers:
            - args:
                - --v=5
                - --csi-address=$(ADDRESS)
                - --kubelet-registration-path=/var/lib/kubelet/harvester-plugins/driver.harvesterhci.io/csi.sock
              env:
                - name: ADDRESS
                  value: /csi/csi.sock
              image: longhornio/csi-node-driver-registrar:v2.12.0
              lifecycle:
                preStop:
                  exec:
                    command:
                      - /bin/sh
                      - -c
                      - rm -rf /registration/driver.harvesterhci.io-reg.sock
                        /csi//*
              name: node-driver-registrar
              securityContext:
                privileged: true
              volumeMounts:
                - mountPath: /csi/
                  name: socket-dir
                - mountPath: /registration
                  name: registration-dir
            - args:
                - --nodeid=$(NODE_ID)
                - --endpoint=$(CSI_ENDPOINT)
                - --kubeconfig=/etc/csi/cloud-config
              env:
                - name: NODE_ID
                  valueFrom:
                    fieldRef:
                      apiVersion: v1
                      fieldPath: spec.nodeName
                - name: CSI_ENDPOINT
                  value: unix:///csi/csi.sock
              image: rancher/harvester-csi-driver:v0.1.6
              imagePullPolicy: Always
              lifecycle:
                preStop:
                  exec:
                    command:
                      - /bin/sh
                      - -c
                      - rm -f /csi//*
              name: harvester-csi-plugin
              securityContext:
                allowPrivilegeEscalation: true
                capabilities:
                  add:
                    - SYS_ADMIN
                privileged: true
              volumeMounts:
                - name: cloud-config
                  mountPath: "/etc/csi"
                  readOnly: true
                - mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
                  mountPropagation: Bidirectional
                  name: kubernetes-csi-dir
                - mountPath: /csi/
                  name: socket-dir
                - mountPath: /var/lib/kubelet/pods
                  mountPropagation: Bidirectional
                  name: pods-mount-dir
                - mountPath: /dev
                  name: host-dev
                - mountPath: /sys
                  name: host-sys
                - mountPath: /rootfs
                  mountPropagation: Bidirectional
                  name: host
                - mountPath: /lib/modules
                  name: lib-modules
                  readOnly: true
          hostPID: true
          serviceAccountName: harvester-csi
          tolerations:
            - effect: NoSchedule
              key: node-role.kubernetes.io/control-plane
              operator: Exists
            - effect: NoSchedule
              key: kubevirt.io/drain
              operator: Exists
          volumes:
            - name: cloud-config
              secret:
                secretName: cloud-config
            - hostPath:
                path: /var/lib/kubelet/plugins/kubernetes.io/csi
                type: DirectoryOrCreate
              name: kubernetes-csi-dir
            - hostPath:
                path: /var/lib/kubelet/plugins_registry
                type: Directory
              name: registration-dir
            - hostPath:
                path: /var/lib/kubelet/harvester-plugins/driver.harvesterhci.io
                type: DirectoryOrCreate
              name: socket-dir
            - hostPath:
                path: /var/lib/kubelet/pods
                type: DirectoryOrCreate
              name: pods-mount-dir
            - hostPath:
                path: /dev
              name: host-dev
            - hostPath:
                path: /sys
              name: host-sys
            - hostPath:
                path: /
              name: host
            - hostPath:
                path: /lib/modules
              name: lib-modules
    ---
    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: harvester-csi
      namespace: kube-system
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRoleBinding
    metadata:
      name: harvester-csi
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: cluster-admin
    subjects:
      - kind: ServiceAccount
        name: harvester-csi
        namespace: kube-system
    ---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: csi-controller
      namespace: kube-system
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: csi-controller
      template:
        metadata:
          labels:
            app: csi-controller
        spec:
          containers:
            - args:
                - --v=5
                - --csi-address=$(ADDRESS)
                - --timeout=2m5s
                - --leader-election
                - --leader-election-namespace=$(POD_NAMESPACE)
              env:
                - name: ADDRESS
                  value: /csi/csi.sock
                - name: POD_NAMESPACE
                  valueFrom:
                    fieldRef:
                      apiVersion: v1
                      fieldPath: metadata.namespace
              image: longhornio/csi-resizer:v1.12.0
              name: csi-resizer
              volumeMounts:
                - mountPath: /csi/
                  name: socket-dir
            - args:
                - --v=5
                - --csi-address=$(ADDRESS)
                - --timeout=2m5s
                - --leader-election
                - --leader-election-namespace=$(POD_NAMESPACE)
              env:
                - name: ADDRESS
                  value: /csi/csi.sock
                - name: POD_NAMESPACE
                  valueFrom:
                    fieldRef:
                      apiVersion: v1
                      fieldPath: metadata.namespace
              image: longhornio/csi-provisioner:v5.1.0
              name: csi-provisioner
              volumeMounts:
                - mountPath: /csi/
                  name: socket-dir
            - args:
                - --v=5
                - --csi-address=$(ADDRESS)
                - --timeout=2m5s
                - --leader-election
                - --leader-election-namespace=$(POD_NAMESPACE)
              env:
                - name: ADDRESS
                  value: /csi/csi.sock
                - name: POD_NAMESPACE
                  valueFrom:
                    fieldRef:
                      apiVersion: v1
                      fieldPath: metadata.namespace
              image: longhornio/csi-attacher:v4.7.0
              name: csi-attacher
              volumeMounts:
                - mountPath: /csi/
                  name: socket-dir
          serviceAccountName: harvester-csi
          tolerations:
            - effect: NoSchedule
              key: node-role.kubernetes.io/control-plane
              operator: Exists
            - effect: NoSchedule
              key: kubevirt.io/drain
              operator: Exists
          volumes:
            - hostPath:
                path: /var/lib/kubelet/harvester-plugins/driver.harvesterhci.io
                type: DirectoryOrCreate
              name: socket-dir
    ---
    apiVersion: storage.k8s.io/v1
    kind: CSIDriver
    metadata:
      name: driver.harvesterhci.io
    spec:
      attachRequired: true
      fsGroupPolicy: ReadWriteOnceWithFSType
      podInfoOnMount: true
      volumeLifecycleModes:
        - Persistent
    ---
    apiVersion: storage.k8s.io/v1
    kind: StorageClass
    metadata:
      name: harvester
      annotation:
        storageclass.kubernetes.io/is-default-class: "true"
    allowVolumeExpansion: true
    provisioner: driver.harvesterhci.io
    reclaimPolicy: Delete
    volumeBindingMode: Immediate
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: harvester-csi-driver-addon
  namespace: ${NAMESPACE}
data:
  harvester-cloud-provider-deploy.yaml: |
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      labels:
        app.kubernetes.io/component: cloud-provider
        app.kubernetes.io/name: harvester-cloud-provider
      name: harvester-cloud-provider
      namespace: kube-system
    spec:
      replicas: 2
      selector:
        matchLabels:
          app.kubernetes.io/component: cloud-provider
          app.kubernetes.io/name: harvester-cloud-provider
      template:
        metadata:
          labels:
            app.kubernetes.io/component: cloud-provider
            app.kubernetes.io/name: harvester-cloud-provider
        spec:
          containers:
          - args:
            - --cloud-config=/etc/kubernetes/cloud-config
            command:
            - harvester-cloud-provider
            image: rancher/harvester-cloud-provider:v0.2.0
            imagePullPolicy: Always
            name: harvester-cloud-provider
            resources: {}
            volumeMounts:
            - mountPath: /etc/kubernetes
              name: cloud-config
          serviceAccountName: harvester-cloud-controller-manager
          tolerations:
          - effect: NoSchedule
            key: node-role.kubernetes.io/control-plane
            operator: Exists
          - effect: NoSchedule
            key: node.cloudprovider.kubernetes.io/uninitialized
            operator: Equal
            value: "true"
          volumes:
            - name: cloud-config
              secret:
                secretName: cloud-config
    ---
    apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: harvester-cloud-controller-manager
      namespace: kube-system
    ---
    apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRole
    metadata:
      name: harvester-cloud-controller-manager
    rules:
    - apiGroups:
      - ""
      resources:
      - services
      - nodes
      - events
      verbs:
      - get
      - list
      - watch
      - create
      - update
      - patch
    - apiGroups:
      - ""
      resources:
      - services/status
      verbs:
      - update
      - patch
    - apiGroups:
      - ""
      resources:
      - nodes/status
      verbs:
      - patch
      - update
    - apiGroups:
      - coordination.k8s.io
      resources:
      - leases
      verbs:
      - get
      - create
      - update
    ---
    kind: ClusterRoleBinding
    apiVersion: rbac.authorization.k8s.io/v1
    metadata:
      name: harvester-cloud-controller-manager
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: harvester-cloud-controller-manager
    subjects:
      - kind: ServiceAccount
        name: harvester-cloud-controller-manager
        namespace: kube-system
    ---
    apiVersion: v1
    kind: Secret
    metadata:
      name: cloud-config
      namespace: kube-system
    type: Opaque
    data:
      cloud-config: ${CLOUD_CONFIG_KUBECONFIG_B64}
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: calico-helm-config
  namespace: ${NAMESPACE}
data:
  calico-helm-config.yaml: |
    apiVersion: helm.cattle.io/v1
    kind: HelmChartConfig
    metadata:
      name: rke2-calico
      namespace: kube-system
    spec:
      valuesContent: |-
        installation:
          controlPlaneTolerations:
            - key: "node-role.kubernetes.io/control-plane"
              operator: Exists
              effect: NoSchedule
            - key: "node-role.kubernetes.io/etcd"
              operator: Exists
              effect: NoExecute
            - key: "node.cloudprovider.kubernetes.io/uninitialized"
              operator: Equal
              value: "true"
              effect: NoSchedule