Feat: VCN Native CNI Changes (#407)

* bug: Update the CRDs for the workrequest feature (#387)

This was left out of the previous feature add.

* changes to uptake go-v1.22.9

* manifests generated with new controller gen

* Feat: API Spec & CRD & Conversion for VCN Native Params

* Feat: NPN CR Reconciliation in OCIMachine_controller

* Fix: Add common label for compatibility & remove non-related crd changes from open source

* Revert "Fix: Add common label for compatibility & remove non-related crd changes from open source"

This reverts commit f816852.

* Fix: Improve NPN CR Reconcile

* Fix: Kube Client Creation

* Fix: Update CNI Type String

* Fix: Return nil upon retry

* Fix: unused var

* Fix: improve logging and error display

* Fix: Use different CR for get and create

* Feat: include the NPN Reconcile into client scope package

* Style: Remove Redudant Const

* Feat: Move CniType to OCICluster

* Image build

* Image build

* Image build

* Logging update

---------

Co-authored-by: Joe Kratzat <[email protected]>
Co-authored-by: Peizhao Li <[email protected]>

Author: 3 people

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.21.8 as builder
+FROM golang:1.22.9 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

Makefile

@@ -369,7 +369,7 @@ envsubst: $(ENVSUBST) ## Build a local copy of envsubst.
 kubectl: $(KUBECTL) ## Build a local copy of kubectl.
 
 $(CONTROLLER_GEN): ## Download controller-gen locally if necessary.
-	GOBIN=$(BIN_DIR)/ $(GO_INSTALL) sigs.k8s.io/controller-tools/cmd/controller-gen $(CONTROLLER_GEN_BIN) v0.13.0
+	GOBIN=$(BIN_DIR)/ $(GO_INSTALL) sigs.k8s.io/controller-tools/cmd/controller-gen $(CONTROLLER_GEN_BIN) v0.14.0
 
 $(CONVERSION_GEN): ## Download controller-gen locally if necessary.
 	GOBIN=$(BIN_DIR)/ $(GO_INSTALL) k8s.io/code-generator/cmd/conversion-gen $(CONVERSION_GEN_BIN) v0.23.1

api/v1beta1/ocicluster_conversion.go

@@ -48,6 +48,7 @@ func (src *OCICluster) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.NetworkSpec.Vcn.InternetGateway.Skip = restored.Spec.NetworkSpec.Vcn.InternetGateway.Skip
 	dst.Spec.NetworkSpec.Vcn.RouteTable.Skip = restored.Spec.NetworkSpec.Vcn.RouteTable.Skip
 	dst.Spec.NetworkSpec.APIServerLB.LoadBalancerType = restored.Spec.NetworkSpec.APIServerLB.LoadBalancerType
+	dst.Spec.NetworkSpec.CniType = restored.Spec.NetworkSpec.CniType
 	dst.Spec.ClientOverrides = restored.Spec.ClientOverrides
 
 	return nil

api/v1beta1/ocimachine_types.go

@@ -145,6 +145,17 @@ type OCIMachineSpec struct {
 	// Specifies whether to delete or preserve the data volumes created during launch when
 	//terminating an instance. When set to true, the data volumes are preserved. The default value is true.
 	PreserveDataVolumesCreatedAtLaunch bool `json:"preserveDataVolumesCreatedAtLaunch,omitempty"`
+
+	// Specifies the list of pod subnets being used for the VCN IP NATIVE CNI type for pod networking.
+	// Set on each NPN CR associated with the OCI Machine.
+	PodSubnetIds []string `json:"podSubnetIds,omitempty"`
+
+	// Specifies the maximum number of pods allowed for each node, decided by the shape
+	MaxPodPerNode int `json:"maxPodCount,omitempty"`
+
+	// Specifies the list of Network Security Groups used for the VCN IP NATIVE CNI type for pod networking.
+	// Set on each NPN CR associated with the OCI Machine.
+	PodNSGIds []string `json:"podNsgIds,omitempty"`
 }
 
 // OCIMachineStatus defines the observed state of OCIMachine.

api/v1beta1/types.go

@@ -1009,6 +1009,9 @@ type NetworkSpec struct {
 	// VCNPeering configuration.
 	// +optional
 	VCNPeering *VCNPeering `json:"vcnPeering,omitempty"`
+
+	// The CNI to be used are OCI_VCN_IP_NATIVE and FLANNEL_OVERLAY
+	CniType CNIOptionEnum `json:"cniType,omitempty"`
 }
 
 // VCNPeering defines the VCN peering details of the workload cluster VCN.

api/v1beta1/zz_generated.conversion.go

@@ -139,6 +139,17 @@ type OCIMachineSpec struct {
 	// Specifies whether to delete or preserve the data volumes created during launch when
 	//terminating an instance. When set to true, the data volumes are preserved. The default value is true.
 	PreserveDataVolumesCreatedAtLaunch bool `json:"preserveDataVolumesCreatedAtLaunch,omitempty"`
+
+	// Specifies the list of pod subnets being used for the VCN IP NATIVE CNI type for pod networking.
+	// Set on each NPN CR associated with the OCI Machine.
+	PodSubnetIds []string `json:"podSubnetIds,omitempty"`
+
+	// Specifies the maximum number of pods allowed for each node, decided by the shape
+	MaxPodPerNode int `json:"maxPodCount,omitempty"`
+
+	// Specifies the list of Network Security Groups used for the VCN IP NATIVE CNI type for pod networking.
+	// Set on each NPN CR associated with the OCI Machine.
+	PodNSGIds []string `json:"podNsgIds,omitempty"`
 }
 
 // OCIMachineStatus defines the observed state of OCIMachine.

api/v1beta1/zz_generated.deepcopy.go

@@ -1018,6 +1018,9 @@ type NetworkSpec struct {
 	// VCNPeering configuration.
 	// +optional
 	VCNPeering *VCNPeering `json:"vcnPeering,omitempty"`
+
+	// The CNI to be used are OCI_VCN_IP_NATIVE and FLANNEL_OVERLAY
+	CniType CNIOptionEnum `json:"cniType,omitempty"`
 }
 
 // VCNPeering defines the VCN peering details of the workload cluster VCN.

api/v1beta2/ocimachine_types.go

@@ -0,0 +1,167 @@
+/*
+ Copyright (c) 2022 Oracle and/or its affiliates.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+      https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package scope
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	npnVersion          = "oci.oraclecloud.com/v1beta1"
+	npnKind             = "NativePodNetwork"
+	apiExtensionVersion = "apiextensions.k8s.io/v1"
+	npnCrdName          = "nativepodnetworks.oci.oraclecloud.com"
+)
+
+func (m *MachineScope) NewWorkloadClient(ctx context.Context) (wlClient client.Client, err error) {
+	u := &unstructured.Unstructured{}
+	u.SetGroupVersionKind(schema.GroupVersionKind{
+		Kind:    "Secret",
+		Version: "v1",
+	})
+	cluster := m.Cluster
+
+	secret_obj := client.ObjectKey{
+		Namespace: cluster.Namespace,
+		Name:      cluster.Spec.InfrastructureRef.Name + "-kubeconfig",
+	}
+	secret := &corev1.Secret{}
+	if err := m.Client.Get(ctx, secret_obj, secret); err != nil {
+		return nil, err
+	}
+	secretData := secret.Data["value"]
+	config, err := clientcmd.RESTConfigFromKubeConfig(secretData)
+	if err != nil {
+		m.Info(fmt.Sprintf("error build config: %s", err))
+		return nil, err
+	}
+
+	wlClient, err = client.New(config, client.Options{})
+
+	return wlClient, err
+}
+
+func (m *MachineScope) DeleteNpn(ctx context.Context) error {
+	m.Info("DELETE NPN CR NOW.")
+
+	wlClient, err := m.NewWorkloadClient(ctx)
+	if err != nil {
+		m.Info(fmt.Sprintf("Failed to initialize kube client set: %s", err))
+		return err
+	}
+	instance, err := m.GetOrCreateMachine(ctx)
+	if err != nil {
+		m.Info(fmt.Sprintf("Failed to get machine: %s", err))
+		return err
+	}
+	npnCr := &unstructured.Unstructured{}
+	slicedId := strings.Split(*instance.Id, ".")
+	instanceSuffix := slicedId[len(slicedId)-1]
+	npnCr.SetName(instanceSuffix)
+	npnCr.SetGroupVersionKind(schema.GroupVersionKind{
+		Version: npnVersion,
+		Kind:    npnKind,
+	})
+	if err := wlClient.Delete(ctx, npnCr); err != nil {
+		m.Info(fmt.Sprintf("Failed to delete NPN CR: %s", err))
+		return err
+	}
+	return nil
+}
+
+func (m *MachineScope) HasNpnCrd(ctx context.Context) (bool, error) {
+	m.Info("Get NPN CRD Now.")
+
+	wlClient, err := m.NewWorkloadClient(ctx)
+	if err != nil {
+		return false, err
+	}
+	npnCrd := &unstructured.Unstructured{}
+	npnCrd.SetGroupVersionKind(schema.GroupVersionKind{
+		Version: apiExtensionVersion,
+		Kind:    "CustomResourceDefinition",
+	})
+
+	err = wlClient.Get(context.Background(), client.ObjectKey{
+		Name: npnCrdName,
+	}, npnCrd)
+	if err != nil {
+		m.Info(fmt.Sprintf("Failed to Get NPN CRD, reason: %v", err))
+		return false, err
+	}
+
+	return true, nil
+
+}
+
+func (m *MachineScope) GetOrCreateNpn(ctx context.Context) (*unstructured.Unstructured, error) {
+
+	m.Info("Get Or Create NPN CR NOW.")
+	instance, err := m.GetOrCreateMachine(ctx)
+	if err != nil {
+		m.Info(fmt.Sprintf("Failed to get machine: %s", err))
+		return nil, err
+	}
+	wlClient, err := m.NewWorkloadClient(ctx)
+	if err != nil {
+		return nil, err
+	}
+	npnCr := &unstructured.Unstructured{}
+	slicedId := strings.Split(*instance.Id, ".")
+	instanceSuffix := slicedId[len(slicedId)-1]
+	npnCr.SetGroupVersionKind(schema.GroupVersionKind{
+		Version: npnVersion,
+		Kind:    npnKind,
+	})
+	err = wlClient.Get(ctx, client.ObjectKey{Name: instanceSuffix}, npnCr)
+	// Return NPN CR Object if it existed
+	if err == nil {
+		m.Info(fmt.Sprintf("Sucessfully Get an Existed NPN CR Object: %s", npnCr))
+		return npnCr, nil
+	}
+	maxPodCount := m.OCIMachine.Spec.MaxPodPerNode
+	podSubnetIds := m.OCIMachine.Spec.PodSubnetIds
+	podNsgIds := m.OCIMachine.Spec.PodNSGIds
+	npnCrCreate := &unstructured.Unstructured{}
+	npnCrCreate.Object = map[string]interface{}{
+		"metadata": map[string]interface{}{
+			"name": instanceSuffix,
+		},
+		"spec": map[string]interface{}{
+			"id":                      *instance.Id,
+			"maxPodCount":             maxPodCount,
+			"podSubnetIds":            podSubnetIds,
+			"networkSecurityGroupIds": podNsgIds,
+		},
+	}
+
+	npnCrCreate.SetGroupVersionKind(schema.GroupVersionKind{
+		Version: npnVersion,
+		Kind:    npnKind,
+	})
+	m.Info(fmt.Sprintf("NPN CR to Create is: %v", npnCrCreate))
+	err = wlClient.Create(ctx, npnCrCreate)
+	return npnCrCreate, err
+}