Merge pull request #2104 from haircommander/minKubeletVersion

OCPNODE-2940: add minimumkubeletversion package

openshift-kube-apiserver/authorization/minimumkubeletversion/minimum_kubelet_version.go

@@ -0,0 +1,90 @@
+package minimumkubeletversion
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/blang/semver/v4"
+	openshiftfeatures "github.com/openshift/api/features"
+	nodelib "github.com/openshift/library-go/pkg/apiserver/node"
+	authorizationv1 "k8s.io/api/authorization/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/util/feature"
+	v1listers "k8s.io/client-go/listers/core/v1"
+	cache "k8s.io/client-go/tools/cache"
+	"k8s.io/component-base/featuregate"
+	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/auth/nodeidentifier"
+)
+
+type minimumKubeletVersionAuth struct {
+	nodeIdentifier          nodeidentifier.NodeIdentifier
+	nodeLister              v1listers.NodeLister
+	minVersion              *semver.Version
+	hasNodeInformerSyncedFn func() bool // factored for unit tests
+}
+
+// Creates a new minimumKubeletVersionAuth object, which is an authorizer that checks
+// whether nodes are new enough to be authorized.
+func NewMinimumKubeletVersion(minVersion *semver.Version,
+	nodeIdentifier nodeidentifier.NodeIdentifier,
+	nodeInformer cache.SharedIndexInformer,
+	nodeLister v1listers.NodeLister,
+) *minimumKubeletVersionAuth {
+	if !feature.DefaultFeatureGate.Enabled(featuregate.Feature(openshiftfeatures.FeatureGateMinimumKubeletVersion)) {
+		minVersion = nil
+	}
+
+	return &minimumKubeletVersionAuth{
+		nodeIdentifier:          nodeIdentifier,
+		nodeLister:              nodeLister,
+		hasNodeInformerSyncedFn: nodeInformer.HasSynced,
+		minVersion:              minVersion,
+	}
+}
+
+func (m *minimumKubeletVersionAuth) Authorize(ctx context.Context, attrs authorizer.Attributes) (authorizer.Decision, string, error) {
+	if m.minVersion == nil {
+		return authorizer.DecisionNoOpinion, "", nil
+	}
+
+	// Short-circut if "subjectaccessreviews", or a "get" or "update" on the node object.
+	// Regardless of kubelet version, it should be allowed to do these things.
+	if attrs.IsResourceRequest() {
+		requestResource := schema.GroupResource{Group: attrs.GetAPIGroup(), Resource: attrs.GetResource()}
+		switch requestResource {
+		case api.Resource("nodes"):
+			if v := attrs.GetVerb(); v == "get" || v == "update" {
+				return authorizer.DecisionNoOpinion, "", nil
+			}
+		case authorizationv1.Resource("subjectaccessreviews"):
+			return authorizer.DecisionNoOpinion, "", nil
+		}
+	}
+
+	nodeName, isNode := m.nodeIdentifier.NodeIdentity(attrs.GetUser())
+	if !isNode {
+		// ignore requests from non-nodes
+		return authorizer.DecisionNoOpinion, "", nil
+	}
+
+	if !m.hasNodeInformerSyncedFn() {
+		return authorizer.DecisionDeny, "", fmt.Errorf("node informer not synced, cannot check if node %s is new enough", nodeName)
+	}
+
+	node, err := m.nodeLister.Get(nodeName)
+	if err != nil {
+		return authorizer.DecisionDeny, "", err
+	}
+
+	if err := nodelib.IsNodeTooOld(node, m.minVersion); err != nil {
+		if errors.Is(err, nodelib.ErrKubeletOutdated) {
+			return authorizer.DecisionDeny, err.Error(), nil
+		}
+		return authorizer.DecisionDeny, "", err
+	}
+
+	return authorizer.DecisionNoOpinion, "", nil
+}

openshift-kube-apiserver/authorization/minimumkubeletversion/minimum_kubelet_version_test.go

@@ -0,0 +1,193 @@
+package minimumkubeletversion
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/blang/semver/v4"
+	authorizationv1 "k8s.io/api/authorization/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/authentication/user"
+	kauthorizer "k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/kubernetes/pkg/auth/nodeidentifier"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+func TestAuthorize(t *testing.T) {
+	nodeUser := &user.DefaultInfo{Name: "system:node:node0", Groups: []string{"system:nodes"}}
+
+	testCases := []struct {
+		name            string
+		minVersion      string
+		attributes      kauthorizer.AttributesRecord
+		expectedAllowed kauthorizer.Decision
+		expectedErr     string
+		expectedMsg     string
+		node            *v1.Node
+	}{
+		{
+			name:            "no version",
+			minVersion:      "",
+			expectedAllowed: kauthorizer.DecisionNoOpinion,
+			expectedErr:     "",
+			node:            &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "name"}},
+		},
+		{
+			name:       "user not a node",
+			minVersion: "1.30.0",
+			attributes: kauthorizer.AttributesRecord{
+				ResourceRequest: true,
+				Namespace:       "ns",
+				User:            &user.DefaultInfo{Name: "name"},
+			},
+			expectedAllowed: kauthorizer.DecisionNoOpinion,
+			node:            &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node0"}},
+		},
+		{
+			name:       "skips if subjectaccessreviews",
+			minVersion: "1.30.0",
+			attributes: kauthorizer.AttributesRecord{
+				ResourceRequest: true,
+				Namespace:       "ns",
+				User:            nodeUser,
+				Resource:        "subjectaccessreviews",
+				APIGroup:        authorizationv1.GroupName,
+			},
+			expectedAllowed: kauthorizer.DecisionNoOpinion,
+			node:            &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node0"}},
+		},
+		{
+			name:       "skips if get node",
+			minVersion: "1.30.0",
+			attributes: kauthorizer.AttributesRecord{
+				ResourceRequest: true,
+				Namespace:       "ns",
+				User:            nodeUser,
+				Resource:        "nodes",
+				Verb:            "get",
+			},
+			expectedAllowed: kauthorizer.DecisionNoOpinion,
+			node:            &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node0"}},
+		},
+		{
+			name:       "skips if update nodes",
+			minVersion: "1.30.0",
+			attributes: kauthorizer.AttributesRecord{
+				ResourceRequest: true,
+				Namespace:       "ns",
+				User:            nodeUser,
+				Resource:        "nodes",
+				Verb:            "update",
+			},
+			expectedAllowed: kauthorizer.DecisionNoOpinion,
+			node:            &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node0"}},
+		},
+		{
+			name:       "fail if update node not found",
+			minVersion: "1.30.0",
+			attributes: kauthorizer.AttributesRecord{
+				ResourceRequest: true,
+				Namespace:       "ns",
+				User:            nodeUser,
+			},
+			expectedAllowed: kauthorizer.DecisionDeny,
+			expectedErr:     `node "node0" not found`,
+			node:            &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1"}},
+		},
+		{
+			name:       "skip if bogus kubelet version",
+			minVersion: "1.30.0",
+			attributes: kauthorizer.AttributesRecord{
+				ResourceRequest: true,
+				Namespace:       "ns",
+				User:            nodeUser,
+			},
+			expectedAllowed: kauthorizer.DecisionDeny,
+			expectedErr:     `failed to parse node version bogus: No Major.Minor.Patch elements found`,
+			node: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node0"},
+				Status: v1.NodeStatus{
+					NodeInfo: v1.NodeSystemInfo{
+						KubeletVersion: "bogus",
+					},
+				}},
+		},
+		{
+			name:       "deny if too low version",
+			minVersion: "1.30.0",
+			attributes: kauthorizer.AttributesRecord{
+				ResourceRequest: true,
+				Namespace:       "ns",
+				User:            nodeUser,
+			},
+			expectedAllowed: kauthorizer.DecisionDeny,
+			expectedMsg:     `kubelet version is outdated: kubelet version is 1.29.8, which is lower than minimumKubeletVersion of 1.30.0`,
+			node: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node0"},
+				Status: v1.NodeStatus{
+					NodeInfo: v1.NodeSystemInfo{
+						KubeletVersion: "v1.29.8-20+15d27f9ba1c119",
+					},
+				}},
+		},
+		{
+			name:       "accept if high enough version",
+			minVersion: "1.30.0",
+			attributes: kauthorizer.AttributesRecord{
+				ResourceRequest: true,
+				Namespace:       "ns",
+				User:            nodeUser,
+			},
+			expectedAllowed: kauthorizer.DecisionNoOpinion,
+			node: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node0"},
+				Status: v1.NodeStatus{
+					NodeInfo: v1.NodeSystemInfo{
+						KubeletVersion: "1.30.0",
+					},
+				}},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			fakeInformerFactory := informers.NewSharedInformerFactory(&fake.Clientset{}, controller.NoResyncPeriodFunc())
+			fakeNodeInformer := fakeInformerFactory.Core().V1().Nodes()
+			fakeNodeInformer.Informer().GetStore().Add(tc.node)
+			var minVersion *semver.Version
+			if tc.minVersion != "" {
+				v := semver.MustParse(tc.minVersion)
+				minVersion = &v
+			}
+
+			authorizer := &minimumKubeletVersionAuth{
+				nodeIdentifier: nodeidentifier.NewDefaultNodeIdentifier(),
+				nodeLister:     fakeNodeInformer.Lister(),
+				minVersion:     minVersion,
+				hasNodeInformerSyncedFn: func() bool {
+					return true
+				},
+			}
+
+			actualAllowed, actualMsg, actualErr := authorizer.Authorize(context.TODO(), tc.attributes)
+			switch {
+			case len(tc.expectedErr) == 0 && actualErr == nil:
+			case len(tc.expectedErr) == 0 && actualErr != nil:
+				t.Errorf("%s: unexpected error: %v", tc.name, actualErr)
+			case len(tc.expectedErr) != 0 && actualErr == nil:
+				t.Errorf("%s: missing error: %v", tc.name, tc.expectedErr)
+			case len(tc.expectedErr) != 0 && actualErr != nil:
+				if !strings.Contains(actualErr.Error(), tc.expectedErr) {
+					t.Errorf("expected %v, got %v", tc.expectedErr, actualErr)
+				}
+			}
+			if tc.expectedMsg != actualMsg {
+				t.Errorf("expected %v, got %v", tc.expectedMsg, actualMsg)
+			}
+			if tc.expectedAllowed != actualAllowed {
+				t.Errorf("expected %v, got %v", tc.expectedAllowed, actualAllowed)
+			}
+		})
+	}
+}

openshift-kube-apiserver/enablement/intialization.go

@@ -87,6 +87,11 @@ func ForceGlobalInitializationForOpenShift() {
 	// we need to have the authorization chain place something before system:masters
 	// SkipSystemMastersAuthorizer disable implicitly added system/master authz, and turn it into another authz mode "SystemMasters", to be added via authorization-mode
 	authorizer.SkipSystemMastersAuthorizer()
+
+	// Set the minimum kubelet version
+	// If the OpenshiftConfig wasn't configured by this point, it's a programming error,
+	// and this should panic.
+	authorizer.SetMinimumKubeletVersion(OpenshiftConfig().MinimumKubeletVersion)
 }
 
 var SCCAdmissionPlugin = sccadmission.NewConstraint()

pkg/features/openshift_features.go

@@ -5,11 +5,16 @@ import (
 )
 
 var RouteExternalCertificate featuregate.Feature = "RouteExternalCertificate"
+var MinimumKubeletVersion featuregate.Feature = "MinimumKubeletVersion"
 
 // registerOpenshiftFeatures injects openshift-specific feature gates
 func registerOpenshiftFeatures() {
 	defaultKubernetesFeatureGates[RouteExternalCertificate] = featuregate.FeatureSpec{
 		Default:    false,
 		PreRelease: featuregate.Alpha,
 	}
+	defaultKubernetesFeatureGates[MinimumKubeletVersion] = featuregate.FeatureSpec{
+		Default:    false,
+		PreRelease: featuregate.Alpha,
+	}
 }

pkg/kubeapiserver/authorizer/modes/patch.go

@@ -2,7 +2,8 @@ package modes
 
 var ModeScope = "Scope"
 var ModeSystemMasters = "SystemMasters"
+var ModeMinimumKubeletVersion = "MinimumKubeletVersion"
 
 func init() {
-	AuthorizationModeChoices = append(AuthorizationModeChoices, ModeScope, ModeSystemMasters)
+	AuthorizationModeChoices = append(AuthorizationModeChoices, ModeScope, ModeSystemMasters, ModeMinimumKubeletVersion)
 }

pkg/kubeapiserver/authorizer/patch.go

@@ -1,8 +1,54 @@
 package authorizer
 
+import (
+	"sync"
+
+	"github.com/blang/semver/v4"
+)
+
 var skipSystemMastersAuthorizer = false
 
 // SkipSystemMastersAuthorizer disable implicitly added system/master authz, and turn it into another authz mode "SystemMasters", to be added via authorization-mode
 func SkipSystemMastersAuthorizer() {
 	skipSystemMastersAuthorizer = true
 }
+
+var (
+	minimumKubeletVersion *semver.Version
+	versionLock           sync.Mutex
+	versionSet            bool
+)
+
+// GetMinimumKubeletVersion retrieves the set global minimum kubelet version in a safe way.
+// It ensures it is only retrieved once, and is set before it's retrieved.
+// The global value should only be gotten through this function.
+// It is valid for the version to be unset. It will be treated the same as explicitly setting version to "".
+// This function (and the corresponding functions/variables) are added to avoid a import cycle between the
+// ./openshift-kube-apiserver/enablement and ./pkg/kubeapiserver/authorizer packages
+func GetMinimumKubeletVersion() *semver.Version {
+	versionLock.Lock()
+	defer versionLock.Unlock()
+	if !versionSet {
+		panic("coding error: MinimumKubeletVersion not set yet")
+	}
+	return minimumKubeletVersion
+}
+
+// SetMinimumKubeletVersion sets the global minimum kubelet version in a safe way.
+// It ensures it is only set once, and the passed version is valid.
+// If will panic on any error.
+// The global value should only be set through this function.
+// Passing an empty string for version is valid, and means there is no minimum version.
+func SetMinimumKubeletVersion(version string) {
+	versionLock.Lock()
+	defer versionLock.Unlock()
+	if versionSet {
+		panic("coding error: MinimumKubeletVersion already set")
+	}
+	versionSet = true
+	if len(version) == 0 {
+		return
+	}
+	v := semver.MustParse(version)
+	minimumKubeletVersion = &v
+}