WIP : [release-4.20] kubelet-config compressible patch

pkg/controller/bootstrap/bootstrap.go

@@ -234,6 +234,13 @@ func (b *Bootstrap) Run(destDir string) error {
 	}
 	klog.Infof("Successfully generated MachineConfigs from kubelet configs.")
 
+	compressibleConfigs, err := kubeletconfig.RunCompressibleBootstrap(pools, cconfig, b.templatesDir, apiServer, fgHandler)
+	if err != nil {
+		return err
+	}
+	configs = append(configs, compressibleConfigs...)
+	klog.Infof("Successfully generated MachineConfigs for compressible kubelet configs.")
+
 	fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig)
 	if err != nil {
 		return err

pkg/controller/kubelet-config/kubelet_config_bootstrap.go

@@ -40,7 +40,7 @@ func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletCon
 			}
 			role := pool.Name
 
-			originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, fgHandler, apiServer)
+			originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, fgHandler, apiServer)
 			if err != nil {
 				return nil, err
 			}

pkg/controller/kubelet-config/kubelet_config_compressible.go

@@ -0,0 +1,155 @@
+package kubeletconfig
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/clarketm/json"
+	configv1 "github.com/openshift/api/config/v1"
+	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
+	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+)
+
+const (
+	// CompressibleMachineConfigNamePrefix is the prefix for compressible machine configs
+	CompressibleMachineConfigNamePrefix = "50-%s-compressible-kubelet-override"
+
+	// KubeletConfPath is the path to the kubelet config file
+	KubeletConfPath = "/etc/kubernetes/kubelet.conf"
+)
+
+// ensureCompressibleMachineConfigs ensures compressible machine configs exist for all pools
+// This is called at controller startup to create compressible MCs for all pools
+func (ctrl *Controller) ensureCompressibleMachineConfigs() error {
+	pools, err := ctrl.mcpLister.List(labels.Everything())
+	if err != nil {
+		return fmt.Errorf("could not list machine config pools: %w", err)
+	}
+
+	cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName)
+	if err != nil {
+		return fmt.Errorf("could not get ControllerConfig: %w", err)
+	}
+
+	apiServer, err := ctrl.apiserverLister.Get(ctrlcommon.APIServerInstanceName)
+	if err != nil && !errors.IsNotFound(err) {
+		return fmt.Errorf("could not get APIServer: %w", err)
+	}
+
+	for _, pool := range pools {
+		// Generate the original kubelet config for this pool
+		// Kubelet config in templates/master/01-master-kubelet/_base/files/kubelet.yaml has parameeters which need
+		// to be resolved
+		_, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, pool.Name, ctrl.fgHandler, apiServer)
+		if err != nil {
+			klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err)
+			continue
+		}
+
+		if err := ctrl.createCompressibleMachineConfigIfNeeded(pool.Name, kubeletContents); err != nil {
+			klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err)
+			// Don't fail startup if compressible MC creation fails for a pool
+			continue
+		}
+	}
+
+	return nil
+}
+
+// createCompressibleMachineConfigIfNeeded creates a compressible machine config if it doesn't exist
+// This function is called from the controller after kubelet config is successfully generated
+func (ctrl *Controller) createCompressibleMachineConfigIfNeeded(poolName string, kubeletContents []byte) error {
+	compressibleKey := fmt.Sprintf(CompressibleMachineConfigNamePrefix, poolName)
+	_, err := ctrl.client.MachineconfigurationV1().MachineConfigs().Get(context.TODO(), compressibleKey, metav1.GetOptions{})
+	compressibleIsNotFound := errors.IsNotFound(err)
+	if err != nil && !compressibleIsNotFound {
+		return err
+	}
+
+	if compressibleIsNotFound {
+		compressibleMC, err := newCompressibleMachineConfig(poolName, kubeletContents)
+		if err != nil {
+			return fmt.Errorf("could not create compressible machine config: %w", err)
+		}
+
+		if err := retry.RetryOnConflict(updateBackoff, func() error {
+			_, err := ctrl.client.MachineconfigurationV1().MachineConfigs().Create(context.TODO(), compressibleMC, metav1.CreateOptions{})
+			return err
+		}); err != nil {
+			return fmt.Errorf("could not create compressible MachineConfig: %w", err)
+		}
+		klog.Infof("Created compressible kubelet configuration %v for pool %v", compressibleKey, poolName)
+	} else {
+		klog.V(4).Infof("Compressible kubelet MachineConfig %v already exists for pool %v, skipping creation", compressibleKey, poolName)
+	}
+
+	return nil
+}
+
+// newCompressibleMachineConfig creates a new machine config for compressible kubelet override
+// from the provided kubelet config contents
+func newCompressibleMachineConfig(poolName string, kubeletContents []byte) (*mcfgv1.MachineConfig, error) {
+	compressibleMCName := fmt.Sprintf(CompressibleMachineConfigNamePrefix, poolName)
+
+	rawCompressibleIgn, err := createCompressibleKubeletIgnConfig(kubeletContents)
+	if err != nil {
+		return nil, fmt.Errorf("could not create compressible kubelet ignition config: %w", err)
+	}
+
+	compressibleMC, err := ctrlcommon.MachineConfigFromRawIgnConfig(poolName, compressibleMCName, rawCompressibleIgn)
+	if err != nil {
+		return nil, fmt.Errorf("could not create machine config from ignition config: %w", err)
+	}
+
+	compressibleMC.ObjectMeta.Annotations = map[string]string{
+		"openshift-patch-reference": "machineConfig-to-override-kubelet-conf-for-compressible-resources",
+	}
+
+	return compressibleMC, nil
+}
+
+// createCompressibleKubeletIgnConfig creates an Ignition config that overrides /etc/kubernetes/kubelet.conf
+// from the provided kubelet config contents
+func createCompressibleKubeletIgnConfig(kubeletContents []byte) ([]byte, error) {
+	// Create an Ignition file that overrides /etc/kubernetes/kubelet.conf
+	compressibleFile := ctrlcommon.NewIgnFileBytesOverwriting(KubeletConfPath, kubeletContents)
+	compressibleIgnConfig := ctrlcommon.NewIgnConfig()
+	compressibleIgnConfig.Storage.Files = append(compressibleIgnConfig.Storage.Files, compressibleFile)
+
+	rawCompressibleIgn, err := json.Marshal(compressibleIgnConfig)
+	if err != nil {
+		return nil, fmt.Errorf("could not marshal ignition config: %w", err)
+	}
+
+	return rawCompressibleIgn, nil
+}
+
+// RunCompressibleBootstrap generates compressible machine configs for all pools during bootstrap
+func RunCompressibleBootstrap(pools []*mcfgv1.MachineConfigPool, cconfig *mcfgv1.ControllerConfig, templatesDir string, apiServer *configv1.APIServer, fgHandler ctrlcommon.FeatureGatesHandler) ([]*mcfgv1.MachineConfig, error) {
+	configs := []*mcfgv1.MachineConfig{}
+
+	for _, pool := range pools {
+		// Generate the original kubelet config for this pool
+		_, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templatesDir, pool.Name, fgHandler, apiServer)
+		if err != nil {
+			klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err)
+			continue
+		}
+
+		// Create compressible MC
+		compressibleMC, err := newCompressibleMachineConfig(pool.Name, kubeletContents)
+		if err != nil {
+			klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err)
+			continue
+		}
+
+		configs = append(configs, compressibleMC)
+	}
+
+	return configs, nil
+}

pkg/controller/kubelet-config/kubelet_config_compressible_test.go

@@ -0,0 +1,82 @@
+package kubeletconfig
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	configv1 "github.com/openshift/api/config/v1"
+	osev1 "github.com/openshift/api/config/v1"
+	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
+	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
+	"github.com/openshift/machine-config-operator/test/helpers"
+)
+
+func TestNewCompressibleMachineConfig(t *testing.T) {
+	kubeletContents := []byte("apiVersion: kubelet.config.k8s.io/v1beta1\nkind: KubeletConfiguration")
+
+	mc, err := newCompressibleMachineConfig("worker", kubeletContents)
+
+	require.NoError(t, err)
+	assert.Equal(t, "50-worker-compressible-kubelet-override", mc.Name)
+	assert.Contains(t, mc.ObjectMeta.Annotations, "openshift-patch-reference")
+
+	ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw)
+	require.NoError(t, err)
+	require.Len(t, ignCfg.Storage.Files, 1)
+	assert.Equal(t, KubeletConfPath, ignCfg.Storage.Files[0].Path)
+
+	contents, err := ctrlcommon.DecodeIgnitionFileContents(ignCfg.Storage.Files[0].Contents.Source, ignCfg.Storage.Files[0].Contents.Compression)
+	require.NoError(t, err)
+	assert.Equal(t, kubeletContents, contents)
+}
+
+func TestRunCompressibleBootstrap(t *testing.T) {
+	cc := newControllerConfig(ctrlcommon.ControllerConfigName, configv1.AWSPlatformType)
+	pools := []*mcfgv1.MachineConfigPool{
+		helpers.NewMachineConfigPool("master", nil, helpers.MasterSelector, "v0"),
+		helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v0"),
+	}
+	fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{}, nil)
+
+	mcs, err := RunCompressibleBootstrap(pools, cc, "../../../templates", nil, fgHandler)
+
+	require.NoError(t, err)
+	require.Len(t, mcs, 2)
+
+	assert.Equal(t, "50-master-compressible-kubelet-override", mcs[0].Name)
+	assert.Equal(t, "50-worker-compressible-kubelet-override", mcs[1].Name)
+
+	for _, mc := range mcs {
+		ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw)
+		require.NoError(t, err)
+		require.Len(t, ignCfg.Storage.Files, 1)
+		assert.Equal(t, KubeletConfPath, ignCfg.Storage.Files[0].Path)
+	}
+}
+
+// This test ensures that the template replacement works as expected
+func TestCompressibleMachineConfigTemplateReplacement(t *testing.T) {
+	cc := newControllerConfig(ctrlcommon.ControllerConfigName, configv1.AWSPlatformType)
+	cc.Spec.ClusterDNSIP = "10.96.0.10"
+
+	fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{}, nil)
+
+	_, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, "../../../templates", "master", fgHandler, nil)
+	require.NoError(t, err)
+
+	mc, err := newCompressibleMachineConfig("master", kubeletContents)
+	require.NoError(t, err)
+
+	ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw)
+	require.NoError(t, err)
+	require.Len(t, ignCfg.Storage.Files, 1)
+
+	contents, err := ctrlcommon.DecodeIgnitionFileContents(ignCfg.Storage.Files[0].Contents.Source, ignCfg.Storage.Files[0].Contents.Compression)
+	require.NoError(t, err)
+
+	contentsStr := string(contents)
+	assert.Contains(t, contentsStr, "10.96.0.10")
+	assert.NotContains(t, contentsStr, "{{.ClusterDNSIP}}")
+}

pkg/controller/kubelet-config/kubelet_config_controller.go

@@ -200,6 +200,18 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
 	klog.Info("Starting MachineConfigController-KubeletConfigController")
 	defer klog.Info("Shutting down MachineConfigController-KubeletConfigController")
 
+	// Wait for ControllerConfig generation to be reconciled before creating compressible machine configs
+	go func() {
+		if err := ctrl.waitForControllerConfig(stopCh); err != nil {
+			klog.Warningf("Failed to wait for ControllerConfig generation reconciliation: %v", err)
+		} else {
+			// Ensure compressible machine configs are created for all pools at startup
+			if err := ctrl.ensureCompressibleMachineConfigs(); err != nil {
+				klog.Warningf("Error ensuring compressible MachineConfigs: %v", err)
+			}
+		}
+	}()
+
 	for i := 0; i < workers; i++ {
 		go wait.Until(ctrl.worker, time.Second, stopCh)
 	}
@@ -214,6 +226,31 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
 
 	<-stopCh
 }
+
+// waitForControllerConfig waits for the ControllerConfig to be reconciled by the template controller
+func (ctrl *Controller) waitForControllerConfig(stopCh <-chan struct{}) error {
+	klog.Info("Waiting for ControllerConfig generation to be reconciled...")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	return wait.PollUntilContextTimeout(ctx, 1*time.Second, 2*time.Minute, true, func(_ context.Context) (bool, error) {
+		select {
+		case <-stopCh:
+			return false, fmt.Errorf("controller stopped while waiting for ControllerConfig reconciliation")
+		default:
+		}
+
+		if err := apihelpers.IsControllerConfigCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil {
+			// If the ControllerConfig is not running, we will encounter an error when generating the
+			// kubeletconfig object.
+			klog.V(1).Infof("ControllerConfig not running: %v", err)
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
 func (ctrl *Controller) filterAPIServer(apiServer *configv1.APIServer) {
 	if apiServer.Name != "cluster" {
 		return
@@ -419,21 +456,22 @@ func (ctrl *Controller) handleFeatureErr(err error, key string) {
 
 // generateOriginalKubeletConfigWithFeatureGates generates a KubeletConfig and ensure the correct feature gates are set
 // based on the given FeatureGate.
-func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer) (*kubeletconfigv1beta1.KubeletConfiguration, error) {
+// It also returns the decoded kubelet config contents for use in creating compressible machine configs.
+func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer) (*kubeletconfigv1beta1.KubeletConfiguration, []byte, error) {
 	originalKubeletIgn, err := generateOriginalKubeletConfigIgn(cc, templatesDir, role, apiServer)
 	if err != nil {
-		return nil, fmt.Errorf("could not generate the original Kubelet config ignition: %w", err)
+		return nil, nil, fmt.Errorf("could not generate the original Kubelet config ignition: %w", err)
 	}
 	if originalKubeletIgn.Contents.Source == nil {
-		return nil, fmt.Errorf("the original Kubelet source string is empty: %w", err)
+		return nil, nil, fmt.Errorf("the original Kubelet source string is empty: %w", err)
 	}
 	contents, err := ctrlcommon.DecodeIgnitionFileContents(originalKubeletIgn.Contents.Source, originalKubeletIgn.Contents.Compression)
 	if err != nil {
-		return nil, fmt.Errorf("could not decode the original Kubelet source string: %w", err)
+		return nil, nil, fmt.Errorf("could not decode the original Kubelet source string: %w", err)
 	}
 	originalKubeConfig, err := DecodeKubeletConfig(contents)
 	if err != nil {
-		return nil, fmt.Errorf("could not deserialize the Kubelet source: %w", err)
+		return nil, nil, fmt.Errorf("could not deserialize the Kubelet source: %w", err)
 	}
 
 	// todo map pointer
@@ -442,10 +480,10 @@ func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig,
 	// Merge in Feature Gates.
 	// If they are the same, this will be a no-op
 	if err := mergo.Merge(&originalKubeConfig.FeatureGates, featureGates, mergo.WithOverride); err != nil {
-		return nil, fmt.Errorf("could not merge feature gates: %w", err)
+		return nil, nil, fmt.Errorf("could not merge feature gates: %w", err)
 	}
 
-	return originalKubeConfig, nil
+	return originalKubeConfig, contents, nil
 }
 
 func generateOriginalKubeletConfigIgn(cc *mcfgv1.ControllerConfig, templatesDir, role string, apiServer *osev1.APIServer) (*ign3types.File, error) {
@@ -616,7 +654,7 @@ func (ctrl *Controller) syncKubeletConfig(key string) error {
 			return fmt.Errorf("could not get ControllerConfig %w", err)
 		}
 
-		originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer)
+		originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer)
 		if err != nil {
 			return ctrl.syncStatusOnly(cfg, err, "could not get original kubelet config: %v", err)
 		}

pkg/controller/kubelet-config/kubelet_config_features.go

@@ -191,7 +191,7 @@ func generateFeatureMap(fgHandler ctrlcommon.FeatureGatesHandler, exclusions ...
 }
 
 func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *osev1.Node, apiServer *osev1.APIServer) ([]byte, error) {
-	originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer)
+	originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer)
 	if err != nil {
 		return nil, err
 	}

pkg/controller/kubelet-config/kubelet_config_features_test.go

@@ -43,7 +43,7 @@ func TestFeatureGateDrift(t *testing.T) {
 			ctrl := f.newController(fgHandler)
 
 			// Generate kubelet config with feature gates applied
-			kubeletConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil)
+			kubeletConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil)
 			require.NoError(t, err)
 
 			t.Logf("Generated Kubelet Config Feature Gates: %v", kubeletConfig.FeatureGates)

pkg/controller/kubelet-config/kubelet_config_nodes.go

@@ -112,7 +112,7 @@ func (ctrl *Controller) syncNodeConfigHandler(key string) error {
 				return err
 			}
 		}
-		originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer)
+		originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer)
 		if err != nil {
 			return err
 		}
@@ -290,7 +290,7 @@ func RunNodeConfigBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGate
 		if err != nil {
 			return nil, err
 		}
-		originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templateDir, role, fgHandler, apiServer)
+		originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templateDir, role, fgHandler, apiServer)
 		if err != nil {
 			return nil, err
 		}