diff --git a/pkg/controller/bootstrap/bootstrap.go b/pkg/controller/bootstrap/bootstrap.go index ac7d92dfcc..e173d53d1f 100644 --- a/pkg/controller/bootstrap/bootstrap.go +++ b/pkg/controller/bootstrap/bootstrap.go @@ -234,6 +234,13 @@ func (b *Bootstrap) Run(destDir string) error { } klog.Infof("Successfully generated MachineConfigs from kubelet configs.") + compressibleConfigs, err := kubeletconfig.RunCompressibleBootstrap(pools, cconfig, b.templatesDir, apiServer, fgHandler) + if err != nil { + return err + } + configs = append(configs, compressibleConfigs...) + klog.Infof("Successfully generated MachineConfigs for compressible kubelet configs.") + fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig) if err != nil { return err diff --git a/pkg/controller/kubelet-config/kubelet_config_bootstrap.go b/pkg/controller/kubelet-config/kubelet_config_bootstrap.go index 2dced82bab..2715ed70bd 100644 --- a/pkg/controller/kubelet-config/kubelet_config_bootstrap.go +++ b/pkg/controller/kubelet-config/kubelet_config_bootstrap.go @@ -40,7 +40,7 @@ func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletCon } role := pool.Name - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, fgHandler, apiServer) if err != nil { return nil, err } diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go new file mode 100644 index 0000000000..7178a308be --- /dev/null +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -0,0 +1,155 @@ +package kubeletconfig + +import ( + "context" + "fmt" + + "github.com/clarketm/json" + configv1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" +) + +const ( + // CompressibleMachineConfigNamePrefix is the prefix for compressible machine configs + CompressibleMachineConfigNamePrefix = "50-%s-compressible-kubelet-override" + + // KubeletConfPath is the path to the kubelet config file + KubeletConfPath = "/etc/kubernetes/kubelet.conf" +) + +// ensureCompressibleMachineConfigs ensures compressible machine configs exist for all pools +// This is called at controller startup to create compressible MCs for all pools +func (ctrl *Controller) ensureCompressibleMachineConfigs() error { + pools, err := ctrl.mcpLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("could not list machine config pools: %w", err) + } + + cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) + if err != nil { + return fmt.Errorf("could not get ControllerConfig: %w", err) + } + + apiServer, err := ctrl.apiserverLister.Get(ctrlcommon.APIServerInstanceName) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("could not get APIServer: %w", err) + } + + for _, pool := range pools { + // Generate the original kubelet config for this pool + // Kubelet config in templates/master/01-master-kubelet/_base/files/kubelet.yaml has parameeters which need + // to be resolved + _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, pool.Name, ctrl.fgHandler, apiServer) + if err != nil { + klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err) + continue + } + + if err := ctrl.createCompressibleMachineConfigIfNeeded(pool.Name, kubeletContents); err != nil { + klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err) + // Don't fail startup if compressible MC creation fails for a pool + continue + } + } + + return nil +} + +// createCompressibleMachineConfigIfNeeded creates a compressible machine config if it doesn't exist +// This function is called from the controller after kubelet config is successfully generated +func (ctrl *Controller) createCompressibleMachineConfigIfNeeded(poolName string, kubeletContents []byte) error { + compressibleKey := fmt.Sprintf(CompressibleMachineConfigNamePrefix, poolName) + _, err := ctrl.client.MachineconfigurationV1().MachineConfigs().Get(context.TODO(), compressibleKey, metav1.GetOptions{}) + compressibleIsNotFound := errors.IsNotFound(err) + if err != nil && !compressibleIsNotFound { + return err + } + + if compressibleIsNotFound { + compressibleMC, err := newCompressibleMachineConfig(poolName, kubeletContents) + if err != nil { + return fmt.Errorf("could not create compressible machine config: %w", err) + } + + if err := retry.RetryOnConflict(updateBackoff, func() error { + _, err := ctrl.client.MachineconfigurationV1().MachineConfigs().Create(context.TODO(), compressibleMC, metav1.CreateOptions{}) + return err + }); err != nil { + return fmt.Errorf("could not create compressible MachineConfig: %w", err) + } + klog.Infof("Created compressible kubelet configuration %v for pool %v", compressibleKey, poolName) + } else { + klog.V(4).Infof("Compressible kubelet MachineConfig %v already exists for pool %v, skipping creation", compressibleKey, poolName) + } + + return nil +} + +// newCompressibleMachineConfig creates a new machine config for compressible kubelet override +// from the provided kubelet config contents +func newCompressibleMachineConfig(poolName string, kubeletContents []byte) (*mcfgv1.MachineConfig, error) { + compressibleMCName := fmt.Sprintf(CompressibleMachineConfigNamePrefix, poolName) + + rawCompressibleIgn, err := createCompressibleKubeletIgnConfig(kubeletContents) + if err != nil { + return nil, fmt.Errorf("could not create compressible kubelet ignition config: %w", err) + } + + compressibleMC, err := ctrlcommon.MachineConfigFromRawIgnConfig(poolName, compressibleMCName, rawCompressibleIgn) + if err != nil { + return nil, fmt.Errorf("could not create machine config from ignition config: %w", err) + } + + compressibleMC.ObjectMeta.Annotations = map[string]string{ + "openshift-patch-reference": "machineConfig-to-override-kubelet-conf-for-compressible-resources", + } + + return compressibleMC, nil +} + +// createCompressibleKubeletIgnConfig creates an Ignition config that overrides /etc/kubernetes/kubelet.conf +// from the provided kubelet config contents +func createCompressibleKubeletIgnConfig(kubeletContents []byte) ([]byte, error) { + // Create an Ignition file that overrides /etc/kubernetes/kubelet.conf + compressibleFile := ctrlcommon.NewIgnFileBytesOverwriting(KubeletConfPath, kubeletContents) + compressibleIgnConfig := ctrlcommon.NewIgnConfig() + compressibleIgnConfig.Storage.Files = append(compressibleIgnConfig.Storage.Files, compressibleFile) + + rawCompressibleIgn, err := json.Marshal(compressibleIgnConfig) + if err != nil { + return nil, fmt.Errorf("could not marshal ignition config: %w", err) + } + + return rawCompressibleIgn, nil +} + +// RunCompressibleBootstrap generates compressible machine configs for all pools during bootstrap +func RunCompressibleBootstrap(pools []*mcfgv1.MachineConfigPool, cconfig *mcfgv1.ControllerConfig, templatesDir string, apiServer *configv1.APIServer, fgHandler ctrlcommon.FeatureGatesHandler) ([]*mcfgv1.MachineConfig, error) { + configs := []*mcfgv1.MachineConfig{} + + for _, pool := range pools { + // Generate the original kubelet config for this pool + _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templatesDir, pool.Name, fgHandler, apiServer) + if err != nil { + klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err) + continue + } + + // Create compressible MC + compressibleMC, err := newCompressibleMachineConfig(pool.Name, kubeletContents) + if err != nil { + klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err) + continue + } + + configs = append(configs, compressibleMC) + } + + return configs, nil +} diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible_test.go b/pkg/controller/kubelet-config/kubelet_config_compressible_test.go new file mode 100644 index 0000000000..c8420f3e77 --- /dev/null +++ b/pkg/controller/kubelet-config/kubelet_config_compressible_test.go @@ -0,0 +1,82 @@ +package kubeletconfig + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + configv1 "github.com/openshift/api/config/v1" + osev1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "github.com/openshift/machine-config-operator/test/helpers" +) + +func TestNewCompressibleMachineConfig(t *testing.T) { + kubeletContents := []byte("apiVersion: kubelet.config.k8s.io/v1beta1\nkind: KubeletConfiguration") + + mc, err := newCompressibleMachineConfig("worker", kubeletContents) + + require.NoError(t, err) + assert.Equal(t, "50-worker-compressible-kubelet-override", mc.Name) + assert.Contains(t, mc.ObjectMeta.Annotations, "openshift-patch-reference") + + ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw) + require.NoError(t, err) + require.Len(t, ignCfg.Storage.Files, 1) + assert.Equal(t, KubeletConfPath, ignCfg.Storage.Files[0].Path) + + contents, err := ctrlcommon.DecodeIgnitionFileContents(ignCfg.Storage.Files[0].Contents.Source, ignCfg.Storage.Files[0].Contents.Compression) + require.NoError(t, err) + assert.Equal(t, kubeletContents, contents) +} + +func TestRunCompressibleBootstrap(t *testing.T) { + cc := newControllerConfig(ctrlcommon.ControllerConfigName, configv1.AWSPlatformType) + pools := []*mcfgv1.MachineConfigPool{ + helpers.NewMachineConfigPool("master", nil, helpers.MasterSelector, "v0"), + helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v0"), + } + fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{}, nil) + + mcs, err := RunCompressibleBootstrap(pools, cc, "../../../templates", nil, fgHandler) + + require.NoError(t, err) + require.Len(t, mcs, 2) + + assert.Equal(t, "50-master-compressible-kubelet-override", mcs[0].Name) + assert.Equal(t, "50-worker-compressible-kubelet-override", mcs[1].Name) + + for _, mc := range mcs { + ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw) + require.NoError(t, err) + require.Len(t, ignCfg.Storage.Files, 1) + assert.Equal(t, KubeletConfPath, ignCfg.Storage.Files[0].Path) + } +} + +// This test ensures that the template replacement works as expected +func TestCompressibleMachineConfigTemplateReplacement(t *testing.T) { + cc := newControllerConfig(ctrlcommon.ControllerConfigName, configv1.AWSPlatformType) + cc.Spec.ClusterDNSIP = "10.96.0.10" + + fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{}, nil) + + _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, "../../../templates", "master", fgHandler, nil) + require.NoError(t, err) + + mc, err := newCompressibleMachineConfig("master", kubeletContents) + require.NoError(t, err) + + ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw) + require.NoError(t, err) + require.Len(t, ignCfg.Storage.Files, 1) + + contents, err := ctrlcommon.DecodeIgnitionFileContents(ignCfg.Storage.Files[0].Contents.Source, ignCfg.Storage.Files[0].Contents.Compression) + require.NoError(t, err) + + contentsStr := string(contents) + assert.Contains(t, contentsStr, "10.96.0.10") + assert.NotContains(t, contentsStr, "{{.ClusterDNSIP}}") +} diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 2316c0827d..68955efc4c 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -200,6 +200,18 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { klog.Info("Starting MachineConfigController-KubeletConfigController") defer klog.Info("Shutting down MachineConfigController-KubeletConfigController") + // Wait for ControllerConfig generation to be reconciled before creating compressible machine configs + go func() { + if err := ctrl.waitForControllerConfig(stopCh); err != nil { + klog.Warningf("Failed to wait for ControllerConfig generation reconciliation: %v", err) + } else { + // Ensure compressible machine configs are created for all pools at startup + if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { + klog.Warningf("Error ensuring compressible MachineConfigs: %v", err) + } + } + }() + for i := 0; i < workers; i++ { go wait.Until(ctrl.worker, time.Second, stopCh) } @@ -214,6 +226,31 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { <-stopCh } + +// waitForControllerConfig waits for the ControllerConfig to be reconciled by the template controller +func (ctrl *Controller) waitForControllerConfig(stopCh <-chan struct{}) error { + klog.Info("Waiting for ControllerConfig generation to be reconciled...") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + return wait.PollUntilContextTimeout(ctx, 1*time.Second, 2*time.Minute, true, func(_ context.Context) (bool, error) { + select { + case <-stopCh: + return false, fmt.Errorf("controller stopped while waiting for ControllerConfig reconciliation") + default: + } + + if err := apihelpers.IsControllerConfigCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + // If the ControllerConfig is not running, we will encounter an error when generating the + // kubeletconfig object. + klog.V(1).Infof("ControllerConfig not running: %v", err) + return false, nil + } + return true, nil + }) +} + func (ctrl *Controller) filterAPIServer(apiServer *configv1.APIServer) { if apiServer.Name != "cluster" { return @@ -419,21 +456,22 @@ func (ctrl *Controller) handleFeatureErr(err error, key string) { // generateOriginalKubeletConfigWithFeatureGates generates a KubeletConfig and ensure the correct feature gates are set // based on the given FeatureGate. -func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer) (*kubeletconfigv1beta1.KubeletConfiguration, error) { +// It also returns the decoded kubelet config contents for use in creating compressible machine configs. +func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer) (*kubeletconfigv1beta1.KubeletConfiguration, []byte, error) { originalKubeletIgn, err := generateOriginalKubeletConfigIgn(cc, templatesDir, role, apiServer) if err != nil { - return nil, fmt.Errorf("could not generate the original Kubelet config ignition: %w", err) + return nil, nil, fmt.Errorf("could not generate the original Kubelet config ignition: %w", err) } if originalKubeletIgn.Contents.Source == nil { - return nil, fmt.Errorf("the original Kubelet source string is empty: %w", err) + return nil, nil, fmt.Errorf("the original Kubelet source string is empty: %w", err) } contents, err := ctrlcommon.DecodeIgnitionFileContents(originalKubeletIgn.Contents.Source, originalKubeletIgn.Contents.Compression) if err != nil { - return nil, fmt.Errorf("could not decode the original Kubelet source string: %w", err) + return nil, nil, fmt.Errorf("could not decode the original Kubelet source string: %w", err) } originalKubeConfig, err := DecodeKubeletConfig(contents) if err != nil { - return nil, fmt.Errorf("could not deserialize the Kubelet source: %w", err) + return nil, nil, fmt.Errorf("could not deserialize the Kubelet source: %w", err) } // todo map pointer @@ -442,10 +480,10 @@ func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, // Merge in Feature Gates. // If they are the same, this will be a no-op if err := mergo.Merge(&originalKubeConfig.FeatureGates, featureGates, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("could not merge feature gates: %w", err) + return nil, nil, fmt.Errorf("could not merge feature gates: %w", err) } - return originalKubeConfig, nil + return originalKubeConfig, contents, nil } func generateOriginalKubeletConfigIgn(cc *mcfgv1.ControllerConfig, templatesDir, role string, apiServer *osev1.APIServer) (*ign3types.File, error) { @@ -616,7 +654,7 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { return fmt.Errorf("could not get ControllerConfig %w", err) } - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) if err != nil { return ctrl.syncStatusOnly(cfg, err, "could not get original kubelet config: %v", err) } diff --git a/pkg/controller/kubelet-config/kubelet_config_features.go b/pkg/controller/kubelet-config/kubelet_config_features.go index 4dc62d4dca..32ce8800d3 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features.go +++ b/pkg/controller/kubelet-config/kubelet_config_features.go @@ -191,7 +191,7 @@ func generateFeatureMap(fgHandler ctrlcommon.FeatureGatesHandler, exclusions ... } func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *osev1.Node, apiServer *osev1.APIServer) ([]byte, error) { - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer) if err != nil { return nil, err } diff --git a/pkg/controller/kubelet-config/kubelet_config_features_test.go b/pkg/controller/kubelet-config/kubelet_config_features_test.go index 2f8ea4c21f..0fa665a9a7 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_features_test.go @@ -43,7 +43,7 @@ func TestFeatureGateDrift(t *testing.T) { ctrl := f.newController(fgHandler) // Generate kubelet config with feature gates applied - kubeletConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil) + kubeletConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil) require.NoError(t, err) t.Logf("Generated Kubelet Config Feature Gates: %v", kubeletConfig.FeatureGates) diff --git a/pkg/controller/kubelet-config/kubelet_config_nodes.go b/pkg/controller/kubelet-config/kubelet_config_nodes.go index 4e4ccc37d8..c407b25a3c 100644 --- a/pkg/controller/kubelet-config/kubelet_config_nodes.go +++ b/pkg/controller/kubelet-config/kubelet_config_nodes.go @@ -112,7 +112,7 @@ func (ctrl *Controller) syncNodeConfigHandler(key string) error { return err } } - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) if err != nil { return err } @@ -290,7 +290,7 @@ func RunNodeConfigBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGate if err != nil { return nil, err } - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templateDir, role, fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templateDir, role, fgHandler, apiServer) if err != nil { return nil, err }