From 0637b9d69c33a7d20a92b7c657d84282ee79b742 Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Thu, 13 Nov 2025 12:49:10 +0530 Subject: [PATCH 01/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- pkg/controller/bootstrap/bootstrap.go | 7 + .../kubelet_config_bootstrap.go | 2 +- .../kubelet_config_compressible.go | 159 ++++++++++++++++++ .../kubelet_config_controller.go | 22 ++- .../kubelet-config/kubelet_config_features.go | 2 +- .../kubelet_config_features_test.go | 2 +- .../kubelet-config/kubelet_config_nodes.go | 4 +- 7 files changed, 185 insertions(+), 13 deletions(-) create mode 100644 pkg/controller/kubelet-config/kubelet_config_compressible.go diff --git a/pkg/controller/bootstrap/bootstrap.go b/pkg/controller/bootstrap/bootstrap.go index ac7d92dfcc..40cce1b64c 100644 --- a/pkg/controller/bootstrap/bootstrap.go +++ b/pkg/controller/bootstrap/bootstrap.go @@ -208,6 +208,13 @@ func (b *Bootstrap) Run(destDir string) error { } klog.Infof("Successfully generated MachineConfigs from feature gates.") + compressibleConfigs, err := kubeletconfig.RunCompressibleBootstrap(pools, cconfig, b.templatesDir, apiServer, fgHandler) + if err != nil { + return err + } + configs = append(configs, compressibleConfigs...) + klog.Infof("Successfully generated MachineConfigs for compressible kubelet configs.") + if nodeConfig == nil { nodeConfig = &apicfgv1.Node{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/controller/kubelet-config/kubelet_config_bootstrap.go b/pkg/controller/kubelet-config/kubelet_config_bootstrap.go index 2dced82bab..2715ed70bd 100644 --- a/pkg/controller/kubelet-config/kubelet_config_bootstrap.go +++ b/pkg/controller/kubelet-config/kubelet_config_bootstrap.go @@ -40,7 +40,7 @@ func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletCon } role := pool.Name - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, fgHandler, apiServer) if err != nil { return nil, err } diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go new file mode 100644 index 0000000000..ae5c4d5897 --- /dev/null +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -0,0 +1,159 @@ +package kubeletconfig + +import ( + "context" + "fmt" + + "github.com/clarketm/json" + configv1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" +) + +const ( + // CompressibleMachineConfigNamePrefix is the prefix for compressible machine configs + CompressibleMachineConfigNamePrefix = "50-%s-compressible-kubelet-override" + + // KubeletConfPath is the path to the kubelet config file + KubeletConfPath = "/etc/kubernetes/kubelet.conf" +) + +// ensureCompressibleMachineConfigs ensures compressible machine configs exist for all pools +// This is called at controller startup to create compressible MCs for all pools +func (ctrl *Controller) ensureCompressibleMachineConfigs() error { + // Get all pools + pools, err := ctrl.mcpLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("could not list machine config pools: %w", err) + } + + // Get ControllerConfig + cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) + if err != nil { + return fmt.Errorf("could not get ControllerConfig: %w", err) + } + + // Get APIServer + apiServer, err := ctrl.apiserverLister.Get(ctrlcommon.APIServerInstanceName) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("could not get APIServer: %w", err) + } + + // Create compressible MC for each pool + for _, pool := range pools { + // Generate the original kubelet config for this pool + _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, pool.Name, ctrl.fgHandler, apiServer) + if err != nil { + klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err) + continue + } + + // Create compressible MC + if err := ctrl.createCompressibleMachineConfigIfNeeded(pool.Name, kubeletContents); err != nil { + klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err) + // Don't fail startup if compressible MC creation fails for a pool + continue + } + } + + return nil +} + +// createCompressibleMachineConfigIfNeeded creates a compressible machine config if it doesn't exist +// This function is called from the controller after kubelet config is successfully generated +func (ctrl *Controller) createCompressibleMachineConfigIfNeeded(poolName string, kubeletContents []byte) error { + compressibleKey := fmt.Sprintf(CompressibleMachineConfigNamePrefix, poolName) + _, err := ctrl.client.MachineconfigurationV1().MachineConfigs().Get(context.TODO(), compressibleKey, metav1.GetOptions{}) + compressibleIsNotFound := errors.IsNotFound(err) + if err != nil && !compressibleIsNotFound { + return err + } + + if compressibleIsNotFound { + compressibleMC, err := newCompressibleMachineConfig(poolName, kubeletContents) + if err != nil { + return fmt.Errorf("could not create compressible machine config: %w", err) + } + + if err := retry.RetryOnConflict(updateBackoff, func() error { + _, err := ctrl.client.MachineconfigurationV1().MachineConfigs().Create(context.TODO(), compressibleMC, metav1.CreateOptions{}) + return err + }); err != nil { + return fmt.Errorf("could not create compressible MachineConfig: %w", err) + } + klog.Infof("Created compressible kubelet configuration %v for pool %v", compressibleKey, poolName) + } else { + klog.V(4).Infof("Compressible kubelet MachineConfig %v already exists for pool %v, skipping creation", compressibleKey, poolName) + } + + return nil +} + +// RunCompressibleBootstrap generates compressible machine configs for all pools during bootstrap +func RunCompressibleBootstrap(pools []*mcfgv1.MachineConfigPool, cconfig *mcfgv1.ControllerConfig, templatesDir string, apiServer *configv1.APIServer, fgHandler ctrlcommon.FeatureGatesHandler) ([]*mcfgv1.MachineConfig, error) { + configs := []*mcfgv1.MachineConfig{} + + for _, pool := range pools { + // Generate the original kubelet config for this pool + _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templatesDir, pool.Name, fgHandler, apiServer) + if err != nil { + klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err) + continue + } + + // Create compressible MC + compressibleMC, err := newCompressibleMachineConfig(pool.Name, kubeletContents) + if err != nil { + klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err) + continue + } + + configs = append(configs, compressibleMC) + } + + return configs, nil +} + +// newCompressibleMachineConfig creates a new machine config for compressible kubelet override +// from the provided kubelet config contents +func newCompressibleMachineConfig(poolName string, kubeletContents []byte) (*mcfgv1.MachineConfig, error) { + compressibleMCName := fmt.Sprintf(CompressibleMachineConfigNamePrefix, poolName) + ignConfig := ctrlcommon.NewIgnConfig() + compressibleMC, err := ctrlcommon.MachineConfigFromIgnConfig(poolName, compressibleMCName, ignConfig) + if err != nil { + return nil, fmt.Errorf("could not create machine config from ignition config: %w", err) + } + + rawCompressibleIgn, err := createCompressibleKubeletIgnConfig(kubeletContents) + if err != nil { + return nil, fmt.Errorf("could not create compressible kubelet ignition config: %w", err) + } + + compressibleMC.Spec.Config.Raw = rawCompressibleIgn + compressibleMC.ObjectMeta.Annotations = map[string]string{ + "openshift-patch-reference": "machineConfig-to-override-kubelet-conf-for-compressible-resources", + } + + return compressibleMC, nil +} + +// createCompressibleKubeletIgnConfig creates an Ignition config that overrides /etc/kubernetes/kubelet.conf +// from the provided kubelet config contents +func createCompressibleKubeletIgnConfig(kubeletContents []byte) ([]byte, error) { + // Create an Ignition file that overrides /etc/kubernetes/kubelet.conf + compressibleFile := ctrlcommon.NewIgnFileBytesOverwriting(KubeletConfPath, kubeletContents) + compressibleIgnConfig := ctrlcommon.NewIgnConfig() + compressibleIgnConfig.Storage.Files = append(compressibleIgnConfig.Storage.Files, compressibleFile) + + rawCompressibleIgn, err := json.Marshal(compressibleIgnConfig) + if err != nil { + return nil, fmt.Errorf("could not marshal ignition config: %w", err) + } + + return rawCompressibleIgn, nil +} diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 2316c0827d..e5b8d1f66e 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -200,6 +200,11 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { klog.Info("Starting MachineConfigController-KubeletConfigController") defer klog.Info("Shutting down MachineConfigController-KubeletConfigController") + // Ensure compressible machine configs are created for all pools at startup + if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { + klog.Warningf("Error ensuring compressible MachineConfigs: %v", err) + } + for i := 0; i < workers; i++ { go wait.Until(ctrl.worker, time.Second, stopCh) } @@ -419,21 +424,22 @@ func (ctrl *Controller) handleFeatureErr(err error, key string) { // generateOriginalKubeletConfigWithFeatureGates generates a KubeletConfig and ensure the correct feature gates are set // based on the given FeatureGate. -func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer) (*kubeletconfigv1beta1.KubeletConfiguration, error) { +// It also returns the decoded kubelet config contents for use in creating compressible machine configs. +func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer) (*kubeletconfigv1beta1.KubeletConfiguration, []byte, error) { originalKubeletIgn, err := generateOriginalKubeletConfigIgn(cc, templatesDir, role, apiServer) if err != nil { - return nil, fmt.Errorf("could not generate the original Kubelet config ignition: %w", err) + return nil, nil, fmt.Errorf("could not generate the original Kubelet config ignition: %w", err) } if originalKubeletIgn.Contents.Source == nil { - return nil, fmt.Errorf("the original Kubelet source string is empty: %w", err) + return nil, nil, fmt.Errorf("the original Kubelet source string is empty: %w", err) } contents, err := ctrlcommon.DecodeIgnitionFileContents(originalKubeletIgn.Contents.Source, originalKubeletIgn.Contents.Compression) if err != nil { - return nil, fmt.Errorf("could not decode the original Kubelet source string: %w", err) + return nil, nil, fmt.Errorf("could not decode the original Kubelet source string: %w", err) } originalKubeConfig, err := DecodeKubeletConfig(contents) if err != nil { - return nil, fmt.Errorf("could not deserialize the Kubelet source: %w", err) + return nil, nil, fmt.Errorf("could not deserialize the Kubelet source: %w", err) } // todo map pointer @@ -442,10 +448,10 @@ func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, // Merge in Feature Gates. // If they are the same, this will be a no-op if err := mergo.Merge(&originalKubeConfig.FeatureGates, featureGates, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("could not merge feature gates: %w", err) + return nil, nil, fmt.Errorf("could not merge feature gates: %w", err) } - return originalKubeConfig, nil + return originalKubeConfig, contents, nil } func generateOriginalKubeletConfigIgn(cc *mcfgv1.ControllerConfig, templatesDir, role string, apiServer *osev1.APIServer) (*ign3types.File, error) { @@ -616,7 +622,7 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { return fmt.Errorf("could not get ControllerConfig %w", err) } - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) if err != nil { return ctrl.syncStatusOnly(cfg, err, "could not get original kubelet config: %v", err) } diff --git a/pkg/controller/kubelet-config/kubelet_config_features.go b/pkg/controller/kubelet-config/kubelet_config_features.go index 4dc62d4dca..32ce8800d3 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features.go +++ b/pkg/controller/kubelet-config/kubelet_config_features.go @@ -191,7 +191,7 @@ func generateFeatureMap(fgHandler ctrlcommon.FeatureGatesHandler, exclusions ... } func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *osev1.Node, apiServer *osev1.APIServer) ([]byte, error) { - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer) if err != nil { return nil, err } diff --git a/pkg/controller/kubelet-config/kubelet_config_features_test.go b/pkg/controller/kubelet-config/kubelet_config_features_test.go index 2f8ea4c21f..0fa665a9a7 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_features_test.go @@ -43,7 +43,7 @@ func TestFeatureGateDrift(t *testing.T) { ctrl := f.newController(fgHandler) // Generate kubelet config with feature gates applied - kubeletConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil) + kubeletConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil) require.NoError(t, err) t.Logf("Generated Kubelet Config Feature Gates: %v", kubeletConfig.FeatureGates) diff --git a/pkg/controller/kubelet-config/kubelet_config_nodes.go b/pkg/controller/kubelet-config/kubelet_config_nodes.go index 4e4ccc37d8..c407b25a3c 100644 --- a/pkg/controller/kubelet-config/kubelet_config_nodes.go +++ b/pkg/controller/kubelet-config/kubelet_config_nodes.go @@ -112,7 +112,7 @@ func (ctrl *Controller) syncNodeConfigHandler(key string) error { return err } } - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer) if err != nil { return err } @@ -290,7 +290,7 @@ func RunNodeConfigBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGate if err != nil { return nil, err } - originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templateDir, role, fgHandler, apiServer) + originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templateDir, role, fgHandler, apiServer) if err != nil { return nil, err } From 5e5800ee83d1935ad30604329cdf9c72e487aef1 Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Mon, 17 Nov 2025 10:14:38 +0530 Subject: [PATCH 02/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- pkg/controller/kubelet-config/kubelet_config_compressible.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go index ae5c4d5897..1164709c23 100644 --- a/pkg/controller/kubelet-config/kubelet_config_compressible.go +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -9,8 +9,8 @@ import ( mcfgv1 "github.com/openshift/api/machineconfiguration/v1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" ) From 4daa59ea40542c1d79b50a66013d597ce9973166 Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Mon, 17 Nov 2025 14:27:29 +0530 Subject: [PATCH 03/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- .../kubelet_config_compressible.go | 7 ++ .../kubelet_config_controller.go | 70 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go index 1164709c23..c2c4e2a285 100644 --- a/pkg/controller/kubelet-config/kubelet_config_compressible.go +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -7,6 +7,7 @@ import ( "github.com/clarketm/json" configv1 "github.com/openshift/api/config/v1" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + "github.com/openshift/machine-config-operator/pkg/apihelpers" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,6 +27,12 @@ const ( // ensureCompressibleMachineConfigs ensures compressible machine configs exist for all pools // This is called at controller startup to create compressible MCs for all pools func (ctrl *Controller) ensureCompressibleMachineConfigs() error { + // Wait to create compressible configs if the controller config is not completed + if err := apihelpers.IsControllerConfigCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + klog.V(4).Infof("ControllerConfig not ready, skipping compressible machine config creation: %v", err) + return nil + } + // Get all pools pools, err := ctrl.mcpLister.List(labels.Everything()) if err != nil { diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index e5b8d1f66e..d48d4d49ac 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -144,6 +144,15 @@ func New( DeleteFunc: ctrl.deleteKubeletConfig, }) + ccInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: ctrl.addControllerConfig, + UpdateFunc: ctrl.updateControllerConfig, + }) + + mcpInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: ctrl.addMachineConfigPool, + }) + featInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: ctrl.addFeature, UpdateFunc: ctrl.updateFeature, @@ -275,6 +284,67 @@ func (ctrl *Controller) deleteAPIServer(obj interface{}) { ctrl.filterAPIServer(apiServer) } +func (ctrl *Controller) addControllerConfig(obj interface{}) { + cc := obj.(*mcfgv1.ControllerConfig) + klog.V(4).Infof("ControllerConfig %s added", cc.Name) + // Try to ensure compressible MCs are created when ControllerConfig becomes available + if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { + klog.Warningf("Error ensuring compressible MachineConfigs after ControllerConfig add: %v", err) + } +} + +func (ctrl *Controller) updateControllerConfig(old, cur interface{}) { + oldCC := old.(*mcfgv1.ControllerConfig) + newCC := cur.(*mcfgv1.ControllerConfig) + + // Check if ControllerConfig just became completed + if oldCC.Status.ObservedGeneration != oldCC.Generation && + newCC.Status.ObservedGeneration == newCC.Generation { + klog.V(4).Infof("ControllerConfig %s became ready", newCC.Name) + // Try to ensure compressible MCs are created when ControllerConfig becomes ready + if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { + klog.Warningf("Error ensuring compressible MachineConfigs after ControllerConfig update: %v", err) + } + } +} + +func (ctrl *Controller) addMachineConfigPool(obj interface{}) { + pool := obj.(*mcfgv1.MachineConfigPool) + klog.V(4).Infof("MachineConfigPool %s added, ensuring compressible MachineConfig exists", pool.Name) + + // Check if ControllerConfig is ready + if err := apihelpers.IsControllerConfigCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + klog.V(4).Infof("ControllerConfig not ready, will create compressible MC for pool %s later", pool.Name) + return + } + + // Get ControllerConfig + cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) + if err != nil { + klog.Warningf("Could not get ControllerConfig for new pool %s: %v", pool.Name, err) + return + } + + // Get APIServer + apiServer, err := ctrl.apiserverLister.Get(ctrlcommon.APIServerInstanceName) + if err != nil && !macherrors.IsNotFound(err) { + klog.Warningf("Could not get APIServer for new pool %s: %v", pool.Name, err) + return + } + + // Generate kubelet config for this pool + _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, pool.Name, ctrl.fgHandler, apiServer) + if err != nil { + klog.Warningf("Failed to generate kubelet config for new pool %s: %v", pool.Name, err) + return + } + + // Create compressible MC for this pool + if err := ctrl.createCompressibleMachineConfigIfNeeded(pool.Name, kubeletContents); err != nil { + klog.Warningf("Failed to create compressible machine config for new pool %s: %v", pool.Name, err) + } +} + func kubeletConfigTriggerObjectChange(old, newKubeletConfig *mcfgv1.KubeletConfig) bool { if old.DeletionTimestamp != newKubeletConfig.DeletionTimestamp { return true From 23fcfd311eb4cdef7b88b46a57fbe48060637af5 Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Tue, 18 Nov 2025 17:44:55 +0530 Subject: [PATCH 04/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- pkg/controller/bootstrap/bootstrap.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/controller/bootstrap/bootstrap.go b/pkg/controller/bootstrap/bootstrap.go index 40cce1b64c..e173d53d1f 100644 --- a/pkg/controller/bootstrap/bootstrap.go +++ b/pkg/controller/bootstrap/bootstrap.go @@ -208,13 +208,6 @@ func (b *Bootstrap) Run(destDir string) error { } klog.Infof("Successfully generated MachineConfigs from feature gates.") - compressibleConfigs, err := kubeletconfig.RunCompressibleBootstrap(pools, cconfig, b.templatesDir, apiServer, fgHandler) - if err != nil { - return err - } - configs = append(configs, compressibleConfigs...) - klog.Infof("Successfully generated MachineConfigs for compressible kubelet configs.") - if nodeConfig == nil { nodeConfig = &apicfgv1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -241,6 +234,13 @@ func (b *Bootstrap) Run(destDir string) error { } klog.Infof("Successfully generated MachineConfigs from kubelet configs.") + compressibleConfigs, err := kubeletconfig.RunCompressibleBootstrap(pools, cconfig, b.templatesDir, apiServer, fgHandler) + if err != nil { + return err + } + configs = append(configs, compressibleConfigs...) + klog.Infof("Successfully generated MachineConfigs for compressible kubelet configs.") + fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig) if err != nil { return err From 3d62d418b82f2c2f74a16a2525d45e174f4a9d6d Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Tue, 18 Nov 2025 19:39:25 +0530 Subject: [PATCH 05/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- .../kubelet-config/kubelet_config_compressible.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go index c2c4e2a285..b396f0b968 100644 --- a/pkg/controller/kubelet-config/kubelet_config_compressible.go +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -29,10 +29,11 @@ const ( func (ctrl *Controller) ensureCompressibleMachineConfigs() error { // Wait to create compressible configs if the controller config is not completed if err := apihelpers.IsControllerConfigCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { - klog.V(4).Infof("ControllerConfig not ready, skipping compressible machine config creation: %v", err) - return nil + klog.V(1).Infof("ControllerConfig not ready, skipping compressible machine config creation: %v", err) + return err } + klog.V(1).Infof("ensureCompressibleMachineConfigs - ngopalak - 1") // Get all pools pools, err := ctrl.mcpLister.List(labels.Everything()) if err != nil { @@ -59,7 +60,7 @@ func (ctrl *Controller) ensureCompressibleMachineConfigs() error { klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err) continue } - + klog.V(1).Infof("ensureCompressibleMachineConfigs - ngopalak - 2") // Create compressible MC if err := ctrl.createCompressibleMachineConfigIfNeeded(pool.Name, kubeletContents); err != nil { klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err) @@ -68,6 +69,7 @@ func (ctrl *Controller) ensureCompressibleMachineConfigs() error { } } + klog.V(1).Infof("ensureCompressibleMachineConfigs - ngopalak - 3") return nil } From 938fe397f4248193f521ecf9f38c1711d1bdced3 Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Tue, 18 Nov 2025 21:25:57 +0530 Subject: [PATCH 06/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- pkg/apihelpers/apihelpers.go | 18 ++++++++++++++++++ .../kubelet_config_compressible.go | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/apihelpers/apihelpers.go b/pkg/apihelpers/apihelpers.go index bb8ea94de9..8dd18683e1 100644 --- a/pkg/apihelpers/apihelpers.go +++ b/pkg/apihelpers/apihelpers.go @@ -325,6 +325,24 @@ func IsControllerConfigCompleted(ccName string, ccGetter func(string) (*mcfgv1.C return fmt.Errorf("ControllerConfig has not completed: completed(%v) running(%v) failing(%v)", completed, running, failing) } +func IsControllerConfigRunning(ccName string, ccGetter func(string) (*mcfgv1.ControllerConfig, error)) error { + cur, err := ccGetter(ccName) + if err != nil { + return err + } + + if cur.Generation != cur.Status.ObservedGeneration { + return fmt.Errorf("status for ControllerConfig %s is being reported for %d, expecting it for %d", ccName, cur.Status.ObservedGeneration, cur.Generation) + } + + running := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerRunning) + failing := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerFailing) + if !running { + return fmt.Errorf("ControllerConfig has not running: running(%v) failing(%v)", running, failing) + } + return nil +} + // AreMCGeneratingSubControllersCompleted checks whether all MC producing sub-controllers are completed func AreMCGeneratingSubControllersCompletedForPool(crcLister func(labels.Selector) ([]*mcfgv1.ContainerRuntimeConfig, error), mckLister func(labels.Selector) ([]*mcfgv1.KubeletConfig, error), poolLabels map[string]string) error { diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go index b396f0b968..0dd12c0948 100644 --- a/pkg/controller/kubelet-config/kubelet_config_compressible.go +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -28,8 +28,8 @@ const ( // This is called at controller startup to create compressible MCs for all pools func (ctrl *Controller) ensureCompressibleMachineConfigs() error { // Wait to create compressible configs if the controller config is not completed - if err := apihelpers.IsControllerConfigCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { - klog.V(1).Infof("ControllerConfig not ready, skipping compressible machine config creation: %v", err) + if err := apihelpers.IsControllerConfigRunning(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + klog.V(1).Infof("ControllerConfig not running, skipping compressible machine config creation: %v", err) return err } From b5e7665888afbe62df885b0741ccc549897f8182 Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Wed, 19 Nov 2025 09:02:28 +0530 Subject: [PATCH 07/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- pkg/apihelpers/apihelpers.go | 10 ++- .../kubelet_config_compressible.go | 75 ++++++++--------- .../kubelet_config_compressible_test.go | 82 +++++++++++++++++++ .../kubelet_config_controller.go | 70 ---------------- 4 files changed, 126 insertions(+), 111 deletions(-) create mode 100644 pkg/controller/kubelet-config/kubelet_config_compressible_test.go diff --git a/pkg/apihelpers/apihelpers.go b/pkg/apihelpers/apihelpers.go index 8dd18683e1..5af630399a 100644 --- a/pkg/apihelpers/apihelpers.go +++ b/pkg/apihelpers/apihelpers.go @@ -325,6 +325,8 @@ func IsControllerConfigCompleted(ccName string, ccGetter func(string) (*mcfgv1.C return fmt.Errorf("ControllerConfig has not completed: completed(%v) running(%v) failing(%v)", completed, running, failing) } +// IsControllerConfigRunning checks whether a ControllerConfig is running. While the bootstrap is not +// finished, the ControllerConfig is considered running func IsControllerConfigRunning(ccName string, ccGetter func(string) (*mcfgv1.ControllerConfig, error)) error { cur, err := ccGetter(ccName) if err != nil { @@ -337,9 +339,15 @@ func IsControllerConfigRunning(ccName string, ccGetter func(string) (*mcfgv1.Con running := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerRunning) failing := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerFailing) + + if failing { + return fmt.Errorf("ControllerConfig %s is failing: running(%v) failing(%v)", ccName, running, failing) + } + if !running { - return fmt.Errorf("ControllerConfig has not running: running(%v) failing(%v)", running, failing) + return fmt.Errorf("ControllerConfig %s not running: running(%v) failing(%v)", ccName, running, failing) } + return nil } diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go index 0dd12c0948..8e7e37f78f 100644 --- a/pkg/controller/kubelet-config/kubelet_config_compressible.go +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -27,41 +27,38 @@ const ( // ensureCompressibleMachineConfigs ensures compressible machine configs exist for all pools // This is called at controller startup to create compressible MCs for all pools func (ctrl *Controller) ensureCompressibleMachineConfigs() error { - // Wait to create compressible configs if the controller config is not completed if err := apihelpers.IsControllerConfigRunning(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + // If the ControllerConfig is not running, we will encounter an error when generating the + // kubeletconfig object. klog.V(1).Infof("ControllerConfig not running, skipping compressible machine config creation: %v", err) return err } - klog.V(1).Infof("ensureCompressibleMachineConfigs - ngopalak - 1") - // Get all pools pools, err := ctrl.mcpLister.List(labels.Everything()) if err != nil { return fmt.Errorf("could not list machine config pools: %w", err) } - // Get ControllerConfig cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) if err != nil { return fmt.Errorf("could not get ControllerConfig: %w", err) } - // Get APIServer apiServer, err := ctrl.apiserverLister.Get(ctrlcommon.APIServerInstanceName) if err != nil && !errors.IsNotFound(err) { return fmt.Errorf("could not get APIServer: %w", err) } - // Create compressible MC for each pool for _, pool := range pools { // Generate the original kubelet config for this pool + // Kubelet config in templates/master/01-master-kubelet/_base/files/kubelet.yaml has parameeters which need + // to be resolved _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, pool.Name, ctrl.fgHandler, apiServer) if err != nil { klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err) continue } - klog.V(1).Infof("ensureCompressibleMachineConfigs - ngopalak - 2") - // Create compressible MC + if err := ctrl.createCompressibleMachineConfigIfNeeded(pool.Name, kubeletContents); err != nil { klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err) // Don't fail startup if compressible MC creation fails for a pool @@ -69,7 +66,6 @@ func (ctrl *Controller) ensureCompressibleMachineConfigs() error { } } - klog.V(1).Infof("ensureCompressibleMachineConfigs - ngopalak - 3") return nil } @@ -103,47 +99,21 @@ func (ctrl *Controller) createCompressibleMachineConfigIfNeeded(poolName string, return nil } -// RunCompressibleBootstrap generates compressible machine configs for all pools during bootstrap -func RunCompressibleBootstrap(pools []*mcfgv1.MachineConfigPool, cconfig *mcfgv1.ControllerConfig, templatesDir string, apiServer *configv1.APIServer, fgHandler ctrlcommon.FeatureGatesHandler) ([]*mcfgv1.MachineConfig, error) { - configs := []*mcfgv1.MachineConfig{} - - for _, pool := range pools { - // Generate the original kubelet config for this pool - _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templatesDir, pool.Name, fgHandler, apiServer) - if err != nil { - klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err) - continue - } - - // Create compressible MC - compressibleMC, err := newCompressibleMachineConfig(pool.Name, kubeletContents) - if err != nil { - klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err) - continue - } - - configs = append(configs, compressibleMC) - } - - return configs, nil -} - // newCompressibleMachineConfig creates a new machine config for compressible kubelet override // from the provided kubelet config contents func newCompressibleMachineConfig(poolName string, kubeletContents []byte) (*mcfgv1.MachineConfig, error) { compressibleMCName := fmt.Sprintf(CompressibleMachineConfigNamePrefix, poolName) - ignConfig := ctrlcommon.NewIgnConfig() - compressibleMC, err := ctrlcommon.MachineConfigFromIgnConfig(poolName, compressibleMCName, ignConfig) - if err != nil { - return nil, fmt.Errorf("could not create machine config from ignition config: %w", err) - } rawCompressibleIgn, err := createCompressibleKubeletIgnConfig(kubeletContents) if err != nil { return nil, fmt.Errorf("could not create compressible kubelet ignition config: %w", err) } - compressibleMC.Spec.Config.Raw = rawCompressibleIgn + compressibleMC, err := ctrlcommon.MachineConfigFromRawIgnConfig(poolName, compressibleMCName, rawCompressibleIgn) + if err != nil { + return nil, fmt.Errorf("could not create machine config from ignition config: %w", err) + } + compressibleMC.ObjectMeta.Annotations = map[string]string{ "openshift-patch-reference": "machineConfig-to-override-kubelet-conf-for-compressible-resources", } @@ -166,3 +136,28 @@ func createCompressibleKubeletIgnConfig(kubeletContents []byte) ([]byte, error) return rawCompressibleIgn, nil } + +// RunCompressibleBootstrap generates compressible machine configs for all pools during bootstrap +func RunCompressibleBootstrap(pools []*mcfgv1.MachineConfigPool, cconfig *mcfgv1.ControllerConfig, templatesDir string, apiServer *configv1.APIServer, fgHandler ctrlcommon.FeatureGatesHandler) ([]*mcfgv1.MachineConfig, error) { + configs := []*mcfgv1.MachineConfig{} + + for _, pool := range pools { + // Generate the original kubelet config for this pool + _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templatesDir, pool.Name, fgHandler, apiServer) + if err != nil { + klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err) + continue + } + + // Create compressible MC + compressibleMC, err := newCompressibleMachineConfig(pool.Name, kubeletContents) + if err != nil { + klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err) + continue + } + + configs = append(configs, compressibleMC) + } + + return configs, nil +} diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible_test.go b/pkg/controller/kubelet-config/kubelet_config_compressible_test.go new file mode 100644 index 0000000000..c8420f3e77 --- /dev/null +++ b/pkg/controller/kubelet-config/kubelet_config_compressible_test.go @@ -0,0 +1,82 @@ +package kubeletconfig + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + configv1 "github.com/openshift/api/config/v1" + osev1 "github.com/openshift/api/config/v1" + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" + "github.com/openshift/machine-config-operator/test/helpers" +) + +func TestNewCompressibleMachineConfig(t *testing.T) { + kubeletContents := []byte("apiVersion: kubelet.config.k8s.io/v1beta1\nkind: KubeletConfiguration") + + mc, err := newCompressibleMachineConfig("worker", kubeletContents) + + require.NoError(t, err) + assert.Equal(t, "50-worker-compressible-kubelet-override", mc.Name) + assert.Contains(t, mc.ObjectMeta.Annotations, "openshift-patch-reference") + + ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw) + require.NoError(t, err) + require.Len(t, ignCfg.Storage.Files, 1) + assert.Equal(t, KubeletConfPath, ignCfg.Storage.Files[0].Path) + + contents, err := ctrlcommon.DecodeIgnitionFileContents(ignCfg.Storage.Files[0].Contents.Source, ignCfg.Storage.Files[0].Contents.Compression) + require.NoError(t, err) + assert.Equal(t, kubeletContents, contents) +} + +func TestRunCompressibleBootstrap(t *testing.T) { + cc := newControllerConfig(ctrlcommon.ControllerConfigName, configv1.AWSPlatformType) + pools := []*mcfgv1.MachineConfigPool{ + helpers.NewMachineConfigPool("master", nil, helpers.MasterSelector, "v0"), + helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v0"), + } + fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{}, nil) + + mcs, err := RunCompressibleBootstrap(pools, cc, "../../../templates", nil, fgHandler) + + require.NoError(t, err) + require.Len(t, mcs, 2) + + assert.Equal(t, "50-master-compressible-kubelet-override", mcs[0].Name) + assert.Equal(t, "50-worker-compressible-kubelet-override", mcs[1].Name) + + for _, mc := range mcs { + ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw) + require.NoError(t, err) + require.Len(t, ignCfg.Storage.Files, 1) + assert.Equal(t, KubeletConfPath, ignCfg.Storage.Files[0].Path) + } +} + +// This test ensures that the template replacement works as expected +func TestCompressibleMachineConfigTemplateReplacement(t *testing.T) { + cc := newControllerConfig(ctrlcommon.ControllerConfigName, configv1.AWSPlatformType) + cc.Spec.ClusterDNSIP = "10.96.0.10" + + fgHandler := ctrlcommon.NewFeatureGatesHardcodedHandler([]osev1.FeatureGateName{}, nil) + + _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, "../../../templates", "master", fgHandler, nil) + require.NoError(t, err) + + mc, err := newCompressibleMachineConfig("master", kubeletContents) + require.NoError(t, err) + + ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw) + require.NoError(t, err) + require.Len(t, ignCfg.Storage.Files, 1) + + contents, err := ctrlcommon.DecodeIgnitionFileContents(ignCfg.Storage.Files[0].Contents.Source, ignCfg.Storage.Files[0].Contents.Compression) + require.NoError(t, err) + + contentsStr := string(contents) + assert.Contains(t, contentsStr, "10.96.0.10") + assert.NotContains(t, contentsStr, "{{.ClusterDNSIP}}") +} diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index d48d4d49ac..e5b8d1f66e 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -144,15 +144,6 @@ func New( DeleteFunc: ctrl.deleteKubeletConfig, }) - ccInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: ctrl.addControllerConfig, - UpdateFunc: ctrl.updateControllerConfig, - }) - - mcpInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: ctrl.addMachineConfigPool, - }) - featInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: ctrl.addFeature, UpdateFunc: ctrl.updateFeature, @@ -284,67 +275,6 @@ func (ctrl *Controller) deleteAPIServer(obj interface{}) { ctrl.filterAPIServer(apiServer) } -func (ctrl *Controller) addControllerConfig(obj interface{}) { - cc := obj.(*mcfgv1.ControllerConfig) - klog.V(4).Infof("ControllerConfig %s added", cc.Name) - // Try to ensure compressible MCs are created when ControllerConfig becomes available - if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { - klog.Warningf("Error ensuring compressible MachineConfigs after ControllerConfig add: %v", err) - } -} - -func (ctrl *Controller) updateControllerConfig(old, cur interface{}) { - oldCC := old.(*mcfgv1.ControllerConfig) - newCC := cur.(*mcfgv1.ControllerConfig) - - // Check if ControllerConfig just became completed - if oldCC.Status.ObservedGeneration != oldCC.Generation && - newCC.Status.ObservedGeneration == newCC.Generation { - klog.V(4).Infof("ControllerConfig %s became ready", newCC.Name) - // Try to ensure compressible MCs are created when ControllerConfig becomes ready - if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { - klog.Warningf("Error ensuring compressible MachineConfigs after ControllerConfig update: %v", err) - } - } -} - -func (ctrl *Controller) addMachineConfigPool(obj interface{}) { - pool := obj.(*mcfgv1.MachineConfigPool) - klog.V(4).Infof("MachineConfigPool %s added, ensuring compressible MachineConfig exists", pool.Name) - - // Check if ControllerConfig is ready - if err := apihelpers.IsControllerConfigCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { - klog.V(4).Infof("ControllerConfig not ready, will create compressible MC for pool %s later", pool.Name) - return - } - - // Get ControllerConfig - cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) - if err != nil { - klog.Warningf("Could not get ControllerConfig for new pool %s: %v", pool.Name, err) - return - } - - // Get APIServer - apiServer, err := ctrl.apiserverLister.Get(ctrlcommon.APIServerInstanceName) - if err != nil && !macherrors.IsNotFound(err) { - klog.Warningf("Could not get APIServer for new pool %s: %v", pool.Name, err) - return - } - - // Generate kubelet config for this pool - _, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, pool.Name, ctrl.fgHandler, apiServer) - if err != nil { - klog.Warningf("Failed to generate kubelet config for new pool %s: %v", pool.Name, err) - return - } - - // Create compressible MC for this pool - if err := ctrl.createCompressibleMachineConfigIfNeeded(pool.Name, kubeletContents); err != nil { - klog.Warningf("Failed to create compressible machine config for new pool %s: %v", pool.Name, err) - } -} - func kubeletConfigTriggerObjectChange(old, newKubeletConfig *mcfgv1.KubeletConfig) bool { if old.DeletionTimestamp != newKubeletConfig.DeletionTimestamp { return true From 79d4ae6d0c168c0d995accb6c70464f05d2f55c1 Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Wed, 19 Nov 2025 10:54:43 +0530 Subject: [PATCH 08/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- .../kubelet_config_controller.go | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index e5b8d1f66e..869be85e1f 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -186,6 +186,45 @@ func New( return ctrl } +// waitForTemplateGeneration waits for the ControllerConfig to be reconciled by the template controller +func (ctrl *Controller) waitForTemplateGeneration(stopCh <-chan struct{}) error { + klog.Info("Waiting for ControllerConfig generation to be reconciled...") + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + return wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(_ context.Context) (bool, error) { + select { + case <-stopCh: + return false, fmt.Errorf("controller stopped while waiting for ControllerConfig reconciliation") + default: + } + + cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) + if err != nil { + klog.V(4).Infof("Error getting ControllerConfig: %v", err) + return false, nil + } + + if cc.Generation != cc.Status.ObservedGeneration { + klog.V(4).Infof("Waiting for ControllerConfig reconciliation: Generation=%d, ObservedGeneration=%d", + cc.Generation, cc.Status.ObservedGeneration) + return false, nil + } + + running := apihelpers.IsControllerConfigStatusConditionTrue(cc.Status.Conditions, mcfgv1.TemplateControllerRunning) + completed := apihelpers.IsControllerConfigStatusConditionTrue(cc.Status.Conditions, mcfgv1.TemplateControllerCompleted) + + if running || completed { + klog.Info("ControllerConfig generation reconciled") + return true, nil + } + + klog.V(4).Infof("Waiting for ControllerConfig reconciliation: running=%v, completed=%v", running, completed) + return false, nil + }) +} + // Run executes the kubelet config controller. func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() @@ -200,9 +239,14 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { klog.Info("Starting MachineConfigController-KubeletConfigController") defer klog.Info("Shutting down MachineConfigController-KubeletConfigController") - // Ensure compressible machine configs are created for all pools at startup - if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { - klog.Warningf("Error ensuring compressible MachineConfigs: %v", err) + // Wait for ControllerConfig generation to be reconciled before creating compressible machine configs + if err := ctrl.waitForTemplateGeneration(stopCh); err != nil { + klog.Warningf("Failed to wait for ControllerConfig generation reconciliation: %v", err) + } else { + // Ensure compressible machine configs are created for all pools at startup + if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { + klog.Warningf("Error ensuring compressible MachineConfigs: %v", err) + } } for i := 0; i < workers; i++ { From a40f8f92f56f658960a7647c279224aedf961def Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Wed, 19 Nov 2025 12:13:01 +0530 Subject: [PATCH 09/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- .../kubelet_config_controller.go | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 869be85e1f..3d2c95d612 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -200,28 +200,13 @@ func (ctrl *Controller) waitForTemplateGeneration(stopCh <-chan struct{}) error default: } - cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) - if err != nil { - klog.V(4).Infof("Error getting ControllerConfig: %v", err) - return false, nil - } - - if cc.Generation != cc.Status.ObservedGeneration { - klog.V(4).Infof("Waiting for ControllerConfig reconciliation: Generation=%d, ObservedGeneration=%d", - cc.Generation, cc.Status.ObservedGeneration) + if err := apihelpers.IsControllerConfigRunning(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + // If the ControllerConfig is not running, we will encounter an error when generating the + // kubeletconfig object. + klog.V(1).Infof("ControllerConfig not running: %v", err) return false, nil } - - running := apihelpers.IsControllerConfigStatusConditionTrue(cc.Status.Conditions, mcfgv1.TemplateControllerRunning) - completed := apihelpers.IsControllerConfigStatusConditionTrue(cc.Status.Conditions, mcfgv1.TemplateControllerCompleted) - - if running || completed { - klog.Info("ControllerConfig generation reconciled") - return true, nil - } - - klog.V(4).Infof("Waiting for ControllerConfig reconciliation: running=%v, completed=%v", running, completed) - return false, nil + return true, nil }) } From 454600b59e8b6054634121a50f6d541e38f70c7b Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Wed, 19 Nov 2025 13:04:01 +0530 Subject: [PATCH 10/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- pkg/apihelpers/apihelpers.go | 13 +++++++------ .../kubelet-config/kubelet_config_compressible.go | 2 +- .../kubelet-config/kubelet_config_controller.go | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/apihelpers/apihelpers.go b/pkg/apihelpers/apihelpers.go index 5af630399a..e36700adec 100644 --- a/pkg/apihelpers/apihelpers.go +++ b/pkg/apihelpers/apihelpers.go @@ -325,9 +325,9 @@ func IsControllerConfigCompleted(ccName string, ccGetter func(string) (*mcfgv1.C return fmt.Errorf("ControllerConfig has not completed: completed(%v) running(%v) failing(%v)", completed, running, failing) } -// IsControllerConfigRunning checks whether a ControllerConfig is running. While the bootstrap is not +// IsControllerConfigRunningOrCompleted checks whether a ControllerConfig is running. While the bootstrap is not // finished, the ControllerConfig is considered running -func IsControllerConfigRunning(ccName string, ccGetter func(string) (*mcfgv1.ControllerConfig, error)) error { +func IsControllerConfigRunningOrCompleted(ccName string, ccGetter func(string) (*mcfgv1.ControllerConfig, error)) error { cur, err := ccGetter(ccName) if err != nil { return err @@ -337,15 +337,16 @@ func IsControllerConfigRunning(ccName string, ccGetter func(string) (*mcfgv1.Con return fmt.Errorf("status for ControllerConfig %s is being reported for %d, expecting it for %d", ccName, cur.Status.ObservedGeneration, cur.Generation) } - running := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerRunning) failing := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerFailing) if failing { - return fmt.Errorf("ControllerConfig %s is failing: running(%v) failing(%v)", ccName, running, failing) + return fmt.Errorf("ControllerConfig %s is failing: failing(%v)", ccName, failing) } - if !running { - return fmt.Errorf("ControllerConfig %s not running: running(%v) failing(%v)", ccName, running, failing) + running := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerRunning) + completed := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerCompleted) + if !completed && !running { + return fmt.Errorf("ControllerConfig %s not running: completed (%v) running(%v) failing(%v)", ccName, completed, running, failing) } return nil diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go index 8e7e37f78f..dc50fc98c2 100644 --- a/pkg/controller/kubelet-config/kubelet_config_compressible.go +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -27,7 +27,7 @@ const ( // ensureCompressibleMachineConfigs ensures compressible machine configs exist for all pools // This is called at controller startup to create compressible MCs for all pools func (ctrl *Controller) ensureCompressibleMachineConfigs() error { - if err := apihelpers.IsControllerConfigRunning(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + if err := apihelpers.IsControllerConfigRunningOrCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { // If the ControllerConfig is not running, we will encounter an error when generating the // kubeletconfig object. klog.V(1).Infof("ControllerConfig not running, skipping compressible machine config creation: %v", err) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 3d2c95d612..6a81928c1c 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -200,7 +200,7 @@ func (ctrl *Controller) waitForTemplateGeneration(stopCh <-chan struct{}) error default: } - if err := apihelpers.IsControllerConfigRunning(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + if err := apihelpers.IsControllerConfigRunningOrCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { // If the ControllerConfig is not running, we will encounter an error when generating the // kubeletconfig object. klog.V(1).Infof("ControllerConfig not running: %v", err) From 7c5e77db70acf39b21bdd128788b85cb78873810 Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Wed, 19 Nov 2025 14:36:10 +0530 Subject: [PATCH 11/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- pkg/apihelpers/apihelpers.go | 5 ++--- .../kubelet-config/kubelet_config_compressible.go | 2 +- .../kubelet-config/kubelet_config_controller.go | 10 +++++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/apihelpers/apihelpers.go b/pkg/apihelpers/apihelpers.go index e36700adec..341dfce9ca 100644 --- a/pkg/apihelpers/apihelpers.go +++ b/pkg/apihelpers/apihelpers.go @@ -325,8 +325,7 @@ func IsControllerConfigCompleted(ccName string, ccGetter func(string) (*mcfgv1.C return fmt.Errorf("ControllerConfig has not completed: completed(%v) running(%v) failing(%v)", completed, running, failing) } -// IsControllerConfigRunningOrCompleted checks whether a ControllerConfig is running. While the bootstrap is not -// finished, the ControllerConfig is considered running +// IsControllerConfigRunningOrCompleted checks whether a ControllerConfig is completed or running func IsControllerConfigRunningOrCompleted(ccName string, ccGetter func(string) (*mcfgv1.ControllerConfig, error)) error { cur, err := ccGetter(ccName) if err != nil { @@ -346,7 +345,7 @@ func IsControllerConfigRunningOrCompleted(ccName string, ccGetter func(string) ( running := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerRunning) completed := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerCompleted) if !completed && !running { - return fmt.Errorf("ControllerConfig %s not running: completed (%v) running(%v) failing(%v)", ccName, completed, running, failing) + return fmt.Errorf("ControllerConfig %s not running or completed: completed (%v) running(%v) failing(%v)", ccName, completed, running, failing) } return nil diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go index dc50fc98c2..0ebeb975c9 100644 --- a/pkg/controller/kubelet-config/kubelet_config_compressible.go +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -30,7 +30,7 @@ func (ctrl *Controller) ensureCompressibleMachineConfigs() error { if err := apihelpers.IsControllerConfigRunningOrCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { // If the ControllerConfig is not running, we will encounter an error when generating the // kubeletconfig object. - klog.V(1).Infof("ControllerConfig not running, skipping compressible machine config creation: %v", err) + klog.V(1).Infof("ControllerConfig not running or completed: %v", err) return err } diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 6a81928c1c..df3a03f426 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -186,14 +186,14 @@ func New( return ctrl } -// waitForTemplateGeneration waits for the ControllerConfig to be reconciled by the template controller -func (ctrl *Controller) waitForTemplateGeneration(stopCh <-chan struct{}) error { +// waitForControllerConfig waits for the ControllerConfig to be reconciled by the template controller +func (ctrl *Controller) waitForControllerConfig(stopCh <-chan struct{}) error { klog.Info("Waiting for ControllerConfig generation to be reconciled...") - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - return wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(_ context.Context) (bool, error) { + return wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(_ context.Context) (bool, error) { select { case <-stopCh: return false, fmt.Errorf("controller stopped while waiting for ControllerConfig reconciliation") @@ -225,7 +225,7 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { defer klog.Info("Shutting down MachineConfigController-KubeletConfigController") // Wait for ControllerConfig generation to be reconciled before creating compressible machine configs - if err := ctrl.waitForTemplateGeneration(stopCh); err != nil { + if err := ctrl.waitForControllerConfig(stopCh); err != nil { klog.Warningf("Failed to wait for ControllerConfig generation reconciliation: %v", err) } else { // Ensure compressible machine configs are created for all pools at startup From b6094ff37c68c8167784754aed3e71ba40489dff Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Wed, 19 Nov 2025 14:41:44 +0530 Subject: [PATCH 12/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- .../kubelet_config_controller.go | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index df3a03f426..6e33f9e9fa 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -186,30 +186,6 @@ func New( return ctrl } -// waitForControllerConfig waits for the ControllerConfig to be reconciled by the template controller -func (ctrl *Controller) waitForControllerConfig(stopCh <-chan struct{}) error { - klog.Info("Waiting for ControllerConfig generation to be reconciled...") - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - return wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(_ context.Context) (bool, error) { - select { - case <-stopCh: - return false, fmt.Errorf("controller stopped while waiting for ControllerConfig reconciliation") - default: - } - - if err := apihelpers.IsControllerConfigRunningOrCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { - // If the ControllerConfig is not running, we will encounter an error when generating the - // kubeletconfig object. - klog.V(1).Infof("ControllerConfig not running: %v", err) - return false, nil - } - return true, nil - }) -} - // Run executes the kubelet config controller. func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() @@ -248,6 +224,31 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { <-stopCh } + +// waitForControllerConfig waits for the ControllerConfig to be reconciled by the template controller +func (ctrl *Controller) waitForControllerConfig(stopCh <-chan struct{}) error { + klog.Info("Waiting for ControllerConfig generation to be reconciled...") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + return wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(_ context.Context) (bool, error) { + select { + case <-stopCh: + return false, fmt.Errorf("controller stopped while waiting for ControllerConfig reconciliation") + default: + } + + if err := apihelpers.IsControllerConfigRunningOrCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + // If the ControllerConfig is not running, we will encounter an error when generating the + // kubeletconfig object. + klog.V(1).Infof("ControllerConfig not running: %v", err) + return false, nil + } + return true, nil + }) +} + func (ctrl *Controller) filterAPIServer(apiServer *configv1.APIServer) { if apiServer.Name != "cluster" { return From ea7a0338e568eaf6d8464cb40421c20a844e0baa Mon Sep 17 00:00:00 2001 From: Neeraj Krishna Gopalakrishna Date: Wed, 19 Nov 2025 16:42:34 +0530 Subject: [PATCH 13/13] OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade --- pkg/apihelpers/apihelpers.go | 26 ------------------- .../kubelet_config_compressible.go | 8 ------ .../kubelet_config_controller.go | 20 +++++++------- 3 files changed, 11 insertions(+), 43 deletions(-) diff --git a/pkg/apihelpers/apihelpers.go b/pkg/apihelpers/apihelpers.go index 341dfce9ca..bb8ea94de9 100644 --- a/pkg/apihelpers/apihelpers.go +++ b/pkg/apihelpers/apihelpers.go @@ -325,32 +325,6 @@ func IsControllerConfigCompleted(ccName string, ccGetter func(string) (*mcfgv1.C return fmt.Errorf("ControllerConfig has not completed: completed(%v) running(%v) failing(%v)", completed, running, failing) } -// IsControllerConfigRunningOrCompleted checks whether a ControllerConfig is completed or running -func IsControllerConfigRunningOrCompleted(ccName string, ccGetter func(string) (*mcfgv1.ControllerConfig, error)) error { - cur, err := ccGetter(ccName) - if err != nil { - return err - } - - if cur.Generation != cur.Status.ObservedGeneration { - return fmt.Errorf("status for ControllerConfig %s is being reported for %d, expecting it for %d", ccName, cur.Status.ObservedGeneration, cur.Generation) - } - - failing := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerFailing) - - if failing { - return fmt.Errorf("ControllerConfig %s is failing: failing(%v)", ccName, failing) - } - - running := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerRunning) - completed := IsControllerConfigStatusConditionTrue(cur.Status.Conditions, mcfgv1.TemplateControllerCompleted) - if !completed && !running { - return fmt.Errorf("ControllerConfig %s not running or completed: completed (%v) running(%v) failing(%v)", ccName, completed, running, failing) - } - - return nil -} - // AreMCGeneratingSubControllersCompleted checks whether all MC producing sub-controllers are completed func AreMCGeneratingSubControllersCompletedForPool(crcLister func(labels.Selector) ([]*mcfgv1.ContainerRuntimeConfig, error), mckLister func(labels.Selector) ([]*mcfgv1.KubeletConfig, error), poolLabels map[string]string) error { diff --git a/pkg/controller/kubelet-config/kubelet_config_compressible.go b/pkg/controller/kubelet-config/kubelet_config_compressible.go index 0ebeb975c9..7178a308be 100644 --- a/pkg/controller/kubelet-config/kubelet_config_compressible.go +++ b/pkg/controller/kubelet-config/kubelet_config_compressible.go @@ -7,7 +7,6 @@ import ( "github.com/clarketm/json" configv1 "github.com/openshift/api/config/v1" mcfgv1 "github.com/openshift/api/machineconfiguration/v1" - "github.com/openshift/machine-config-operator/pkg/apihelpers" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,13 +26,6 @@ const ( // ensureCompressibleMachineConfigs ensures compressible machine configs exist for all pools // This is called at controller startup to create compressible MCs for all pools func (ctrl *Controller) ensureCompressibleMachineConfigs() error { - if err := apihelpers.IsControllerConfigRunningOrCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { - // If the ControllerConfig is not running, we will encounter an error when generating the - // kubeletconfig object. - klog.V(1).Infof("ControllerConfig not running or completed: %v", err) - return err - } - pools, err := ctrl.mcpLister.List(labels.Everything()) if err != nil { return fmt.Errorf("could not list machine config pools: %w", err) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 6e33f9e9fa..68955efc4c 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -201,14 +201,16 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) { defer klog.Info("Shutting down MachineConfigController-KubeletConfigController") // Wait for ControllerConfig generation to be reconciled before creating compressible machine configs - if err := ctrl.waitForControllerConfig(stopCh); err != nil { - klog.Warningf("Failed to wait for ControllerConfig generation reconciliation: %v", err) - } else { - // Ensure compressible machine configs are created for all pools at startup - if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { - klog.Warningf("Error ensuring compressible MachineConfigs: %v", err) + go func() { + if err := ctrl.waitForControllerConfig(stopCh); err != nil { + klog.Warningf("Failed to wait for ControllerConfig generation reconciliation: %v", err) + } else { + // Ensure compressible machine configs are created for all pools at startup + if err := ctrl.ensureCompressibleMachineConfigs(); err != nil { + klog.Warningf("Error ensuring compressible MachineConfigs: %v", err) + } } - } + }() for i := 0; i < workers; i++ { go wait.Until(ctrl.worker, time.Second, stopCh) @@ -232,14 +234,14 @@ func (ctrl *Controller) waitForControllerConfig(stopCh <-chan struct{}) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - return wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(_ context.Context) (bool, error) { + return wait.PollUntilContextTimeout(ctx, 1*time.Second, 2*time.Minute, true, func(_ context.Context) (bool, error) { select { case <-stopCh: return false, fmt.Errorf("controller stopped while waiting for ControllerConfig reconciliation") default: } - if err := apihelpers.IsControllerConfigRunningOrCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { + if err := apihelpers.IsControllerConfigCompleted(ctrlcommon.ControllerConfigName, ctrl.ccLister.Get); err != nil { // If the ControllerConfig is not running, we will encounter an error when generating the // kubeletconfig object. klog.V(1).Infof("ControllerConfig not running: %v", err)