feat: VPA InPlace updateMode

vertical-pod-autoscaler/charts/vertical-pod-autoscaler/crds/vpa-v1-crd-gen.yaml

@@ -475,6 +475,7 @@ spec:
                     - Initial
                     - Recreate
                     - InPlaceOrRecreate
+                    - InPlace
                     - Auto
                     type: string
                 type: object

vertical-pod-autoscaler/deploy/vpa-v1-crd-gen.yaml

@@ -475,6 +475,7 @@ spec:
                     - Initial
                     - Recreate
                     - InPlaceOrRecreate
+                    - InPlace
                     - Auto
                     type: string
                 type: object

vertical-pod-autoscaler/docs/flags.md

@@ -14,7 +14,7 @@ This document is auto-generated from the flag definitions in the VPA admission-c
 | `address` | string |  ":8944" | The address to expose Prometheus metrics.  |
 | `alsologtostderr` |  |  | log to standard error as well as files (no effect when -logtostderr=true) |
 | `client-ca-file` | string |  "/etc/tls-certs/caCert.pem" | Path to CA PEM file.  |
-| `feature-gates` | mapStringBool |  | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:<br>AllAlpha=true\|false (ALPHA - default=false)<br>AllBeta=true\|false (BETA - default=false)<br>InPlaceOrRecreate=true\|false (BETA - default=true)<br>PerVPAConfig=true\|false (ALPHA - default=false) |
+| `feature-gates` | mapStringBool |  | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:<br>AllAlpha=true\|false (ALPHA - default=false)<br>AllBeta=true\|false (BETA - default=false)<br>InPlace=true\|false (ALPHA - default=false)<br>InPlaceOrRecreate=true\|false (BETA - default=true)<br>PerVPAConfig=true\|false (ALPHA - default=false) |
 | `ignored-vpa-object-namespaces` | string |  | A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector. |
 | `kube-api-burst` | float |  100 | QPS burst limit when making requests to Kubernetes apiserver  |
 | `kube-api-qps` | float |  50 | QPS limit when making requests to Kubernetes apiserver  |
@@ -68,7 +68,7 @@ This document is auto-generated from the flag definitions in the VPA recommender
 | `cpu-integer-post-processor-enabled` |  |  | Enable the cpu-integer recommendation post processor. The post processor will round up CPU recommendations to a whole CPU for pods which were opted in by setting an appropriate label on VPA object (experimental) |
 | `external-metrics-cpu-metric` | string |  | ALPHA.  Metric to use with external metrics provider for CPU usage. |
 | `external-metrics-memory-metric` | string |  | ALPHA.  Metric to use with external metrics provider for memory usage. |
-| `feature-gates` | mapStringBool |  | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:<br>AllAlpha=true\|false (ALPHA - default=false)<br>AllBeta=true\|false (BETA - default=false)<br>InPlaceOrRecreate=true\|false (BETA - default=true)<br>PerVPAConfig=true\|false (ALPHA - default=false) |
+| `feature-gates` | mapStringBool |  | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:<br>AllAlpha=true\|false (ALPHA - default=false)<br>AllBeta=true\|false (BETA - default=false)<br>InPlace=true\|false (ALPHA - default=false)<br>InPlaceOrRecreate=true\|false (BETA - default=true)<br>PerVPAConfig=true\|false (ALPHA - default=false) |
 | `history-length` | string |  "8d" | How much time back prometheus have to be queried to get historical metrics  |
 | `history-resolution` | string |  "1h" | Resolution at which Prometheus is queried for historical metrics  |
 | `humanize-memory` |  |  | DEPRECATED: Convert memory values in recommendations to the highest appropriate SI unit with up to 2 decimal places for better readability. This flag is deprecated and will be removed in a future version. Use --round-memory-bytes instead. |
@@ -144,7 +144,7 @@ This document is auto-generated from the flag definitions in the VPA updater cod
 | `eviction-rate-burst` | int |  1 | Burst of pods that can be evicted.  |
 | `eviction-rate-limit` | float |  | Number of pods that can be evicted per seconds. A rate limit set to 0 or -1 will disable<br>the rate limiter. (default -1) |
 | `eviction-tolerance` | float |  0.5 | Fraction of replica count that can be evicted for update, if more than one pod can be evicted.  |
-| `feature-gates` | mapStringBool |  | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:<br>AllAlpha=true\|false (ALPHA - default=false)<br>AllBeta=true\|false (BETA - default=false)<br>InPlaceOrRecreate=true\|false (BETA - default=true)<br>PerVPAConfig=true\|false (ALPHA - default=false) |
+| `feature-gates` | mapStringBool |  | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:<br>AllAlpha=true\|false (ALPHA - default=false)<br>AllBeta=true\|false (BETA - default=false)<br>InPlace=true\|false (ALPHA - default=false)<br>InPlaceOrRecreate=true\|false (BETA - default=true)<br>PerVPAConfig=true\|false (ALPHA - default=false) |
 | `ignored-vpa-object-namespaces` | string |  | A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector. |
 | `in-recommendation-bounds-eviction-lifetime-threshold` |  |  12h0m0s | duration   Pods that live for at least that long can be evicted even if their request is within the [MinRecommended...MaxRecommended] range  |
 | `kube-api-burst` | float |  100 | QPS burst limit when making requests to Kubernetes apiserver  |
@@ -174,4 +174,3 @@ This document is auto-generated from the flag definitions in the VPA updater cod
 | `v,` |  | : 4 | , --v Level                                                         set the log level verbosity  (default 4) |
 | `vmodule` | moduleSpec |  | comma-separated list of pattern=N settings for file-filtered logging |
 | `vpa-object-namespace` | string |  | Specifies the namespace to search for VPA objects. Leave empty to include all namespaces. If provided, the garbage collector will only clean this namespace. |
-

vertical-pod-autoscaler/e2e/utils/common.go

@@ -237,7 +237,7 @@ func NewNHamstersDeployment(f *framework.Framework, n int) *appsv1.Deployment {
 		DefaultHamsterReplicas,                     /*replicas*/
 		HamsterLabels,                              /*podLabels*/
 		GetHamsterContainerNameByIndex(0),          /*imageName*/
-		"registry.k8s.io/ubuntu-slim:0.14",         /*image*/
+		"ubuntu:latest",                            /*image*/
 		appsv1.RollingUpdateDeploymentStrategyType, /*strategyType*/
 	)
 	d.ObjectMeta.Namespace = f.Namespace.Name

vertical-pod-autoscaler/e2e/v1/updater.go

@@ -23,9 +23,11 @@ import (
 
 	autoscaling "k8s.io/api/autoscaling/v1"
 	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils"
 	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
+	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/status"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -63,7 +65,7 @@ var _ = UpdaterE2eDescribe("Updater", func() {
 		}()
 		statusUpdater.Run(stopCh)
 
-		podList := setupPodsForUpscalingEviction(f)
+		podList := setupPodsForUpscalingEviction(f, vpa_types.UpdateModeRecreate)
 
 		ginkgo.By("Waiting for pods to be evicted")
 		err := WaitForPodsEvicted(f, podList)
@@ -94,7 +96,7 @@ var _ = UpdaterE2eDescribe("Updater", func() {
 		}()
 		statusUpdater.Run(stopCh)
 
-		podList := setupPodsForDownscalingEviction(f, nil)
+		podList := setupPodsForDownscalingEviction(f, nil, vpa_types.UpdateModeRecreate)
 
 		ginkgo.By("Waiting for pods to be evicted")
 		err := WaitForPodsEvicted(f, podList)
@@ -129,14 +131,14 @@ var _ = UpdaterE2eDescribe("Updater", func() {
 				ChangeRequirement: vpa_types.TargetHigherThanRequests,
 			},
 		}
-		podList := setupPodsForDownscalingEviction(f, er)
+		podList := setupPodsForDownscalingEviction(f, er, vpa_types.UpdateModeRecreate)
 
 		ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", VpaEvictionTimeout.String()))
 		CheckNoPodsEvicted(f, MakePodSet(podList))
 	})
 	// FIXME todo(adrianmoisey): This test seems to be flaky after running in parallel, unsure why, see if it's possible to fix
 	framework.It("doesn't evict pods when Admission Controller status unavailable", framework.WithSerial(), func() {
-		podList := setupPodsForUpscalingEviction(f)
+		podList := setupPodsForUpscalingEviction(f, vpa_types.UpdateModeInPlaceOrRecreate)
 
 		ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", VpaEvictionTimeout.String()))
 		CheckNoPodsEvicted(f, MakePodSet(podList))
@@ -166,7 +168,7 @@ var _ = UpdaterE2eDescribe("Updater", func() {
 		}()
 		statusUpdater.Run(stopCh)
 
-		podList := setupPodsForUpscalingInPlace(f)
+		podList := setupPodsForUpscalingInPlace(f, vpa_types.UpdateModeInPlaceOrRecreate)
 		initialPods := podList.DeepCopy()
 
 		ginkgo.By("Waiting for pods to be in-place updated")
@@ -198,24 +200,165 @@ var _ = UpdaterE2eDescribe("Updater", func() {
 		}()
 		statusUpdater.Run(stopCh)
 
-		podList := setupPodsForDownscalingInPlace(f, nil)
+		podList := setupPodsForDownscalingInPlace(f, nil, vpa_types.UpdateModeInPlaceOrRecreate)
 		initialPods := podList.DeepCopy()
 
 		ginkgo.By("Waiting for pods to be in-place downscaled")
 		err := WaitForPodsUpdatedWithoutEviction(f, initialPods)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	})
+
+	// Sets up a lease object updated periodically to signal - requires WithSerial()
+	framework.It("In-place updates pods with InPlace mode when update succeeds", framework.WithSerial(), framework.WithFeatureGate(features.InPlace), func() {
+		const statusUpdateInterval = 10 * time.Second
+
+		ginkgo.By("Setting up the Admission Controller status")
+		stopCh := make(chan struct{})
+		statusUpdater := status.NewUpdater(
+			f.ClientSet,
+			status.AdmissionControllerStatusName,
+			utils.VpaNamespace,
+			statusUpdateInterval,
+			"e2e test",
+		)
+		defer func() {
+			// Schedule a cleanup of the Admission Controller status.
+			// Status is created outside the test namespace.
+			ginkgo.By("Deleting the Admission Controller status")
+			close(stopCh)
+			err := f.ClientSet.CoordinationV1().Leases(utils.VpaNamespace).
+				Delete(context.TODO(), status.AdmissionControllerStatusName, metav1.DeleteOptions{})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}()
+		statusUpdater.Run(stopCh)
+
+		podList := setupPodsForUpscalingInPlace(f, vpa_types.UpdateModeInPlace)
+		initialPods := podList.DeepCopy()
+
+		ginkgo.By("Waiting for pods to be in-place updated with InPlace mode")
+		err := WaitForPodsUpdatedWithoutEviction(f, initialPods)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	})
+	framework.It("does not evicts pods for downscaling with InPlace mode", framework.WithSerial(), func() {
+		const statusUpdateInterval = 10 * time.Second
+
+		ginkgo.By("Setting up the Admission Controller status")
+		stopCh := make(chan struct{})
+		statusUpdater := status.NewUpdater(
+			f.ClientSet,
+			status.AdmissionControllerStatusName,
+			utils.VpaNamespace,
+			statusUpdateInterval,
+			"e2e test",
+		)
+		defer func() {
+			// Schedule a cleanup of the Admission Controller status.
+			// Status is created outside the test namespace.
+			ginkgo.By("Deleting the Admission Controller status")
+			close(stopCh)
+			err := f.ClientSet.CoordinationV1().Leases(utils.VpaNamespace).
+				Delete(context.TODO(), status.AdmissionControllerStatusName, metav1.DeleteOptions{})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}()
+		statusUpdater.Run(stopCh)
+
+		podList := setupPodsForDownscalingEviction(f, nil, vpa_types.UpdateModeInPlace)
+
+		ginkgo.By(fmt.Sprintf("Waiting for pods to be evicted, hoping it won't happen, sleep for %s", VpaEvictionTimeout.String()))
+		CheckNoPodsEvicted(f, MakePodSet(podList))
+	})
+	framework.It("InPlace mode retries when recommendations change", framework.WithSerial(), framework.WithFeatureGate(features.InPlace), func() {
+		const statusUpdateInterval = 10 * time.Second
+
+		ginkgo.By("Setting up the Admission Controller status")
+		stopCh := make(chan struct{})
+		statusUpdater := status.NewUpdater(
+			f.ClientSet,
+			status.AdmissionControllerStatusName,
+			utils.VpaNamespace,
+			statusUpdateInterval,
+			"e2e test",
+		)
+		defer func() {
+			ginkgo.By("Deleting the Admission Controller status")
+			close(stopCh)
+			err := f.ClientSet.CoordinationV1().Leases(utils.VpaNamespace).
+				Delete(context.TODO(), status.AdmissionControllerStatusName, metav1.DeleteOptions{})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}()
+		statusUpdater.Run(stopCh)
+
+		// Set up pods with initial recommendation (100m -> 200m)
+		podList := setupPodsForUpscalingInPlace(f, vpa_types.UpdateModeInPlace)
+		initialPodSet := MakePodSet(podList)
+		initialPods := podList.DeepCopy()
+
+		ginkgo.By("Waiting for initial in-place update")
+		err := WaitForPodsUpdatedWithoutEviction(f, initialPods)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Recording current pod state after first update")
+		podListAfterFirstUpdate, err := GetHamsterPods(f)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		podsAfterFirstUpdate := podListAfterFirstUpdate.DeepCopy()
+
+		ginkgo.By("Updating VPA with new recommendations (300m)")
+		containerName := utils.GetHamsterContainerNameByIndex(0)
+		vpaClientSet := utils.GetVpaClientSet(f)
+
+		vpaCRD, err := vpaClientSet.AutoscalingV1().VerticalPodAutoscalers(f.Namespace.Name).
+			Get(context.TODO(), "hamster-vpa", metav1.GetOptions{})
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		vpaCRD.Status.Recommendation = &vpa_types.RecommendedPodResources{
+			ContainerRecommendations: []vpa_types.RecommendedContainerResources{
+				{
+					ContainerName: containerName,
+					Target: apiv1.ResourceList{
+						apiv1.ResourceCPU:    resource.MustParse("300m"),
+						apiv1.ResourceMemory: resource.MustParse("200Mi"),
+					},
+					LowerBound: apiv1.ResourceList{
+						apiv1.ResourceCPU:    resource.MustParse("300m"),
+						apiv1.ResourceMemory: resource.MustParse("200Mi"),
+					},
+					UpperBound: apiv1.ResourceList{
+						apiv1.ResourceCPU:    resource.MustParse("300m"),
+						apiv1.ResourceMemory: resource.MustParse("200Mi"),
+					},
+				},
+			},
+		}
+
+		_, err = vpaClientSet.AutoscalingV1().VerticalPodAutoscalers(f.Namespace.Name).
+			UpdateStatus(context.TODO(), vpaCRD, metav1.UpdateOptions{})
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Waiting for pods to be updated again with new recommendations")
+		err = WaitForPodsUpdatedWithoutEviction(f, podsAfterFirstUpdate)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Verifying no pods were evicted during the process")
+		currentPods, err := GetHamsterPods(f)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		currentPodSet := MakePodSet(currentPods)
+
+		// Verify no pods were evicted by checking UIDs remain the same
+		evictedCount := GetEvictedPodsCount(currentPodSet, initialPodSet)
+		gomega.Expect(evictedCount).To(gomega.Equal(0),
+			"No pods should be evicted when using InPlace mode")
+	})
 })
 
-func setupPodsForUpscalingEviction(f *framework.Framework) *apiv1.PodList {
-	return setupPodsForEviction(f, "100m", "100Mi", nil)
+func setupPodsForUpscalingEviction(f *framework.Framework, updateMode vpa_types.UpdateMode) *apiv1.PodList {
+	return setupPodsForEviction(f, "100m", "100Mi", nil, updateMode)
 }
 
-func setupPodsForDownscalingEviction(f *framework.Framework, er []*vpa_types.EvictionRequirement) *apiv1.PodList {
-	return setupPodsForEviction(f, "500m", "500Mi", er)
+func setupPodsForDownscalingEviction(f *framework.Framework, er []*vpa_types.EvictionRequirement, updateMode vpa_types.UpdateMode) *apiv1.PodList {
+	return setupPodsForEviction(f, "500m", "500Mi", er, updateMode)
 }
 
-func setupPodsForEviction(f *framework.Framework, hamsterCPU, hamsterMemory string, er []*vpa_types.EvictionRequirement) *apiv1.PodList {
+func setupPodsForEviction(f *framework.Framework, hamsterCPU, hamsterMemory string, er []*vpa_types.EvictionRequirement, updateMode vpa_types.UpdateMode) *apiv1.PodList {
 	controller := &autoscaling.CrossVersionObjectReference{
 		APIVersion: "apps/v1",
 		Kind:       "Deployment",
@@ -232,7 +375,7 @@ func setupPodsForEviction(f *framework.Framework, hamsterCPU, hamsterMemory stri
 		WithName("hamster-vpa").
 		WithNamespace(f.Namespace.Name).
 		WithTargetRef(controller).
-		WithUpdateMode(vpa_types.UpdateModeRecreate).
+		WithUpdateMode(updateMode).
 		WithEvictionRequirements(er).
 		WithContainer(containerName).
 		AppendRecommendation(
@@ -249,15 +392,15 @@ func setupPodsForEviction(f *framework.Framework, hamsterCPU, hamsterMemory stri
 	return podList
 }
 
-func setupPodsForUpscalingInPlace(f *framework.Framework) *apiv1.PodList {
-	return setupPodsForInPlace(f, "100m", "100Mi", nil, true)
+func setupPodsForUpscalingInPlace(f *framework.Framework, updateMode vpa_types.UpdateMode) *apiv1.PodList {
+	return setupPodsForInPlace(f, "100m", "100Mi", nil, true, updateMode)
 }
 
-func setupPodsForDownscalingInPlace(f *framework.Framework, er []*vpa_types.EvictionRequirement) *apiv1.PodList {
-	return setupPodsForInPlace(f, "500m", "500Mi", er, true)
+func setupPodsForDownscalingInPlace(f *framework.Framework, er []*vpa_types.EvictionRequirement, updateMode vpa_types.UpdateMode) *apiv1.PodList {
+	return setupPodsForInPlace(f, "500m", "500Mi", er, true, updateMode)
 }
 
-func setupPodsForInPlace(f *framework.Framework, hamsterCPU, hamsterMemory string, er []*vpa_types.EvictionRequirement, withRecommendation bool) *apiv1.PodList {
+func setupPodsForInPlace(f *framework.Framework, hamsterCPU, hamsterMemory string, er []*vpa_types.EvictionRequirement, withRecommendation bool, updateMode vpa_types.UpdateMode) *apiv1.PodList {
 	controller := &autoscaling.CrossVersionObjectReference{
 		APIVersion: "apps/v1",
 		Kind:       "Deployment",
@@ -274,7 +417,7 @@ func setupPodsForInPlace(f *framework.Framework, hamsterCPU, hamsterMemory strin
 		WithName("hamster-vpa").
 		WithNamespace(f.Namespace.Name).
 		WithTargetRef(controller).
-		WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
+		WithUpdateMode(updateMode).
 		WithEvictionRequirements(er).
 		WithContainer(containerName)
 

vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler.go

@@ -118,7 +118,9 @@ func ValidateVPA(vpa *vpa_types.VerticalPodAutoscaler, isCreate bool) error {
 		if (*mode == vpa_types.UpdateModeInPlaceOrRecreate) && !features.Enabled(features.InPlaceOrRecreate) && isCreate {
 			return fmt.Errorf("in order to use UpdateMode %s, you must enable feature gate %s in the admission-controller args", vpa_types.UpdateModeInPlaceOrRecreate, features.InPlaceOrRecreate)
 		}
-
+		if (*mode == vpa_types.UpdateModeInPlace) && !features.Enabled(features.InPlace) && isCreate {
+			return fmt.Errorf("in order to use UpdateMode %s, you must enable feature gate %s in the admission-controller args", vpa_types.UpdateModeInPlace, features.InPlace)
+		}
 		if minReplicas := vpa.Spec.UpdatePolicy.MinReplicas; minReplicas != nil && *minReplicas <= 0 {
 			return fmt.Errorf("minReplicas has to be positive, got %v", *minReplicas)
 		}