diff --git a/pkg/apis/autoscaling/types.go b/pkg/apis/autoscaling/types.go index b8802a992b2..dfbc0474d99 100644 --- a/pkg/apis/autoscaling/types.go +++ b/pkg/apis/autoscaling/types.go @@ -433,6 +433,8 @@ const ( // ScalingLimited indicates that the calculated scale based on metrics would be above or // below the range for the HPA, and has thus been capped. ScalingLimited HorizontalPodAutoscalerConditionType = "ScalingLimited" + // ScaledToZero indicates that the HPA controller scaled the workload to zero. + ScaledToZero HorizontalPodAutoscalerConditionType = "ScaledToZero" ) // HorizontalPodAutoscalerCondition describes the state of diff --git a/pkg/controller/podautoscaler/horizontal.go b/pkg/controller/podautoscaler/horizontal.go index def33660183..e676d5d48ae 100644 --- a/pkg/controller/podautoscaler/horizontal.go +++ b/pkg/controller/podautoscaler/horizontal.go @@ -756,6 +756,20 @@ func (a *HorizontalController) recordInitialRecommendation(currentReplicas int32 } } +// shouldComputeMetricsForZeroReplicas determines the scaling behavior when current replicas is zero. +// Returns: +// - needsMetricComputation: true if metrics should be computed to determine desired replicas +// - shouldDisable: true if scaling should be disabled (workload was manually scaled to zero) +func (a *HorizontalController) shouldComputeMetricsForZeroReplicas( + minReplicas int32, + scaledToZeroCondition, canScaleFromZero bool, +) (needsMetricComputation bool, shouldDisable bool) { + if (minReplicas != 0 && scaledToZeroCondition) || canScaleFromZero { + return true, false + } + return false, true +} + func (a *HorizontalController) reconcileAutoscaler(ctx context.Context, hpaShared *autoscalingv2.HorizontalPodAutoscaler, key string) (retErr error) { // actionLabel is used to report which actions this reconciliation has taken. actionLabel := monitor.ActionLabelNone @@ -838,18 +852,37 @@ func (a *HorizontalController) reconcileAutoscaler(ctx context.Context, hpaShare rescale := true logger := klog.FromContext(ctx) - if currentReplicas == 0 && minReplicas != 0 { - // Autoscaling is disabled for this resource - desiredReplicas = 0 - rescale = false - setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "ScalingDisabled", "scaling is disabled since the replica count of the target is zero") + // Pre-compute scale-to-zero related conditions + scaleToZeroFeatureEnabled := utilfeature.DefaultFeatureGate.Enabled(features.HPAScaleToZero) + hasObjectOrExtMetrics := hasObjectOrExternalMetrics(hpa) + scaledToZeroCondition := scaleToZeroFeatureEnabled && getScaledToZeroConditionStatus(hpa) + canScaleFromZero := scaledToZeroCondition && hasObjectOrExtMetrics + + needsMetricComputation := true + + if currentReplicas == 0 { + var shouldDisable bool + needsMetricComputation, shouldDisable = a.shouldComputeMetricsForZeroReplicas(minReplicas, scaledToZeroCondition, canScaleFromZero) + if shouldDisable { + desiredReplicas = 0 + rescale = false + setCondition(hpa, autoscalingv2.ScalingActive, v1.ConditionFalse, "ScalingDisabled", "scaling is disabled since the replica count of the target is zero") + if !scaleToZeroFeatureEnabled { + removeCondition(hpa, autoscalingv2.ScaledToZero) + } + } } else if currentReplicas > hpa.Spec.MaxReplicas { rescaleReason = "Current number of replicas above Spec.MaxReplicas" desiredReplicas = hpa.Spec.MaxReplicas + needsMetricComputation = false } else if currentReplicas < minReplicas { rescaleReason = "Current number of replicas below Spec.MinReplicas" desiredReplicas = minReplicas - } else { + needsMetricComputation = false + } + + // Compute metrics and normalize desired replicas for cases that require metric-based scaling + if needsMetricComputation { var metricTimestamp time.Time metricDesiredReplicas, metricName, metricStatuses, metricTimestamp, err = a.computeReplicasForMetrics(ctx, hpa, scale, hpa.Spec.Metrics) // computeReplicasForMetrics may return both non-zero metricDesiredReplicas and an error. @@ -867,7 +900,11 @@ func (a *HorizontalController) reconcileAutoscaler(ctx context.Context, hpaShare retErr = err } - logger.V(4).Info("Proposing desired replicas", + logMessage := "Proposing desired replicas" + if currentReplicas == 0 { + logMessage = "Proposing desired replicas from zero" + } + logger.V(4).Info(logMessage, "desiredReplicas", metricDesiredReplicas, "metric", metricName, "tolerances", a.tolerancesForHpa(hpa), @@ -890,6 +927,13 @@ func (a *HorizontalController) reconcileAutoscaler(ctx context.Context, hpaShare } else { desiredReplicas = a.normalizeDesiredReplicasWithBehaviors(hpa, key, currentReplicas, desiredReplicas, minReplicas) } + // Ensure we scale to at least minReplicas when scaling from zero with increased minReplicas + if currentReplicas == 0 && minReplicas != 0 && scaledToZeroCondition && desiredReplicas < minReplicas { + desiredReplicas = minReplicas + if rescaleReason == "" { + rescaleReason = "Current number of replicas below Spec.MinReplicas" + } + } rescale = desiredReplicas != currentReplicas } if rescale { @@ -940,6 +984,17 @@ func (a *HorizontalController) reconcileAutoscaler(ctx context.Context, hpaShare "desiredReplicas", desiredReplicas, "reason", rescaleReason) + // Set ScaledToZero condition on every rescale so the condition is never stale. + if scaleToZeroFeatureEnabled { + if currentReplicas > 0 && desiredReplicas == 0 && minReplicas == 0 && hasObjectOrExtMetrics { + setCondition(hpa, autoscalingv2.ScaledToZero, v1.ConditionTrue, "ScaledToZero", "the HPA controller scaled the workload to zero") + } else { + setCondition(hpa, autoscalingv2.ScaledToZero, v1.ConditionFalse, "NotScaledToZero", "the HPA controller did not scale the workload to zero") + } + } else { + removeCondition(hpa, autoscalingv2.ScaledToZero) + } + if desiredReplicas > currentReplicas { actionLabel = monitor.ActionLabelScaleUp } else { @@ -1479,6 +1534,36 @@ func setCondition(hpa *autoscalingv2.HorizontalPodAutoscaler, conditionType auto hpa.Status.Conditions = setConditionInList(hpa.Status.Conditions, conditionType, status, reason, message, args...) } +func removeCondition(hpa *autoscalingv2.HorizontalPodAutoscaler, conditionType autoscalingv2.HorizontalPodAutoscalerConditionType) { + filtered := hpa.Status.Conditions[:0] + for _, c := range hpa.Status.Conditions { + if c.Type != conditionType { + filtered = append(filtered, c) + } + } + hpa.Status.Conditions = filtered +} + +// hasObjectOrExternalMetrics checks if the HPA has at least one object or external metric. +func hasObjectOrExternalMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler) bool { + for _, metric := range hpa.Spec.Metrics { + if metric.Type == autoscalingv2.ObjectMetricSourceType || metric.Type == autoscalingv2.ExternalMetricSourceType { + return true + } + } + return false +} + +// getScaledToZeroConditionStatus returns true if the ScaledToZero condition exists and is True. +func getScaledToZeroConditionStatus(hpa *autoscalingv2.HorizontalPodAutoscaler) bool { + for _, condition := range hpa.Status.Conditions { + if condition.Type == autoscalingv2.ScaledToZero { + return condition.Status == v1.ConditionTrue + } + } + return false +} + // setConditionInList sets the specific condition type on the given HPA to the specified value with the given // reason and message. The message and args are treated like a format string. The condition will be added if // it is not present. The new list will be returned. diff --git a/pkg/controller/podautoscaler/horizontal_test.go b/pkg/controller/podautoscaler/horizontal_test.go index 7a98fb93076..a568ac8dfb7 100644 --- a/pkg/controller/podautoscaler/horizontal_test.go +++ b/pkg/controller/podautoscaler/horizontal_test.go @@ -172,8 +172,9 @@ type testCase struct { testEMClient *emfake.FakeExternalMetricsClient testScaleClient *scalefake.FakeScaleClient - recommendations []timestampedRecommendation - hpaSelectors *selectors.BiMultimap + recommendations []timestampedRecommendation + hpaSelectors *selectors.BiMultimap + initialConditions []autoscalingv2.HorizontalPodAutoscalerCondition verifyReconciliationDuration bool verifyMetricComputationDurations bool @@ -257,6 +258,7 @@ func (tc *testCase) prepareTestClient(t *testing.T) (*fake.Clientset, *metricsfa CurrentReplicas: tc.specReplicas, DesiredReplicas: tc.specReplicas, LastScaleTime: tc.lastScaleTime, + Conditions: tc.initialConditions, }, } // Initialize default values @@ -1434,85 +1436,147 @@ func TestScaleUpCMObject(t *testing.T) { } func TestScaleUpFromZeroCMObject(t *testing.T) { - targetValue := resource.MustParse("15.0") - tc := testCase{ - minReplicas: 0, - maxReplicas: 6, - specReplicas: 0, - statusReplicas: 0, - expectedDesiredReplicas: 2, - CPUTarget: 0, - metricsTarget: []autoscalingv2.MetricSpec{ - { - Type: autoscalingv2.ObjectMetricSourceType, - Object: &autoscalingv2.ObjectMetricSource{ - DescribedObject: autoscalingv2.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "some-deployment", - }, - Metric: autoscalingv2.MetricIdentifier{ - Name: "qps", - }, - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.ValueMetricType, - Value: &targetValue, + for _, fgEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("HPAScaleToZero=%v", fgEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, fgEnabled) + targetValue := resource.MustParse("15.0") + tc := testCase{ + minReplicas: 0, + maxReplicas: 6, + specReplicas: 0, + statusReplicas: 0, + CPUTarget: 0, + metricsTarget: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ObjectMetricSourceType, + Object: &autoscalingv2.ObjectMetricSource{ + DescribedObject: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "some-deployment", + }, + Metric: autoscalingv2.MetricIdentifier{ + Name: "qps", + }, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.ValueMetricType, + Value: &targetValue, + }, + }, }, }, - }, - }, - reportedLevels: []uint64{20000}, - expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp, - expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, - expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ - autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleUp, - }, - expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ - autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone, - }, + reportedLevels: []uint64{20000}, + initialConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{ + { + Type: autoscalingv2.ScaledToZero, + Status: v1.ConditionTrue, + Reason: "ScaledToZero", + }, + }, + } + if fgEnabled { + tc.expectedDesiredReplicas = 2 + tc.expectedConditions = []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.ScaledToZero, Status: v1.ConditionFalse, Reason: "NotScaledToZero"}, + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededRescale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionTrue, Reason: "ValidMetricFound"}, + {Type: autoscalingv2.ScalingLimited, Status: v1.ConditionFalse, Reason: "DesiredWithinRange"}, + } + tc.expectedReportedReconciliationActionLabel = monitor.ActionLabelScaleUp + tc.expectedReportedReconciliationErrorLabel = monitor.ErrorLabelNone + tc.expectedReportedMetricComputationActionLabels = map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ + autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleUp, + } + tc.expectedReportedMetricComputationErrorLabels = map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ + autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone, + } + } else { + // FG off: scaledToZeroCondition=false, canScaleFromZero=false → scaling disabled + tc.expectedDesiredReplicas = 0 + tc.expectedConditions = []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "ScalingDisabled"}, + } + tc.expectedReportedReconciliationActionLabel = monitor.ActionLabelNone + tc.expectedReportedReconciliationErrorLabel = monitor.ErrorLabelNone + tc.expectedReportedMetricComputationActionLabels = map[autoscalingv2.MetricSourceType]monitor.ActionLabel{} + tc.expectedReportedMetricComputationErrorLabels = map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{} + } + tc.runTest(t) + }) } - tc.runTest(t) } func TestScaleUpFromZeroIgnoresToleranceCMObject(t *testing.T) { - targetValue := resource.MustParse("1.0") - tc := testCase{ - minReplicas: 0, - maxReplicas: 6, - specReplicas: 0, - statusReplicas: 0, - expectedDesiredReplicas: 1, - CPUTarget: 0, - metricsTarget: []autoscalingv2.MetricSpec{ - { - Type: autoscalingv2.ObjectMetricSourceType, - Object: &autoscalingv2.ObjectMetricSource{ - DescribedObject: autoscalingv2.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "some-deployment", - }, - Metric: autoscalingv2.MetricIdentifier{ - Name: "qps", - }, - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.ValueMetricType, - Value: &targetValue, + for _, fgEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("HPAScaleToZero=%v", fgEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, fgEnabled) + targetValue := resource.MustParse("1.0") + tc := testCase{ + minReplicas: 0, + maxReplicas: 6, + specReplicas: 0, + statusReplicas: 0, + CPUTarget: 0, + metricsTarget: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ObjectMetricSourceType, + Object: &autoscalingv2.ObjectMetricSource{ + DescribedObject: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "some-deployment", + }, + Metric: autoscalingv2.MetricIdentifier{ + Name: "qps", + }, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.ValueMetricType, + Value: &targetValue, + }, + }, }, }, - }, - }, - reportedLevels: []uint64{1000}, - expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp, - expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, - expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ - autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleUp, - }, - expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ - autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone, - }, + reportedLevels: []uint64{1000}, + initialConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{ + { + Type: autoscalingv2.ScaledToZero, + Status: v1.ConditionTrue, + Reason: "ScaledToZero", + }, + }, + } + if fgEnabled { + tc.expectedDesiredReplicas = 1 + tc.expectedConditions = []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.ScaledToZero, Status: v1.ConditionFalse, Reason: "NotScaledToZero"}, + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededRescale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionTrue, Reason: "ValidMetricFound"}, + {Type: autoscalingv2.ScalingLimited, Status: v1.ConditionFalse, Reason: "DesiredWithinRange"}, + } + tc.expectedReportedReconciliationActionLabel = monitor.ActionLabelScaleUp + tc.expectedReportedReconciliationErrorLabel = monitor.ErrorLabelNone + tc.expectedReportedMetricComputationActionLabels = map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ + autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleUp, + } + tc.expectedReportedMetricComputationErrorLabels = map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ + autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone, + } + } else { + // FG off: scaledToZeroCondition=false, canScaleFromZero=false → scaling disabled + tc.expectedDesiredReplicas = 0 + tc.expectedConditions = []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "ScalingDisabled"}, + } + tc.expectedReportedReconciliationActionLabel = monitor.ActionLabelNone + tc.expectedReportedReconciliationErrorLabel = monitor.ErrorLabelNone + tc.expectedReportedMetricComputationActionLabels = map[autoscalingv2.MetricSourceType]monitor.ActionLabel{} + tc.expectedReportedMetricComputationErrorLabels = map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{} + } + tc.runTest(t) + }) } - tc.runTest(t) } func TestScaleUpPerPodCMObject(t *testing.T) { @@ -1744,36 +1808,39 @@ func TestScaleUpOneMetricInvalid(t *testing.T) { } func TestScaleUpFromZeroOneMetricInvalid(t *testing.T) { - tc := testCase{ - minReplicas: 0, - maxReplicas: 6, - specReplicas: 0, - statusReplicas: 0, - expectedDesiredReplicas: 4, - CPUTarget: 30, - verifyCPUCurrent: true, - metricsTarget: []autoscalingv2.MetricSpec{ - { - Type: "CheddarCheese", - }, - }, - reportedLevels: []uint64{300, 400, 500}, - reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, - recommendations: []timestampedRecommendation{}, - expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp, - expectedReportedReconciliationErrorLabel: monitor.ErrorLabelInternal, - expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ - autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp, - // Actually, such an invalid type should be validated in the kube-apiserver and invalid metric type shouldn't be recorded. - "CheddarCheese": monitor.ActionLabelNone, - }, - expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ - autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone, - // Actually, such an invalid type should be validated in the kube-apiserver and invalid metric type shouldn't be recorded. - "CheddarCheese": monitor.ErrorLabelSpec, - }, + for _, fgEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("HPAScaleToZero=%v", fgEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, fgEnabled) + // This test uses CPU metrics, not object/external, so it can't scale from zero + // regardless of feature gate state (only object/external metrics support scale-from-zero per KEP-2021) + tc := testCase{ + minReplicas: 0, + maxReplicas: 6, + specReplicas: 0, + statusReplicas: 0, + expectedDesiredReplicas: 0, + CPUTarget: 30, + verifyCPUCurrent: false, + metricsTarget: []autoscalingv2.MetricSpec{ + { + Type: "CheddarCheese", + }, + }, + reportedLevels: []uint64{300, 400, 500}, + reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + recommendations: []timestampedRecommendation{}, + expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "ScalingDisabled"}, + }, + expectedReportedReconciliationActionLabel: monitor.ActionLabelNone, + expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, + expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{}, + expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{}, + } + tc.runTest(t) + }) } - tc.runTest(t) } func TestScaleUpBothMetricsEmpty(t *testing.T) { // Switch to missing @@ -1925,46 +1992,62 @@ func TestScaleDownCMObject(t *testing.T) { } func TestScaleDownToZeroCMObject(t *testing.T) { - targetValue := resource.MustParse("20.0") - tc := testCase{ - minReplicas: 0, - maxReplicas: 6, - specReplicas: 5, - statusReplicas: 5, - expectedDesiredReplicas: 0, - CPUTarget: 0, - metricsTarget: []autoscalingv2.MetricSpec{ - { - Type: autoscalingv2.ObjectMetricSourceType, - Object: &autoscalingv2.ObjectMetricSource{ - DescribedObject: autoscalingv2.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: "some-deployment", - }, - Metric: autoscalingv2.MetricIdentifier{ - Name: "qps", - }, - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.ValueMetricType, - Value: &targetValue, + for _, fgEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("HPAScaleToZero=%v", fgEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, fgEnabled) + targetValue := resource.MustParse("20.0") + tc := testCase{ + minReplicas: 0, + maxReplicas: 6, + specReplicas: 5, + statusReplicas: 5, + expectedDesiredReplicas: 0, + CPUTarget: 0, + metricsTarget: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ObjectMetricSourceType, + Object: &autoscalingv2.ObjectMetricSource{ + DescribedObject: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "some-deployment", + }, + Metric: autoscalingv2.MetricIdentifier{ + Name: "qps", + }, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.ValueMetricType, + Value: &targetValue, + }, + }, }, }, - }, - }, - reportedLevels: []uint64{0}, - reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, - recommendations: []timestampedRecommendation{}, - expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown, - expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, - expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ - autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleDown, - }, - expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ - autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone, - }, + reportedLevels: []uint64{0}, + reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + recommendations: []timestampedRecommendation{}, + expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown, + expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, + expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ + autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleDown, + }, + expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ + autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone, + }, + } + if fgEnabled { + // FG on: ScaledToZero condition set on scale-down to zero + tc.expectedConditions = statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ + Type: autoscalingv2.ScaledToZero, + Status: v1.ConditionTrue, + Reason: "ScaledToZero", + }) + } else { + // FG off: no ScaledToZero condition, but scale-down still happens + tc.expectedConditions = statusOkWithOverrides() + } + tc.runTest(t) + }) } - tc.runTest(t) } func TestScaleDownPerPodCMObject(t *testing.T) { @@ -2047,39 +2130,252 @@ func TestScaleDownCMExternal(t *testing.T) { } func TestScaleDownToZeroCMExternal(t *testing.T) { - tc := testCase{ - minReplicas: 0, - maxReplicas: 6, - specReplicas: 5, - statusReplicas: 5, - expectedDesiredReplicas: 0, - metricsTarget: []autoscalingv2.MetricSpec{ - { - Type: autoscalingv2.ExternalMetricSourceType, - External: &autoscalingv2.ExternalMetricSource{ - Metric: autoscalingv2.MetricIdentifier{ - Name: "qps", - Selector: &metav1.LabelSelector{}, - }, - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.ValueMetricType, - Value: resource.NewMilliQuantity(14400, resource.DecimalSI), + for _, fgEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("HPAScaleToZero=%v", fgEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, fgEnabled) + tc := testCase{ + minReplicas: 0, + maxReplicas: 6, + specReplicas: 5, + statusReplicas: 5, + expectedDesiredReplicas: 0, + metricsTarget: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ExternalMetricSourceType, + External: &autoscalingv2.ExternalMetricSource{ + Metric: autoscalingv2.MetricIdentifier{ + Name: "qps", + Selector: &metav1.LabelSelector{}, + }, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.ValueMetricType, + Value: resource.NewMilliQuantity(14400, resource.DecimalSI), + }, + }, }, }, - }, - }, - reportedLevels: []uint64{0}, - recommendations: []timestampedRecommendation{}, - expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown, - expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, - expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ - autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelScaleDown, - }, - expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ - autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelNone, - }, + reportedLevels: []uint64{0}, + recommendations: []timestampedRecommendation{}, + expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleDown, + expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, + expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ + autoscalingv2.ExternalMetricSourceType: monitor.ActionLabelScaleDown, + }, + expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ + autoscalingv2.ExternalMetricSourceType: monitor.ErrorLabelNone, + }, + } + if fgEnabled { + tc.expectedConditions = statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ + Type: autoscalingv2.ScaledToZero, + Status: v1.ConditionTrue, + Reason: "ScaledToZero", + }) + } else { + tc.expectedConditions = statusOkWithOverrides() + } + tc.runTest(t) + }) + } +} + +func TestScaleUpFromZeroWithoutCondition(t *testing.T) { + for _, fgEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("HPAScaleToZero=%v", fgEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, fgEnabled) + // Without ScaledToZero condition, scaling from zero is disabled regardless of feature gate + targetValue := resource.MustParse("15.0") + tc := testCase{ + minReplicas: 0, + maxReplicas: 6, + specReplicas: 0, + statusReplicas: 0, + expectedDesiredReplicas: 0, + CPUTarget: 0, + metricsTarget: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ObjectMetricSourceType, + Object: &autoscalingv2.ObjectMetricSource{ + DescribedObject: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "some-deployment", + }, + Metric: autoscalingv2.MetricIdentifier{ + Name: "qps", + }, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.ValueMetricType, + Value: &targetValue, + }, + }, + }, + }, + reportedLevels: []uint64{20000}, + expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "ScalingDisabled"}, + }, + expectedReportedReconciliationActionLabel: monitor.ActionLabelNone, + expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, + expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{}, + expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{}, + } + tc.runTest(t) + }) + } +} + +func TestScaleUpFromZeroWhenMinReplicasIncreased(t *testing.T) { + for _, fgEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("HPAScaleToZero=%v", fgEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, fgEnabled) + tc := testCase{ + minReplicas: 2, + maxReplicas: 6, + specReplicas: 0, + statusReplicas: 0, + CPUTarget: 0, + metricsTarget: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ObjectMetricSourceType, + Object: &autoscalingv2.ObjectMetricSource{ + DescribedObject: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "some-deployment", + }, + Metric: autoscalingv2.MetricIdentifier{ + Name: "qps", + }, + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.ValueMetricType, + Value: resource.NewMilliQuantity(15000, resource.DecimalSI), + }, + }, + }, + }, + reportedLevels: []uint64{10000}, + initialConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{ + { + Type: autoscalingv2.ScaledToZero, + Status: v1.ConditionTrue, + Reason: "ScaledToZero", + }, + }, + } + if fgEnabled { + // HPA previously scaled to zero, minReplicas increased to 2 → scale up to minReplicas + tc.expectedDesiredReplicas = 2 + tc.expectedConditions = []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.ScaledToZero, Status: v1.ConditionFalse, Reason: "NotScaledToZero"}, + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededRescale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionTrue, Reason: "ValidMetricFound"}, + {Type: autoscalingv2.ScalingLimited, Status: v1.ConditionTrue, Reason: "TooFewReplicas"}, + } + tc.expectedReportedReconciliationActionLabel = monitor.ActionLabelScaleUp + tc.expectedReportedReconciliationErrorLabel = monitor.ErrorLabelNone + tc.expectedReportedMetricComputationActionLabels = map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ + autoscalingv2.ObjectMetricSourceType: monitor.ActionLabelScaleUp, + } + tc.expectedReportedMetricComputationErrorLabels = map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ + autoscalingv2.ObjectMetricSourceType: monitor.ErrorLabelNone, + } + } else { + // FG off: scaledToZeroCondition=false → treated as manual scale-down, scaling disabled + tc.expectedDesiredReplicas = 0 + tc.expectedConditions = []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "ScalingDisabled"}, + } + tc.expectedReportedReconciliationActionLabel = monitor.ActionLabelNone + tc.expectedReportedReconciliationErrorLabel = monitor.ErrorLabelNone + tc.expectedReportedMetricComputationActionLabels = map[autoscalingv2.MetricSourceType]monitor.ActionLabel{} + tc.expectedReportedMetricComputationErrorLabels = map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{} + } + tc.runTest(t) + }) + } +} + +func TestScaledToZeroConditionHandledOnNormalRescale(t *testing.T) { + for _, fgEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("HPAScaleToZero=%v", fgEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, fgEnabled) + tc := testCase{ + minReplicas: 1, + maxReplicas: 6, + specReplicas: 3, + statusReplicas: 3, + expectedDesiredReplicas: 5, + CPUTarget: 30, + reportedLevels: []uint64{300, 500, 700}, + reportedCPURequests: []resource.Quantity{resource.MustParse("1.0"), resource.MustParse("1.0"), resource.MustParse("1.0")}, + useMetricsAPI: true, + initialConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{ + { + Type: autoscalingv2.ScaledToZero, + Status: v1.ConditionTrue, + Reason: "ScaledToZero", + }, + }, + expectedReportedReconciliationActionLabel: monitor.ActionLabelScaleUp, + expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, + expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{ + autoscalingv2.ResourceMetricSourceType: monitor.ActionLabelScaleUp, + }, + expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{ + autoscalingv2.ResourceMetricSourceType: monitor.ErrorLabelNone, + }, + } + if fgEnabled { + // FG on: ScaledToZero condition set to False on rescale + tc.expectedConditions = []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.ScaledToZero, Status: v1.ConditionFalse, Reason: "NotScaledToZero"}, + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededRescale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionTrue, Reason: "ValidMetricFound"}, + {Type: autoscalingv2.ScalingLimited, Status: v1.ConditionFalse, Reason: "DesiredWithinRange"}, + } + } else { + // FG off: stale ScaledToZero condition removed on rescale + tc.expectedConditions = statusOkWithOverrides(autoscalingv2.HorizontalPodAutoscalerCondition{ + Type: autoscalingv2.ScalingLimited, Status: v1.ConditionFalse, + Reason: "DesiredWithinRange", + }) + } + tc.runTest(t) + }) + } +} + +func TestManualScaleToZeroDisablesHPA(t *testing.T) { + for _, fgEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("HPAScaleToZero=%v", fgEnabled), func(t *testing.T) { + // With minReplicas >= 1, a workload manually scaled to zero (no ScaledToZero condition) + // should cause HPA to pause (ScalingDisabled), regardless of feature gate + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HPAScaleToZero, fgEnabled) + tc := testCase{ + minReplicas: 1, + maxReplicas: 6, + specReplicas: 0, + statusReplicas: 0, + expectedDesiredReplicas: 0, + CPUTarget: 50, + reportedLevels: []uint64{}, + reportedCPURequests: []resource.Quantity{}, + useMetricsAPI: true, + expectedConditions: []autoscalingv2.HorizontalPodAutoscalerCondition{ + {Type: autoscalingv2.AbleToScale, Status: v1.ConditionTrue, Reason: "SucceededGetScale"}, + {Type: autoscalingv2.ScalingActive, Status: v1.ConditionFalse, Reason: "ScalingDisabled"}, + }, + expectedReportedReconciliationActionLabel: monitor.ActionLabelNone, + expectedReportedReconciliationErrorLabel: monitor.ErrorLabelNone, + expectedReportedMetricComputationActionLabels: map[autoscalingv2.MetricSourceType]monitor.ActionLabel{}, + expectedReportedMetricComputationErrorLabels: map[autoscalingv2.MetricSourceType]monitor.ErrorLabel{}, + } + tc.runTest(t) + }) } - tc.runTest(t) } func TestScaleDownPerPodCMExternal(t *testing.T) { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 9798965ba47..60df04dd423 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -360,7 +360,8 @@ const ( // Enables support of configurable HPA scale-up and scale-down tolerances. HPAConfigurableTolerance featuregate.Feature = "HPAConfigurableTolerance" - // owner: @dxist + // owner: @johanneswuerbach + // kep: https://kep.k8s.io/2021 // // Enables support of HPA scaling to zero pods when an object or custom metric is configured. HPAScaleToZero featuregate.Feature = "HPAScaleToZero" diff --git a/staging/src/k8s.io/api/autoscaling/v1/types.go b/staging/src/k8s.io/api/autoscaling/v1/types.go index d09faa423c0..97222afc080 100644 --- a/staging/src/k8s.io/api/autoscaling/v1/types.go +++ b/staging/src/k8s.io/api/autoscaling/v1/types.go @@ -414,6 +414,8 @@ const ( // ScalingLimited indicates that the calculated scale based on metrics would be above or // below the range for the HPA, and has thus been capped. ScalingLimited HorizontalPodAutoscalerConditionType = "ScalingLimited" + // ScaledToZero indicates that the HPA controller scaled the workload to zero. + ScaledToZero HorizontalPodAutoscalerConditionType = "ScaledToZero" ) // HorizontalPodAutoscalerCondition describes the state of diff --git a/staging/src/k8s.io/api/autoscaling/v2/types.go b/staging/src/k8s.io/api/autoscaling/v2/types.go index 166fb96be3b..ad10169a01c 100644 --- a/staging/src/k8s.io/api/autoscaling/v2/types.go +++ b/staging/src/k8s.io/api/autoscaling/v2/types.go @@ -450,6 +450,8 @@ const ( // ScalingLimited indicates that the calculated scale based on metrics would be above or // below the range for the HPA, and has thus been capped. ScalingLimited HorizontalPodAutoscalerConditionType = "ScalingLimited" + // ScaledToZero indicates that the HPA controller scaled the workload to zero. + ScaledToZero HorizontalPodAutoscalerConditionType = "ScaledToZero" ) // HorizontalPodAutoscalerCondition describes the state of diff --git a/test/e2e/autoscaling/horizontal_pod_autosclaing_external_metrics.go b/test/e2e/autoscaling/horizontal_pod_autosclaing_external_metrics.go index b0d87f3f369..22c3f64429d 100644 --- a/test/e2e/autoscaling/horizontal_pod_autosclaing_external_metrics.go +++ b/test/e2e/autoscaling/horizontal_pod_autosclaing_external_metrics.go @@ -22,7 +22,11 @@ import ( "time" "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" v2 "k8s.io/api/autoscaling/v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2eautoscaling "k8s.io/kubernetes/test/e2e/framework/autoscaling" @@ -84,3 +88,80 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (external metrics)" ginkgo.DeferCleanup(e2eautoscaling.DeleteHorizontalPodAutoscaler, rc, hpa.Name) }) }) + +var _ = SIGDescribe(feature.HPA, framework.WithFeatureGate(features.HPAScaleToZero), + "Horizontal pod autoscaling (scale to zero)", func() { + var ( + rc *e2eautoscaling.ResourceConsumer + metricsController *e2eautoscaling.ExternalMetricsController + ) + + waitBuffer := 1 * time.Minute + + f := framework.NewDefaultFramework("horizontal-pod-autoscaling-scale-to-zero") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + + ginkgo.BeforeEach(func(ctx context.Context) { + ginkgo.By("Setting up the external metrics server") + metricsController = e2eautoscaling.RunExternalMetricsServer(ctx, f.ClientSet, f.Namespace.Name, "external-metrics-server", nil) + }) + ginkgo.AfterEach(func(ctx context.Context) { + if metricsController != nil { + e2eautoscaling.CleanupExternalMetricsServer(ctx, f.ClientSet, f.Namespace.Name, "external-metrics-server") + } + }) + + ginkgo.It("should scale down to zero and back up based on external metric value", func(ctx context.Context) { + ginkgo.By("Creating the resource consumer deployment") + initPods := 1 + rc = e2eautoscaling.NewDynamicResourceConsumer(ctx, + hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, + 0, 0, 0, + int64(podCPURequest), 200, + f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, + nil) + ginkgo.DeferCleanup(rc.CleanUp) + rc.WaitForReplicas(ctx, initPods, maxResourceConsumerDelay+waitBuffer) + + metricName := "queue_messages_ready" + + ginkgo.By(fmt.Sprintf("Creating an HPA with minReplicas=0 based on external metric %s", metricName)) + stabilizationWindowZero := int32(0) + behavior := &v2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &v2.HPAScalingRules{ + StabilizationWindowSeconds: &stabilizationWindowZero, + }, + } + hpa := e2eautoscaling.CreateExternalHorizontalPodAutoscalerWithBehavior(ctx, + rc, metricName, nil, v2.ValueMetricType, 50, + 0, 3, behavior) + ginkgo.DeferCleanup(e2eautoscaling.DeleteHPAWithBehavior, rc, hpa.Name) + + ginkgo.By(fmt.Sprintf("Setting %s metric value to 0 to trigger scale to zero", metricName)) + err := metricsController.SetMetricValue(ctx, metricName, 0, nil) + framework.ExpectNoError(err) + + ginkgo.By("Waiting for HPA to scale down to zero replicas") + rc.WaitForReplicas(ctx, 0, maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) + + ginkgo.By("Verifying the ScaledToZero condition is True") + updatedHPA, err := f.ClientSet.AutoscalingV2().HorizontalPodAutoscalers(f.Namespace.Name).Get(ctx, hpa.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + var scaledToZeroCondition *v2.HorizontalPodAutoscalerCondition + for i := range updatedHPA.Status.Conditions { + if updatedHPA.Status.Conditions[i].Type == v2.ScaledToZero { + scaledToZeroCondition = &updatedHPA.Status.Conditions[i] + break + } + } + gomega.Expect(scaledToZeroCondition).NotTo(gomega.BeNil(), "expected ScaledToZero condition to be present") + gomega.Expect(scaledToZeroCondition.Status).To(gomega.Equal(v1.ConditionTrue), "expected ScaledToZero condition to be True") + + ginkgo.By(fmt.Sprintf("Setting %s metric value to 200 to trigger scale from zero", metricName)) + err = metricsController.SetMetricValue(ctx, metricName, 200, nil) + framework.ExpectNoError(err) + + ginkgo.By("Waiting for HPA to scale back up from zero") + rc.WaitForReplicas(ctx, int(hpa.Spec.MaxReplicas), maxHPAReactionTime+maxResourceConsumerDelay+waitBuffer) + }) + })