diff --git a/pkg/event/event.go b/pkg/event/event.go index a68944c3..e0af1ba8 100644 --- a/pkg/event/event.go +++ b/pkg/event/event.go @@ -20,4 +20,5 @@ const ( RestartDeployment = "RestartDeployment" WarningHittingHardMaxReplicaLimit = "HitHardMaxReplicaLimit" + WarningReplicaValidation = "ReplicaValidationWarning" ) diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index 099a00a1..e9e43b8c 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -412,24 +412,44 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet return nil, tortoise, fmt.Errorf("get maxReplicas recommendation: %w", err) } + // Apply user-specified maxReplicas limit if set if tortoise.Spec.MaxReplicas != nil && recommendMax > *tortoise.Spec.MaxReplicas { recommendMax = *tortoise.Spec.MaxReplicas } + // Apply cluster-wide maxReplicas limit if recommendMax > c.maximumMaxReplica { c.recorder.Event(tortoise, corev1.EventTypeWarning, event.WarningHittingHardMaxReplicaLimit, fmt.Sprintf("MaxReplica (%v) suggested from Tortoise (%s/%s) hits a cluster-wide maximum replica number (%v). It wouldn't be a problem until the replica number actually grows to %v though, you may want to reach out to your cluster admin.", recommendMax, tortoise.Namespace, tortoise.Name, c.maximumMaxReplica, c.maximumMaxReplica)) recommendMax = c.maximumMaxReplica } - hpa.Spec.MaxReplicas = recommendMax + // Ensure maxReplicas is at least minimumMinReplicas + if recommendMax < c.minimumMinReplicas { + c.recorder.Event(tortoise, corev1.EventTypeWarning, event.WarningReplicaValidation, fmt.Sprintf("MaxReplica (%v) is below minimum allowed replicas (%v), adjusting to minimum", recommendMax, c.minimumMinReplicas)) + recommendMax = c.minimumMinReplicas + } recommendMin, err := GetReplicasRecommendation(tortoise.Status.Recommendations.Horizontal.MinReplicas, now) if err != nil { return nil, tortoise, fmt.Errorf("get minReplicas recommendation: %w", err) } + + // Apply cluster-wide minReplicas limit if recommendMin > c.maximumMinReplica { + c.recorder.Event(tortoise, corev1.EventTypeWarning, event.WarningReplicaValidation, fmt.Sprintf("MinReplica (%v) suggested from Tortoise (%s/%s) hits a cluster-wide maximum min replica number (%v), capping to maximum", recommendMin, tortoise.Namespace, tortoise.Name, c.maximumMinReplica)) recommendMin = c.maximumMinReplica - // We don't change the maxReplica because it's dangerous to limit. + } + + // Ensure minReplicas is at least minimumMinReplicas + if recommendMin < c.minimumMinReplicas { + c.recorder.Event(tortoise, corev1.EventTypeWarning, event.WarningReplicaValidation, fmt.Sprintf("MinReplica (%v) is below minimum allowed replicas (%v), adjusting to minimum", recommendMin, c.minimumMinReplicas)) + recommendMin = c.minimumMinReplicas + } + + // Ensure minReplicas doesn't exceed maxReplicas + if recommendMin > recommendMax { + c.recorder.Event(tortoise, corev1.EventTypeWarning, event.WarningReplicaValidation, fmt.Sprintf("MinReplica (%v) exceeds MaxReplica (%v), adjusting MaxReplica to MinReplica", recommendMin, recommendMax)) + recommendMax = recommendMin } if recordMetrics { @@ -459,6 +479,31 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet minToActuallyApply = recommendMin } + // Final validation: Ensure minReplicas doesn't exceed maxReplicas after phase-specific adjustments + if minToActuallyApply > recommendMax { + if tortoise.Status.TortoisePhase == autoscalingv1beta3.TortoisePhaseBackToNormal { + // During BackToNormal, temporarily increase maxReplicas to allow safe gradual reduction + // This preserves the safety mechanism while ensuring HPA validity + originalMaxReplicas := recommendMax + recommendMax = minToActuallyApply + c.recorder.Event(tortoise, corev1.EventTypeNormal, event.Working, + fmt.Sprintf("Temporarily increased MaxReplicas from %v to %v during BackToNormal phase to allow safe gradual MinReplicas reduction", + originalMaxReplicas, minToActuallyApply)) + } else { + // For other phases, cap minReplicas to maxReplicas + minToActuallyApply = recommendMax + c.recorder.Event(tortoise, corev1.EventTypeWarning, event.WarningReplicaValidation, + fmt.Sprintf("MinReplicas was capped to MaxReplicas (%v) to prevent HPA validation error", recommendMax)) + } + } + + // Ensure final minReplicas is at least minimumMinReplicas + if minToActuallyApply < c.minimumMinReplicas { + c.recorder.Event(tortoise, corev1.EventTypeWarning, event.WarningReplicaValidation, fmt.Sprintf("Final MinReplicas (%v) is below minimum allowed replicas (%v), adjusting to minimum", minToActuallyApply, c.minimumMinReplicas)) + minToActuallyApply = c.minimumMinReplicas + } + + hpa.Spec.MaxReplicas = recommendMax hpa.Spec.MinReplicas = &minToActuallyApply if tortoise.Spec.UpdateMode != autoscalingv1beta3.UpdateModeOff && recordMetrics { // We don't want to record applied* metric when UpdateMode is Off. diff --git a/pkg/hpa/service_test.go b/pkg/hpa/service_test.go index e6a02298..364b02ba 100644 --- a/pkg/hpa/service_test.go +++ b/pkg/hpa/service_test.go @@ -2746,6 +2746,983 @@ func TestClient_UpdateHPAFromTortoiseRecommendation(t *testing.T) { }, wantErr: false, }, + { + name: "MinReplicas below minimum allowed replicas should be adjusted", + args: args{ + ctx: context.Background(), + tortoise: &v1beta3.Tortoise{ + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + Conditions: v1beta3.Conditions{ + TortoiseConditions: []v1beta3.TortoiseCondition{ + { + Type: v1beta3.TortoiseConditionTypeHPATargetUtilizationUpdated, + Status: v1.ConditionTrue, + LastUpdateTime: metav1.NewTime(now.Add(-3 * time.Hour)), + LastTransitionTime: metav1.NewTime(now.Add(-3 * time.Hour)), + Reason: "HPATargetUtilizationUpdated", + Message: "HPA target utilization is updated", + }, + }, + }, + ContainerResourcePhases: []v1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]v1beta3.ResourcePhase{ + v1.ResourceMemory: { + Phase: v1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: v1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + Recommendations: v1beta3.Recommendations{ + Horizontal: v1beta3.HorizontalRecommendations{ + TargetUtilizations: []v1beta3.HPATargetUtilizationRecommendationPerContainer{ + { + ContainerName: "app", + TargetUtilization: map[v1.ResourceName]int32{ + v1.ResourceMemory: 90, + }, + }, + }, + MaxReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 10, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + MinReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 1, // Below minimum (3) + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + }, + }, + }, + }, + now: now.Time, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(3), + MaxReplicas: 10, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](60), + }, + Container: "app", + }, + }, + }, + }, + }, + want: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + Behavior: defaultHPABehaviorValue.DeepCopy(), + MinReplicas: ptrInt32(3), // Should be adjusted to minimum + MaxReplicas: 10, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](90), + }, + Container: "app", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "MaxReplicas below minimum allowed replicas should be adjusted", + args: args{ + ctx: context.Background(), + tortoise: &v1beta3.Tortoise{ + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + Conditions: v1beta3.Conditions{ + TortoiseConditions: []v1beta3.TortoiseCondition{ + { + Type: v1beta3.TortoiseConditionTypeHPATargetUtilizationUpdated, + Status: v1.ConditionTrue, + LastUpdateTime: metav1.NewTime(now.Add(-3 * time.Hour)), + LastTransitionTime: metav1.NewTime(now.Add(-3 * time.Hour)), + Reason: "HPATargetUtilizationUpdated", + Message: "HPA target utilization is updated", + }, + }, + }, + ContainerResourcePhases: []v1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]v1beta3.ResourcePhase{ + v1.ResourceMemory: { + Phase: v1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: v1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + Recommendations: v1beta3.Recommendations{ + Horizontal: v1beta3.HorizontalRecommendations{ + TargetUtilizations: []v1beta3.HPATargetUtilizationRecommendationPerContainer{ + { + ContainerName: "app", + TargetUtilization: map[v1.ResourceName]int32{ + v1.ResourceMemory: 90, + }, + }, + }, + MaxReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 1, // Below minimum (3) + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + MinReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 3, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + }, + }, + }, + }, + now: now.Time, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(3), + MaxReplicas: 10, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](60), + }, + Container: "app", + }, + }, + }, + }, + }, + want: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + Behavior: defaultHPABehaviorValue.DeepCopy(), + MinReplicas: ptrInt32(3), + MaxReplicas: 3, // Should be adjusted to minimum (3) since minReplicas is 3 + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](90), + }, + Container: "app", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "MinReplicas exceeds MaxReplicas should adjust MaxReplicas", + args: args{ + ctx: context.Background(), + tortoise: &v1beta3.Tortoise{ + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + Conditions: v1beta3.Conditions{ + TortoiseConditions: []v1beta3.TortoiseCondition{ + { + Type: v1beta3.TortoiseConditionTypeHPATargetUtilizationUpdated, + Status: v1.ConditionTrue, + LastUpdateTime: metav1.NewTime(now.Add(-3 * time.Hour)), + LastTransitionTime: metav1.NewTime(now.Add(-3 * time.Hour)), + Reason: "HPATargetUtilizationUpdated", + Message: "HPA target utilization is updated", + }, + }, + }, + ContainerResourcePhases: []v1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]v1beta3.ResourcePhase{ + v1.ResourceMemory: { + Phase: v1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: v1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + Recommendations: v1beta3.Recommendations{ + Horizontal: v1beta3.HorizontalRecommendations{ + TargetUtilizations: []v1beta3.HPATargetUtilizationRecommendationPerContainer{ + { + ContainerName: "app", + TargetUtilization: map[v1.ResourceName]int32{ + v1.ResourceMemory: 90, + }, + }, + }, + MaxReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 5, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + MinReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 8, // Exceeds maxReplicas (5) + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + }, + }, + }, + }, + now: now.Time, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(3), + MaxReplicas: 10, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](60), + }, + Container: "app", + }, + }, + }, + }, + }, + want: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + Behavior: defaultHPABehaviorValue.DeepCopy(), + MinReplicas: ptrInt32(8), + MaxReplicas: 8, // Should be adjusted to minReplicas (8) + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](90), + }, + Container: "app", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "BackToNormal phase should temporarily increase MaxReplicas for safety", + args: args{ + ctx: context.Background(), + tortoise: &v1beta3.Tortoise{ + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseBackToNormal, + AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + Conditions: v1beta3.Conditions{ + TortoiseConditions: []v1beta3.TortoiseCondition{ + { + Type: v1beta3.TortoiseConditionTypeHPATargetUtilizationUpdated, + Status: v1.ConditionTrue, + LastUpdateTime: metav1.NewTime(now.Add(-3 * time.Hour)), + LastTransitionTime: metav1.NewTime(now.Add(-3 * time.Hour)), + Reason: "HPATargetUtilizationUpdated", + Message: "HPA target utilization is updated", + }, + }, + }, + ContainerResourcePhases: []v1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]v1beta3.ResourcePhase{ + v1.ResourceMemory: { + Phase: v1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: v1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + Recommendations: v1beta3.Recommendations{ + Horizontal: v1beta3.HorizontalRecommendations{ + TargetUtilizations: []v1beta3.HPATargetUtilizationRecommendationPerContainer{ + { + ContainerName: "app", + TargetUtilization: map[v1.ResourceName]int32{ + v1.ResourceMemory: 90, + }, + }, + }, + MaxReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 5, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + MinReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 3, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + }, + }, + }, + }, + now: now.Time, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(10), // Current min is high (from emergency) + MaxReplicas: 5, // But max is low + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](60), + }, + Container: "app", + }, + }, + }, + }, + }, + want: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + Behavior: defaultHPABehaviorValue.DeepCopy(), + MinReplicas: ptrInt32(9), // Reduced by 0.95 factor (10 * 0.95 = 9.5, truncated to 9) + MaxReplicas: 9, // Temporarily increased to accommodate minReplicas + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](90), + }, + Container: "app", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "Emergency phase should set minReplicas to maxReplicas", + args: args{ + ctx: context.Background(), + tortoise: &v1beta3.Tortoise{ + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseEmergency, + AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + Conditions: v1beta3.Conditions{ + TortoiseConditions: []v1beta3.TortoiseCondition{ + { + Type: v1beta3.TortoiseConditionTypeHPATargetUtilizationUpdated, + Status: v1.ConditionTrue, + LastUpdateTime: metav1.NewTime(now.Add(-3 * time.Hour)), + LastTransitionTime: metav1.NewTime(now.Add(-3 * time.Hour)), + Reason: "HPATargetUtilizationUpdated", + Message: "HPA target utilization is updated", + }, + }, + }, + ContainerResourcePhases: []v1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]v1beta3.ResourcePhase{ + v1.ResourceMemory: { + Phase: v1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: v1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + Recommendations: v1beta3.Recommendations{ + Horizontal: v1beta3.HorizontalRecommendations{ + TargetUtilizations: []v1beta3.HPATargetUtilizationRecommendationPerContainer{ + { + ContainerName: "app", + TargetUtilization: map[v1.ResourceName]int32{ + v1.ResourceMemory: 90, + }, + }, + }, + MaxReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 10, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + MinReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 3, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + }, + }, + }, + }, + now: now.Time, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(3), + MaxReplicas: 10, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](60), + }, + Container: "app", + }, + }, + }, + }, + }, + want: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + Behavior: defaultHPABehaviorValue.DeepCopy(), + MinReplicas: ptrInt32(10), // Should be set to maxReplicas in emergency + MaxReplicas: 10, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](90), + }, + Container: "app", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "User-specified MaxReplicas limit should be respected", + args: args{ + ctx: context.Background(), + tortoise: &v1beta3.Tortoise{ + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + MaxReplicas: ptr.To[int32](5), // User-specified limit + }, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + Conditions: v1beta3.Conditions{ + TortoiseConditions: []v1beta3.TortoiseCondition{ + { + Type: v1beta3.TortoiseConditionTypeHPATargetUtilizationUpdated, + Status: v1.ConditionTrue, + LastUpdateTime: metav1.NewTime(now.Add(-3 * time.Hour)), + LastTransitionTime: metav1.NewTime(now.Add(-3 * time.Hour)), + Reason: "HPATargetUtilizationUpdated", + Message: "HPA target utilization is updated", + }, + }, + }, + ContainerResourcePhases: []v1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]v1beta3.ResourcePhase{ + v1.ResourceMemory: { + Phase: v1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: v1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + Recommendations: v1beta3.Recommendations{ + Horizontal: v1beta3.HorizontalRecommendations{ + TargetUtilizations: []v1beta3.HPATargetUtilizationRecommendationPerContainer{ + { + ContainerName: "app", + TargetUtilization: map[v1.ResourceName]int32{ + v1.ResourceMemory: 90, + }, + }, + }, + MaxReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 10, // Exceeds user-specified limit (5) + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + MinReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 3, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + }, + }, + }, + }, + now: now.Time, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(3), + MaxReplicas: 10, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](60), + }, + Container: "app", + }, + }, + }, + }, + }, + want: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + Behavior: defaultHPABehaviorValue.DeepCopy(), + MinReplicas: ptrInt32(3), + MaxReplicas: 5, // Should be capped to user-specified limit + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](90), + }, + Container: "app", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "MinReplicas above cluster-wide maximum should be capped", + args: args{ + ctx: context.Background(), + tortoise: &v1beta3.Tortoise{ + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + Conditions: v1beta3.Conditions{ + TortoiseConditions: []v1beta3.TortoiseCondition{ + { + Type: v1beta3.TortoiseConditionTypeHPATargetUtilizationUpdated, + Status: v1.ConditionTrue, + LastUpdateTime: metav1.NewTime(now.Add(-3 * time.Hour)), + LastTransitionTime: metav1.NewTime(now.Add(-3 * time.Hour)), + Reason: "HPATargetUtilizationUpdated", + Message: "HPA target utilization is updated", + }, + }, + }, + ContainerResourcePhases: []v1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]v1beta3.ResourcePhase{ + v1.ResourceMemory: { + Phase: v1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: v1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + Recommendations: v1beta3.Recommendations{ + Horizontal: v1beta3.HorizontalRecommendations{ + TargetUtilizations: []v1beta3.HPATargetUtilizationRecommendationPerContainer{ + { + ContainerName: "app", + TargetUtilization: map[v1.ResourceName]int32{ + v1.ResourceMemory: 90, + }, + }, + }, + MaxReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 10, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + MinReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 1000, // Above cluster-wide maximum (1000) + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + }, + }, + }, + }, + now: now.Time, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(3), + MaxReplicas: 10, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](60), + }, + Container: "app", + }, + }, + }, + }, + }, + want: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + Behavior: defaultHPABehaviorValue.DeepCopy(), + MinReplicas: ptrInt32(1000), // Should be capped to cluster-wide maximum + MaxReplicas: 1000, // Should also be adjusted to match + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](90), + }, + Container: "app", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "Final validation should ensure minReplicas doesn't exceed maxReplicas after BackToNormal", + args: args{ + ctx: context.Background(), + tortoise: &v1beta3.Tortoise{ + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseBackToNormal, + AutoscalingPolicy: []v1beta3.ContainerAutoscalingPolicy{ + { + ContainerName: "app", + Policy: map[v1.ResourceName]v1beta3.AutoscalingType{ + v1.ResourceMemory: v1beta3.AutoscalingTypeHorizontal, + }, + }, + }, + Conditions: v1beta3.Conditions{ + TortoiseConditions: []v1beta3.TortoiseCondition{ + { + Type: v1beta3.TortoiseConditionTypeHPATargetUtilizationUpdated, + Status: v1.ConditionTrue, + LastUpdateTime: metav1.NewTime(now.Add(-3 * time.Hour)), + LastTransitionTime: metav1.NewTime(now.Add(-3 * time.Hour)), + Reason: "HPATargetUtilizationUpdated", + Message: "HPA target utilization is updated", + }, + }, + }, + ContainerResourcePhases: []v1beta3.ContainerResourcePhases{ + { + ContainerName: "app", + ResourcePhases: map[v1.ResourceName]v1beta3.ResourcePhase{ + v1.ResourceMemory: { + Phase: v1beta3.ContainerResourcePhaseWorking, + }, + }, + }, + }, + Targets: v1beta3.TargetsStatus{ + HorizontalPodAutoscaler: "hpa", + }, + Recommendations: v1beta3.Recommendations{ + Horizontal: v1beta3.HorizontalRecommendations{ + TargetUtilizations: []v1beta3.HPATargetUtilizationRecommendationPerContainer{ + { + ContainerName: "app", + TargetUtilization: map[v1.ResourceName]int32{ + v1.ResourceMemory: 90, + }, + }, + }, + MaxReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 5, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + MinReplicas: []v1beta3.ReplicasRecommendation{ + { + From: 0, + To: 2, + Value: 3, + UpdatedAt: now, + WeekDay: ptr.To(now.Weekday().String()), + }, + }, + }, + }, + }, + }, + now: now.Time, + }, + initialHPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(20), // Very high current min (from emergency) + MaxReplicas: 5, // But max is low + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](60), + }, + Container: "app", + }, + }, + }, + }, + }, + want: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hpa", + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + Behavior: defaultHPABehaviorValue.DeepCopy(), + MinReplicas: ptrInt32(19), // Reduced by 0.95 factor (20 * 0.95 = 19) + MaxReplicas: 19, // Temporarily increased to accommodate minReplicas + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceMemory, + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](90), + }, + Container: "app", + }, + }, + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests {