Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enforce provisioning requests processing even if all pods are new #7688

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions cluster-autoscaler/core/static_autoscaler.go
Original file line number Diff line number Diff line change
Expand Up @@ -520,13 +520,15 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr
return false, nil
}

forceScaleUp := a.processors.ScaleUpEnforcer.ShouldForceScaleUp(unschedulablePodsToHelp)

if len(unschedulablePodsToHelp) == 0 {
scaleUpStatus.Result = status.ScaleUpNotNeeded
klog.V(1).Info("No unschedulable pods")
} else if a.MaxNodesTotal > 0 && len(readyNodes) >= a.MaxNodesTotal {
} else if a.MaxNodesTotal > 0 && len(readyNodes) >= a.MaxNodesTotal && !forceScaleUp {
scaleUpStatus.Result = status.ScaleUpNoOptionsAvailable
klog.V(1).Infof("Max total nodes in cluster reached: %v. Current number of ready nodes: %v", a.MaxNodesTotal, len(readyNodes))
} else if len(a.BypassedSchedulers) == 0 && allPodsAreNew(unschedulablePodsToHelp, currentTime) {
} else if len(a.BypassedSchedulers) == 0 && !forceScaleUp && allPodsAreNew(unschedulablePodsToHelp, currentTime) {
// The assumption here is that these pods have been created very recently and probably there
// is more pods to come. In theory we could check the newest pod time but then if pod were created
// slowly but at the pace of 1 every 2 seconds then no scale up would be triggered for long time.
Expand Down
2 changes: 2 additions & 0 deletions cluster-autoscaler/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,8 @@ func buildAutoscaler(context ctx.Context, debuggingSnapshotter debuggingsnapshot
opts.LoopStartNotifier = loopstart.NewObserversList([]loopstart.Observer{provreqProcesor})

podListProcessor.AddProcessor(provreqProcesor)

opts.Processors.ScaleUpEnforcer = provreq.NewProvisioningRequestScaleUpEnforcer()
}

if *proactiveScaleupEnabled {
Expand Down
38 changes: 38 additions & 0 deletions cluster-autoscaler/processors/pods/scaleup_enforcer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
Copyright 2025 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pods

import apiv1 "k8s.io/api/core/v1"

// ScaleUpEnforcer can force scale up even if all pods are new or MaxNodesTotal was achieved.
type ScaleUpEnforcer interface {
ShouldForceScaleUp(unschedulablePods []*apiv1.Pod) bool
}

// NoOpScaleUpEnforcer returns false by default in case of ProvisioningRequests disabled.
type NoOpScaleUpEnforcer struct {
}

// NewDefaultScaleUpEnforcer creates an instance of ScaleUpEnforcer.
func NewDefaultScaleUpEnforcer() ScaleUpEnforcer {
return &NoOpScaleUpEnforcer{}
}

// ShouldForceScaleUp returns false by default.
func (p *NoOpScaleUpEnforcer) ShouldForceScaleUp(unschedulablePods []*apiv1.Pod) bool {
return false
}
34 changes: 34 additions & 0 deletions cluster-autoscaler/processors/pods/scaleup_enforcer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
Copyright 2025 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pods

import (
"testing"

apiv1 "k8s.io/api/core/v1"
testutils "k8s.io/autoscaler/cluster-autoscaler/utils/test"
)

func TestDefaultScaleUpEnforcer(t *testing.T) {
p1 := testutils.BuildTestPod("p1", 40, 0)
unschedulablePods := []*apiv1.Pod{p1}
scaleUpEnforcer := NewDefaultScaleUpEnforcer()
forceScaleUp := scaleUpEnforcer.ShouldForceScaleUp(unschedulablePods)
if forceScaleUp {
t.Errorf("Error: scaleUpEnforcer should not force scale up by default")
}
}
3 changes: 3 additions & 0 deletions cluster-autoscaler/processors/processors.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ type AutoscalingProcessors struct {
ScaleStateNotifier *nodegroupchange.NodeGroupChangeObserversList
// AsyncNodeGroupChecker checks if node group is upcoming or not
AsyncNodeGroupStateChecker asyncnodegroups.AsyncNodeGroupStateChecker
// ScaleUpEnforcer can force scale up even if all pods are new or MaxNodesTotal was achieved.
ScaleUpEnforcer pods.ScaleUpEnforcer
}

// DefaultProcessors returns default set of processors.
Expand All @@ -100,6 +102,7 @@ func DefaultProcessors(options config.AutoscalingOptions) *AutoscalingProcessors
TemplateNodeInfoProvider: nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false),
ScaleDownCandidatesNotifier: scaledowncandidates.NewObserversList(),
ScaleStateNotifier: nodegroupchange.NewNodeGroupChangeObserversList(),
ScaleUpEnforcer: pods.NewDefaultScaleUpEnforcer(),
}
}

Expand Down
2 changes: 1 addition & 1 deletion cluster-autoscaler/processors/provreq/pods_filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (

apiv1 "k8s.io/api/core/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
v1 "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
"k8s.io/autoscaler/cluster-autoscaler/context"
"k8s.io/autoscaler/cluster-autoscaler/processors/pods"
provreqpods "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/pods"
Expand Down
41 changes: 41 additions & 0 deletions cluster-autoscaler/processors/provreq/scaleup_enforcer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
Copyright 2025 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provreq

import (
apiv1 "k8s.io/api/core/v1"
"k8s.io/autoscaler/cluster-autoscaler/processors/pods"
)

// ProvisioningRequestScaleUpEnforcer forces scale up if there is any unschedulable pod that belongs to ProvisioningRequest.
type ProvisioningRequestScaleUpEnforcer struct {
}

// NewProvisioningRequestScaleUpEnforcer creates a ProvisioningRequest scale up enforcer.
func NewProvisioningRequestScaleUpEnforcer() pods.ScaleUpEnforcer {
return &ProvisioningRequestScaleUpEnforcer{}
}

// ShouldForceScaleUp forces scale up if there is any unschedulable pod that belongs to ProvisioningRequest.
func (p *ProvisioningRequestScaleUpEnforcer) ShouldForceScaleUp(unschedulablePods []*apiv1.Pod) bool {
for _, pod := range unschedulablePods {
if _, ok := provisioningRequestName(pod); ok {
return true
}
}
return false
}
66 changes: 66 additions & 0 deletions cluster-autoscaler/processors/provreq/scaleup_enforcer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/*
Copyright 2025 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provreq

import (
"testing"

"github.com/stretchr/testify/assert"
apiv1 "k8s.io/api/core/v1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1"
"k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/pods"
testutils "k8s.io/autoscaler/cluster-autoscaler/utils/test"
)

func TestProvisioningRequestScaleUpEnforcer(t *testing.T) {
prPod1 := testutils.BuildTestPod("pr-pod-1", 500, 10)
prPod1.Annotations[v1.ProvisioningRequestPodAnnotationKey] = "pr-class"

prPod2 := testutils.BuildTestPod("pr-pod-2", 500, 10)
prPod2.Annotations[pods.DeprecatedProvisioningRequestPodAnnotationKey] = "pr-class-2"

pod1 := testutils.BuildTestPod("pod-1", 500, 10)
pod2 := testutils.BuildTestPod("pod-2", 500, 10)

testCases := map[string]struct {
unschedulablePods []*apiv1.Pod
want bool
}{
"Any pod with ProvisioningRequest annotation key forces scale up": {
unschedulablePods: []*corev1.Pod{prPod1, pod1},
want: true,
},
"Any pod with ProvisioningRequest deprecated annotation key forces scale up": {
unschedulablePods: []*corev1.Pod{prPod2, pod1},
want: true,
},
"Pod without ProvisioningRequest annotation key don't force scale up": {
unschedulablePods: []*corev1.Pod{pod1, pod2},
want: false,
},
"No pods don't force scale up": {
unschedulablePods: []*corev1.Pod{},
want: false,
},
}
for _, test := range testCases {
scaleUpEnforcer := NewProvisioningRequestScaleUpEnforcer()
got := scaleUpEnforcer.ShouldForceScaleUp(test.unschedulablePods)
assert.Equal(t, got, test.want)
}
}
2 changes: 2 additions & 0 deletions cluster-autoscaler/processors/test/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset"
"k8s.io/autoscaler/cluster-autoscaler/processors/nodeinfosprovider"
"k8s.io/autoscaler/cluster-autoscaler/processors/nodes"
"k8s.io/autoscaler/cluster-autoscaler/processors/pods"
"k8s.io/autoscaler/cluster-autoscaler/processors/scaledowncandidates"
"k8s.io/autoscaler/cluster-autoscaler/processors/status"
"k8s.io/autoscaler/cluster-autoscaler/simulator/scheduling"
Expand All @@ -56,5 +57,6 @@ func NewTestProcessors(context *context.AutoscalingContext) *processors.Autoscal
ScaleDownCandidatesNotifier: scaledowncandidates.NewObserversList(),
ScaleStateNotifier: nodegroupchange.NewNodeGroupChangeObserversList(),
AsyncNodeGroupStateChecker: asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker(),
ScaleUpEnforcer: pods.NewDefaultScaleUpEnforcer(),
}
}
Loading