diff --git a/cluster-autoscaler/clusterstate/clusterstate.go b/cluster-autoscaler/clusterstate/clusterstate.go index 9665d83a431e..823b527356f3 100644 --- a/cluster-autoscaler/clusterstate/clusterstate.go +++ b/cluster-autoscaler/clusterstate/clusterstate.go @@ -28,6 +28,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/clusterstate/api" "k8s.io/autoscaler/cluster-autoscaler/clusterstate/utils" "k8s.io/autoscaler/cluster-autoscaler/metrics" + node_processors "k8s.io/autoscaler/cluster-autoscaler/processors/nodes" "k8s.io/autoscaler/cluster-autoscaler/utils/backoff" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" "k8s.io/autoscaler/cluster-autoscaler/utils/taints" @@ -136,6 +137,9 @@ type ClusterStateRegistry struct { cloudProviderNodeInstancesCache *utils.CloudProviderNodeInstancesCache interrupt chan struct{} maxNodeProvisionTimeProvider maxNodeProvisionTimeProvider + // scaleDownDelayStatus is for tracking nodegroups which should be + // in scaledown cooldown + scaleDownDelayStatus *node_processors.ScaleDownDelayStatus // scaleUpFailures contains information about scale-up failures for each node group. It should be // cleared periodically to avoid unnecessary accumulation. @@ -168,6 +172,11 @@ func NewClusterStateRegistry(cloudProvider cloudprovider.CloudProvider, config C interrupt: make(chan struct{}), scaleUpFailures: make(map[string][]ScaleUpFailure), maxNodeProvisionTimeProvider: maxNodeProvisionTimeProvider, + scaleDownDelayStatus: &node_processors.ScaleDownDelayStatus{ + ScaleDownDelayAfterAdd: node_processors.ScaleDownDelay{ + PerNG: make(map[string]time.Time), + }, + }, } } @@ -193,6 +202,13 @@ func (csr *ClusterStateRegistry) RegisterOrUpdateScaleUp(nodeGroup cloudprovider csr.registerOrUpdateScaleUpNoLock(nodeGroup, delta, currentTime) } +// UpdateScaleDownDelayStatus updates ScaleDownDelayStatus to keep track of nodegroups which were recently scaled up +func (csr *ClusterStateRegistry) UpdateScaleDownDelayStatus(nodeGroup cloudprovider.NodeGroup, currentTime time.Time) { + csr.Lock() + defer csr.Unlock() + csr.scaleDownDelayStatus.ScaleDownDelayAfterAdd.PerNG[nodeGroup.Id()] = currentTime +} + // MaxNodeProvisionTime returns MaxNodeProvisionTime value that should be used for the given NodeGroup. func (csr *ClusterStateRegistry) MaxNodeProvisionTime(nodeGroup cloudprovider.NodeGroup) (time.Duration, error) { return csr.maxNodeProvisionTimeProvider.GetMaxNodeProvisionTime(nodeGroup) @@ -1211,3 +1227,8 @@ func (csr *ClusterStateRegistry) GetScaleUpFailures() map[string][]ScaleUpFailur } return result } + +// GetScaleDownDelayStatus returns scale-down delay status. +func (csr *ClusterStateRegistry) GetScaleDownDelayStatus() *node_processors.ScaleDownDelayStatus { + return csr.scaleDownDelayStatus +} diff --git a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go index b851a9e5996b..1bb29e66967d 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go @@ -591,6 +591,7 @@ func (o *ScaleUpOrchestrator) executeScaleUp( o.clusterStateRegistry.RegisterFailedScaleUp(info.Group, metrics.FailedScaleUpReason(string(aerr.Type())), now) return aerr } + o.clusterStateRegistry.UpdateScaleDownDelayStatus(info.Group, time.Now()) o.clusterStateRegistry.RegisterOrUpdateScaleUp( info.Group, increase, diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index ec4d7b96d896..02e952ee4a1c 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -47,7 +47,6 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/expander" "k8s.io/autoscaler/cluster-autoscaler/metrics" ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors" - node_processors "k8s.io/autoscaler/cluster-autoscaler/processors/nodes" "k8s.io/autoscaler/cluster-autoscaler/processors/status" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" @@ -92,7 +91,6 @@ type StaticAutoscaler struct { processorCallbacks *staticAutoscalerProcessorCallbacks initialized bool taintConfig taints.TaintConfig - scaleDownDelayStatus *node_processors.ScaleDownDelayStatus } type staticAutoscalerProcessorCallbacks struct { @@ -208,9 +206,6 @@ func NewStaticAutoscaler( processorCallbacks: processorCallbacks, clusterStateRegistry: clusterStateRegistry, taintConfig: taintConfig, - scaleDownDelayStatus: &node_processors.ScaleDownDelayStatus{ - ScaleDownDelayAfterAdd: node_processors.ScaleDownDelay{}, - }, } } @@ -594,7 +589,7 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr } else { var err caerrors.AutoscalerError scaleDownCandidates, err = a.processors.ScaleDownNodeProcessor.GetScaleDownCandidates( - autoscalingContext, allNodes, a.scaleDownDelayStatus) + autoscalingContext, allNodes, a.clusterStateRegistry.GetScaleDownDelayStatus()) if err != nil { klog.Error(err) return err @@ -625,6 +620,10 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr a.lastScaleDownFailTime.Add(a.ScaleDownDelayAfterFailure).After(currentTime) || a.lastScaleDownDeleteTime.Add(a.ScaleDownDelayAfterDelete).After(currentTime) + if !a.ScaleDownDelayTypeLocal { + scaleDownInCooldown = scaleDownInCooldown || a.lastScaleUpTime.Add(a.ScaleDownDelayAfterAdd).After(currentTime) + } + klog.V(4).Infof("Scale down status: lastScaleUpTime=%s lastScaleDownDeleteTime=%v "+ "lastScaleDownFailTime=%s scaleDownForbidden=%v scaleDownInCooldown=%v", a.lastScaleUpTime, a.lastScaleDownDeleteTime, a.lastScaleDownFailTime, diff --git a/cluster-autoscaler/processors/nodes/types.go b/cluster-autoscaler/processors/nodes/types.go index ff0fbb3fa024..e4d53ff6094f 100644 --- a/cluster-autoscaler/processors/nodes/types.go +++ b/cluster-autoscaler/processors/nodes/types.go @@ -17,6 +17,8 @@ limitations under the License. package nodes import ( + "time" + apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/context" @@ -48,6 +50,5 @@ type ScaleDownDelayStatus struct { } type ScaleDownDelay struct { - PerNG map[string]bool - Aggregated bool + PerNG map[string]time.Time } diff --git a/cluster-autoscaler/processors/scaledowncandidates/scale_down_candidates_delay_processor.go b/cluster-autoscaler/processors/scaledowncandidates/scale_down_candidates_delay_processor.go index c7e5cee9069d..89afae70d327 100644 --- a/cluster-autoscaler/processors/scaledowncandidates/scale_down_candidates_delay_processor.go +++ b/cluster-autoscaler/processors/scaledowncandidates/scale_down_candidates_delay_processor.go @@ -18,6 +18,7 @@ package scaledowncandidates import ( "reflect" + "time" apiv1 "k8s.io/api/core/v1" "k8s.io/klog/v2" @@ -54,10 +55,13 @@ func (p *ScaleDownCandidatesDelayProcessor) GetScaleDownCandidates(ctx *context. continue } - if _, ok := scaleDownDelayStatus.ScaleDownDelayAfterAdd.PerNG[nodeGroup.Id()]; ok { - klog.V(4).Infof("Skipping scale down on node group %s because it was scaled up recently", nodeGroup.Id()) + lastScaledUpTime, ok := scaleDownDelayStatus.ScaleDownDelayAfterAdd.PerNG[nodeGroup.Id()] + + if ok && lastScaledUpTime.Add(ctx.ScaleDownDelayAfterAdd).Before(time.Now()) { + klog.V(4).Infof("Skipping scale down on node group %s because it was scaled up recently at %v", nodeGroup.Id(), lastScaledUpTime.String()) continue } + result = append(result, node) } return result, nil diff --git a/cluster-autoscaler/processors/scaledowncandidates/scale_down_candidates_sorting_processor.go b/cluster-autoscaler/processors/scaledowncandidates/scale_down_candidates_sorting_processor.go index bca043429084..7729b8f77f34 100644 --- a/cluster-autoscaler/processors/scaledowncandidates/scale_down_candidates_sorting_processor.go +++ b/cluster-autoscaler/processors/scaledowncandidates/scale_down_candidates_sorting_processor.go @@ -40,8 +40,8 @@ func (p *ScaleDownCandidatesSortingProcessor) GetPodDestinationCandidates(ctx *c // GetScaleDownCandidates returns filter nodes and move previous scale down candidates to the beginning of the list. func (p *ScaleDownCandidatesSortingProcessor) GetScaleDownCandidates(ctx *context.AutoscalingContext, - nodes []*apiv1.Node, _ *nodes.ScaleDownDelayStatus) ([]*apiv1.Node, errors.AutoscalerError) { - candidates, err := p.preFilter.GetScaleDownCandidates(ctx, nodes) + nodes []*apiv1.Node, scaleDownDelayStatus *nodes.ScaleDownDelayStatus) ([]*apiv1.Node, errors.AutoscalerError) { + candidates, err := p.preFilter.GetScaleDownCandidates(ctx, nodes, scaleDownDelayStatus) if err != nil { return candidates, err }