Add suport to --cordon-node-before-terminating autoscaler flag

This feature was added on k8s autoscaler on Oct 2020: kubernetes/autoscaler#3649 but kops didn't provide support to add it via the autoscaler addon This PR adds it
dcfranca · Sep 6, 2022 · a594b53 · a594b53
1 parent 202c333
commit a594b53
Show file tree

Hide file tree

Showing 20 changed files with 32 additions and 7 deletions.
diff --git a/docs/releases/1.25-NOTES.md b/docs/releases/1.25-NOTES.md
@@ -9,6 +9,7 @@ This is a document to gather the release notes prior to the release.
 
 ## Other significant changes
 
+* Add support to --cordon-node-before-terminating on the cluster autoscaler addon (CordonNodeBeforeTerminating)
 
 # Breaking changes
 

diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -660,6 +660,10 @@ spec:
                     description: 'BalanceSimilarNodeGroups makes cluster autoscaler
                       treat similar node groups as one. Default: false'
                     type: boolean
+                  cordonNodeBeforeTerminating:
+                    description: 'CordonNodeBeforeTerminating should CA cordon nodes
+                      before terminating during downscale process Default: false'
+                    type: boolean
                   cpuRequest:
                     anyOf:
                     - type: integer

diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go
@@ -1013,6 +1013,9 @@ type ClusterAutoscalerConfig struct {
 	// ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes
 	// Default: 10m0s
 	ScaleDownDelayAfterAdd *string `json:"scaleDownDelayAfterAdd,omitempty"`
+	// CordonNodeBeforeTerminating should CA cordon nodes before terminating during downscale process
+	// Default: false
+	CordonNodeBeforeTerminating *bool `json:"cordonNodeBeforeTerminating,omitempty"`
 	// Image is the docker container used.
 	// Default: the latest supported image for the specified kubernetes version.
 	Image *string `json:"image,omitempty"`

diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go
@@ -1039,6 +1039,9 @@ type ClusterAutoscalerConfig struct {
 	// ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes
 	// Default: 10m0s
 	ScaleDownDelayAfterAdd *string `json:"scaleDownDelayAfterAdd,omitempty"`
+	// CordonNodeBeforeTerminating should CA cordon nodes before terminating during downscale process
+	// Default: false
+	CordonNodeBeforeTerminating *bool `json:"cordonNodeBeforeTerminating,omitempty"`
 	// Image is the docker container used.
 	// Default: the latest supported image for the specified kubernetes version.
 	Image *string `json:"image,omitempty"`

diff --git a/pkg/apis/kops/v1alpha3/componentconfig.go b/pkg/apis/kops/v1alpha3/componentconfig.go
@@ -1010,6 +1010,9 @@ type ClusterAutoscalerConfig struct {
 	// ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes
 	// Default: 10m0s
 	ScaleDownDelayAfterAdd *string `json:"scaleDownDelayAfterAdd,omitempty"`
+	// CordonNodeBeforeTerminating should CA cordon nodes before terminating during downscale process
+	// Default: false
+	CordonNodeBeforeTerminating *bool `json:"cordonNodeBeforeTerminating,omitempty"`
 	// Image is the docker container used.
 	// Default: the latest supported image for the specified kubernetes version.
 	Image *string `json:"image,omitempty"`

diff --git a/...ster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...ster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: c20d5158b4c6343a85e02495db2ae251e06f1ada95bfde99b471dc4b8c447092
+    manifestHash: 35fe99c65eeb24b151986c6dc4cf255cc3012d6e089083972a48777a293a902a
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -330,6 +330,7 @@ spec:
         - --scale-down-delay-after-add=10m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
+        - --cordon-node-before-terminating="true"
         - --logtostderr=true
         - --stderrthreshold=info
         - --v=4

diff --git a/...er/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...er/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: abb81e68a6726627a4b2b0d7baf06e73ea73be8c7965fec9d4ea85424ad0b8b2
+    manifestHash: f9b26aa3a37f54436809ebe832096cbe3af212774a60d0b9e17498fdd4a06dd7
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -330,6 +330,7 @@ spec:
         - --scale-down-delay-after-add=10m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
+        - --cordon-node-before-terminating="true"
         - --logtostderr=true
         - --stderrthreshold=info
         - --v=4

diff --git a/...er/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...er/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 32c2d0d535f2a11e9034cf857011b214402f08c16b6bc1cf22cfbd54f2a62a5a
+    manifestHash: faa9596e64df6bbf071647f3a74acb0230cccb801a330bccb67821251eb73140
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -330,6 +330,7 @@ spec:
         - --scale-down-delay-after-add=10m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
+        - --cordon-node-before-terminating="true"
         - --logtostderr=true
         - --stderrthreshold=info
         - --v=4

diff --git a/...er/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...er/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 32c2d0d535f2a11e9034cf857011b214402f08c16b6bc1cf22cfbd54f2a62a5a
+    manifestHash: faa9596e64df6bbf071647f3a74acb0230cccb801a330bccb67821251eb73140
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -330,6 +330,7 @@ spec:
         - --scale-down-delay-after-add=10m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
+        - --cordon-node-before-terminating="true"
         - --logtostderr=true
         - --stderrthreshold=info
         - --v=4

diff --git a/...er/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...er/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 32c2d0d535f2a11e9034cf857011b214402f08c16b6bc1cf22cfbd54f2a62a5a
+    manifestHash: faa9596e64df6bbf071647f3a74acb0230cccb801a330bccb67821251eb73140
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -330,6 +330,7 @@ spec:
         - --scale-down-delay-after-add=10m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
+        - --cordon-node-before-terminating="true"
         - --logtostderr=true
         - --stderrthreshold=info
         - --v=4

diff --git a/...e_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...e_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: b0ff8e67871acaff74c97c66a2db69b8b532a133176a8ce851891f4bb5ffaed3
+    manifestHash: 5c85e5d23ecb0579b1da6d1faa0f10e08b4f1255f260c33dd739ec0b8a8df9d1
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -333,6 +333,7 @@ spec:
         - --scale-down-delay-after-add=10m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
+        - --cordon-node-before-terminating="true"
         - --logtostderr=true
         - --stderrthreshold=info
         - --v=4

diff --git a/...pdate_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...pdate_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 0b7796d3ad2a8f30ea9c082d3148825ce8f729d2b41b3894fff1b73f0698c350
+    manifestHash: d013e73853c14acc8e433b9a8819f5027d695cdefbaa7f7d369210a0a23db8e9
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -334,6 +334,7 @@ spec:
         - --scale-down-delay-after-add=10m0s
         - --new-pod-scale-up-delay=0s
         - --max-node-provision-time=15m0s
+        - --cordon-node-before-terminating="true"
         - --logtostderr=true
         - --stderrthreshold=info
         - --v=4

diff --git a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template
@@ -329,6 +329,10 @@ spec:
             - --scale-down-delay-after-add={{ .ScaleDownDelayAfterAdd }}
             - --new-pod-scale-up-delay={{ .NewPodScaleUpDelay }}
             - --max-node-provision-time={{ .MaxNodeProvisionTime }}
+            # This flag does not exist before CAS 1.21
+            {{ if IsKubernetesGTE "1.21" }}
+            - --cordon-node-before-terminating="{{ WithDefaultBool .CordonNodeBeforeTerminating true }}"
+            {{ end }}
             - --logtostderr=true
             - --stderrthreshold=info
             - --v=4
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,6 +9,7 @@ This is a document to gather the release notes prior to the release.

		## Other significant changes

		* Add support to --cordon-node-before-terminating on the cluster autoscaler addon (CordonNodeBeforeTerminating)

		# Breaking changes

Expand Down