diff --git a/docs/releases/1.25-NOTES.md b/docs/releases/1.25-NOTES.md index 92e46d093fe6c..7763e9d65d3d0 100644 --- a/docs/releases/1.25-NOTES.md +++ b/docs/releases/1.25-NOTES.md @@ -9,6 +9,7 @@ This is a document to gather the release notes prior to the release. ## Other significant changes +* Add support to --cordon-node-before-terminating on the cluster autoscaler addon (CordonNodeBeforeTerminating) # Breaking changes diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index c7efe3bc216f1..1017d908a8c67 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -660,6 +660,10 @@ spec: description: 'BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one. Default: false' type: boolean + cordonNodeBeforeTerminating: + description: 'CordonNodeBeforeTerminating should CA cordon nodes + before terminating during downscale process Default: false' + type: boolean cpuRequest: anyOf: - type: integer diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 159e10dc9bf91..410b6c1057d4d 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -1013,6 +1013,9 @@ type ClusterAutoscalerConfig struct { // ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes // Default: 10m0s ScaleDownDelayAfterAdd *string `json:"scaleDownDelayAfterAdd,omitempty"` + // CordonNodeBeforeTerminating should CA cordon nodes before terminating during downscale process + // Default: false + CordonNodeBeforeTerminating *bool `json:"cordonNodeBeforeTerminating,omitempty"` // Image is the docker container used. // Default: the latest supported image for the specified kubernetes version. Image *string `json:"image,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 622fb3107054b..1d20c6697b0ba 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -1039,6 +1039,9 @@ type ClusterAutoscalerConfig struct { // ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes // Default: 10m0s ScaleDownDelayAfterAdd *string `json:"scaleDownDelayAfterAdd,omitempty"` + // CordonNodeBeforeTerminating should CA cordon nodes before terminating during downscale process + // Default: false + CordonNodeBeforeTerminating *bool `json:"cordonNodeBeforeTerminating,omitempty"` // Image is the docker container used. // Default: the latest supported image for the specified kubernetes version. Image *string `json:"image,omitempty"` diff --git a/pkg/apis/kops/v1alpha3/componentconfig.go b/pkg/apis/kops/v1alpha3/componentconfig.go index 8d43b5eb837fc..6b2a18eafab5b 100644 --- a/pkg/apis/kops/v1alpha3/componentconfig.go +++ b/pkg/apis/kops/v1alpha3/componentconfig.go @@ -1010,6 +1010,9 @@ type ClusterAutoscalerConfig struct { // ScaleDownDelayAfterAdd determines the time after scale up that scale down evaluation resumes // Default: 10m0s ScaleDownDelayAfterAdd *string `json:"scaleDownDelayAfterAdd,omitempty"` + // CordonNodeBeforeTerminating should CA cordon nodes before terminating during downscale process + // Default: false + CordonNodeBeforeTerminating *bool `json:"cordonNodeBeforeTerminating,omitempty"` // Image is the docker container used. // Default: the latest supported image for the specified kubernetes version. Image *string `json:"image,omitempty"` diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index d30f830d0131b..b493eb09bd504 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -41,7 +41,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: c20d5158b4c6343a85e02495db2ae251e06f1ada95bfde99b471dc4b8c447092 + manifestHash: 35fe99c65eeb24b151986c6dc4cf255cc3012d6e089083972a48777a293a902a name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index 7671f9de297e9..58c8df2baf72f 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -330,6 +330,7 @@ spec: - --scale-down-delay-after-add=10m0s - --new-pod-scale-up-delay=0s - --max-node-provision-time=15m0s + - --cordon-node-before-terminating="true" - --logtostderr=true - --stderrthreshold=info - --v=4 diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 67b5a8fa0262b..77c77e1087107 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -48,7 +48,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: abb81e68a6726627a4b2b0d7baf06e73ea73be8c7965fec9d4ea85424ad0b8b2 + manifestHash: f9b26aa3a37f54436809ebe832096cbe3af212774a60d0b9e17498fdd4a06dd7 name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index d0b22e1bdd39e..ed930e10e15e4 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa23/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -330,6 +330,7 @@ spec: - --scale-down-delay-after-add=10m0s - --new-pod-scale-up-delay=0s - --max-node-provision-time=15m0s + - --cordon-node-before-terminating="true" - --logtostderr=true - --stderrthreshold=info - --v=4 diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 28c34066e5272..c770a659731ef 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -48,7 +48,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: 32c2d0d535f2a11e9034cf857011b214402f08c16b6bc1cf22cfbd54f2a62a5a + manifestHash: faa9596e64df6bbf071647f3a74acb0230cccb801a330bccb67821251eb73140 name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index 7a2cd091790da..9654b6b3379c5 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa24/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -330,6 +330,7 @@ spec: - --scale-down-delay-after-add=10m0s - --new-pod-scale-up-delay=0s - --max-node-provision-time=15m0s + - --cordon-node-before-terminating="true" - --logtostderr=true - --stderrthreshold=info - --v=4 diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 3a5bdae6a1e6d..b207e46352827 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -48,7 +48,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: 32c2d0d535f2a11e9034cf857011b214402f08c16b6bc1cf22cfbd54f2a62a5a + manifestHash: faa9596e64df6bbf071647f3a74acb0230cccb801a330bccb67821251eb73140 name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index 7a2cd091790da..9654b6b3379c5 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -330,6 +330,7 @@ spec: - --scale-down-delay-after-add=10m0s - --new-pod-scale-up-delay=0s - --max-node-provision-time=15m0s + - --cordon-node-before-terminating="true" - --logtostderr=true - --stderrthreshold=info - --v=4 diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 0f7732ef71ffd..4d4473b15fa48 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -41,7 +41,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: 32c2d0d535f2a11e9034cf857011b214402f08c16b6bc1cf22cfbd54f2a62a5a + manifestHash: faa9596e64df6bbf071647f3a74acb0230cccb801a330bccb67821251eb73140 name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index 7a2cd091790da..9654b6b3379c5 100644 --- a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -330,6 +330,7 @@ spec: - --scale-down-delay-after-add=10m0s - --new-pod-scale-up-delay=0s - --max-node-provision-time=15m0s + - --cordon-node-before-terminating="true" - --logtostderr=true - --stderrthreshold=info - --v=4 diff --git a/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 96f3317cea46f..fcee55f5e9123 100644 --- a/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -41,7 +41,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: b0ff8e67871acaff74c97c66a2db69b8b532a133176a8ce851891f4bb5ffaed3 + manifestHash: 5c85e5d23ecb0579b1da6d1faa0f10e08b4f1255f260c33dd739ec0b8a8df9d1 name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index a59507beca6af..1340c185ad07a 100644 --- a/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -333,6 +333,7 @@ spec: - --scale-down-delay-after-add=10m0s - --new-pod-scale-up-delay=0s - --max-node-provision-time=15m0s + - --cordon-node-before-terminating="true" - --logtostderr=true - --stderrthreshold=info - --v=4 diff --git a/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-bootstrap_content index 68de60966c6c2..ae1d0f56e90cc 100644 --- a/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-bootstrap_content @@ -41,7 +41,7 @@ spec: version: 9.99.0 - id: k8s-1.15 manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml - manifestHash: 0b7796d3ad2a8f30ea9c082d3148825ce8f729d2b41b3894fff1b73f0698c350 + manifestHash: d013e73853c14acc8e433b9a8819f5027d695cdefbaa7f7d369210a0a23db8e9 name: cluster-autoscaler.addons.k8s.io selector: k8s-addon: cluster-autoscaler.addons.k8s.io diff --git a/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content index 278757ebae41f..86579f802d7ec 100644 --- a/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content +++ b/tests/integration/update_cluster/many-addons/data/aws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content @@ -334,6 +334,7 @@ spec: - --scale-down-delay-after-add=10m0s - --new-pod-scale-up-delay=0s - --max-node-provision-time=15m0s + - --cordon-node-before-terminating="true" - --logtostderr=true - --stderrthreshold=info - --v=4 diff --git a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template index ce5e1c0221795..e93d05cc7871e 100644 --- a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template @@ -329,6 +329,10 @@ spec: - --scale-down-delay-after-add={{ .ScaleDownDelayAfterAdd }} - --new-pod-scale-up-delay={{ .NewPodScaleUpDelay }} - --max-node-provision-time={{ .MaxNodeProvisionTime }} + # This flag does not exist before CAS 1.21 + {{ if IsKubernetesGTE "1.21" }} + - --cordon-node-before-terminating="{{ WithDefaultBool .CordonNodeBeforeTerminating true }}" + {{ end }} - --logtostderr=true - --stderrthreshold=info - --v=4