chore: Include Updates to the v1 API for v0.36.x (#1506)

Makefile

@@ -75,7 +75,6 @@ verify: ## Verify code. Includes codegen, docgen, dependencies, linting, formatt
 	hack/validation/requirements.sh
 	hack/validation/labels.sh
 	hack/validation/resources.sh
-	hack/mutation/nodepool.sh
 	hack/dependabot.sh
 	@# Use perl instead of sed due to https://stackoverflow.com/questions/4247068/sed-command-with-i-option-failing-on-mac-but-works-on-linux
 	@# We need to do this "sed replace" until controller-tools fixes this parameterized types issue: https://github.com/kubernetes-sigs/controller-tools/issues/756

pkg/apis/crds/karpenter.sh_nodeclaims.yaml

@@ -628,7 +628,7 @@ spec:
                       - key
                       - operator
                     type: object
-                  maxItems: 30
+                  maxItems: 100
                   type: array
                   x-kubernetes-validations:
                     - message: requirements with operator 'In' must have a value defined

pkg/apis/crds/karpenter.sh_nodepools.yaml

@@ -145,13 +145,13 @@ spec:
                       pattern: ^(([0-9]+(s|m|h))+)|(Never)$
                       type: string
                     consolidationPolicy:
-                      default: WhenUnderutilized
+                      default: WhenEmptyOrUnderutilized
                       description: |-
                         ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation
-                        algorithm. This policy defaults to "WhenUnderutilized" if not specified
+                        algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified
                       enum:
                         - WhenEmpty
-                        - WhenUnderutilized
+                        - WhenEmptyOrUnderutilized
                       type: string
                   required:
                     - consolidateAfter
@@ -206,7 +206,10 @@ spec:
                               rule: self.all(x, x != "kubernetes.io/hostname")
                       type: object
                     spec:
-                      description: NodeClaimSpec describes the desired state of the NodeClaim
+                      description: |-
+                        NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool
+                        NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since
+                        users are not able to set resource requests in the NodePool.
                       properties:
                         expireAfter:
                           default: 720h
@@ -867,7 +870,7 @@ spec:
                               - key
                               - operator
                             type: object
-                          maxItems: 30
+                          maxItems: 100
                           type: array
                           x-kubernetes-validations:
                             - message: requirements with operator 'In' must have a value defined

pkg/apis/v1/nodeclaim_conversion_test.go

@@ -50,7 +50,7 @@ var _ = Describe("Convert v1 to v1beta1 NodeClaim API", func() {
 			ObjectMeta: test.ObjectMeta(),
 			Spec: NodePoolSpec{
 				Template: NodeClaimTemplate{
-					Spec: NodeClaimSpec{
+					Spec: NodeClaimTemplateSpec{
 						NodeClassRef: &NodeClassReference{
 							Name: "default",
 						},
@@ -269,7 +269,7 @@ var _ = Describe("Convert V1beta1 to V1 NodeClaim API", func() {
 			ObjectMeta: test.ObjectMeta(),
 			Spec: NodePoolSpec{
 				Template: NodeClaimTemplate{
-					Spec: NodeClaimSpec{
+					Spec: NodeClaimTemplateSpec{
 						NodeClassRef: &NodeClassReference{
 							Name: "default",
 						},

pkg/apis/v1/nodeclaim_validation.go

@@ -60,7 +60,7 @@ type taintKeyEffect struct {
 	Effect   v1.TaintEffect
 }
 
-func (in *NodeClaimSpec) validateTaints() (errs *apis.FieldError) {
+func (in *NodeClaimTemplateSpec) validateTaints() (errs *apis.FieldError) {
 	existing := map[taintKeyEffect]struct{}{}
 	errs = errs.Also(validateTaintsField(in.Taints, existing, "taints"))
 	errs = errs.Also(validateTaintsField(in.StartupTaints, existing, "startupTaints"))
@@ -104,7 +104,7 @@ func validateTaintsField(taints []v1.Taint, existing map[taintKeyEffect]struct{}
 // This function is used by the NodeClaim validation webhook to verify the nodepool requirements.
 // When this function is called, the nodepool's requirements do not include the requirements from labels.
 // NodeClaim requirements only support well known labels.
-func (in *NodeClaimSpec) validateRequirements() (errs *apis.FieldError) {
+func (in *NodeClaimTemplateSpec) validateRequirements() (errs *apis.FieldError) {
 	for i, requirement := range in.Requirements {
 		if err := ValidateRequirement(requirement); err != nil {
 			errs = errs.Also(apis.ErrInvalidArrayValue(err, "requirements", i))

pkg/apis/v1/nodepool.go

@@ -68,9 +68,9 @@ type Disruption struct {
 	// +required
 	ConsolidateAfter NillableDuration `json:"consolidateAfter"`
 	// ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation
-	// algorithm. This policy defaults to "WhenUnderutilized" if not specified
-	// +kubebuilder:default:="WhenUnderutilized"
-	// +kubebuilder:validation:Enum:={WhenEmpty,WhenUnderutilized}
+	// algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified
+	// +kubebuilder:default:="WhenEmptyOrUnderutilized"
+	// +kubebuilder:validation:Enum:={WhenEmpty,WhenEmptyOrUnderutilized}
 	// +optional
 	ConsolidationPolicy ConsolidationPolicy `json:"consolidationPolicy,omitempty"`
 	// Budgets is a list of Budgets.
@@ -124,8 +124,8 @@ type Budget struct {
 type ConsolidationPolicy string
 
 const (
-	ConsolidationPolicyWhenEmpty         ConsolidationPolicy = "WhenEmpty"
-	ConsolidationPolicyWhenUnderutilized ConsolidationPolicy = "WhenUnderutilized"
+	ConsolidationPolicyWhenEmpty                ConsolidationPolicy = "WhenEmpty"
+	ConsolidationPolicyWhenEmptyOrUnderutilized ConsolidationPolicy = "WhenEmptyOrUnderutilized"
 )
 
 // DisruptionReason defines valid reasons for disruption budgets.
@@ -162,7 +162,77 @@ func (l Limits) ExceededBy(resources v1.ResourceList) error {
 type NodeClaimTemplate struct {
 	ObjectMeta `json:"metadata,omitempty"`
 	// +required
-	Spec NodeClaimSpec `json:"spec"`
+	Spec NodeClaimTemplateSpec `json:"spec"`
+}
+
+// NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool
+// NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since
+// users are not able to set resource requests in the NodePool.
+type NodeClaimTemplateSpec struct {
+	// Taints will be applied to the NodeClaim's node.
+	// +optional
+	Taints []v1.Taint `json:"taints,omitempty"`
+	// StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically
+	// within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by
+	// daemonsets to allow initialization and enforce startup ordering.  StartupTaints are ignored for provisioning
+	// purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them.
+	// +optional
+	StartupTaints []v1.Taint `json:"startupTaints,omitempty"`
+	// Requirements are layered with GetLabels and applied to every node.
+	// +kubebuilder:validation:XValidation:message="requirements with operator 'In' must have a value defined",rule="self.all(x, x.operator == 'In' ? x.values.size() != 0 : true)"
+	// +kubebuilder:validation:XValidation:message="requirements operator 'Gt' or 'Lt' must have a single positive integer value",rule="self.all(x, (x.operator == 'Gt' || x.operator == 'Lt') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)"
+	// +kubebuilder:validation:XValidation:message="requirements with 'minValues' must have at least that many values specified in the 'values' field",rule="self.all(x, (x.operator == 'In' && has(x.minValues)) ? x.values.size() >= x.minValues : true)"
+	// +kubebuilder:validation:MaxItems:=100
+	// +required
+	Requirements []NodeSelectorRequirementWithMinValues `json:"requirements" hash:"ignore"`
+	// NodeClassRef is a reference to an object that defines provider specific configuration
+	// +required
+	NodeClassRef *NodeClassReference `json:"nodeClassRef"`
+	// TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated.
+	//
+	// Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation.
+	//
+	// This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period.
+	// When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached.
+	//
+	// Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod.
+	// If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout,
+	// that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds.
+	//
+	// The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks.
+	// If left undefined, the controller will wait indefinitely for pods to be drained.
+	// +kubebuilder:validation:Pattern=`^([0-9]+(s|m|h))+$`
+	// +kubebuilder:validation:Type="string"
+	// +optional
+	TerminationGracePeriod *metav1.Duration `json:"terminationGracePeriod,omitempty"`
+	// ExpireAfter is the duration the controller will wait
+	// before terminating a node, measured from when the node is created. This
+	// is useful to implement features like eventually consistent node upgrade,
+	// memory leak protection, and disruption testing.
+	// +kubebuilder:default:="720h"
+	// +kubebuilder:validation:Pattern=`^(([0-9]+(s|m|h))+)|(Never)$`
+	// +kubebuilder:validation:Type="string"
+	// +kubebuilder:validation:Schemaless
+	// +optional
+	ExpireAfter NillableDuration `json:"expireAfter,omitempty"`
+}
+
+// This is used to convert between the NodeClaim's NodeClaimSpec to the Nodepool NodeClaimTemplate's NodeClaimSpec.
+func (in *NodeClaimTemplate) ToNodeClaim() *NodeClaim {
+	return &NodeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Labels:      in.ObjectMeta.Labels,
+			Annotations: in.ObjectMeta.Annotations,
+		},
+		Spec: NodeClaimSpec{
+			Taints:                 in.Spec.Taints,
+			StartupTaints:          in.Spec.StartupTaints,
+			Requirements:           in.Spec.Requirements,
+			NodeClassRef:           in.Spec.NodeClassRef,
+			TerminationGracePeriod: in.Spec.TerminationGracePeriod,
+			ExpireAfter:            in.Spec.ExpireAfter,
+		},
+	}
 }
 
 type ObjectMeta struct {

pkg/apis/v1/nodepool_conversion.go

@@ -54,9 +54,10 @@ func (in *NodePoolSpec) convertTo(ctx context.Context, v1beta1np *v1beta1.NodePo
 }
 
 func (in *Disruption) convertTo(v1beta1np *v1beta1.Disruption) {
-	v1beta1np.ConsolidationPolicy = v1beta1.ConsolidationPolicy(in.ConsolidationPolicy)
-	// If the v1 nodepool is WhenUnderutilized, the v1beta1 nodepool should have an unset consolidateAfter
-	v1beta1np.ConsolidateAfter = lo.Ternary(in.ConsolidationPolicy == ConsolidationPolicyWhenUnderutilized,
+	v1beta1np.ConsolidationPolicy = lo.Ternary(in.ConsolidationPolicy == ConsolidationPolicyWhenEmptyOrUnderutilized,
+		v1beta1.ConsolidationPolicyWhenUnderutilized, v1beta1.ConsolidationPolicy(in.ConsolidationPolicy))
+	// If the v1 nodepool is WhenEmptyOrUnderutilized, the v1beta1 nodepool should have an unset consolidateAfter
+	v1beta1np.ConsolidateAfter = lo.Ternary(in.ConsolidationPolicy == ConsolidationPolicyWhenEmptyOrUnderutilized,
 		nil, (*v1beta1.NillableDuration)(lo.ToPtr(in.ConsolidateAfter)))
 	v1beta1np.Budgets = lo.Map(in.Budgets, func(v1budget Budget, _ int) v1beta1.Budget {
 		return v1beta1.Budget{
@@ -147,7 +148,8 @@ func (in *Disruption) convertFrom(v1beta1np *v1beta1.Disruption) {
 	// if consolidationPolicy is WhenUnderutilized, set the v1 duration to 0, otherwise, set to the value of consolidateAfter.
 	in.ConsolidateAfter = lo.Ternary(v1beta1np.ConsolidationPolicy == v1beta1.ConsolidationPolicyWhenUnderutilized,
 		NillableDuration{Duration: lo.ToPtr(time.Duration(0))}, (NillableDuration)(lo.FromPtr(v1beta1np.ConsolidateAfter)))
-	in.ConsolidationPolicy = ConsolidationPolicy(v1beta1np.ConsolidationPolicy)
+	in.ConsolidationPolicy = lo.Ternary(v1beta1np.ConsolidationPolicy == v1beta1.ConsolidationPolicyWhenUnderutilized,
+		ConsolidationPolicyWhenEmptyOrUnderutilized, ConsolidationPolicy(v1beta1np.ConsolidationPolicy))
 	in.Budgets = lo.Map(v1beta1np.Budgets, func(v1beta1budget v1beta1.Budget, _ int) Budget {
 		return Budget{
 			Nodes:    v1beta1budget.Nodes,

pkg/apis/v1/nodepool_conversion_test.go

@@ -46,7 +46,7 @@ var _ = Describe("Convert V1 to V1beta1 NodePool API", func() {
 		v1nodepool = &NodePool{
 			Spec: NodePoolSpec{
 				Template: NodeClaimTemplate{
-					Spec: NodeClaimSpec{
+					Spec: NodeClaimTemplateSpec{
 						NodeClassRef: &NodeClassReference{
 							Name:  "test",
 							Kind:  "test",
@@ -201,8 +201,8 @@ var _ = Describe("Convert V1 to V1beta1 NodePool API", func() {
 			})
 		})
 		Context("Disruption", func() {
-			It("should convert v1 nodepool consolidateAfter to nil with WhenUnderutilized", func() {
-				v1nodepool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenUnderutilized
+			It("should convert v1 nodepool consolidateAfter to nil with WhenEmptyOrUnderutilized", func() {
+				v1nodepool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenEmptyOrUnderutilized
 				v1nodepool.Spec.Disruption.ConsolidateAfter = NillableDuration{Duration: lo.ToPtr(time.Second * 2121)}
 				Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed())
 				Expect(v1beta1nodepool.Spec.Disruption.ConsolidateAfter).To(BeNil())
@@ -276,7 +276,7 @@ var _ = Describe("Convert V1beta1 to V1 NodePool API", func() {
 		v1nodepool = &NodePool{
 			Spec: NodePoolSpec{
 				Template: NodeClaimTemplate{
-					Spec: NodeClaimSpec{
+					Spec: NodeClaimTemplateSpec{
 						NodeClassRef: &NodeClassReference{
 							Name:  "test",
 							Kind:  "test",

pkg/apis/v1/nodepool_validation_cel_test.go

@@ -44,7 +44,7 @@ var _ = Describe("CEL/Validation", func() {
 			ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())},
 			Spec: NodePoolSpec{
 				Template: NodeClaimTemplate{
-					Spec: NodeClaimSpec{
+					Spec: NodeClaimTemplateSpec{
 						NodeClassRef: &NodeClassReference{
 							Kind: "NodeClaim",
 							Name: "default",
@@ -93,14 +93,14 @@ var _ = Describe("CEL/Validation", func() {
 			nodePool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenEmpty
 			Expect(env.Client.Create(ctx, nodePool)).To(Succeed())
 		})
-		It("should succeed when setting consolidateAfter with consolidationPolicy=WhenUnderutilized", func() {
+		It("should succeed when setting consolidateAfter with consolidationPolicy=WhenEmptyOrUnderutilized", func() {
 			nodePool.Spec.Disruption.ConsolidateAfter = NillableDuration{Duration: lo.ToPtr(lo.Must(time.ParseDuration("30s")))}
-			nodePool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenUnderutilized
+			nodePool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenEmptyOrUnderutilized
 			Expect(env.Client.Create(ctx, nodePool)).To(Succeed())
 		})
-		It("should succeed when setting consolidateAfter to 'Never' with consolidationPolicy=WhenUnderutilized", func() {
+		It("should succeed when setting consolidateAfter to 'Never' with consolidationPolicy=WhenEmptyOrUnderutilized", func() {
 			nodePool.Spec.Disruption.ConsolidateAfter = NillableDuration{Duration: nil}
-			nodePool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenUnderutilized
+			nodePool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenEmptyOrUnderutilized
 			Expect(env.Client.Create(ctx, nodePool)).To(Succeed())
 		})
 		It("should succeed when setting consolidateAfter to 'Never' with consolidationPolicy=WhenEmpty", func() {

pkg/apis/v1/suite_test.go

@@ -70,7 +70,7 @@ var _ = Describe("OrderByWeight", func() {
 				Spec: v1.NodePoolSpec{
 					Weight: lo.ToPtr[int32](int32(rand.Intn(100) + 1)), //nolint:gosec
 					Template: v1.NodeClaimTemplate{
-						Spec: v1.NodeClaimSpec{
+						Spec: v1.NodeClaimTemplateSpec{
 							NodeClassRef: &v1.NodeClassReference{
 								Name: "default",
 							},
@@ -101,7 +101,7 @@ var _ = Describe("OrderByWeight", func() {
 				Spec: v1.NodePoolSpec{
 					Weight: lo.ToPtr[int32](10), //nolint:gosec
 					Template: v1.NodeClaimTemplate{
-						Spec: v1.NodeClaimSpec{
+						Spec: v1.NodeClaimTemplateSpec{
 							NodeClassRef: &v1.NodeClassReference{
 								Name: "default",
 							},

pkg/apis/v1beta1/nodeclaim.go

@@ -37,7 +37,7 @@ type NodeClaimSpec struct {
 	// +kubebuilder:validation:XValidation:message="requirements with operator 'In' must have a value defined",rule="self.all(x, x.operator == 'In' ? x.values.size() != 0 : true)"
 	// +kubebuilder:validation:XValidation:message="requirements operator 'Gt' or 'Lt' must have a single positive integer value",rule="self.all(x, (x.operator == 'Gt' || x.operator == 'Lt') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)"
 	// +kubebuilder:validation:XValidation:message="requirements with 'minValues' must have at least that many values specified in the 'values' field",rule="self.all(x, (x.operator == 'In' && has(x.minValues)) ? x.values.size() >= x.minValues : true)"
-	// +kubebuilder:validation:MaxItems:=30
+	// +kubebuilder:validation:MaxItems:=100
 	// +required
 	Requirements []NodeSelectorRequirementWithMinValues `json:"requirements" hash:"ignore"`
 	// Resources models the resource requirements for the NodeClaim to launch