From 681fef7cf3b9a35d6da047e3fbeb733870167ddc Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Fri, 30 Aug 2024 16:23:24 -0700 Subject: [PATCH 1/6] fix: enable webhooks by default --- pkg/operator/options/options.go | 2 +- pkg/operator/options/suite_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 81168d66e5..ae11851977 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -83,7 +83,7 @@ func (fs *FlagSet) BoolVarWithEnv(p *bool, name string, envVar string, val bool, func (o *Options) AddFlags(fs *FlagSet) { fs.StringVar(&o.ServiceName, "karpenter-service", env.WithDefaultString("KARPENTER_SERVICE", ""), "The Karpenter Service name for the dynamic webhook certificate") - fs.BoolVarWithEnv(&o.DisableWebhook, "disable-webhook", "DISABLE_WEBHOOK", true, "Disable the admission and validation webhooks") + fs.BoolVarWithEnv(&o.DisableWebhook, "disable-webhook", "DISABLE_WEBHOOK", false, "Disable the admission and validation webhooks") fs.IntVar(&o.WebhookPort, "webhook-port", env.WithDefaultInt("WEBHOOK_PORT", 8443), "The port the webhook endpoint binds to for validation and mutation of resources") fs.IntVar(&o.MetricsPort, "metrics-port", env.WithDefaultInt("METRICS_PORT", 8000), "The port the metric endpoint binds to for operating metrics about the controller itself") fs.IntVar(&o.WebhookMetricsPort, "webhook-metrics-port", env.WithDefaultInt("WEBHOOK_METRICS_PORT", 8001), "The port the webhook metric endpoing binds to for operating metrics about the webhook") diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go index ef9307fb8e..0a530de9c2 100644 --- a/pkg/operator/options/suite_test.go +++ b/pkg/operator/options/suite_test.go @@ -97,7 +97,7 @@ var _ = Describe("Options", func() { Expect(err).To(BeNil()) expectOptionsMatch(opts, test.Options(test.OptionsFields{ ServiceName: lo.ToPtr(""), - DisableWebhook: lo.ToPtr(true), + DisableWebhook: lo.ToPtr(false), WebhookPort: lo.ToPtr(8443), MetricsPort: lo.ToPtr(8000), WebhookMetricsPort: lo.ToPtr(8001), @@ -255,7 +255,7 @@ var _ = Describe("Options", func() { Entry("explicit true", "--disable-webhook=true", true), Entry("explicit false", "--disable-webhook=false", false), Entry("implicit true", "--disable-webhook", true), - Entry("implicit true", "", true), + Entry("implicit false", "", false), ) }) From eb788f3fbb94e5ccdaefa28b28e7e999499346a5 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Fri, 30 Aug 2024 16:48:01 -0700 Subject: [PATCH 2/6] add conversion webhook injection script --- Makefile | 1 + hack/mutation/conversion_webhook_injection.sh | 5 +++++ pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 11 +++++++++++ pkg/apis/crds/karpenter.sh_nodepools.yaml | 11 +++++++++++ 4 files changed, 28 insertions(+) create mode 100755 hack/mutation/conversion_webhook_injection.sh diff --git a/Makefile b/Makefile index 8101d9490b..61ae639771 100644 --- a/Makefile +++ b/Makefile @@ -91,6 +91,7 @@ verify: ## Verify code. Includes codegen, docgen, dependencies, linting, formatt hack/validation/labels.sh hack/validation/resources.sh hack/validation/status.sh + hack/mutation/conversion_webhook_injection.sh hack/dependabot.sh @# Use perl instead of sed due to https://stackoverflow.com/questions/4247068/sed-command-with-i-option-failing-on-mac-but-works-on-linux @# We need to do this "sed replace" until controller-tools fixes this parameterized types issue: https://github.com/kubernetes-sigs/controller-tools/issues/756 diff --git a/hack/mutation/conversion_webhook_injection.sh b/hack/mutation/conversion_webhook_injection.sh new file mode 100755 index 0000000000..3bd151c0cd --- /dev/null +++ b/hack/mutation/conversion_webhook_injection.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Add the conversion stanza to the CRD spec to enable conversion via webhook +yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml +yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 62a37c41ad..2f05befa19 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -825,3 +825,14 @@ spec: storage: true subresources: status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index b867b465fc..caff97ef92 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -1011,3 +1011,14 @@ spec: storage: true subresources: status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 From aca7faa8336b18e73e460fbb902085740e847e1e Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Fri, 30 Aug 2024 16:49:06 -0700 Subject: [PATCH 3/6] add space for eof --- hack/mutation/conversion_webhook_injection.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/mutation/conversion_webhook_injection.sh b/hack/mutation/conversion_webhook_injection.sh index 3bd151c0cd..4af5c3abde 100755 --- a/hack/mutation/conversion_webhook_injection.sh +++ b/hack/mutation/conversion_webhook_injection.sh @@ -2,4 +2,4 @@ # Add the conversion stanza to the CRD spec to enable conversion via webhook yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml -yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file +yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodepools.yaml From 01c180e604e0176898ff6d5176b9c7edeb63ad49 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Fri, 30 Aug 2024 18:21:13 -0700 Subject: [PATCH 4/6] address new golint issue --- pkg/controllers/disruption/validation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/disruption/validation.go b/pkg/controllers/disruption/validation.go index 5316d10845..da00b5b348 100644 --- a/pkg/controllers/disruption/validation.go +++ b/pkg/controllers/disruption/validation.go @@ -165,7 +165,7 @@ func (v *Validation) ValidateCommand(ctx context.Context, cmd Command, candidate return fmt.Errorf("simluating scheduling, %w", err) } if !results.AllNonPendingPodsScheduled() { - return NewValidationError(fmt.Errorf(results.NonPendingPodSchedulingErrors())) + return NewValidationError(fmt.Errorf("%s", results.NonPendingPodSchedulingErrors())) } // We want to ensure that the re-simulated scheduling using the current cluster state produces the same result. From a4bb730db6b8c704af7753f32c4f58014ab39732 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Fri, 30 Aug 2024 19:06:57 -0700 Subject: [PATCH 5/6] add latest from kubebuilder --- .../kwok.karpenter.sh_kwoknodeclasses.yaml | 9 ++------- pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 20 +++---------------- pkg/apis/crds/karpenter.sh_nodepools.yaml | 13 ++---------- .../karpenter.test.sh_testnodeclasses.yaml | 9 ++------- 4 files changed, 9 insertions(+), 42 deletions(-) diff --git a/kwok/apis/crds/kwok.karpenter.sh_kwoknodeclasses.yaml b/kwok/apis/crds/kwok.karpenter.sh_kwoknodeclasses.yaml index a124a2d298..5cb8667411 100644 --- a/kwok/apis/crds/kwok.karpenter.sh_kwoknodeclasses.yaml +++ b/kwok/apis/crds/kwok.karpenter.sh_kwoknodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.2 name: kwoknodeclasses.kwok.karpenter.sh spec: group: kwok.karpenter.sh @@ -96,12 +96,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 2f05befa19..862c8299b7 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.2 name: nodeclaims.karpenter.sh spec: group: karpenter.sh @@ -259,19 +259,15 @@ spec: description: |- TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. - Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. - This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. - Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. - The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. If left undefined, the controller will wait indefinitely for pods to be drained. pattern: ^([0-9]+(s|m|h))+$ @@ -347,12 +343,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -793,12 +784,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index caff97ef92..46b2fd4dc9 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.2 name: nodepools.karpenter.sh spec: group: karpenter.sh @@ -387,19 +387,15 @@ spec: description: |- TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. - Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. - This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. - Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. - The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. If left undefined, the controller will wait indefinitely for pods to be drained. pattern: ^([0-9]+(s|m|h))+$ @@ -471,12 +467,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string diff --git a/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml b/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml index 3907c2cafd..fb3c15c292 100644 --- a/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml +++ b/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 + controller-gen.kubebuilder.io/version: v0.16.2 name: testnodeclasses.karpenter.test.sh spec: group: karpenter.test.sh @@ -84,12 +84,7 @@ spec: - Unknown type: string type: - description: |- - type of condition in CamelCase or in foo.example.com/CamelCase. - --- - Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be - useful (see .node.status.conditions), the ability to deconflict is important. - The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + description: type of condition in CamelCase or in foo.example.com/CamelCase. maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string From 1c2a85cdf36b8629ee15a3ebf9fcc126b533b358 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Tue, 3 Sep 2024 10:10:30 -0700 Subject: [PATCH 6/6] remove flake and unused test --- .../disruption/consolidation_test.go | 174 ------------------ 1 file changed, 174 deletions(-) diff --git a/pkg/controllers/disruption/consolidation_test.go b/pkg/controllers/disruption/consolidation_test.go index b5c8f9bb58..ab3771551a 100644 --- a/pkg/controllers/disruption/consolidation_test.go +++ b/pkg/controllers/disruption/consolidation_test.go @@ -4136,180 +4136,6 @@ var _ = Describe("Consolidation", func() { ExpectExists(ctx, env.Client, nodes[1]) }) }) - - Context("Timeout", func() { - It("should return the last valid command when multi-nodeclaim consolidation times out", func() { - numNodes := 20 - nodeClaims, nodes := test.NodeClaimsAndNodes(numNodes, v1beta1.NodeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelInstanceTypeStable: leastExpensiveInstance.Name, - v1beta1.CapacityTypeLabelKey: leastExpensiveOffering.Requirements.Get(v1beta1.CapacityTypeLabelKey).Any(), - v1.LabelTopologyZone: leastExpensiveOffering.Requirements.Get(v1.LabelTopologyZone).Any(), - }, - }, - Status: v1beta1.NodeClaimStatus{ - Allocatable: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("32"), - v1.ResourcePods: resource.MustParse("100"), - }, - }}, - ) - // create our RS so we can link a pod to it - rs := test.ReplicaSet() - ExpectApplied(ctx, env.Client, rs) - pods := test.Pods(numNodes, test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: rs.Name, - UID: rs.UID, - Controller: lo.ToPtr(true), - BlockOwnerDeletion: lo.ToPtr(true), - }, - }}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - // Make the resource requests small so that many nodes can be consolidated at once. - v1.ResourceCPU: resource.MustParse("10m"), - }, - }, - }) - - ExpectApplied(ctx, env.Client, rs, nodePool) - for _, nodeClaim := range nodeClaims { - ExpectApplied(ctx, env.Client, nodeClaim) - } - for _, node := range nodes { - ExpectApplied(ctx, env.Client, node) - } - for i, pod := range pods { - ExpectApplied(ctx, env.Client, pod) - ExpectManualBinding(ctx, env.Client, pod, nodes[i]) - } - - // inform cluster state about nodes and nodeClaims - ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, nodes, nodeClaims) - - var wg sync.WaitGroup - wg.Add(1) - finished := atomic.Bool{} - go func() { - defer GinkgoRecover() - defer wg.Done() - defer finished.Store(true) - ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{}) - }() - - // advance the clock so that the timeout expires - fakeClock.Step(disruption.MultiNodeConsolidationTimeoutDuration) - - // wait for the controller to block on the validation timeout - Eventually(fakeClock.HasWaiters, time.Second*10).Should(BeTrue()) - - ExpectTriggerVerifyAction(&wg) - - // controller should be blocking during the timeout - Expect(finished.Load()).To(BeFalse()) - - // and the node should not be deleted yet - for i := range nodeClaims { - ExpectExists(ctx, env.Client, nodeClaims[i]) - } - - // controller should finish - Eventually(finished.Load, 10*time.Second).Should(BeTrue()) - wg.Wait() - - ExpectReconcileSucceeded(ctx, queue, types.NamespacedName{}) - - // should have at least two nodes deleted from multi nodeClaim consolidation - Expect(len(ExpectNodeClaims(ctx, env.Client))).To(BeNumerically("<=", numNodes-2)) - }) - It("should exit single-nodeclaim consolidation if it times out", func() { - numNodes := 25 - nodeClaims, nodes := test.NodeClaimsAndNodes(numNodes, v1beta1.NodeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - v1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelInstanceTypeStable: leastExpensiveInstance.Name, - v1beta1.CapacityTypeLabelKey: leastExpensiveOffering.Requirements.Get(v1beta1.CapacityTypeLabelKey).Any(), - v1.LabelTopologyZone: leastExpensiveOffering.Requirements.Get(v1.LabelTopologyZone).Any(), - }, - }, - Status: v1beta1.NodeClaimStatus{ - Allocatable: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("32"), - v1.ResourcePods: resource.MustParse("100"), - }, - }}, - ) - // create our RS so we can link a pod to it - rs := test.ReplicaSet() - ExpectApplied(ctx, env.Client, rs) - pods := test.Pods(numNodes, test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{Labels: labels, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: rs.Name, - UID: rs.UID, - Controller: lo.ToPtr(true), - BlockOwnerDeletion: lo.ToPtr(true), - }, - }}, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - // Make the pods more than half of the allocatable so that only one nodeclaim can be done at any time - v1.ResourceCPU: resource.MustParse("20"), - }, - }, - }) - - ExpectApplied(ctx, env.Client, rs, nodePool) - for _, nodeClaim := range nodeClaims { - ExpectApplied(ctx, env.Client, nodeClaim) - } - for _, node := range nodes { - ExpectApplied(ctx, env.Client, node) - } - for i, pod := range pods { - ExpectApplied(ctx, env.Client, pod) - ExpectManualBinding(ctx, env.Client, pod, nodes[i]) - } - - // inform cluster state about nodes and nodeClaims - ExpectMakeNodesAndNodeClaimsInitializedAndStateUpdated(ctx, env.Client, nodeStateController, nodeClaimStateController, nodes, nodeClaims) - - var wg sync.WaitGroup - wg.Add(1) - finished := atomic.Bool{} - go func() { - defer GinkgoRecover() - defer wg.Done() - defer finished.Store(true) - ExpectReconcileSucceeded(ctx, disruptionController, client.ObjectKey{}) - }() - - // advance the clock so that the timeout expires for multi-nodeClaim - fakeClock.Step(disruption.MultiNodeConsolidationTimeoutDuration) - // advance the clock so that the timeout expires for single-nodeClaim - fakeClock.Step(disruption.SingleNodeConsolidationTimeoutDuration) - - ExpectTriggerVerifyAction(&wg) - - // controller should finish - Eventually(finished.Load, 10*time.Second).Should(BeTrue()) - wg.Wait() - - // should have no nodeClaims deleted from single nodeClaim consolidation - Expect(ExpectNodeClaims(ctx, env.Client)).To(HaveLen(numNodes)) - }) - }) Context("Multi-NodeClaim", func() { var nodeClaims, spotNodeClaims []*v1beta1.NodeClaim var nodes, spotNodes []*v1.Node