From ad657c3886e865fa822a9e8614c24f97d51b6dcb Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Tue, 20 Aug 2024 01:26:11 +0800 Subject: [PATCH 01/14] refactor BuildTestDeployment method --- test/e2e/e2e_duplicatepods_test.go | 43 +---------- test/e2e/e2e_failedpods_test.go | 3 +- test/e2e/e2e_leaderelection_test.go | 45 +---------- test/e2e/e2e_test.go | 76 ++++++++++++++++++- test/e2e/e2e_toomanyrestarts_test.go | 51 ++----------- test/e2e/e2e_topologyspreadconstraint_test.go | 4 +- test/run-e2e-tests.sh | 4 +- test/test_utils.go | 75 ------------------ 8 files changed, 86 insertions(+), 215 deletions(-) diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index 92672de188..56abfd1661 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -62,48 +62,7 @@ func TestRemoveDuplicates(t *testing.T) { defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) t.Log("Creating duplicates pods") - - deploymentObj := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "duplicate-pod", - Namespace: testNamespace.Name, - Labels: map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, - }, - Spec: v1.PodSpec{ - SecurityContext: &v1.PodSecurityContext{ - RunAsNonRoot: utilptr.To(true), - RunAsUser: utilptr.To[int64](1000), - RunAsGroup: utilptr.To[int64](1000), - SeccompProfile: &v1.SeccompProfile{ - Type: v1.SeccompProfileTypeRuntimeDefault, - }, - }, - Containers: []v1.Container{{ - Name: "pause", - ImagePullPolicy: "Always", - Image: "registry.k8s.io/pause", - Ports: []v1.ContainerPort{{ContainerPort: 80}}, - SecurityContext: &v1.SecurityContext{ - AllowPrivilegeEscalation: utilptr.To(false), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{ - "ALL", - }, - }, - }, - }}, - }, - }, - }, - } + deploymentObj := buildTestDeployment("duplicate-pod", testNamespace.Name, 0, map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, nil) tests := []struct { description string diff --git a/test/e2e/e2e_failedpods_test.go b/test/e2e/e2e_failedpods_test.go index b056830d93..7ff28956e6 100644 --- a/test/e2e/e2e_failedpods_test.go +++ b/test/e2e/e2e_failedpods_test.go @@ -23,7 +23,6 @@ import ( "sigs.k8s.io/descheduler/pkg/framework/plugins/removefailedpods" frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing" frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" - "sigs.k8s.io/descheduler/test" ) var oneHourPodLifetimeSeconds uint = 3600 @@ -134,7 +133,7 @@ func TestFailedPods(t *testing.T) { } func initFailedJob(name, namespace string) *batchv1.Job { - podSpec := test.MakePodSpec("", nil) + podSpec := makePodSpec("", nil) podSpec.Containers[0].Command = []string{"/bin/false"} podSpec.RestartPolicy = v1.RestartPolicyNever labelsSet := labels.Set{"test": name, "name": name} diff --git a/test/e2e/e2e_leaderelection_test.go b/test/e2e/e2e_leaderelection_test.go index 5ed4e76bdf..9e1807f6df 100644 --- a/test/e2e/e2e_leaderelection_test.go +++ b/test/e2e/e2e_leaderelection_test.go @@ -31,7 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" clientset "k8s.io/client-go/kubernetes" - utilptr "k8s.io/utils/ptr" + "sigs.k8s.io/descheduler/cmd/descheduler/app/options" "sigs.k8s.io/descheduler/pkg/descheduler" ) @@ -166,48 +166,7 @@ func TestLeaderElection(t *testing.T) { } func createDeployment(ctx context.Context, clientSet clientset.Interface, namespace string, replicas int32, t *testing.T) (*appsv1.Deployment, error) { - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "leaderelection", - Namespace: namespace, - Labels: map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: utilptr.To[int32](replicas), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, - }, - Spec: v1.PodSpec{ - SecurityContext: &v1.PodSecurityContext{ - RunAsNonRoot: utilptr.To(true), - RunAsUser: utilptr.To[int64](1000), - RunAsGroup: utilptr.To[int64](1000), - SeccompProfile: &v1.SeccompProfile{ - Type: v1.SeccompProfileTypeRuntimeDefault, - }, - }, - Containers: []v1.Container{{ - Name: "pause", - ImagePullPolicy: "Always", - Image: "registry.k8s.io/pause", - Ports: []v1.ContainerPort{{ContainerPort: 80}}, - SecurityContext: &v1.SecurityContext{ - AllowPrivilegeEscalation: utilptr.To(false), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{ - "ALL", - }, - }, - }, - }}, - }, - }, - }, - } + deployment := buildTestDeployment("leaderelection", namespace, replicas, map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, nil) t.Logf("Creating deployment %v for namespace %s", deployment.Name, deployment.Namespace) deployment, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 0f8b81ef29..0d9262cb76 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -63,7 +63,6 @@ import ( frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing" frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" "sigs.k8s.io/descheduler/pkg/utils" - "sigs.k8s.io/descheduler/test" ) func isClientRateLimiterError(err error) bool { @@ -297,7 +296,7 @@ func RcByNameContainer(name, namespace string, replicas int32, labels map[string ObjectMeta: metav1.ObjectMeta{ Labels: labels, }, - Spec: test.MakePodSpec(priorityClassName, gracePeriod), + Spec: makePodSpec(priorityClassName, gracePeriod), }, }, } @@ -329,12 +328,83 @@ func DsByNameContainer(name, namespace string, labels map[string]string, gracePe ObjectMeta: metav1.ObjectMeta{ Labels: labels, }, - Spec: test.MakePodSpec("", gracePeriod), + Spec: makePodSpec("", gracePeriod), }, }, } } +func buildTestDeployment(name, namespace string, replicas int32, testLabel map[string]string, apply func(deployment *appsv1.Deployment)) *appsv1.Deployment { + deployment := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: testLabel, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: utilptr.To[int32](replicas), + Selector: &metav1.LabelSelector{ + MatchLabels: testLabel, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: testLabel, + }, + Spec: makePodSpec("", utilptr.To[int64](0)), + }, + }, + } + + if apply != nil { + apply(deployment) + } + + return deployment +} + +func makePodSpec(priorityClassName string, gracePeriod *int64) v1.PodSpec { + return v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + RunAsNonRoot: utilptr.To(true), + RunAsUser: utilptr.To[int64](1000), + RunAsGroup: utilptr.To[int64](1000), + SeccompProfile: &v1.SeccompProfile{ + Type: v1.SeccompProfileTypeRuntimeDefault, + }, + }, + Containers: []v1.Container{{ + Name: "pause", + ImagePullPolicy: "IfNotPresent", + Image: "registry.k8s.io/pause", + Ports: []v1.ContainerPort{{ContainerPort: 80}}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: utilptr.To(false), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{ + "ALL", + }, + }, + }, + }}, + PriorityClassName: priorityClassName, + TerminationGracePeriodSeconds: gracePeriod, + } +} + func initializeClient(ctx context.Context, t *testing.T) (clientset.Interface, informers.SharedInformerFactory, listersv1.NodeLister, podutil.GetPodsAssignedToNodeFunc) { clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") if err != nil { diff --git a/test/e2e/e2e_toomanyrestarts_test.go b/test/e2e/e2e_toomanyrestarts_test.go index a635d4bb0c..c10cfad2d0 100644 --- a/test/e2e/e2e_toomanyrestarts_test.go +++ b/test/e2e/e2e_toomanyrestarts_test.go @@ -32,8 +32,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" componentbaseconfig "k8s.io/component-base/config" - utilptr "k8s.io/utils/ptr" - + "sigs.k8s.io/descheduler/cmd/descheduler/app/options" "sigs.k8s.io/descheduler/pkg/api" apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" @@ -104,50 +103,10 @@ func TestTooManyRestarts(t *testing.T) { } defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) - deploymentObj := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "restart-pod", - Namespace: testNamespace.Name, - Labels: map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"}, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: utilptr.To[int32](deploymentReplicas), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"}, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"}, - }, - Spec: v1.PodSpec{ - SecurityContext: &v1.PodSecurityContext{ - RunAsNonRoot: utilptr.To(true), - RunAsUser: utilptr.To[int64](1000), - RunAsGroup: utilptr.To[int64](1000), - SeccompProfile: &v1.SeccompProfile{ - Type: v1.SeccompProfileTypeRuntimeDefault, - }, - }, - Containers: []v1.Container{{ - Name: "pause", - ImagePullPolicy: "Always", - Image: "registry.k8s.io/pause", - Command: []string{"/bin/sh"}, - Args: []string{"-c", "sleep 1s && exit 1"}, - Ports: []v1.ContainerPort{{ContainerPort: 80}}, - SecurityContext: &v1.SecurityContext{ - AllowPrivilegeEscalation: utilptr.To(false), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{ - "ALL", - }, - }, - }, - }}, - }, - }, - }, - } + deploymentObj := buildTestDeployment("restart-pod", testNamespace.Name, deploymentReplicas, map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"}, func(deployment *appsv1.Deployment) { + deployment.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh"} + deployment.Spec.Template.Spec.Containers[0].Args = []string{"-c", "sleep 1s && exit 1"} + }) t.Logf("Creating deployment %v", deploymentObj.Name) _, err = clientSet.AppsV1().Deployments(deploymentObj.Namespace).Create(ctx, deploymentObj, metav1.CreateOptions{}) diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index 7de8070aea..5379e4532a 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -126,7 +126,7 @@ func TestTopologySpreadConstraint(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { t.Logf("Creating Deployment %s with %d replicas", name, tc.replicaCount) - deployment := test.BuildTestDeployment(name, testNamespace.Name, int32(tc.replicaCount), tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels, func(d *appsv1.Deployment) { + deployment := buildTestDeployment(name, testNamespace.Name, int32(tc.replicaCount), tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels, func(d *appsv1.Deployment) { d.Spec.Template.Spec.TopologySpreadConstraints = []v1.TopologySpreadConstraint{tc.topologySpreadConstraint} }) if _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}); err != nil { @@ -138,7 +138,7 @@ func TestTopologySpreadConstraint(t *testing.T) { // Create a "Violator" Deployment that has the same label and is forced to be on the same node using a nodeSelector violatorDeploymentName := name + "-violator" violatorCount := tc.topologySpreadConstraint.MaxSkew + 1 - violatorDeployment := test.BuildTestDeployment(violatorDeploymentName, testNamespace.Name, violatorCount, tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels, func(d *appsv1.Deployment) { + violatorDeployment := buildTestDeployment(violatorDeploymentName, testNamespace.Name, violatorCount, tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels, func(d *appsv1.Deployment) { d.Spec.Template.Spec.NodeSelector = map[string]string{zoneTopologyKey: workerNodes[0].Labels[zoneTopologyKey]} }) if _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, violatorDeployment, metav1.CreateOptions{}); err != nil { diff --git a/test/run-e2e-tests.sh b/test/run-e2e-tests.sh index e49c01128a..b3001a276a 100755 --- a/test/run-e2e-tests.sh +++ b/test/run-e2e-tests.sh @@ -23,7 +23,7 @@ SKIP_INSTALL=${SKIP_INSTALL:-} KIND_E2E=${KIND_E2E:-} # Build a descheduler image -IMAGE_TAG=$(git describe --tags --match v0*) +IMAGE_TAG=v$(date +%Y%m%d)-$(git describe --tags) BASEDIR=$(dirname "$0") VERSION="${IMAGE_TAG}" make -C ${BASEDIR}/.. image @@ -54,4 +54,4 @@ fi kubectl apply -f kubernetes/base/rbac.yaml PRJ_PREFIX="sigs.k8s.io/descheduler" -go test ${PRJ_PREFIX}/test/e2e/ -v +go test ${PRJ_PREFIX}/test/e2e/ -v -timeout 0 diff --git a/test/test_utils.go b/test/test_utils.go index c43a942ff7..1d4268b7f8 100644 --- a/test/test_utils.go +++ b/test/test_utils.go @@ -34,42 +34,6 @@ import ( utilptr "k8s.io/utils/ptr" ) -func BuildTestDeployment(name, namespace string, replicas int32, labels map[string]string, apply func(deployment *appsv1.Deployment)) *appsv1.Deployment { - // Add "name": name to the labels, overwriting if it exists. - labels["name"] = name - - deployment := &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: utilptr.To[int32](replicas), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": name, - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: MakePodSpec("", utilptr.To[int64](0)), - }, - }, - } - - if apply != nil { - apply(deployment) - } - - return deployment -} - // BuildTestPod creates a test pod with given parameters. func BuildTestPod(name string, cpu, memory int64, nodeName string, apply func(*v1.Pod)) *v1.Pod { pod := &v1.Pod{ @@ -171,45 +135,6 @@ func BuildTestNode(name string, millicpu, mem, pods int64, apply func(*v1.Node)) return node } -func MakePodSpec(priorityClassName string, gracePeriod *int64) v1.PodSpec { - return v1.PodSpec{ - SecurityContext: &v1.PodSecurityContext{ - RunAsNonRoot: utilptr.To(true), - RunAsUser: utilptr.To[int64](1000), - RunAsGroup: utilptr.To[int64](1000), - SeccompProfile: &v1.SeccompProfile{ - Type: v1.SeccompProfileTypeRuntimeDefault, - }, - }, - Containers: []v1.Container{{ - Name: "pause", - ImagePullPolicy: "Never", - Image: "registry.k8s.io/pause", - Ports: []v1.ContainerPort{{ContainerPort: 80}}, - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("200Mi"), - }, - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - }, - }, - SecurityContext: &v1.SecurityContext{ - AllowPrivilegeEscalation: utilptr.To(false), - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{ - "ALL", - }, - }, - }, - }}, - PriorityClassName: priorityClassName, - TerminationGracePeriodSeconds: gracePeriod, - } -} - // MakeBestEffortPod makes the given pod a BestEffort pod func MakeBestEffortPod(pod *v1.Pod) { pod.Spec.Containers[0].Resources.Requests = nil From 1e23dbcc2669f69288ec6c9dbcd51fa7b3c1c8e5 Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Tue, 20 Aug 2024 01:35:53 +0800 Subject: [PATCH 02/14] refactor waitForPodsRunning method and add new waitForPodsToDisappear method to remove duplicated method --- test/e2e/e2e_duplicatepods_test.go | 3 +- test/e2e/e2e_test.go | 141 +++++++----------- test/e2e/e2e_toomanyrestarts_test.go | 14 +- test/e2e/e2e_topologyspreadconstraint_test.go | 21 ++- test/test_utils.go | 24 --- 5 files changed, 72 insertions(+), 131 deletions(-) diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index 56abfd1661..8beb099a48 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -159,7 +159,8 @@ func TestRemoveDuplicates(t *testing.T) { t.Log("Running removeduplicates plugin") plugin.(frameworktypes.BalancePlugin).Balance(ctx, workerNodes) - waitForTerminatingPodsToDisappear(ctx, t, clientSet, testNamespace.Name) + waitForPodsToDisappear(ctx, t, clientSet, map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, testNamespace.Name) + actualEvictedPodCount := podEvictor.TotalEvicted() if actualEvictedPodCount != tc.expectedEvictedPodCount { t.Errorf("Test error for description: %s. Unexpected number of pods have been evicted, got %v, expected %v", tc.description, actualEvictedPodCount, tc.expectedEvictedPodCount) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 0d9262cb76..41092b7773 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -194,67 +194,6 @@ func printPodLogs(ctx context.Context, t *testing.T, kubeClient clientset.Interf } } -func waitForDeschedulerPodRunning(t *testing.T, ctx context.Context, kubeClient clientset.Interface, testName string) string { - deschedulerPodName := "" - if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { - podList, err := kubeClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"app": "descheduler", "test": testName})).String(), - }) - if err != nil { - t.Logf("Unable to list pods: %v", err) - if isClientRateLimiterError(err) { - return false, nil - } - return false, err - } - - runningPods := []*v1.Pod{} - for _, item := range podList.Items { - if item.Status.Phase != v1.PodRunning { - continue - } - pod := item - runningPods = append(runningPods, &pod) - } - - if len(runningPods) != 1 { - t.Logf("Expected a single running pod, got %v instead", len(runningPods)) - return false, nil - } - - deschedulerPodName = runningPods[0].Name - t.Logf("Found a descheduler pod running: %v", deschedulerPodName) - return true, nil - }); err != nil { - t.Fatalf("Error waiting for a running descheduler: %v", err) - } - return deschedulerPodName -} - -func waitForDeschedulerPodAbsent(t *testing.T, ctx context.Context, kubeClient clientset.Interface, testName string) { - if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { - podList, err := kubeClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"app": "descheduler", "test": testName})).String(), - }) - if err != nil { - t.Logf("Unable to list pods: %v", err) - if isClientRateLimiterError(err) { - return false, nil - } - return false, err - } - - if len(podList.Items) > 0 { - t.Logf("Found a descheduler pod. Waiting until it gets deleted") - return false, nil - } - - return true, nil - }); err != nil { - t.Fatalf("Error waiting for a descheduler to disapear: %v", err) - } -} - func TestMain(m *testing.M) { if os.Getenv("DESCHEDULER_IMAGE") == "" { klog.Errorf("DESCHEDULER_IMAGE env is not set") @@ -680,7 +619,7 @@ func TestLowNodeUtilization(t *testing.T) { } plugin.(frameworktypes.BalancePlugin).Balance(ctx, workerNodes) - waitForTerminatingPodsToDisappear(ctx, t, clientSet, rc.Namespace) + waitForPodsToDisappear(ctx, t, clientSet, rc.Spec.Template.Labels, rc.Namespace) podFilter, err = podutil.NewOptions().WithFilter(handle.EvictorFilterImpl.Filter).BuildFilterFunc() if err != nil { @@ -1379,7 +1318,7 @@ func TestPodLifeTimeOldestEvicted(t *testing.T) { t.Log("Finished PodLifetime plugin") t.Logf("Wait for terminating pod to disappear") - waitForTerminatingPodsToDisappear(ctx, t, clientSet, rc.Namespace) + waitForPodsToDisappear(ctx, t, clientSet, rc.Spec.Template.Labels, rc.Namespace) podList, err = clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rc.Spec.Template.Labels).String()}) if err != nil { @@ -1453,24 +1392,6 @@ func waitForRCPodsRunning(ctx context.Context, t *testing.T, clientSet clientset } } -func waitForTerminatingPodsToDisappear(ctx context.Context, t *testing.T, clientSet clientset.Interface, namespace string) { - if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { - podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) - if err != nil { - return false, err - } - for _, pod := range podList.Items { - if pod.DeletionTimestamp != nil { - t.Logf("Pod %v still terminating", pod.Name) - return false, nil - } - } - return true, nil - }); err != nil { - t.Fatalf("Error waiting for terminating pods to disappear: %v", err) - } -} - func deleteDS(ctx context.Context, t *testing.T, clientSet clientset.Interface, ds *appsv1.DaemonSet) { // adds nodeselector to avoid any nodes by setting an unused label dsDeepCopy := ds.DeepCopy() @@ -1775,6 +1696,10 @@ func waitForPodRunning(ctx context.Context, t *testing.T, clientSet clientset.In if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { podItem, err := clientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) if err != nil { + t.Logf("Unable to list pods: %v", err) + if isClientRateLimiterError(err) { + return false, nil + } return false, err } @@ -1789,27 +1714,65 @@ func waitForPodRunning(ctx context.Context, t *testing.T, clientSet clientset.In } } -func waitForPodsRunning(ctx context.Context, t *testing.T, clientSet clientset.Interface, labelMap map[string]string, desireRunningPodNum int, namespace string) { - if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { +func waitForPodsRunning(ctx context.Context, t *testing.T, clientSet clientset.Interface, labelMap map[string]string, desireRunningPodNum int, namespace string) string { + runningPodName := "" + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ LabelSelector: labels.SelectorFromSet(labelMap).String(), }) if err != nil { + t.Logf("Unable to list pods: %v", err) + if isClientRateLimiterError(err) { + return false, nil + } return false, err } - if len(podList.Items) != desireRunningPodNum { - t.Logf("Waiting for %v pods to be running, got %v instead", desireRunningPodNum, len(podList.Items)) + + runningPods := []*v1.Pod{} + for _, item := range podList.Items { + if item.Status.Phase != v1.PodRunning { + continue + } + pod := item + runningPods = append(runningPods, &pod) + } + + if len(runningPods) != desireRunningPodNum { + t.Logf("Waiting for %v pods to be running, got %v instead", desireRunningPodNum, len(runningPods)) return false, nil } - for _, pod := range podList.Items { - if pod.Status.Phase != v1.PodRunning { - t.Logf("Pod %v not running yet, is %v instead", pod.Name, pod.Status.Phase) + + if desireRunningPodNum == 1 { + runningPodName = runningPods[0].Name + } + + return true, nil + }); err != nil { + t.Fatalf("Error waiting for pods running: %v", err) + } + return runningPodName +} + +func waitForPodsToDisappear(ctx context.Context, t *testing.T, clientSet clientset.Interface, labelMap map[string]string, namespace string) { + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labelMap).String(), + }) + if err != nil { + t.Logf("Unable to list pods: %v", err) + if isClientRateLimiterError(err) { return false, nil } + return false, err + } + + if len(podList.Items) > 0 { + t.Logf("Found a existing pod. Waiting until it gets deleted") + return false, nil } return true, nil }); err != nil { - t.Fatalf("Error waiting for pods running: %v", err) + t.Fatalf("Error waiting for pods to disappear: %v", err) } } diff --git a/test/e2e/e2e_toomanyrestarts_test.go b/test/e2e/e2e_toomanyrestarts_test.go index c10cfad2d0..714769085e 100644 --- a/test/e2e/e2e_toomanyrestarts_test.go +++ b/test/e2e/e2e_toomanyrestarts_test.go @@ -32,7 +32,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" componentbaseconfig "k8s.io/component-base/config" - + "sigs.k8s.io/descheduler/cmd/descheduler/app/options" "sigs.k8s.io/descheduler/pkg/api" apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" @@ -101,7 +101,6 @@ func TestTooManyRestarts(t *testing.T) { if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil { t.Fatalf("Unable to create ns %v", testNamespace.Name) } - defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) deploymentObj := buildTestDeployment("restart-pod", testNamespace.Name, deploymentReplicas, map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"}, func(deployment *appsv1.Deployment) { deployment.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh"} @@ -119,7 +118,11 @@ func TestTooManyRestarts(t *testing.T) { } return } - defer clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{}) + defer func() { + clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{}) + waitForPodsToDisappear(ctx, t, clientSet, deploymentObj.Labels, deploymentObj.Namespace) + clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) + }() // Wait for 3 restarts waitPodRestartCount(ctx, clientSet, testNamespace.Name, t, 3) @@ -187,11 +190,11 @@ func TestTooManyRestarts(t *testing.T) { if err != nil { t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) } - waitForDeschedulerPodAbsent(t, ctx, clientSet, testNamespace.Name) + waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) }() t.Logf("Waiting for the descheduler pod running") - deschedulerPodName = waitForDeschedulerPodRunning(t, ctx, clientSet, testNamespace.Name) + deschedulerPodName = waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) // Run RemovePodsHavingTooManyRestarts strategy if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 20*time.Second, true, func(ctx context.Context) (bool, error) { @@ -208,7 +211,6 @@ func TestTooManyRestarts(t *testing.T) { }); err != nil { t.Errorf("Error waiting for descheduler running: %v", err) } - waitForTerminatingPodsToDisappear(ctx, t, clientSet, testNamespace.Name) }) } } diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index 5379e4532a..dacf599c85 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -20,7 +20,6 @@ import ( "sigs.k8s.io/descheduler/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint" frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing" frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" - "sigs.k8s.io/descheduler/test" ) const zoneTopologyKey string = "topology.kubernetes.io/zone" @@ -132,8 +131,11 @@ func TestTopologySpreadConstraint(t *testing.T) { if _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}); err != nil { t.Fatalf("Error creating Deployment %s %v", name, err) } - defer test.DeleteDeployment(ctx, t, clientSet, deployment) - test.WaitForDeploymentPodsRunning(ctx, t, clientSet, deployment) + defer func() { + clientSet.AppsV1().Deployments(deployment.Namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{}) + waitForPodsToDisappear(ctx, t, clientSet, deployment.Labels, deployment.Namespace) + }() + waitForPodsRunning(ctx, t, clientSet, deployment.Labels, tc.replicaCount, deployment.Namespace) // Create a "Violator" Deployment that has the same label and is forced to be on the same node using a nodeSelector violatorDeploymentName := name + "-violator" @@ -144,8 +146,11 @@ func TestTopologySpreadConstraint(t *testing.T) { if _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, violatorDeployment, metav1.CreateOptions{}); err != nil { t.Fatalf("Error creating Deployment %s: %v", violatorDeploymentName, err) } - defer test.DeleteDeployment(ctx, t, clientSet, violatorDeployment) - test.WaitForDeploymentPodsRunning(ctx, t, clientSet, violatorDeployment) + defer func() { + clientSet.AppsV1().Deployments(violatorDeployment.Namespace).Delete(ctx, violatorDeployment.Name, metav1.DeleteOptions{}) + waitForPodsToDisappear(ctx, t, clientSet, violatorDeployment.Labels, violatorDeployment.Namespace) + }() + waitForPodsRunning(ctx, t, clientSet, violatorDeployment.Labels, int(violatorCount), violatorDeployment.Namespace) evictionPolicyGroupVersion, err := eutils.SupportEviction(clientSet) if err != nil || len(evictionPolicyGroupVersion) == 0 { @@ -181,9 +186,6 @@ func TestTopologySpreadConstraint(t *testing.T) { plugin.(frameworktypes.BalancePlugin).Balance(ctx, workerNodes) t.Logf("Finished RemovePodsViolatingTopologySpreadConstraint strategy for %s", name) - t.Logf("Wait for terminating pods of %s to disappear", name) - waitForTerminatingPodsToDisappear(ctx, t, clientSet, deployment.Namespace) - if totalEvicted := podEvictor.TotalEvicted(); totalEvicted == tc.expectedEvictedCount { t.Logf("Total of %d Pods were evicted for %s", totalEvicted, name) } else { @@ -194,9 +196,6 @@ func TestTopologySpreadConstraint(t *testing.T) { return } - // Ensure recently evicted Pod are rescheduled and running before asserting for a balanced topology spread - test.WaitForDeploymentPodsRunning(ctx, t, clientSet, deployment) - listOptions := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(tc.topologySpreadConstraint.LabelSelector.MatchLabels).String()} pods, err := clientSet.CoreV1().Pods(testNamespace.Name).List(ctx, listOptions) if err != nil { diff --git a/test/test_utils.go b/test/test_utils.go index 1d4268b7f8..58485c117b 100644 --- a/test/test_utils.go +++ b/test/test_utils.go @@ -241,30 +241,6 @@ func DeleteDeployment(ctx context.Context, t *testing.T, clientSet clientset.Int } } -func WaitForDeploymentPodsRunning(ctx context.Context, t *testing.T, clientSet clientset.Interface, deployment *appsv1.Deployment) { - if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(c context.Context) (bool, error) { - podList, err := clientSet.CoreV1().Pods(deployment.Namespace).List(ctx, metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(deployment.Spec.Template.ObjectMeta.Labels).String(), - }) - if err != nil { - return false, err - } - if len(podList.Items) != int(*deployment.Spec.Replicas) { - t.Logf("Waiting for %v pods to be created, got %v instead", *deployment.Spec.Replicas, len(podList.Items)) - return false, nil - } - for _, pod := range podList.Items { - if pod.Status.Phase != v1.PodRunning { - t.Logf("Pod %v not running yet, is %v instead", pod.Name, pod.Status.Phase) - return false, nil - } - } - return true, nil - }); err != nil { - t.Fatalf("Error waiting for pods running: %v", err) - } -} - func SetPodAntiAffinity(inputPod *v1.Pod, labelKey, labelValue string) { inputPod.Spec.Affinity = &v1.Affinity{ PodAntiAffinity: &v1.PodAntiAffinity{ From a3941a6ce4e3625bd4d323efa05390b905a36756 Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Tue, 20 Aug 2024 02:02:05 +0800 Subject: [PATCH 03/14] migrate e2e_duplicatepods --- test/e2e/e2e_duplicatepods_test.go | 182 ++++++++++++++++++++++------- 1 file changed, 137 insertions(+), 45 deletions(-) diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index 8beb099a48..28feae49ff 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -21,30 +21,68 @@ import ( "os" "strings" "testing" + "time" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" componentbaseconfig "k8s.io/component-base/config" utilptr "k8s.io/utils/ptr" "sigs.k8s.io/descheduler/pkg/api" + apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" "sigs.k8s.io/descheduler/pkg/descheduler/client" - eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/framework/plugins/removeduplicates" - frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing" - frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" ) +func removeDuplicatesPolicy(removeDuplicatesArgs *removeduplicates.RemoveDuplicatesArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy { + return &apiv1alpha2.DeschedulerPolicy{ + Profiles: []apiv1alpha2.DeschedulerProfile{ + { + Name: removeduplicates.PluginName + "Profile", + PluginConfigs: []apiv1alpha2.PluginConfig{ + { + Name: removeduplicates.PluginName, + Args: runtime.RawExtension{ + Object: removeDuplicatesArgs, + }, + }, + { + Name: defaultevictor.PluginName, + Args: runtime.RawExtension{ + Object: evictorArgs, + }, + }, + }, + Plugins: apiv1alpha2.Plugins{ + Filter: apiv1alpha2.PluginSet{ + Enabled: []string{ + defaultevictor.PluginName, + }, + }, + Balance: apiv1alpha2.PluginSet{ + Enabled: []string{ + removeduplicates.PluginName, + }, + }, + }, + }, + }, + } +} + func TestRemoveDuplicates(t *testing.T) { ctx := context.Background() clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") if err != nil { - t.Errorf("Error during client creation with %v", err) + t.Errorf("Error during kubernetes client creation with %v", err) } nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) @@ -62,26 +100,33 @@ func TestRemoveDuplicates(t *testing.T) { defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) t.Log("Creating duplicates pods") - deploymentObj := buildTestDeployment("duplicate-pod", testNamespace.Name, 0, map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, nil) + testLabel := map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"} + deploymentObj := buildTestDeployment("duplicate-pod", testNamespace.Name, 0, testLabel, nil) tests := []struct { - description string + name string replicasNum int beforeFunc func(deployment *appsv1.Deployment) - expectedEvictedPodCount uint - minReplicas uint + expectedEvictedPodCount int + removeDuplicatesArgs *removeduplicates.RemoveDuplicatesArgs + evictorArgs *defaultevictor.DefaultEvictorArgs }{ { - description: "Evict Pod even Pods schedule to specific node", + name: "Evict Pod even Pods schedule to specific node", replicasNum: 4, beforeFunc: func(deployment *appsv1.Deployment) { deployment.Spec.Replicas = utilptr.To[int32](4) deployment.Spec.Template.Spec.NodeName = workerNodes[0].Name }, expectedEvictedPodCount: 2, + removeDuplicatesArgs: &removeduplicates.RemoveDuplicatesArgs{}, + evictorArgs: &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + MinReplicas: 3, + }, }, { - description: "Evict Pod even Pods with local storage", + name: "Evict Pod even Pods with local storage", replicasNum: 5, beforeFunc: func(deployment *appsv1.Deployment) { deployment.Spec.Replicas = utilptr.To[int32](5) @@ -97,19 +142,28 @@ func TestRemoveDuplicates(t *testing.T) { } }, expectedEvictedPodCount: 2, + removeDuplicatesArgs: &removeduplicates.RemoveDuplicatesArgs{}, + evictorArgs: &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + MinReplicas: 3, + }, }, { - description: "Ignores eviction with minReplicas of 4", + name: "Ignores eviction with minReplicas of 4", replicasNum: 3, beforeFunc: func(deployment *appsv1.Deployment) { deployment.Spec.Replicas = utilptr.To[int32](3) }, expectedEvictedPodCount: 0, - minReplicas: 4, + removeDuplicatesArgs: &removeduplicates.RemoveDuplicatesArgs{}, + evictorArgs: &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + MinReplicas: 4, + }, }, } for _, tc := range tests { - t.Run(tc.description, func(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { t.Logf("Creating deployment %v in %v namespace", deploymentObj.Name, deploymentObj.Namespace) tc.beforeFunc(deploymentObj) @@ -117,53 +171,91 @@ func TestRemoveDuplicates(t *testing.T) { if err != nil { t.Logf("Error creating deployment: %v", err) if err = clientSet.AppsV1().Deployments(deploymentObj.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"})).String(), + LabelSelector: labels.SelectorFromSet(deploymentObj.Labels).String(), }); err != nil { t.Fatalf("Unable to delete deployment: %v", err) } return } - defer clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{}) - waitForPodsRunning(ctx, t, clientSet, map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, tc.replicasNum, testNamespace.Name) + defer func() { + clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{}) + waitForPodsToDisappear(ctx, t, clientSet, deploymentObj.Labels, deploymentObj.Namespace) + }() + waitForPodsRunning(ctx, t, clientSet, deploymentObj.Labels, tc.replicasNum, deploymentObj.Namespace) + + preRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...) - // Run removeduplicates plugin - evictionPolicyGroupVersion, err := eutils.SupportEviction(clientSet) - if err != nil || len(evictionPolicyGroupVersion) == 0 { - t.Fatalf("Error creating eviction policy group %v", err) + // Deploy the descheduler with the configured policy + tc.removeDuplicatesArgs.Namespaces = &api.Namespaces{ + Include: []string{testNamespace.Name}, + } + tc.evictorArgs.MinPodAge = &metav1.Duration{Duration: 1 * time.Second} + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs)) + if err != nil { + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - handle, podEvictor, err := frameworktesting.InitFrameworkHandle( - ctx, - clientSet, - nil, - defaultevictor.DefaultEvictorArgs{ - EvictLocalStoragePods: true, - MinReplicas: tc.minReplicas, - }, - nil, - ) + t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name) + _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) if err != nil { - t.Fatalf("Unable to initialize a framework handle: %v", err) + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - plugin, err := removeduplicates.New(&removeduplicates.RemoveDuplicatesArgs{ - Namespaces: &api.Namespaces{ - Include: []string{testNamespace.Name}, - }, - }, - handle, - ) + defer func() { + t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) + err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + }() + + deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) + t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) + _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { - t.Fatalf("Unable to initialize the plugin: %v", err) + t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) } - t.Log("Running removeduplicates plugin") - plugin.(frameworktypes.BalancePlugin).Balance(ctx, workerNodes) - waitForPodsToDisappear(ctx, t, clientSet, map[string]string{"app": "test-duplicate", "name": "test-duplicatePods"}, testNamespace.Name) + deschedulerPodName := "" + defer func() { + if deschedulerPodName != "" { + printPodLogs(ctx, t, clientSet, deschedulerPodName) + } + + t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) + err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + + waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) + }() + + t.Logf("Waiting for the descheduler pod running") + deschedulerPodName = waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) + + // Run RemoveDuplicates strategy + var meetsExpectations bool + var actualEvictedPodCount int + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + currentRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...) + actualEvictedPod := preRunNames.Difference(currentRunNames) + actualEvictedPodCount = actualEvictedPod.Len() + t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", preRunNames.List(), currentRunNames.List(), actualEvictedPodCount) + if actualEvictedPodCount != tc.expectedEvictedPodCount { + t.Logf("Expecting %v number of pods evicted, got %v instead", tc.expectedEvictedPodCount, actualEvictedPodCount) + return false, nil + } + meetsExpectations = true + return true, nil + }); err != nil { + t.Errorf("Error waiting for descheduler running: %v", err) + } - actualEvictedPodCount := podEvictor.TotalEvicted() - if actualEvictedPodCount != tc.expectedEvictedPodCount { - t.Errorf("Test error for description: %s. Unexpected number of pods have been evicted, got %v, expected %v", tc.description, actualEvictedPodCount, tc.expectedEvictedPodCount) + if !meetsExpectations { + t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedPodCount, tc.expectedEvictedPodCount) + } else { + t.Logf("Total of %d Pods were evicted for %s", actualEvictedPodCount, tc.name) } }) } From 8b76062318b29629a3a59de2f9c2fa48cf3e3f8f Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Tue, 20 Aug 2024 02:04:16 +0800 Subject: [PATCH 04/14] migrate e2e_failedpods --- test/e2e/e2e_failedpods_test.go | 208 ++++++++++++++++++++++---------- 1 file changed, 145 insertions(+), 63 deletions(-) diff --git a/test/e2e/e2e_failedpods_test.go b/test/e2e/e2e_failedpods_test.go index 7ff28956e6..b5b56e6eb4 100644 --- a/test/e2e/e2e_failedpods_test.go +++ b/test/e2e/e2e_failedpods_test.go @@ -11,122 +11,204 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" componentbaseconfig "k8s.io/component-base/config" utilptr "k8s.io/utils/ptr" + "sigs.k8s.io/descheduler/pkg/api" + apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" "sigs.k8s.io/descheduler/pkg/descheduler/client" - "sigs.k8s.io/descheduler/pkg/descheduler/evictions" - eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/framework/plugins/removefailedpods" - frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing" - frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" ) var oneHourPodLifetimeSeconds uint = 3600 +var oneSecondPodLifetimeSeconds uint = 1 + +func removeFailedPodsPolicy(removeFailedPodsArgs *removefailedpods.RemoveFailedPodsArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy { + return &apiv1alpha2.DeschedulerPolicy{ + Profiles: []apiv1alpha2.DeschedulerProfile{ + { + Name: removefailedpods.PluginName + "Profile", + PluginConfigs: []apiv1alpha2.PluginConfig{ + { + Name: removefailedpods.PluginName, + Args: runtime.RawExtension{ + Object: removeFailedPodsArgs, + }, + }, + { + Name: defaultevictor.PluginName, + Args: runtime.RawExtension{ + Object: evictorArgs, + }, + }, + }, + Plugins: apiv1alpha2.Plugins{ + Filter: apiv1alpha2.PluginSet{ + Enabled: []string{ + defaultevictor.PluginName, + }, + }, + Deschedule: apiv1alpha2.PluginSet{ + Enabled: []string{ + removefailedpods.PluginName, + }, + }, + }, + }, + }, + } +} func TestFailedPods(t *testing.T) { ctx := context.Background() clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") if err != nil { - t.Errorf("Error during client creation with %v", err) + t.Errorf("Error during kubernetes client creation with %v", err) } - nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { - t.Errorf("Error listing node with %v", err) - } - nodes, _ := splitNodesAndWorkerNodes(nodeList.Items) t.Log("Creating testing namespace") testNamespace := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "e2e-" + strings.ToLower(t.Name())}} if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil { t.Fatalf("Unable to create ns %v", testNamespace.Name) } defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) - testCases := map[string]struct { - expectedEvictedCount uint - args *removefailedpods.RemoveFailedPodsArgs + + tests := []struct { + name string + expectedEvictedPodCount int + removeFailedPodsArgs *removefailedpods.RemoveFailedPodsArgs }{ - "test-failed-pods-default-args": { - expectedEvictedCount: 1, - args: &removefailedpods.RemoveFailedPodsArgs{}, + { + name: "test-failed-pods-default-args", + expectedEvictedPodCount: 1, + removeFailedPodsArgs: &removefailedpods.RemoveFailedPodsArgs{ + MinPodLifetimeSeconds: &oneSecondPodLifetimeSeconds, + }, }, - "test-failed-pods-reason-unmatched": { - expectedEvictedCount: 0, - args: &removefailedpods.RemoveFailedPodsArgs{ - Reasons: []string{"ReasonDoesNotMatch"}, + { + name: "test-failed-pods-reason-unmatched", + expectedEvictedPodCount: 0, + removeFailedPodsArgs: &removefailedpods.RemoveFailedPodsArgs{ + Reasons: []string{"ReasonDoesNotMatch"}, + MinPodLifetimeSeconds: &oneSecondPodLifetimeSeconds, }, }, - "test-failed-pods-min-age-unmet": { - expectedEvictedCount: 0, - args: &removefailedpods.RemoveFailedPodsArgs{ + { + name: "test-failed-pods-min-age-unmet", + expectedEvictedPodCount: 0, + removeFailedPodsArgs: &removefailedpods.RemoveFailedPodsArgs{ MinPodLifetimeSeconds: &oneHourPodLifetimeSeconds, }, }, - "test-failed-pods-exclude-job-kind": { - expectedEvictedCount: 0, - args: &removefailedpods.RemoveFailedPodsArgs{ - ExcludeOwnerKinds: []string{"Job"}, + { + name: "test-failed-pods-exclude-job-kind", + expectedEvictedPodCount: 0, + removeFailedPodsArgs: &removefailedpods.RemoveFailedPodsArgs{ + ExcludeOwnerKinds: []string{"Job"}, + MinPodLifetimeSeconds: &oneSecondPodLifetimeSeconds, }, }, } - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - job := initFailedJob(name, testNamespace.Namespace) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + job := initFailedJob(tc.name, testNamespace.Namespace) t.Logf("Creating job %s in %s namespace", job.Name, job.Namespace) jobClient := clientSet.BatchV1().Jobs(testNamespace.Name) if _, err := jobClient.Create(ctx, job, metav1.CreateOptions{}); err != nil { - t.Fatalf("Error creating Job %s: %v", name, err) + t.Fatalf("Error creating Job %s: %v", tc.name, err) } deletePropagationPolicy := metav1.DeletePropagationForeground - defer jobClient.Delete(ctx, job.Name, metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}) + defer func() { + jobClient.Delete(ctx, job.Name, metav1.DeleteOptions{PropagationPolicy: &deletePropagationPolicy}) + waitForPodsToDisappear(ctx, t, clientSet, job.Labels, job.Namespace) + }() waitForJobPodPhase(ctx, t, clientSet, job, v1.PodFailed) - evictionPolicyGroupVersion, err := eutils.SupportEviction(clientSet) - if err != nil || len(evictionPolicyGroupVersion) == 0 { - t.Fatalf("Error detecting eviction policy group: %v", err) + preRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...) + + // Deploy the descheduler with the configured policy + evictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + tc.removeFailedPodsArgs.Namespaces = &api.Namespaces{ + Include: []string{testNamespace.Name}, } - handle, podEvictor, err := frameworktesting.InitFrameworkHandle( - ctx, - clientSet, - evictions.NewOptions(). - WithPolicyGroupVersion(evictionPolicyGroupVersion), - defaultevictor.DefaultEvictorArgs{ - EvictLocalStoragePods: true, - }, - nil, - ) + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs)) if err != nil { - t.Fatalf("Unable to initialize a framework handle: %v", err) + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - t.Logf("Running RemoveFailedPods strategy for %s", name) + t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name) + _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } - plugin, err := removefailedpods.New(&removefailedpods.RemoveFailedPodsArgs{ - Reasons: tc.args.Reasons, - MinPodLifetimeSeconds: tc.args.MinPodLifetimeSeconds, - IncludingInitContainers: tc.args.IncludingInitContainers, - ExcludeOwnerKinds: tc.args.ExcludeOwnerKinds, - LabelSelector: tc.args.LabelSelector, - Namespaces: tc.args.Namespaces, - }, - handle, - ) + defer func() { + t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) + err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + }() + + deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) + t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) + _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { - t.Fatalf("Unable to initialize the plugin: %v", err) + t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) } - plugin.(frameworktypes.DeschedulePlugin).Deschedule(ctx, nodes) - t.Logf("Finished RemoveFailedPods strategy for %s", name) + deschedulerPodName := "" + defer func() { + if deschedulerPodName != "" { + printPodLogs(ctx, t, clientSet, deschedulerPodName) + } + + t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) + err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + + waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) + }() + + t.Logf("Waiting for the descheduler pod running") + deschedulerPodName = waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) + + // Run RemoveDuplicates strategy + var meetsExpectations bool + var actualEvictedPodCount int + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + currentRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...) + actualEvictedPod := preRunNames.Difference(currentRunNames) + actualEvictedPodCount = actualEvictedPod.Len() + t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", preRunNames.List(), currentRunNames.List(), actualEvictedPodCount) + if actualEvictedPodCount != tc.expectedEvictedPodCount { + t.Logf("Expecting %v number of pods evicted, got %v instead", tc.expectedEvictedPodCount, actualEvictedPodCount) + return false, nil + } + meetsExpectations = true + return true, nil + }); err != nil { + t.Errorf("Error waiting for descheduler running: %v", err) + } - if actualEvictedCount := podEvictor.TotalEvicted(); actualEvictedCount == tc.expectedEvictedCount { - t.Logf("Total of %d Pods were evicted for %s", actualEvictedCount, name) + if !meetsExpectations { + t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedPodCount, tc.expectedEvictedPodCount) } else { - t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedCount, tc.expectedEvictedCount) + t.Logf("Total of %d Pods were evicted for %s", actualEvictedPodCount, tc.name) } }) } From dbf4d37a90d5a24a322e2b1811e3308a6b82835d Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Tue, 20 Aug 2024 02:08:27 +0800 Subject: [PATCH 05/14] refactor e2e_toomanyrestarts --- test/e2e/e2e_toomanyrestarts_test.go | 77 ++++++++++++++++------------ 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/test/e2e/e2e_toomanyrestarts_test.go b/test/e2e/e2e_toomanyrestarts_test.go index 714769085e..6af8830a62 100644 --- a/test/e2e/e2e_toomanyrestarts_test.go +++ b/test/e2e/e2e_toomanyrestarts_test.go @@ -33,7 +33,6 @@ import ( clientset "k8s.io/client-go/kubernetes" componentbaseconfig "k8s.io/component-base/config" - "sigs.k8s.io/descheduler/cmd/descheduler/app/options" "sigs.k8s.io/descheduler/pkg/api" apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" "sigs.k8s.io/descheduler/pkg/descheduler/client" @@ -43,30 +42,24 @@ import ( const deploymentReplicas = 4 -func tooManyRestartsPolicy(targetNamespace string, podRestartThresholds int32, includingInitContainers bool) *apiv1alpha2.DeschedulerPolicy { +var testRestartLabel = map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"} + +func tooManyRestartsPolicy(restartsArgs *removepodshavingtoomanyrestarts.RemovePodsHavingTooManyRestartsArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy { return &apiv1alpha2.DeschedulerPolicy{ Profiles: []apiv1alpha2.DeschedulerProfile{ { - Name: "TooManyRestartsProfile", + Name: removepodshavingtoomanyrestarts.PluginName + "Profile", PluginConfigs: []apiv1alpha2.PluginConfig{ { Name: removepodshavingtoomanyrestarts.PluginName, Args: runtime.RawExtension{ - Object: &removepodshavingtoomanyrestarts.RemovePodsHavingTooManyRestartsArgs{ - PodRestartThreshold: podRestartThresholds, - IncludingInitContainers: includingInitContainers, - Namespaces: &api.Namespaces{ - Include: []string{targetNamespace}, - }, - }, + Object: restartsArgs, }, }, { Name: defaultevictor.PluginName, Args: runtime.RawExtension{ - Object: &defaultevictor.DefaultEvictorArgs{ - EvictLocalStoragePods: true, - }, + Object: evictorArgs, }, }, }, @@ -89,7 +82,6 @@ func tooManyRestartsPolicy(targetNamespace string, podRestartThresholds int32, i func TestTooManyRestarts(t *testing.T) { ctx := context.Background() - initPluginRegistry() clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") if err != nil { @@ -102,7 +94,7 @@ func TestTooManyRestarts(t *testing.T) { t.Fatalf("Unable to create ns %v", testNamespace.Name) } - deploymentObj := buildTestDeployment("restart-pod", testNamespace.Name, deploymentReplicas, map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"}, func(deployment *appsv1.Deployment) { + deploymentObj := buildTestDeployment("restart-pod", testNamespace.Name, deploymentReplicas, testRestartLabel, func(deployment *appsv1.Deployment) { deployment.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh"} deployment.Spec.Template.Spec.Containers[0].Args = []string{"-c", "sleep 1s && exit 1"} }) @@ -112,7 +104,7 @@ func TestTooManyRestarts(t *testing.T) { if err != nil { t.Logf("Error creating deployment: %v", err) if err = clientSet.AppsV1().Deployments(deploymentObj.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"})).String(), + LabelSelector: labels.SelectorFromSet(deploymentObj.Labels).String(), }); err != nil { t.Fatalf("Unable to delete deployment: %v", err) } @@ -129,35 +121,46 @@ func TestTooManyRestarts(t *testing.T) { tests := []struct { name string - policy *apiv1alpha2.DeschedulerPolicy - expectedEvictedPodCount uint + expectedEvictedPodCount int + restartsArgs *removepodshavingtoomanyrestarts.RemovePodsHavingTooManyRestartsArgs }{ { name: "test-no-evictions", - policy: tooManyRestartsPolicy(testNamespace.Name, 10000, true), expectedEvictedPodCount: 0, + restartsArgs: &removepodshavingtoomanyrestarts.RemovePodsHavingTooManyRestartsArgs{ + PodRestartThreshold: 10000, + IncludingInitContainers: true, + }, }, { name: "test-one-evictions", - policy: tooManyRestartsPolicy(testNamespace.Name, 3, true), expectedEvictedPodCount: 4, + restartsArgs: &removepodshavingtoomanyrestarts.RemovePodsHavingTooManyRestartsArgs{ + PodRestartThreshold: 3, + IncludingInitContainers: true, + }, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - rs, err := options.NewDeschedulerServer() - if err != nil { - t.Fatalf("Unable to initialize server: %v\n", err) - } - rs.Client = clientSet - rs.EventClient = clientSet - preRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...) + // Deploy the descheduler with the configured policy - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(tc.policy) + evictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + tc.restartsArgs.Namespaces = &api.Namespaces{ + Include: []string{testNamespace.Name}, + } + + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(tooManyRestartsPolicy(tc.restartsArgs, evictorArgs)) if err != nil { t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } + t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name) _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) if err != nil { @@ -197,20 +200,28 @@ func TestTooManyRestarts(t *testing.T) { deschedulerPodName = waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) // Run RemovePodsHavingTooManyRestarts strategy - if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 20*time.Second, true, func(ctx context.Context) (bool, error) { + var meetsExpectations bool + var actualEvictedPodCount int + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { currentRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...) actualEvictedPod := preRunNames.Difference(currentRunNames) - actualEvictedPodCount := uint(actualEvictedPod.Len()) + actualEvictedPodCount = actualEvictedPod.Len() t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", preRunNames.List(), currentRunNames.List(), actualEvictedPodCount) - if actualEvictedPodCount < tc.expectedEvictedPodCount { + if actualEvictedPodCount != tc.expectedEvictedPodCount { t.Logf("Expecting %v number of pods evicted, got %v instead", tc.expectedEvictedPodCount, actualEvictedPodCount) return false, nil } - + meetsExpectations = true return true, nil }); err != nil { t.Errorf("Error waiting for descheduler running: %v", err) } + + if !meetsExpectations { + t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedPodCount, tc.expectedEvictedPodCount) + } else { + t.Logf("Total of %d Pods were evicted for %s", actualEvictedPodCount, tc.name) + } }) } } @@ -218,7 +229,7 @@ func TestTooManyRestarts(t *testing.T) { func waitPodRestartCount(ctx context.Context, clientSet clientset.Interface, namespace string, t *testing.T, expectedNumberOfRestarts int) { if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"test": "restart-pod", "name": "test-toomanyrestarts"})).String(), + LabelSelector: labels.SelectorFromSet(testRestartLabel).String(), }) if err != nil { t.Fatalf("Unexpected err: %v", err) From 5464f377a10f217fe4c35a756fcdf43ca680a7a2 Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Tue, 20 Aug 2024 02:10:02 +0800 Subject: [PATCH 06/14] migrate e2e_topologyspreadconstraint --- test/e2e/e2e_topologyspreadconstraint_test.go | 186 ++++++++++++------ 1 file changed, 129 insertions(+), 57 deletions(-) diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index dacf599c85..a3cc27b29b 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -6,29 +6,68 @@ import ( "os" "strings" "testing" + "time" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" componentbaseconfig "k8s.io/component-base/config" + "sigs.k8s.io/descheduler/pkg/api" + apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" "sigs.k8s.io/descheduler/pkg/descheduler/client" - "sigs.k8s.io/descheduler/pkg/descheduler/evictions" - eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" "sigs.k8s.io/descheduler/pkg/framework/plugins/removepodsviolatingtopologyspreadconstraint" - frameworktesting "sigs.k8s.io/descheduler/pkg/framework/testing" - frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types" ) const zoneTopologyKey string = "topology.kubernetes.io/zone" +func topologySpreadConstraintPolicy(constraintArgs *removepodsviolatingtopologyspreadconstraint.RemovePodsViolatingTopologySpreadConstraintArgs, + evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy { + return &apiv1alpha2.DeschedulerPolicy{ + Profiles: []apiv1alpha2.DeschedulerProfile{ + { + Name: removepodsviolatingtopologyspreadconstraint.PluginName + "Profile", + PluginConfigs: []apiv1alpha2.PluginConfig{ + { + Name: removepodsviolatingtopologyspreadconstraint.PluginName, + Args: runtime.RawExtension{ + Object: constraintArgs, + }, + }, + { + Name: defaultevictor.PluginName, + Args: runtime.RawExtension{ + Object: evictorArgs, + }, + }, + }, + Plugins: apiv1alpha2.Plugins{ + Filter: apiv1alpha2.PluginSet{ + Enabled: []string{ + defaultevictor.PluginName, + }, + }, + Balance: apiv1alpha2.PluginSet{ + Enabled: []string{ + removepodsviolatingtopologyspreadconstraint.PluginName, + }, + }, + }, + }, + }, + } +} + func TestTopologySpreadConstraint(t *testing.T) { ctx := context.Background() + clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") if err != nil { - t.Errorf("Error during client creation with %v", err) + t.Errorf("Error during kubernetes client creation with %v", err) } nodeList, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) @@ -125,7 +164,9 @@ func TestTopologySpreadConstraint(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { t.Logf("Creating Deployment %s with %d replicas", name, tc.replicaCount) - deployment := buildTestDeployment(name, testNamespace.Name, int32(tc.replicaCount), tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels, func(d *appsv1.Deployment) { + deployLabels := tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels + deployLabels["name"] = name + deployment := buildTestDeployment(name, testNamespace.Name, int32(tc.replicaCount), deployLabels, func(d *appsv1.Deployment) { d.Spec.Template.Spec.TopologySpreadConstraints = []v1.TopologySpreadConstraint{tc.topologySpreadConstraint} }) if _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}); err != nil { @@ -140,7 +181,9 @@ func TestTopologySpreadConstraint(t *testing.T) { // Create a "Violator" Deployment that has the same label and is forced to be on the same node using a nodeSelector violatorDeploymentName := name + "-violator" violatorCount := tc.topologySpreadConstraint.MaxSkew + 1 - violatorDeployment := buildTestDeployment(violatorDeploymentName, testNamespace.Name, violatorCount, tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels, func(d *appsv1.Deployment) { + violatorDeployLabels := tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels + violatorDeployLabels["name"] = violatorDeploymentName + violatorDeployment := buildTestDeployment(violatorDeploymentName, testNamespace.Name, violatorCount, violatorDeployLabels, func(d *appsv1.Deployment) { d.Spec.Template.Spec.NodeSelector = map[string]string{zoneTopologyKey: workerNodes[0].Labels[zoneTopologyKey]} }) if _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, violatorDeployment, metav1.CreateOptions{}); err != nil { @@ -152,76 +195,105 @@ func TestTopologySpreadConstraint(t *testing.T) { }() waitForPodsRunning(ctx, t, clientSet, violatorDeployment.Labels, int(violatorCount), violatorDeployment.Namespace) - evictionPolicyGroupVersion, err := eutils.SupportEviction(clientSet) - if err != nil || len(evictionPolicyGroupVersion) == 0 { - t.Fatalf("Error detecting eviction policy group: %v", err) - } + // Run TopologySpreadConstraint strategy + t.Logf("Running RemovePodsViolatingTopologySpreadConstraint strategy for %s", name) - handle, podEvictor, err := frameworktesting.InitFrameworkHandle( - ctx, - clientSet, - evictions.NewOptions(). - WithPolicyGroupVersion(evictionPolicyGroupVersion), - defaultevictor.DefaultEvictorArgs{ - EvictLocalStoragePods: true, + evictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + } + constraintArgs := &removepodsviolatingtopologyspreadconstraint.RemovePodsViolatingTopologySpreadConstraintArgs{ + Constraints: []v1.UnsatisfiableConstraintAction{tc.topologySpreadConstraint.WhenUnsatisfiable}, + Namespaces: &api.Namespaces{ + Include: []string{testNamespace.Name}, }, - nil, - ) + } + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(topologySpreadConstraintPolicy(constraintArgs, evictorArgs)) if err != nil { - t.Fatalf("Unable to initialize a framework handle: %v", err) + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - // Run TopologySpreadConstraint strategy - t.Logf("Running RemovePodsViolatingTopologySpreadConstraint strategy for %s", name) + t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name) + _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } - plugin, err := removepodsviolatingtopologyspreadconstraint.New(&removepodsviolatingtopologyspreadconstraint.RemovePodsViolatingTopologySpreadConstraintArgs{ - Constraints: []v1.UnsatisfiableConstraintAction{tc.topologySpreadConstraint.WhenUnsatisfiable}, - }, - handle, - ) + defer func() { + t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) + err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + }() + deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) + t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) + _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { - t.Fatalf("Unable to initialize the plugin: %v", err) + t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) } - plugin.(frameworktypes.BalancePlugin).Balance(ctx, workerNodes) - t.Logf("Finished RemovePodsViolatingTopologySpreadConstraint strategy for %s", name) + deschedulerPodName := "" + defer func() { + if deschedulerPodName != "" { + printPodLogs(ctx, t, clientSet, deschedulerPodName) + } - if totalEvicted := podEvictor.TotalEvicted(); totalEvicted == tc.expectedEvictedCount { - t.Logf("Total of %d Pods were evicted for %s", totalEvicted, name) - } else { - t.Fatalf("Expected %d evictions but got %d for %s TopologySpreadConstraint", tc.expectedEvictedCount, totalEvicted, name) - } + t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) + err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) + }() - if tc.expectedEvictedCount == 0 { - return - } + t.Logf("Waiting for the descheduler pod running") + deschedulerPodName = waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - listOptions := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(tc.topologySpreadConstraint.LabelSelector.MatchLabels).String()} - pods, err := clientSet.CoreV1().Pods(testNamespace.Name).List(ctx, listOptions) - if err != nil { - t.Errorf("Error listing pods for %s: %v", name, err) - } + // Run RemovePodsHavingTooManyRestarts strategy + var meetsExpectations bool + var skewVal int + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + listOptions := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(tc.topologySpreadConstraint.LabelSelector.MatchLabels).String()} + pods, err := clientSet.CoreV1().Pods(testNamespace.Name).List(ctx, listOptions) + if err != nil { + t.Errorf("Error listing pods for %s: %v", name, err) + } - nodePodCountMap := make(map[string]int) - for _, pod := range pods.Items { - nodePodCountMap[pod.Spec.NodeName]++ - } + nodePodCountMap := make(map[string]int) + for _, pod := range pods.Items { + nodePodCountMap[pod.Spec.NodeName]++ + } - if len(nodePodCountMap) != len(workerNodes) { - t.Errorf("%s Pods were scheduled on only '%d' nodes and were not properly distributed on the nodes", name, len(nodePodCountMap)) - } + if len(nodePodCountMap) != len(workerNodes) { + t.Errorf("%s Pods were scheduled on only '%d' nodes and were not properly distributed on the nodes", name, len(nodePodCountMap)) + return false, nil + } + + skewVal = getSkewValPodDistribution(nodePodCountMap) + if skewVal > int(tc.topologySpreadConstraint.MaxSkew) { + t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", name, tc.topologySpreadConstraint.MaxSkew, skewVal) + return false, nil + } - min, max := getMinAndMaxPodDistribution(nodePodCountMap) - if max-min > int(tc.topologySpreadConstraint.MaxSkew) { - t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", name, tc.topologySpreadConstraint.MaxSkew, max-min) + meetsExpectations = true + return true, nil + }); err != nil { + t.Errorf("Error waiting for descheduler running: %v", err) } - t.Logf("Pods for %s were distributed in line with max skew of %d", name, tc.topologySpreadConstraint.MaxSkew) + if !meetsExpectations { + t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", name, tc.topologySpreadConstraint.MaxSkew, skewVal) + } else { + t.Logf("Pods for %s were distributed in line with max skew of %d", name, tc.topologySpreadConstraint.MaxSkew) + } }) } } -func getMinAndMaxPodDistribution(nodePodCountMap map[string]int) (int, int) { +func getSkewValPodDistribution(nodePodCountMap map[string]int) int { min := math.MaxInt32 max := math.MinInt32 for _, podCount := range nodePodCountMap { @@ -233,7 +305,7 @@ func getMinAndMaxPodDistribution(nodePodCountMap map[string]int) (int, int) { } } - return min, max + return max - min } func nodeInclusionPolicyRef(policy v1.NodeInclusionPolicy) *v1.NodeInclusionPolicy { From 59b020368aee7820f8c8efd19c2aa975cd20611b Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Tue, 20 Aug 2024 02:19:17 +0800 Subject: [PATCH 07/14] migrate e2e_leaderelection --- test/e2e/e2e_leaderelection_test.go | 210 ++++++++++++++++---------- test/e2e/policy_leaderelection_a.yaml | 15 -- test/e2e/policy_leaderelection_b.yaml | 15 -- 3 files changed, 128 insertions(+), 112 deletions(-) delete mode 100644 test/e2e/policy_leaderelection_a.yaml delete mode 100644 test/e2e/policy_leaderelection_b.yaml diff --git a/test/e2e/e2e_leaderelection_test.go b/test/e2e/e2e_leaderelection_test.go index 9e1807f6df..6c54314de4 100644 --- a/test/e2e/e2e_leaderelection_test.go +++ b/test/e2e/e2e_leaderelection_test.go @@ -30,17 +30,60 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" clientset "k8s.io/client-go/kubernetes" + componentbaseconfig "k8s.io/component-base/config" - "sigs.k8s.io/descheduler/cmd/descheduler/app/options" - "sigs.k8s.io/descheduler/pkg/descheduler" + "sigs.k8s.io/descheduler/pkg/api" + apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" + "sigs.k8s.io/descheduler/pkg/descheduler/client" + "sigs.k8s.io/descheduler/pkg/framework/plugins/defaultevictor" + "sigs.k8s.io/descheduler/pkg/framework/plugins/podlifetime" ) +func podlifetimePolicy(podLifeTimeArgs *podlifetime.PodLifeTimeArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy { + return &apiv1alpha2.DeschedulerPolicy{ + Profiles: []apiv1alpha2.DeschedulerProfile{ + { + Name: podlifetime.PluginName + "Profile", + PluginConfigs: []apiv1alpha2.PluginConfig{ + { + Name: podlifetime.PluginName, + Args: runtime.RawExtension{ + Object: podLifeTimeArgs, + }, + }, + { + Name: defaultevictor.PluginName, + Args: runtime.RawExtension{ + Object: evictorArgs, + }, + }, + }, + Plugins: apiv1alpha2.Plugins{ + Filter: apiv1alpha2.PluginSet{ + Enabled: []string{ + defaultevictor.PluginName, + }, + }, + Deschedule: apiv1alpha2.PluginSet{ + Enabled: []string{ + podlifetime.PluginName, + }, + }, + }, + }, + }, + } +} + func TestLeaderElection(t *testing.T) { - descheduler.SetupPlugins() ctx := context.Background() - clientSet, _, _, _ := initializeClient(ctx, t) + clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{Kubeconfig: os.Getenv("KUBECONFIG")}, "") + if err != nil { + t.Errorf("Error during kubernetes client creation with %v", err) + } ns1 := "e2e-" + strings.ToLower(t.Name()+"-a") ns2 := "e2e-" + strings.ToLower(t.Name()+"-b") @@ -59,91 +102,60 @@ func TestLeaderElection(t *testing.T) { } defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace2.Name, metav1.DeleteOptions{}) - deployment1, err := createDeployment(ctx, clientSet, ns1, 5, t) + testLabel := map[string]string{"test": "leaderelection", "name": "test-leaderelection"} + deployment1 := buildTestDeployment("leaderelection", ns1, 5, testLabel, nil) + err = createDeployment(t, ctx, clientSet, deployment1) if err != nil { t.Fatalf("create deployment 1: %v", err) } - defer clientSet.AppsV1().Deployments(deployment1.Namespace).Delete(ctx, deployment1.Name, metav1.DeleteOptions{}) - deployment2, err := createDeployment(ctx, clientSet, ns2, 5, t) + deployment2 := buildTestDeployment("leaderelection", ns2, 5, testLabel, nil) + err = createDeployment(t, ctx, clientSet, deployment2) if err != nil { t.Fatalf("create deployment 2: %v", err) } - defer clientSet.AppsV1().Deployments(deployment2.Namespace).Delete(ctx, deployment2.Name, metav1.DeleteOptions{}) - - waitForPodsRunning(ctx, t, clientSet, map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, 5, ns1) - - podListAOrg := getPodNameList(ctx, clientSet, ns1, t) - - waitForPodsRunning(ctx, t, clientSet, map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, 5, ns2) - - podListBOrg := getPodNameList(ctx, clientSet, ns2, t) + defer func() { + clientSet.AppsV1().Deployments(deployment1.Namespace).Delete(ctx, deployment1.Name, metav1.DeleteOptions{}) + clientSet.AppsV1().Deployments(deployment2.Namespace).Delete(ctx, deployment2.Name, metav1.DeleteOptions{}) + }() - s1, err := options.NewDeschedulerServer() - if err != nil { - t.Fatalf("unable to initialize server: %v", err) - } - s1.Client = clientSet - s1.DeschedulingInterval = 5 * time.Second - s1.LeaderElection.LeaderElect = true - s1.LeaderElection.RetryPeriod = metav1.Duration{ - Duration: time.Second, - } - s1.ClientConnection.Kubeconfig = os.Getenv("KUBECONFIG") - s1.PolicyConfigFile = "./policy_leaderelection_a.yaml" + waitForPodsRunning(ctx, t, clientSet, deployment1.Labels, 5, deployment1.Namespace) + podListAOrg := getCurrentPodNames(t, ctx, clientSet, ns1) - s2, err := options.NewDeschedulerServer() - if err != nil { - t.Fatalf("unable to initialize server: %v", err) - } - s2.Client = clientSet - s2.DeschedulingInterval = 5 * time.Second - s2.LeaderElection.LeaderElect = true - s2.LeaderElection.RetryPeriod = metav1.Duration{ - Duration: time.Second, - } - s2.ClientConnection.Kubeconfig = os.Getenv("KUBECONFIG") - s2.PolicyConfigFile = "./policy_leaderelection_b.yaml" + waitForPodsRunning(ctx, t, clientSet, deployment2.Labels, 5, deployment2.Namespace) + podListBOrg := getCurrentPodNames(t, ctx, clientSet, ns2) + t.Log("Starting deschedulers") // Delete the descheduler lease err = clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{}) - if err != nil { - if !apierrors.IsNotFound(err) { - t.Fatalf("Unable to remove kube-system/descheduler lease: %v", err) - } + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("Unable to remove kube-system/descheduler lease: %v", err) } t.Logf("Removed kube-system/descheduler lease") - - t.Log("starting deschedulers") - - go func() { - err := descheduler.Run(ctx, s1) - if err != nil { - t.Errorf("unable to start descheduler: %v", err) - return - } - }() - + descheduler1 := startDeschedulerServer(t, ctx, clientSet, ns1) time.Sleep(1 * time.Second) - - go func() { - err := descheduler.Run(ctx, s2) - if err != nil { - t.Errorf("unable to start descheduler: %v", err) - return + descheduler2 := startDeschedulerServer(t, ctx, clientSet, ns2) + defer func() { + for _, deploy := range []*appsv1.Deployment{descheduler1, descheduler2} { + printPodLogs(ctx, t, clientSet, deploy.Name) + t.Logf("Deleting %q deployment...", deploy.Name) + err = clientSet.AppsV1().Deployments(deploy.Namespace).Delete(ctx, deploy.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q deployment: %v", deploy.Name, err) + } + + waitForPodsToDisappear(ctx, t, clientSet, deploy.Labels, deploy.Namespace) } - }() - defer clientSet.CoordinationV1().Leases(s1.LeaderElection.ResourceNamespace).Delete(ctx, s1.LeaderElection.ResourceName, metav1.DeleteOptions{}) - defer clientSet.CoordinationV1().Leases(s2.LeaderElection.ResourceNamespace).Delete(ctx, s2.LeaderElection.ResourceName, metav1.DeleteOptions{}) + clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{}) + }() // wait for a while so all the pods are 5 seconds older time.Sleep(7 * time.Second) // validate only pods from e2e-testleaderelection-a namespace are evicted. - podListA := getPodNameList(ctx, clientSet, ns1, t) - - podListB := getPodNameList(ctx, clientSet, ns2, t) + podListA := getCurrentPodNames(t, ctx, clientSet, ns1) + podListB := getCurrentPodNames(t, ctx, clientSet, ns2) left := reflect.DeepEqual(podListAOrg, podListA) right := reflect.DeepEqual(podListBOrg, podListB) @@ -165,32 +177,66 @@ func TestLeaderElection(t *testing.T) { } } -func createDeployment(ctx context.Context, clientSet clientset.Interface, namespace string, replicas int32, t *testing.T) (*appsv1.Deployment, error) { - deployment := buildTestDeployment("leaderelection", namespace, replicas, map[string]string{"test": "leaderelection", "name": "test-leaderelection"}, nil) - +func createDeployment(t *testing.T, ctx context.Context, clientSet clientset.Interface, deployment *appsv1.Deployment) error { t.Logf("Creating deployment %v for namespace %s", deployment.Name, deployment.Namespace) - deployment, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}) + _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}) if err != nil { t.Logf("Error creating deployment: %v", err) if err = clientSet.AppsV1().Deployments(deployment.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"test": "leaderelection", "name": "test-leaderelection"})).String(), + LabelSelector: labels.SelectorFromSet(deployment.Labels).String(), }); err != nil { t.Fatalf("Unable to delete deployment: %v", err) } - return nil, fmt.Errorf("create deployment %v", err) + return fmt.Errorf("create deployment %v", err) } - return deployment, nil + return nil } -func getPodNameList(ctx context.Context, clientSet clientset.Interface, namespace string, t *testing.T) []string { - podList, err := clientSet.CoreV1().Pods(namespace).List( - ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(map[string]string{"test": "leaderelection", "name": "test-leaderelection"})).String()}) +func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clientset.Interface, testName string) *appsv1.Deployment { + var maxLifeTime uint = 5 + podLifeTimeArgs := &podlifetime.PodLifeTimeArgs{ + MaxPodLifeTimeSeconds: &maxLifeTime, + Namespaces: &api.Namespaces{ + Include: []string{testName}, + }, + } + + // Deploy the descheduler with the configured policy + evictorArgs := &defaultevictor.DefaultEvictorArgs{ + EvictLocalStoragePods: true, + EvictSystemCriticalPods: false, + IgnorePvcPods: false, + EvictFailedBarePods: false, + MinPodAge: &metav1.Duration{Duration: 1 * time.Second}, + } + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(podlifetimePolicy(podLifeTimeArgs, evictorArgs)) if err != nil { - t.Fatalf("Unable to list pods from ns: %s: %v", namespace, err) + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - podNames := make([]string, len(podList.Items)) - for i, pod := range podList.Items { - podNames[i] = pod.Name + + t.Logf("Creating %q policy CM with RemoveDuplicates configured...", deschedulerPolicyConfigMapObj.Name) + _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + + defer func() { + t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) + err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + }() + + deschedulerDeploymentObj := deschedulerDeployment(testName) + deschedulerDeploymentObj.Name = fmt.Sprintf("%s-%s", deschedulerDeploymentObj.Name, testName) + t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) + _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) } - return podNames + + t.Logf("Waiting for the descheduler pod running") + waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) + return deschedulerDeploymentObj } diff --git a/test/e2e/policy_leaderelection_a.yaml b/test/e2e/policy_leaderelection_a.yaml deleted file mode 100644 index 2e7478b756..0000000000 --- a/test/e2e/policy_leaderelection_a.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: "descheduler/v1alpha2" -kind: "DeschedulerPolicy" -profiles: - - name: ProfileName - pluginConfig: - - name: "PodLifeTime" - args: - maxPodLifeTimeSeconds: 5 - namespaces: - include: - - "e2e-testleaderelection-a" - plugins: - deschedule: - enabled: - - "PodLifeTime" diff --git a/test/e2e/policy_leaderelection_b.yaml b/test/e2e/policy_leaderelection_b.yaml deleted file mode 100644 index 58d1ccac8f..0000000000 --- a/test/e2e/policy_leaderelection_b.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: "descheduler/v1alpha2" -kind: "DeschedulerPolicy" -profiles: - - name: ProfileName - pluginConfig: - - name: "PodLifeTime" - args: - maxPodLifeTimeSeconds: 5 - namespaces: - include: - - "e2e-testleaderelection-b" - plugins: - deschedule: - enabled: - - "PodLifeTime" From a4b52b8776504e21c68e19ab07644dd6632e52bd Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Tue, 20 Aug 2024 02:19:59 +0800 Subject: [PATCH 08/14] migrate e2e_clientconnection --- test/e2e/test_clientconnection_test.go | 61 +++++++++++++++++++------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/test/e2e/test_clientconnection_test.go b/test/e2e/test_clientconnection_test.go index e6bcdc9206..2c195242f5 100644 --- a/test/e2e/test_clientconnection_test.go +++ b/test/e2e/test_clientconnection_test.go @@ -5,36 +5,65 @@ import ( "os" "testing" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" componentbaseconfig "k8s.io/component-base/config" - "sigs.k8s.io/descheduler/cmd/descheduler/app/options" - deschedulerapi "sigs.k8s.io/descheduler/pkg/api" - "sigs.k8s.io/descheduler/pkg/descheduler" + + apiv1alpha2 "sigs.k8s.io/descheduler/pkg/api/v1alpha2" "sigs.k8s.io/descheduler/pkg/descheduler/client" - eutils "sigs.k8s.io/descheduler/pkg/descheduler/evictions/utils" ) func TestClientConnectionConfiguration(t *testing.T) { ctx := context.Background() - clientConnection := componentbaseconfig.ClientConnectionConfiguration{ + testName := "test-clientconnection" + clientSet, err := client.CreateClient(componentbaseconfig.ClientConnectionConfiguration{ Kubeconfig: os.Getenv("KUBECONFIG"), QPS: 50, Burst: 100, - } - clientSet, err := client.CreateClient(clientConnection, "") + }, "") if err != nil { - t.Errorf("Error during client creation with %v", err) + t.Errorf("Error during kubernetes client creation with %v", err) } - s, err := options.NewDeschedulerServer() + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(&apiv1alpha2.DeschedulerPolicy{}) if err != nil { - t.Fatalf("Unable to initialize server: %v", err) + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - s.Client = clientSet - evictionPolicyGroupVersion, err := eutils.SupportEviction(s.Client) - if err != nil || len(evictionPolicyGroupVersion) == 0 { - t.Errorf("Error when checking support for eviction: %v", err) + + t.Logf("Creating %q policy CM with RemovePodsHavingTooManyRestarts configured...", deschedulerPolicyConfigMapObj.Name) + _, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - if err := descheduler.RunDeschedulerStrategies(ctx, s, &deschedulerapi.DeschedulerPolicy{}, evictionPolicyGroupVersion); err != nil { - t.Errorf("Error running descheduler strategies: %+v", err) + + defer func() { + t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) + err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) + } + }() + + deschedulerDeploymentObj := deschedulerDeployment(testName) + t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) + _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) } + + deschedulerPodName := "" + defer func() { + if deschedulerPodName != "" { + printPodLogs(ctx, t, clientSet, deschedulerPodName) + } + + t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) + err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) + } + waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) + }() + + t.Logf("Waiting for the descheduler pod running") + deschedulerPodName = waitForPodsRunning(ctx, t, clientSet, map[string]string{"app": "descheduler", "test": testName}, 1, deschedulerDeploymentObj.Namespace) } From 71dd70dd490443a65473da9c534ef3bbe0e57132 Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Tue, 20 Aug 2024 02:31:39 +0800 Subject: [PATCH 09/14] gofumpt e2e_tests code --- test/e2e/e2e_failedpods_test.go | 6 ++++-- test/e2e/e2e_topologyspreadconstraint_test.go | 3 ++- test/run-e2e-tests.sh | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/e2e/e2e_failedpods_test.go b/test/e2e/e2e_failedpods_test.go index b5b56e6eb4..f022483954 100644 --- a/test/e2e/e2e_failedpods_test.go +++ b/test/e2e/e2e_failedpods_test.go @@ -25,8 +25,10 @@ import ( "sigs.k8s.io/descheduler/pkg/framework/plugins/removefailedpods" ) -var oneHourPodLifetimeSeconds uint = 3600 -var oneSecondPodLifetimeSeconds uint = 1 +var ( + oneHourPodLifetimeSeconds uint = 3600 + oneSecondPodLifetimeSeconds uint = 1 +) func removeFailedPodsPolicy(removeFailedPodsArgs *removefailedpods.RemoveFailedPodsArgs, evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy { return &apiv1alpha2.DeschedulerPolicy{ diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index a3cc27b29b..32ac8b25ab 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -26,7 +26,8 @@ import ( const zoneTopologyKey string = "topology.kubernetes.io/zone" func topologySpreadConstraintPolicy(constraintArgs *removepodsviolatingtopologyspreadconstraint.RemovePodsViolatingTopologySpreadConstraintArgs, - evictorArgs *defaultevictor.DefaultEvictorArgs) *apiv1alpha2.DeschedulerPolicy { + evictorArgs *defaultevictor.DefaultEvictorArgs, +) *apiv1alpha2.DeschedulerPolicy { return &apiv1alpha2.DeschedulerPolicy{ Profiles: []apiv1alpha2.DeschedulerProfile{ { diff --git a/test/run-e2e-tests.sh b/test/run-e2e-tests.sh index b3001a276a..2930923e5e 100755 --- a/test/run-e2e-tests.sh +++ b/test/run-e2e-tests.sh @@ -23,7 +23,7 @@ SKIP_INSTALL=${SKIP_INSTALL:-} KIND_E2E=${KIND_E2E:-} # Build a descheduler image -IMAGE_TAG=v$(date +%Y%m%d)-$(git describe --tags) +IMAGE_TAG=$(git describe --tags --match v0*) BASEDIR=$(dirname "$0") VERSION="${IMAGE_TAG}" make -C ${BASEDIR}/.. image From 9befb6ec17c854bf950c5c0db6808670acb74f0a Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Wed, 21 Aug 2024 17:43:56 +0800 Subject: [PATCH 10/14] modify the IMAGE_TAG to ensure that the version is correctly parsed when the descheduler running --- test/run-e2e-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run-e2e-tests.sh b/test/run-e2e-tests.sh index 2930923e5e..b3001a276a 100755 --- a/test/run-e2e-tests.sh +++ b/test/run-e2e-tests.sh @@ -23,7 +23,7 @@ SKIP_INSTALL=${SKIP_INSTALL:-} KIND_E2E=${KIND_E2E:-} # Build a descheduler image -IMAGE_TAG=$(git describe --tags --match v0*) +IMAGE_TAG=v$(date +%Y%m%d)-$(git describe --tags) BASEDIR=$(dirname "$0") VERSION="${IMAGE_TAG}" make -C ${BASEDIR}/.. image From 9f7d8ada960f7cd68185ecf29b83c525acfdfd5f Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Thu, 22 Aug 2024 00:43:20 +0800 Subject: [PATCH 11/14] resolve the issue causing the e2e test failures --- kubernetes/base/rbac.yaml | 5 +- test/e2e/e2e_leaderelection_test.go | 43 ++++---- test/e2e/e2e_test.go | 22 +++- test/e2e/e2e_topologyspreadconstraint_test.go | 104 ++++++++++++------ 4 files changed, 112 insertions(+), 62 deletions(-) diff --git a/kubernetes/base/rbac.yaml b/kubernetes/base/rbac.yaml index 9587b86a3a..60726624d0 100644 --- a/kubernetes/base/rbac.yaml +++ b/kubernetes/base/rbac.yaml @@ -22,13 +22,10 @@ rules: - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["get", "watch", "list"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] resourceNames: ["descheduler"] - verbs: ["get", "patch", "delete"] + verbs: ["get", "patch", "delete","create", "update"] --- apiVersion: v1 kind: ServiceAccount diff --git a/test/e2e/e2e_leaderelection_test.go b/test/e2e/e2e_leaderelection_test.go index 6c54314de4..858b118452 100644 --- a/test/e2e/e2e_leaderelection_test.go +++ b/test/e2e/e2e_leaderelection_test.go @@ -27,7 +27,6 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -126,18 +125,15 @@ func TestLeaderElection(t *testing.T) { podListBOrg := getCurrentPodNames(t, ctx, clientSet, ns2) t.Log("Starting deschedulers") - // Delete the descheduler lease - err = clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - t.Fatalf("Unable to remove kube-system/descheduler lease: %v", err) - } - t.Logf("Removed kube-system/descheduler lease") - descheduler1 := startDeschedulerServer(t, ctx, clientSet, ns1) + pod1Name, deploy1, cm1 := startDeschedulerServer(t, ctx, clientSet, ns1) time.Sleep(1 * time.Second) - descheduler2 := startDeschedulerServer(t, ctx, clientSet, ns2) + pod2Name, deploy2, cm2 := startDeschedulerServer(t, ctx, clientSet, ns2) defer func() { - for _, deploy := range []*appsv1.Deployment{descheduler1, descheduler2} { - printPodLogs(ctx, t, clientSet, deploy.Name) + for _, podName := range []string{pod1Name, pod2Name} { + printPodLogs(ctx, t, clientSet, podName) + } + + for _, deploy := range []*appsv1.Deployment{deploy1, deploy2} { t.Logf("Deleting %q deployment...", deploy.Name) err = clientSet.AppsV1().Deployments(deploy.Namespace).Delete(ctx, deploy.Name, metav1.DeleteOptions{}) if err != nil { @@ -147,7 +143,13 @@ func TestLeaderElection(t *testing.T) { waitForPodsToDisappear(ctx, t, clientSet, deploy.Labels, deploy.Namespace) } - clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{}) + for _, cm := range []*v1.ConfigMap{cm1, cm2} { + t.Logf("Deleting %q CM...", cm.Name) + err = clientSet.CoreV1().ConfigMaps(cm.Namespace).Delete(ctx, cm.Name, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("Unable to delete %q CM: %v", cm.Name, err) + } + } }() // wait for a while so all the pods are 5 seconds older @@ -192,7 +194,7 @@ func createDeployment(t *testing.T, ctx context.Context, clientSet clientset.Int return nil } -func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clientset.Interface, testName string) *appsv1.Deployment { +func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clientset.Interface, testName string) (string, *appsv1.Deployment, *v1.ConfigMap) { var maxLifeTime uint = 5 podLifeTimeArgs := &podlifetime.PodLifeTimeArgs{ MaxPodLifeTimeSeconds: &maxLifeTime, @@ -210,6 +212,7 @@ func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clients MinPodAge: &metav1.Duration{Duration: 1 * time.Second}, } deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(podlifetimePolicy(podLifeTimeArgs, evictorArgs)) + deschedulerPolicyConfigMapObj.Name = fmt.Sprintf("%s-%s", deschedulerPolicyConfigMapObj.Name, testName) if err != nil { t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } @@ -220,16 +223,10 @@ func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clients t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - defer func() { - t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name) - err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) - } - }() - deschedulerDeploymentObj := deschedulerDeployment(testName) deschedulerDeploymentObj.Name = fmt.Sprintf("%s-%s", deschedulerDeploymentObj.Name, testName) + args := deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args + deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = append(args, "--leader-elect") t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { @@ -237,6 +234,6 @@ func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clients } t.Logf("Waiting for the descheduler pod running") - waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - return deschedulerDeploymentObj + podName := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) + return podName, deschedulerDeploymentObj, deschedulerPolicyConfigMapObj } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 41092b7773..6b8450e9af 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -619,7 +619,7 @@ func TestLowNodeUtilization(t *testing.T) { } plugin.(frameworktypes.BalancePlugin).Balance(ctx, workerNodes) - waitForPodsToDisappear(ctx, t, clientSet, rc.Spec.Template.Labels, rc.Namespace) + waitForTerminatingPodsToDisappear(ctx, t, clientSet, rc.Namespace) podFilter, err = podutil.NewOptions().WithFilter(handle.EvictorFilterImpl.Filter).BuildFilterFunc() if err != nil { @@ -1318,7 +1318,7 @@ func TestPodLifeTimeOldestEvicted(t *testing.T) { t.Log("Finished PodLifetime plugin") t.Logf("Wait for terminating pod to disappear") - waitForPodsToDisappear(ctx, t, clientSet, rc.Spec.Template.Labels, rc.Namespace) + waitForTerminatingPodsToDisappear(ctx, t, clientSet, rc.Namespace) podList, err = clientSet.CoreV1().Pods(rc.Namespace).List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(rc.Spec.Template.Labels).String()}) if err != nil { @@ -1392,6 +1392,24 @@ func waitForRCPodsRunning(ctx context.Context, t *testing.T, clientSet clientset } } +func waitForTerminatingPodsToDisappear(ctx context.Context, t *testing.T, clientSet clientset.Interface, namespace string) { + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { + podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + for _, pod := range podList.Items { + if pod.DeletionTimestamp != nil { + t.Logf("Pod %v still terminating", pod.Name) + return false, nil + } + } + return true, nil + }); err != nil { + t.Fatalf("Error waiting for terminating pods to disappear: %v", err) + } +} + func deleteDS(ctx context.Context, t *testing.T, clientSet clientset.Interface, ds *appsv1.DaemonSet) { // adds nodeselector to avoid any nodes by setting an unused label dsDeepCopy := ds.DeepCopy() diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index 32ac8b25ab..a00cb54dfa 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -13,6 +13,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" componentbaseconfig "k8s.io/component-base/config" @@ -83,14 +84,16 @@ func TestTopologySpreadConstraint(t *testing.T) { } defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) - testCases := map[string]struct { - expectedEvictedCount uint + testCases := []struct { + name string + expectedEvictedPodCount int replicaCount int topologySpreadConstraint v1.TopologySpreadConstraint }{ - "test-topology-spread-hard-constraint": { - expectedEvictedCount: 1, - replicaCount: 4, + { + name: "test-topology-spread-hard-constraint", + expectedEvictedPodCount: 1, + replicaCount: 4, topologySpreadConstraint: v1.TopologySpreadConstraint{ LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -102,9 +105,10 @@ func TestTopologySpreadConstraint(t *testing.T) { WhenUnsatisfiable: v1.DoNotSchedule, }, }, - "test-topology-spread-soft-constraint": { - expectedEvictedCount: 1, - replicaCount: 4, + { + name: "test-topology-spread-soft-constraint", + expectedEvictedPodCount: 1, + replicaCount: 4, topologySpreadConstraint: v1.TopologySpreadConstraint{ LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -116,9 +120,10 @@ func TestTopologySpreadConstraint(t *testing.T) { WhenUnsatisfiable: v1.ScheduleAnyway, }, }, - "test-node-taints-policy-honor": { - expectedEvictedCount: 1, - replicaCount: 4, + { + name: "test-node-taints-policy-honor", + expectedEvictedPodCount: 1, + replicaCount: 4, topologySpreadConstraint: v1.TopologySpreadConstraint{ LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -131,9 +136,10 @@ func TestTopologySpreadConstraint(t *testing.T) { WhenUnsatisfiable: v1.DoNotSchedule, }, }, - "test-node-affinity-policy-ignore": { - expectedEvictedCount: 1, - replicaCount: 4, + { + name: "test-node-affinity-policy-ignore", + expectedEvictedPodCount: 1, + replicaCount: 4, topologySpreadConstraint: v1.TopologySpreadConstraint{ LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -146,9 +152,10 @@ func TestTopologySpreadConstraint(t *testing.T) { WhenUnsatisfiable: v1.DoNotSchedule, }, }, - "test-match-label-keys": { - expectedEvictedCount: 0, - replicaCount: 4, + { + name: "test-match-label-keys", + expectedEvictedPodCount: 0, + replicaCount: 4, topologySpreadConstraint: v1.TopologySpreadConstraint{ LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -162,16 +169,16 @@ func TestTopologySpreadConstraint(t *testing.T) { }, }, } - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - t.Logf("Creating Deployment %s with %d replicas", name, tc.replicaCount) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Logf("Creating Deployment %s with %d replicas", tc.name, tc.replicaCount) deployLabels := tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels - deployLabels["name"] = name - deployment := buildTestDeployment(name, testNamespace.Name, int32(tc.replicaCount), deployLabels, func(d *appsv1.Deployment) { + deployLabels["name"] = tc.name + deployment := buildTestDeployment(tc.name, testNamespace.Name, int32(tc.replicaCount), deployLabels, func(d *appsv1.Deployment) { d.Spec.Template.Spec.TopologySpreadConstraints = []v1.TopologySpreadConstraint{tc.topologySpreadConstraint} }) if _, err := clientSet.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}); err != nil { - t.Fatalf("Error creating Deployment %s %v", name, err) + t.Fatalf("Error creating Deployment %s %v", tc.name, err) } defer func() { clientSet.AppsV1().Deployments(deployment.Namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{}) @@ -180,7 +187,7 @@ func TestTopologySpreadConstraint(t *testing.T) { waitForPodsRunning(ctx, t, clientSet, deployment.Labels, tc.replicaCount, deployment.Namespace) // Create a "Violator" Deployment that has the same label and is forced to be on the same node using a nodeSelector - violatorDeploymentName := name + "-violator" + violatorDeploymentName := tc.name + "-violator" violatorCount := tc.topologySpreadConstraint.MaxSkew + 1 violatorDeployLabels := tc.topologySpreadConstraint.LabelSelector.DeepCopy().MatchLabels violatorDeployLabels["name"] = violatorDeploymentName @@ -197,7 +204,9 @@ func TestTopologySpreadConstraint(t *testing.T) { waitForPodsRunning(ctx, t, clientSet, violatorDeployment.Labels, int(violatorCount), violatorDeployment.Namespace) // Run TopologySpreadConstraint strategy - t.Logf("Running RemovePodsViolatingTopologySpreadConstraint strategy for %s", name) + t.Logf("Running RemovePodsViolatingTopologySpreadConstraint strategy for %s", tc.name) + + preRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...) evictorArgs := &defaultevictor.DefaultEvictorArgs{ EvictLocalStoragePods: true, @@ -254,13 +263,42 @@ func TestTopologySpreadConstraint(t *testing.T) { deschedulerPodName = waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) // Run RemovePodsHavingTooManyRestarts strategy - var meetsExpectations bool + var meetsEvictedExpectations bool + var actualEvictedPodCount int + t.Logf("Check whether the number of evicted pods meets the expectation") + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + currentRunNames := sets.NewString(getCurrentPodNames(t, ctx, clientSet, testNamespace.Name)...) + actualEvictedPod := preRunNames.Difference(currentRunNames) + actualEvictedPodCount = actualEvictedPod.Len() + t.Logf("preRunNames: %v, currentRunNames: %v, actualEvictedPodCount: %v\n", preRunNames.List(), currentRunNames.List(), actualEvictedPodCount) + if actualEvictedPodCount != tc.expectedEvictedPodCount { + t.Logf("Expecting %v number of pods evicted, got %v instead", tc.expectedEvictedPodCount, actualEvictedPodCount) + return false, nil + } + meetsEvictedExpectations = true + return true, nil + }); err != nil { + t.Errorf("Error waiting for descheduler running: %v", err) + } + + if !meetsEvictedExpectations { + t.Errorf("Unexpected number of pods have been evicted, got %v, expected %v", actualEvictedPodCount, tc.expectedEvictedPodCount) + } else { + t.Logf("Total of %d Pods were evicted for %s", actualEvictedPodCount, tc.name) + } + + if tc.expectedEvictedPodCount == 0 { + return + } + + var meetsSkewExpectations bool var skewVal int + t.Logf("Check whether the skew meets the expectation") if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { listOptions := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(tc.topologySpreadConstraint.LabelSelector.MatchLabels).String()} pods, err := clientSet.CoreV1().Pods(testNamespace.Name).List(ctx, listOptions) if err != nil { - t.Errorf("Error listing pods for %s: %v", name, err) + t.Errorf("Error listing pods for %s: %v", tc.name, err) } nodePodCountMap := make(map[string]int) @@ -269,26 +307,26 @@ func TestTopologySpreadConstraint(t *testing.T) { } if len(nodePodCountMap) != len(workerNodes) { - t.Errorf("%s Pods were scheduled on only '%d' nodes and were not properly distributed on the nodes", name, len(nodePodCountMap)) + t.Errorf("%s Pods were scheduled on only '%d' nodes and were not properly distributed on the nodes", tc.name, len(nodePodCountMap)) return false, nil } skewVal = getSkewValPodDistribution(nodePodCountMap) if skewVal > int(tc.topologySpreadConstraint.MaxSkew) { - t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", name, tc.topologySpreadConstraint.MaxSkew, skewVal) + t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", tc.name, tc.topologySpreadConstraint.MaxSkew, skewVal) return false, nil } - meetsExpectations = true + meetsSkewExpectations = true return true, nil }); err != nil { t.Errorf("Error waiting for descheduler running: %v", err) } - if !meetsExpectations { - t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", name, tc.topologySpreadConstraint.MaxSkew, skewVal) + if !meetsSkewExpectations { + t.Errorf("Pod distribution for %s is still violating the max skew of %d as it is %d", tc.name, tc.topologySpreadConstraint.MaxSkew, skewVal) } else { - t.Logf("Pods for %s were distributed in line with max skew of %d", name, tc.topologySpreadConstraint.MaxSkew) + t.Logf("Pods for %s were distributed in line with max skew of %d", tc.name, tc.topologySpreadConstraint.MaxSkew) } }) } From 20fcb33055244f5b822c58e04e7a664098c12de9 Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Thu, 22 Aug 2024 14:42:29 +0800 Subject: [PATCH 12/14] abstract deschedulerPolicyConfigMap and deschedulerDeployment methods --- test/e2e/e2e_duplicatepods_test.go | 5 ++-- test/e2e/e2e_failedpods_test.go | 4 +-- test/e2e/e2e_leaderelection_test.go | 28 ++++++++++++++----- test/e2e/e2e_test.go | 17 +++++++++-- test/e2e/e2e_toomanyrestarts_test.go | 4 +-- test/e2e/e2e_topologyspreadconstraint_test.go | 4 +-- test/e2e/test_clientconnection_test.go | 4 +-- 7 files changed, 45 insertions(+), 21 deletions(-) diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index 28feae49ff..5df2d3f14e 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -189,8 +189,7 @@ func TestRemoveDuplicates(t *testing.T) { tc.removeDuplicatesArgs.Namespaces = &api.Namespaces{ Include: []string{testNamespace.Name}, } - tc.evictorArgs.MinPodAge = &metav1.Duration{Duration: 1 * time.Second} - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs)) + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs), nil) if err != nil { t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } @@ -209,7 +208,7 @@ func TestRemoveDuplicates(t *testing.T) { } }() - deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) + deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name, nil) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { diff --git a/test/e2e/e2e_failedpods_test.go b/test/e2e/e2e_failedpods_test.go index f022483954..565f18d89c 100644 --- a/test/e2e/e2e_failedpods_test.go +++ b/test/e2e/e2e_failedpods_test.go @@ -145,7 +145,7 @@ func TestFailedPods(t *testing.T) { Include: []string{testNamespace.Name}, } - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs)) + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs), nil) if err != nil { t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } @@ -164,7 +164,7 @@ func TestFailedPods(t *testing.T) { } }() - deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) + deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name, nil) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { diff --git a/test/e2e/e2e_leaderelection_test.go b/test/e2e/e2e_leaderelection_test.go index 858b118452..d84fe71157 100644 --- a/test/e2e/e2e_leaderelection_test.go +++ b/test/e2e/e2e_leaderelection_test.go @@ -209,10 +209,10 @@ func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clients EvictSystemCriticalPods: false, IgnorePvcPods: false, EvictFailedBarePods: false, - MinPodAge: &metav1.Duration{Duration: 1 * time.Second}, } - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(podlifetimePolicy(podLifeTimeArgs, evictorArgs)) - deschedulerPolicyConfigMapObj.Name = fmt.Sprintf("%s-%s", deschedulerPolicyConfigMapObj.Name, testName) + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(podlifetimePolicy(podLifeTimeArgs, evictorArgs), func(cm *v1.ConfigMap) { + cm.Name = fmt.Sprintf("%s-%s", cm.Name, testName) + }) if err != nil { t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } @@ -223,10 +223,24 @@ func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clients t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } - deschedulerDeploymentObj := deschedulerDeployment(testName) - deschedulerDeploymentObj.Name = fmt.Sprintf("%s-%s", deschedulerDeploymentObj.Name, testName) - args := deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args - deschedulerDeploymentObj.Spec.Template.Spec.Containers[0].Args = append(args, "--leader-elect") + deschedulerDeploymentObj := deschedulerDeployment(testName, func(deployment *appsv1.Deployment) { + deployment.Name = fmt.Sprintf("%s-%s", deployment.Name, testName) + args := deployment.Spec.Template.Spec.Containers[0].Args + deployment.Spec.Template.Spec.Containers[0].Args = append(args, "--leader-elect") + deployment.Spec.Template.Spec.Volumes = []v1.Volume{ + { + Name: "policy-volume", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: deschedulerPolicyConfigMapObj.Name, + }, + }, + }, + }, + } + }) + t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 6b8450e9af..b0df2bdea1 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -69,7 +69,7 @@ func isClientRateLimiterError(err error) bool { return strings.Contains(err.Error(), "client rate limiter") } -func deschedulerPolicyConfigMap(policy *deschedulerapiv1alpha2.DeschedulerPolicy) (*v1.ConfigMap, error) { +func deschedulerPolicyConfigMap(policy *deschedulerapiv1alpha2.DeschedulerPolicy, apply func(cm *v1.ConfigMap)) (*v1.ConfigMap, error) { cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "descheduler-policy-configmap", @@ -83,11 +83,16 @@ func deschedulerPolicyConfigMap(policy *deschedulerapiv1alpha2.DeschedulerPolicy return nil, err } cm.Data = map[string]string{"policy.yaml": string(policyBytes)} + + if apply != nil { + apply(cm) + } + return cm, nil } -func deschedulerDeployment(testName string) *appsv1.Deployment { - return &appsv1.Deployment{ +func deschedulerDeployment(testName string, apply func(deployment *appsv1.Deployment)) *appsv1.Deployment { + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "descheduler", Namespace: "kube-system", @@ -174,6 +179,12 @@ func deschedulerDeployment(testName string) *appsv1.Deployment { }, }, } + + if apply != nil { + apply(deployment) + } + + return deployment } func printPodLogs(ctx context.Context, t *testing.T, kubeClient clientset.Interface, podName string) { diff --git a/test/e2e/e2e_toomanyrestarts_test.go b/test/e2e/e2e_toomanyrestarts_test.go index 6af8830a62..7df2add97a 100644 --- a/test/e2e/e2e_toomanyrestarts_test.go +++ b/test/e2e/e2e_toomanyrestarts_test.go @@ -156,7 +156,7 @@ func TestTooManyRestarts(t *testing.T) { Include: []string{testNamespace.Name}, } - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(tooManyRestartsPolicy(tc.restartsArgs, evictorArgs)) + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(tooManyRestartsPolicy(tc.restartsArgs, evictorArgs), nil) if err != nil { t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } @@ -175,7 +175,7 @@ func TestTooManyRestarts(t *testing.T) { } }() - deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) + deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name, nil) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index a00cb54dfa..d29465a95c 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -220,7 +220,7 @@ func TestTopologySpreadConstraint(t *testing.T) { Include: []string{testNamespace.Name}, }, } - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(topologySpreadConstraintPolicy(constraintArgs, evictorArgs)) + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(topologySpreadConstraintPolicy(constraintArgs, evictorArgs), nil) if err != nil { t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } @@ -238,7 +238,7 @@ func TestTopologySpreadConstraint(t *testing.T) { t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } }() - deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) + deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name, nil) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { diff --git a/test/e2e/test_clientconnection_test.go b/test/e2e/test_clientconnection_test.go index 2c195242f5..901fc97dd6 100644 --- a/test/e2e/test_clientconnection_test.go +++ b/test/e2e/test_clientconnection_test.go @@ -24,7 +24,7 @@ func TestClientConnectionConfiguration(t *testing.T) { t.Errorf("Error during kubernetes client creation with %v", err) } - deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(&apiv1alpha2.DeschedulerPolicy{}) + deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(&apiv1alpha2.DeschedulerPolicy{}, nil) if err != nil { t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err) } @@ -43,7 +43,7 @@ func TestClientConnectionConfiguration(t *testing.T) { } }() - deschedulerDeploymentObj := deschedulerDeployment(testName) + deschedulerDeploymentObj := deschedulerDeployment(testName, nil) t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { From 9ba1c5562758e34378a1fc2a7f1246af69c19773 Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Thu, 22 Aug 2024 15:06:36 +0800 Subject: [PATCH 13/14] resolve the issue with lease operations in e2e_leaderelection_test --- kubernetes/base/rbac.yaml | 5 ++++- test/e2e/e2e_leaderelection_test.go | 14 +++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/kubernetes/base/rbac.yaml b/kubernetes/base/rbac.yaml index 60726624d0..9587b86a3a 100644 --- a/kubernetes/base/rbac.yaml +++ b/kubernetes/base/rbac.yaml @@ -22,10 +22,13 @@ rules: - apiGroups: ["scheduling.k8s.io"] resources: ["priorityclasses"] verbs: ["get", "watch", "list"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] resourceNames: ["descheduler"] - verbs: ["get", "patch", "delete","create", "update"] + verbs: ["get", "patch", "delete"] --- apiVersion: v1 kind: ServiceAccount diff --git a/test/e2e/e2e_leaderelection_test.go b/test/e2e/e2e_leaderelection_test.go index d84fe71157..a0e0ba8650 100644 --- a/test/e2e/e2e_leaderelection_test.go +++ b/test/e2e/e2e_leaderelection_test.go @@ -27,6 +27,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -124,6 +125,15 @@ func TestLeaderElection(t *testing.T) { waitForPodsRunning(ctx, t, clientSet, deployment2.Labels, 5, deployment2.Namespace) podListBOrg := getCurrentPodNames(t, ctx, clientSet, ns2) + // Delete the descheduler lease + err = clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + t.Fatalf("Unable to remove kube-system/descheduler lease: %v", err) + } + } + t.Logf("Removed kube-system/descheduler lease") + t.Log("Starting deschedulers") pod1Name, deploy1, cm1 := startDeschedulerServer(t, ctx, clientSet, ns1) time.Sleep(1 * time.Second) @@ -150,6 +160,8 @@ func TestLeaderElection(t *testing.T) { t.Fatalf("Unable to delete %q CM: %v", cm.Name, err) } } + + clientSet.CoordinationV1().Leases("kube-system").Delete(ctx, "descheduler", metav1.DeleteOptions{}) }() // wait for a while so all the pods are 5 seconds older @@ -226,7 +238,7 @@ func startDeschedulerServer(t *testing.T, ctx context.Context, clientSet clients deschedulerDeploymentObj := deschedulerDeployment(testName, func(deployment *appsv1.Deployment) { deployment.Name = fmt.Sprintf("%s-%s", deployment.Name, testName) args := deployment.Spec.Template.Spec.Containers[0].Args - deployment.Spec.Template.Spec.Containers[0].Args = append(args, "--leader-elect") + deployment.Spec.Template.Spec.Containers[0].Args = append(args, "--leader-elect", "--leader-elect-retry-period", "1s") deployment.Spec.Template.Spec.Volumes = []v1.Volume{ { Name: "policy-volume", From 196ac655c3b3a4ec8e7eed2a0a442f215cc7f35f Mon Sep 17 00:00:00 2001 From: Hao Fan Date: Thu, 22 Aug 2024 18:39:31 +0800 Subject: [PATCH 14/14] add leases update permission --- kubernetes/base/rbac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/base/rbac.yaml b/kubernetes/base/rbac.yaml index 9587b86a3a..b2288e3437 100644 --- a/kubernetes/base/rbac.yaml +++ b/kubernetes/base/rbac.yaml @@ -24,7 +24,7 @@ rules: verbs: ["get", "watch", "list"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - verbs: ["create"] + verbs: ["create", "update"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] resourceNames: ["descheduler"]