test: force delete E2E add check to ensure pods match replicas (#908)

test/e2e/join_and_leave_test.go

@@ -23,6 +23,10 @@ import (
 	"go.goms.io/fleet/pkg/utils"
 )
 
+const (
+	memberAgentName = "member-agent"
+)
+
 // Note that this container cannot run in parallel with other containers.
 var _ = Describe("Test member cluster join and leave flow", Ordered, Serial, func() {
 	crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())
@@ -125,9 +129,7 @@ var _ = Describe("Test member cluster join and leave flow", Ordered, Serial, fun
 var _ = Describe("Test member cluster force delete flow", Ordered, Serial, func() {
 	Context("Test cluster join and leave flow with member agent down and force delete member cluster", Ordered, Serial, func() {
 		It("Simulate the member agent going down in member cluster", func() {
-			Eventually(func() error {
-				return updateMemberAgentDeploymentReplicas(memberCluster3WestProdClient, 0)
-			}, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to simulate member agent going down")
+			updateMemberAgentDeploymentReplicas(memberCluster3WestProdClient, 0)
 		})
 
 		It("Delete member cluster CR associated to the member cluster with member agent down", func() {
@@ -151,21 +153,37 @@ var _ = Describe("Test member cluster force delete flow", Ordered, Serial, func(
 
 	AfterAll(func() {
 		By("Simulate the member agent coming back up")
-		Eventually(func() error {
-			return updateMemberAgentDeploymentReplicas(memberCluster3WestProdClient, 1)
-		}, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to simulate member agent coming back up")
+		updateMemberAgentDeploymentReplicas(memberCluster3WestProdClient, 1)
 
 		createMemberCluster(memberCluster3WestProd.ClusterName, memberCluster3WestProd.PresentingServiceAccountInHubClusterName, labelsByClusterName[memberCluster3WestProd.ClusterName], annotationsByClusterName[memberCluster3WestProd.ClusterName])
 		checkIfMemberClusterHasJoined(memberCluster3WestProd)
 	})
 })
 
-func updateMemberAgentDeploymentReplicas(clusterClient client.Client, replicas int32) error {
-	var d appsv1.Deployment
-	err := clusterClient.Get(ctx, types.NamespacedName{Name: "member-agent", Namespace: fleetSystemNS}, &d)
-	if err != nil {
-		return err
-	}
-	d.Spec.Replicas = ptr.To(replicas)
-	return clusterClient.Update(ctx, &d)
+func updateMemberAgentDeploymentReplicas(clusterClient client.Client, replicas int) {
+	Eventually(func() error {
+		var d appsv1.Deployment
+		err := clusterClient.Get(ctx, types.NamespacedName{Name: memberAgentName, Namespace: fleetSystemNS}, &d)
+		if err != nil {
+			return err
+		}
+		d.Spec.Replicas = ptr.To(int32(replicas))
+		return clusterClient.Update(ctx, &d)
+	}, eventuallyDuration, eventuallyInterval).Should(Succeed())
+
+	Eventually(func() error {
+		var podList corev1.PodList
+		listOpts := []client.ListOption{
+			client.InNamespace(fleetSystemNS),
+			client.MatchingLabels(map[string]string{"app.kubernetes.io/name": memberAgentName}),
+		}
+		err := clusterClient.List(ctx, &podList, listOpts...)
+		if err != nil {
+			return err
+		}
+		if len(podList.Items) != replicas {
+			return fmt.Errorf("member agent pods %d doesn't match replicas specified %d", len(podList.Items), replicas)
+		}
+		return nil
+	}, eventuallyDuration, eventuallyInterval).Should(Succeed())
 }