MGMT-8571: Monitor stateful set properly

This adds a section to monitorOperands which checks the replicas reported in the stateful set status against the desired number set in the spec. The monitor reports unhealthy if any reported replicas don't match the desired number.
openshift · Jan 21, 2022 · b206f2e · b206f2e
1 parent 0bfc49c
commit b206f2e
Show file tree

Hide file tree

Showing 2 changed files with 98 additions and 0 deletions.
diff --git a/internal/controller/controllers/agentserviceconfig_controller.go b/internal/controller/controllers/agentserviceconfig_controller.go
@@ -345,6 +345,32 @@ func (r *AgentServiceConfigReconciler) monitorOperands(ctx context.Context, log
 		}
 	}
 
+	// monitor statefulset
+	ss := &appsv1.StatefulSet{}
+	if err := r.Get(ctx, types.NamespacedName{Name: imageServiceName, Namespace: r.Namespace}, ss); err != nil {
+		return err
+	}
+
+	desiredReplicas := *ss.Spec.Replicas
+	checkReplicas := func(replicas int32, name string) error {
+		if replicas != desiredReplicas {
+			return fmt.Errorf("StatefulSet %s %s replicas does not match desired replicas", imageServiceName, name)
+		}
+		return nil
+	}
+	if err := checkReplicas(ss.Status.Replicas, "created"); err != nil {
+		return err
+	}
+	if err := checkReplicas(ss.Status.ReadyReplicas, "ready"); err != nil {
+		return err
+	}
+	if err := checkReplicas(ss.Status.CurrentReplicas, "current"); err != nil {
+		return err
+	}
+	if err := checkReplicas(ss.Status.UpdatedReplicas, "updated"); err != nil {
+		return err
+	}
+
 	return nil
 }
 

diff --git a/internal/controller/controllers/agentserviceconfig_controller_test.go b/internal/controller/controllers/agentserviceconfig_controller_test.go
@@ -118,11 +118,21 @@ var _ = Describe("agentserviceconfig_controller reconcile", func() {
 				Namespace: testNamespace,
 			},
 		}
+		var replicas int32 = 1
 		imageServiceStatefulSet := &appsv1.StatefulSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "assisted-image-service",
 				Namespace: testNamespace,
 			},
+			Spec: appsv1.StatefulSetSpec{
+				Replicas: &replicas,
+			},
+			Status: appsv1.StatefulSetStatus{
+				Replicas:        1,
+				ReadyReplicas:   1,
+				CurrentReplicas: 1,
+				UpdatedReplicas: 1,
+			},
 		}
 		assistedServiceDeployment = &appsv1.Deployment{
 			ObjectMeta: metav1.ObjectMeta{
@@ -151,11 +161,21 @@ var _ = Describe("agentserviceconfig_controller reconcile", func() {
 				},
 			},
 		}
+		var replicas int32 = 1
 		imageServiceStatefulSet := &appsv1.StatefulSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "assisted-image-service",
 				Namespace: testNamespace,
 			},
+			Spec: appsv1.StatefulSetSpec{
+				Replicas: &replicas,
+			},
+			Status: appsv1.StatefulSetStatus{
+				Replicas:        1,
+				ReadyReplicas:   1,
+				CurrentReplicas: 1,
+				UpdatedReplicas: 1,
+			},
 		}
 		assistedServiceDeployment = &appsv1.Deployment{
 			ObjectMeta: metav1.ObjectMeta{
@@ -185,6 +205,48 @@ var _ = Describe("agentserviceconfig_controller reconcile", func() {
 		Expect(conditionsv1.FindStatusCondition(instance.Status.Conditions, aiv1beta1.ConditionDeploymentsHealthy).Reason).To(Equal(aiv1beta1.ReasonDeploymentFailure))
 	})
 
+	It("should set `DeploymentsHealthy` condition to `False` on AgentServiceConfig when the stateful set replicas are incorrect", func() {
+		agentinstalladmissionDeployment = &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "agentinstalladmission",
+				Namespace: testNamespace,
+			},
+		}
+		var replicas int32 = 1
+		imageServiceStatefulSet := &appsv1.StatefulSet{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "assisted-image-service",
+				Namespace: testNamespace,
+			},
+			Spec: appsv1.StatefulSetSpec{
+				Replicas: &replicas,
+			},
+			Status: appsv1.StatefulSetStatus{
+				Replicas:        1,
+				ReadyReplicas:   0,
+				CurrentReplicas: 1,
+				UpdatedReplicas: 1,
+			},
+		}
+		assistedServiceDeployment = &appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "assisted-service",
+				Namespace: testNamespace,
+			},
+		}
+		ascr = newTestReconciler(asc, ingressCM, route, imageRoute, agentinstalladmissionDeployment, imageServiceStatefulSet, assistedServiceDeployment)
+		result, err := ascr.Reconcile(ctx, newAgentServiceConfigRequest(asc))
+
+		Expect(err).NotTo(Succeed())
+		Expect(result).NotTo(Equal(ctrl.Result{}))
+
+		instance := &aiv1beta1.AgentServiceConfig{}
+		err = ascr.Get(ctx, types.NamespacedName{Name: "agent"}, instance)
+		Expect(err).To(BeNil())
+		Expect(conditionsv1.FindStatusCondition(instance.Status.Conditions, aiv1beta1.ConditionDeploymentsHealthy).Status).To(Equal(corev1.ConditionFalse))
+		Expect(conditionsv1.FindStatusCondition(instance.Status.Conditions, aiv1beta1.ConditionDeploymentsHealthy).Reason).To(Equal(aiv1beta1.ReasonDeploymentFailure))
+	})
+
 	It("should set `DeploymentsHealthy` condition to `True` on AgentServiceConfig when all the deployments are Available", func() {
 		agentinstalladmissionDeployment = &appsv1.Deployment{
 			ObjectMeta: metav1.ObjectMeta{
@@ -200,11 +262,21 @@ var _ = Describe("agentserviceconfig_controller reconcile", func() {
 				},
 			},
 		}
+		var replicas int32 = 1
 		imageServiceStatefulSet := &appsv1.StatefulSet{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "assisted-image-service",
 				Namespace: testNamespace,
 			},
+			Spec: appsv1.StatefulSetSpec{
+				Replicas: &replicas,
+			},
+			Status: appsv1.StatefulSetStatus{
+				Replicas:        1,
+				ReadyReplicas:   1,
+				CurrentReplicas: 1,
+				UpdatedReplicas: 1,
+			},
 		}
 		assistedServiceDeployment = &appsv1.Deployment{
 			ObjectMeta: metav1.ObjectMeta{