From b206f2e1b8b8e4d1a5ab1c3605e324651a665193 Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Fri, 21 Jan 2022 12:08:36 -0500 Subject: [PATCH] MGMT-8571: Monitor stateful set properly This adds a section to monitorOperands which checks the replicas reported in the stateful set status against the desired number set in the spec. The monitor reports unhealthy if any reported replicas don't match the desired number. --- .../agentserviceconfig_controller.go | 26 +++++++ .../agentserviceconfig_controller_test.go | 72 +++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/internal/controller/controllers/agentserviceconfig_controller.go b/internal/controller/controllers/agentserviceconfig_controller.go index 3d84840be9e..8630b9da2af 100644 --- a/internal/controller/controllers/agentserviceconfig_controller.go +++ b/internal/controller/controllers/agentserviceconfig_controller.go @@ -345,6 +345,32 @@ func (r *AgentServiceConfigReconciler) monitorOperands(ctx context.Context, log } } + // monitor statefulset + ss := &appsv1.StatefulSet{} + if err := r.Get(ctx, types.NamespacedName{Name: imageServiceName, Namespace: r.Namespace}, ss); err != nil { + return err + } + + desiredReplicas := *ss.Spec.Replicas + checkReplicas := func(replicas int32, name string) error { + if replicas != desiredReplicas { + return fmt.Errorf("StatefulSet %s %s replicas does not match desired replicas", imageServiceName, name) + } + return nil + } + if err := checkReplicas(ss.Status.Replicas, "created"); err != nil { + return err + } + if err := checkReplicas(ss.Status.ReadyReplicas, "ready"); err != nil { + return err + } + if err := checkReplicas(ss.Status.CurrentReplicas, "current"); err != nil { + return err + } + if err := checkReplicas(ss.Status.UpdatedReplicas, "updated"); err != nil { + return err + } + return nil } diff --git a/internal/controller/controllers/agentserviceconfig_controller_test.go b/internal/controller/controllers/agentserviceconfig_controller_test.go index 459fe23c4ed..2eada26ab24 100644 --- a/internal/controller/controllers/agentserviceconfig_controller_test.go +++ b/internal/controller/controllers/agentserviceconfig_controller_test.go @@ -118,11 +118,21 @@ var _ = Describe("agentserviceconfig_controller reconcile", func() { Namespace: testNamespace, }, } + var replicas int32 = 1 imageServiceStatefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "assisted-image-service", Namespace: testNamespace, }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + }, + Status: appsv1.StatefulSetStatus{ + Replicas: 1, + ReadyReplicas: 1, + CurrentReplicas: 1, + UpdatedReplicas: 1, + }, } assistedServiceDeployment = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -151,11 +161,21 @@ var _ = Describe("agentserviceconfig_controller reconcile", func() { }, }, } + var replicas int32 = 1 imageServiceStatefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "assisted-image-service", Namespace: testNamespace, }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + }, + Status: appsv1.StatefulSetStatus{ + Replicas: 1, + ReadyReplicas: 1, + CurrentReplicas: 1, + UpdatedReplicas: 1, + }, } assistedServiceDeployment = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -185,6 +205,48 @@ var _ = Describe("agentserviceconfig_controller reconcile", func() { Expect(conditionsv1.FindStatusCondition(instance.Status.Conditions, aiv1beta1.ConditionDeploymentsHealthy).Reason).To(Equal(aiv1beta1.ReasonDeploymentFailure)) }) + It("should set `DeploymentsHealthy` condition to `False` on AgentServiceConfig when the stateful set replicas are incorrect", func() { + agentinstalladmissionDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agentinstalladmission", + Namespace: testNamespace, + }, + } + var replicas int32 = 1 + imageServiceStatefulSet := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "assisted-image-service", + Namespace: testNamespace, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + }, + Status: appsv1.StatefulSetStatus{ + Replicas: 1, + ReadyReplicas: 0, + CurrentReplicas: 1, + UpdatedReplicas: 1, + }, + } + assistedServiceDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "assisted-service", + Namespace: testNamespace, + }, + } + ascr = newTestReconciler(asc, ingressCM, route, imageRoute, agentinstalladmissionDeployment, imageServiceStatefulSet, assistedServiceDeployment) + result, err := ascr.Reconcile(ctx, newAgentServiceConfigRequest(asc)) + + Expect(err).NotTo(Succeed()) + Expect(result).NotTo(Equal(ctrl.Result{})) + + instance := &aiv1beta1.AgentServiceConfig{} + err = ascr.Get(ctx, types.NamespacedName{Name: "agent"}, instance) + Expect(err).To(BeNil()) + Expect(conditionsv1.FindStatusCondition(instance.Status.Conditions, aiv1beta1.ConditionDeploymentsHealthy).Status).To(Equal(corev1.ConditionFalse)) + Expect(conditionsv1.FindStatusCondition(instance.Status.Conditions, aiv1beta1.ConditionDeploymentsHealthy).Reason).To(Equal(aiv1beta1.ReasonDeploymentFailure)) + }) + It("should set `DeploymentsHealthy` condition to `True` on AgentServiceConfig when all the deployments are Available", func() { agentinstalladmissionDeployment = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -200,11 +262,21 @@ var _ = Describe("agentserviceconfig_controller reconcile", func() { }, }, } + var replicas int32 = 1 imageServiceStatefulSet := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "assisted-image-service", Namespace: testNamespace, }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + }, + Status: appsv1.StatefulSetStatus{ + Replicas: 1, + ReadyReplicas: 1, + CurrentReplicas: 1, + UpdatedReplicas: 1, + }, } assistedServiceDeployment = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{