controllers: check CSV phase to be succeeded for container readiness

odf-operator creates subscription for odf-dependencies and CRDs will
become available only after odf-dependencies CSV is succeeded. We check
the phase of odf-deps CSV to surface whether odf-op is ready to perform
it's actions or not.

Signed-off-by: Leela Venkaiah G <lgangava@ibm.com>

config/manager/manager.yaml

@@ -50,6 +50,7 @@ spec:
             drop:
               - ALL
           readOnlyRootFilesystem: true
+        # ref https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes
         livenessProbe:
           httpGet:
             path: /healthz
@@ -62,6 +63,7 @@ spec:
             port: 8081
           initialDelaySeconds: 5
           periodSeconds: 10
+          timeoutSeconds: 90
         resources:
           limits:
             cpu: 200m

main.go

@@ -171,7 +171,24 @@ func main() {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)
 	}
-	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+
+	// console plugin is dependent on CRDs that are part of these CSVs and we
+	// can't check for only odf-dependencies CSV as OLM doesn't guarantee
+	// (based on observations) existence of all CRDs brought by OLM dependency
+	// mechanism.
+	csvsToBeSuceeded := []string{
+		controllers.OcsSubscriptionStartingCSV,
+		controllers.RookSubscriptionStartingCSV,
+		controllers.NoobaaSubscriptionStartingCSV,
+	}
+	if err := mgr.AddReadyzCheck(
+		"readyz",
+		util.CheckCSVPhase(
+			mgr.GetClient(),
+			operatorNamespace,
+			csvsToBeSuceeded...,
+		),
+	); err != nil {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}

pkg/util/readiness.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2024 Red Hat OpenShift Data Foundation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"net/http"
+
+	opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
+	"golang.org/x/exp/maps"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+)
+
+func CheckCSVPhase(c client.Client, namespace string, csvNames ...string) healthz.Checker {
+	csvMap := map[string]struct{}{}
+	for _, name := range csvNames {
+		csvMap[name] = struct{}{}
+	}
+	return func(r *http.Request) error {
+		csvList := &opv1a1.ClusterServiceVersionList{}
+		if err := c.List(r.Context(), csvList, client.InNamespace(namespace)); err != nil {
+			return err
+		}
+		for idx := range csvList.Items {
+			csv := &csvList.Items[idx]
+			_, exists := csvMap[csv.Name]
+			if exists {
+				if csv.Status.Phase != opv1a1.CSVPhaseSucceeded {
+					return fmt.Errorf("CSV %s is not in Succeeded phase", csv.Name)
+				} else if csv.Status.Phase == opv1a1.CSVPhaseSucceeded {
+					delete(csvMap, csv.Name)
+				}
+			}
+		}
+		if len(csvMap) != 0 {
+			for csvName := range csvMap {
+				return fmt.Errorf("CSV %s is not found", csvName)
+			}
+		}
+		maps.Clear(csvMap)
+		return nil
+	}
+}