chore: Add logger to track reconcilers invocation (#1921)

* chore: Add logger to track reconcilers invocation

Signed-off-by: Anatolii Bazko <abazko@redhat.com>

bundle/next/eclipse-che/manifests/che-operator.clusterserviceversion.yaml

@@ -104,7 +104,7 @@ metadata:
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
     repository: https://github.com/eclipse-che/che-operator
     support: Eclipse Foundation
-  name: eclipse-che.v7.94.0-888.next
+  name: eclipse-che.v7.94.0-889.next
   namespace: placeholder
 spec:
   apiservicedefinitions: {}
@@ -1035,7 +1035,7 @@ spec:
   minKubeVersion: 1.19.0
   provider:
     name: Eclipse Foundation
-  version: 7.94.0-888.next
+  version: 7.94.0-889.next
   webhookdefinitions:
     - admissionReviewVersions:
         - v1

pkg/deploy/container-build/container_build.go

@@ -47,28 +47,28 @@ func (cb *ContainerBuildReconciler) Reconcile(ctx *chetypes.DeployContext) (reco
 		// The check below to avoid NPE while CheCluster is not updated with defaults.
 		if ctx.CheCluster.IsOpenShiftSecurityContextConstraintSet() {
 			if done, err := cb.syncSCC(ctx); !done {
-				return reconcile.Result{}, false, err
+				return reconcile.Result{Requeue: true}, false, err
 			}
 
 			if done, err := cb.syncRBAC(ctx); !done {
-				return reconcile.Result{}, false, err
+				return reconcile.Result{Requeue: true}, false, err
 			}
 
 			if err := deploy.AppendFinalizer(ctx, cb.getFinalizerName()); err != nil {
-				return reconcile.Result{}, false, err
+				return reconcile.Result{Requeue: true}, false, err
 			}
 		}
 	} else {
 		if done, err := cb.removeRBAC(ctx); !done {
-			return reconcile.Result{}, false, err
+			return reconcile.Result{Requeue: true}, false, err
 		}
 
 		if done, err := cb.removeSCC(ctx); !done {
-			return reconcile.Result{}, false, err
+			return reconcile.Result{Requeue: true}, false, err
 		}
 
 		if err := deploy.DeleteFinalizer(ctx, cb.getFinalizerName()); err != nil {
-			return reconcile.Result{}, false, err
+			return reconcile.Result{Requeue: true}, false, err
 		}
 	}
 

pkg/deploy/dev-workspace-config/dev_workspace_config.go

@@ -67,11 +67,8 @@ func (d *DevWorkspaceConfigReconciler) Reconcile(ctx *chetypes.DeployContext) (r
 		return reconcile.Result{}, false, err
 	}
 
-	if done, err := deploy.Sync(ctx, dwoc); !done {
-		return reconcile.Result{}, false, err
-	}
-
-	return reconcile.Result{}, true, nil
+	done, err := deploy.Sync(ctx, dwoc)
+	return reconcile.Result{Requeue: !done}, done, err
 }
 
 func (d *DevWorkspaceConfigReconciler) Finalize(ctx *chetypes.DeployContext) bool {

pkg/deploy/reconcile_manager.go

@@ -14,13 +14,20 @@ package deploy
 
 import (
 	"fmt"
+	"log"
+	"os"
+
+	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/eclipse-che/che-operator/pkg/common/chetypes"
 	"github.com/eclipse-che/che-operator/pkg/common/constants"
-	"github.com/sirupsen/logrus"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+var (
+	reconcilerLogger = ctrl.Log.WithName("reconciler_manager")
+)
+
 const Finalizer = "cluster-resources." + constants.FinalizerSuffix
 
 type Reconcilable interface {
@@ -33,11 +40,15 @@ type Reconcilable interface {
 type ReconcileManager struct {
 	reconcilers      []Reconcilable
 	failedReconciler Reconcilable
+
+	// track reconciler invocations
+	operationLogger *log.Logger
 }
 
 func NewReconcileManager() *ReconcileManager {
 	return &ReconcileManager{
 		reconcilers:      make([]Reconcilable, 0),
+		operationLogger:  initOperationLogger(),
 		failedReconciler: nil,
 	}
 }
@@ -46,29 +57,44 @@ func (manager *ReconcileManager) RegisterReconciler(reconciler Reconcilable) {
 	manager.reconcilers = append(manager.reconcilers, reconciler)
 }
 
-// Reconcile all objects in a order they have been added
+// ReconcileAll reconciles all objects in an order they have been added.
 // If reconciliation failed then CheCluster status will be updated accordingly.
 func (manager *ReconcileManager) ReconcileAll(ctx *chetypes.DeployContext) (reconcile.Result, bool, error) {
 	if err := AppendFinalizer(ctx, Finalizer); err != nil {
 		return reconcile.Result{}, false, err
 	}
 
 	for _, reconciler := range manager.reconcilers {
+		reconcilerName := GetObjectType(reconciler)
+
+		if manager.operationLogger != nil {
+			manager.operationLogger.Printf("Reconciler [%s] started.", reconcilerName)
+		}
+
 		result, done, err := reconciler.Reconcile(ctx)
+
+		if manager.operationLogger != nil {
+			manager.operationLogger.Printf("Reconciler [%s] done: %t", reconcilerName, done)
+		}
+
 		if err != nil {
+			// set failed reconciler
 			manager.failedReconciler = reconciler
-			reconcilerName := GetObjectType(reconciler)
+
 			errMsg := fmt.Sprintf("Reconciler failed %s, cause: %v", reconcilerName, err)
 			if err := SetStatusDetails(ctx, constants.InstallOrUpdateFailed, errMsg); err != nil {
-				logrus.Errorf("Failed to update checluster status, cause: %v", err)
+				reconcilerLogger.Error(err, "Failed to update checluster status")
 			}
 		} else if manager.failedReconciler == reconciler {
+			// cleanup failed reconciler
 			manager.failedReconciler = nil
+
 			if err := SetStatusDetails(ctx, "", ""); err != nil {
-				logrus.Errorf("Failed to update checluster status, cause: %v", err)
+				reconcilerLogger.Error(err, "Failed to update checluster status")
 			}
 		}
 
+		// don't continue if reconciliation failed
 		if !done {
 			return result, done, err
 		}
@@ -80,20 +106,30 @@ func (manager *ReconcileManager) ReconcileAll(ctx *chetypes.DeployContext) (reco
 func (manager *ReconcileManager) FinalizeAll(ctx *chetypes.DeployContext) (done bool) {
 	done = true
 	for _, reconciler := range manager.reconcilers {
-		if completed := reconciler.Finalize(ctx); !completed {
-			reconcilerName := GetObjectType(reconciler)
-			ctx.CheCluster.Status.Message = fmt.Sprintf("Finalization failed for reconciler: %s", reconcilerName)
-			_ = UpdateCheCRStatus(ctx, "Message", ctx.CheCluster.Status.Message)
+		completed := reconciler.Finalize(ctx)
+		done = done && completed
 
-			done = false
+		if !completed {
+			// don't prevent from invoking other finalizers, just log the error
+			reconcilerLogger.Error(nil, fmt.Sprintf("Finalization failed for reconciler: %s", GetObjectType(reconciler)))
 		}
 	}
 
 	if done {
+		// Removes remaining finalizers not to prevent CheCluster object from being deleted
 		if err := CleanUpAllFinalizers(ctx); err != nil {
 			return false
 		}
 	}
 
 	return done
 }
+
+func initOperationLogger() *log.Logger {
+	logFile, err := os.OpenFile("/tmp/reconciler_manager.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
+	if err == nil {
+		return log.New(logFile, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile)
+	}
+
+	return nil
+}