fix: restart the merge controllers on conflict

Fixes #3861

What this change effectively does is that it changes immediate reconcile
request to an error return, so that controller will be restarted with a
backoff.

More details:

* root cause of the update/teardown conflict is that the finalizer is
still pending on the tearing down resource
* finalizer might not be removed immediately, e.g. if the controller
which put the finalizer is itself in the crash loop
* if the merge controller queues reconcile immediately, it restarts
itself, but the finalizer is still there, so it once again goes into
reconcile loop and that goes forever until the finalizer is removed, so
instead if the controller fails, it will be restarted with exponential
backoff lowering the load on the system

Change is validated with the unit-tests reproducing the conflict.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

internal/app/machined/pkg/controllers/network/address_merge.go

@@ -86,6 +86,8 @@ func (ctrl *AddressMergeController) Run(ctx context.Context, r controller.Runtim
 			addresses[id] = address
 		}
 
+		conflictsDetected := 0
+
 		for id, address := range addresses {
 			address := address
 
@@ -97,12 +99,11 @@ func (ctrl *AddressMergeController) Run(ctx context.Context, r controller.Runtim
 				return nil
 			}); err != nil {
 				if state.IsPhaseConflictError(err) {
-					logger.Debug("conflict detected", zap.String("id", id))
+					// phase conflict, resource is being torn down, skip updating it and trigger reconcile
+					// later by failing the
+					conflictsDetected++
 
 					delete(addresses, id)
-
-					// trigger another reconcile
-					r.QueueReconcile()
 				} else {
 					return fmt.Errorf("error updating resource: %w", err)
 				}
@@ -131,5 +132,9 @@ func (ctrl *AddressMergeController) Run(ctx context.Context, r controller.Runtim
 				}
 			}
 		}
+
+		if conflictsDetected > 0 {
+			return fmt.Errorf("%d conflict(s) detected", conflictsDetected)
+		}
 	}
 }

internal/app/machined/pkg/controllers/network/link_merge.go

@@ -100,6 +100,8 @@ func (ctrl *LinkMergeController) Run(ctx context.Context, r controller.Runtime,
 			}
 		}
 
+		conflictsDetected := 0
+
 		for id, link := range links {
 			link := link
 
@@ -111,12 +113,11 @@ func (ctrl *LinkMergeController) Run(ctx context.Context, r controller.Runtime,
 				return nil
 			}); err != nil {
 				if state.IsPhaseConflictError(err) {
-					logger.Debug("conflict detected", zap.String("id", id))
+					// phase conflict, resource is being torn down, skip updating it and trigger reconcile
+					// later by failing the
+					conflictsDetected++
 
 					delete(links, id)
-
-					// trigger another reconcile
-					r.QueueReconcile()
 				} else {
 					return fmt.Errorf("error updating resource: %w", err)
 				}
@@ -147,5 +148,9 @@ func (ctrl *LinkMergeController) Run(ctx context.Context, r controller.Runtime,
 				}
 			}
 		}
+
+		if conflictsDetected > 0 {
+			return fmt.Errorf("%d conflict(s) detected", conflictsDetected)
+		}
 	}
 }

internal/app/machined/pkg/controllers/network/route_merge.go

@@ -86,6 +86,8 @@ func (ctrl *RouteMergeController) Run(ctx context.Context, r controller.Runtime,
 			routes[id] = route
 		}
 
+		conflictsDetected := 0
+
 		for id, route := range routes {
 			route := route
 
@@ -97,12 +99,11 @@ func (ctrl *RouteMergeController) Run(ctx context.Context, r controller.Runtime,
 				return nil
 			}); err != nil {
 				if state.IsPhaseConflictError(err) {
-					logger.Debug("conflict detected", zap.String("id", id))
+					// phase conflict, resource is being torn down, skip updating it and trigger reconcile
+					// later by failing the
+					conflictsDetected++
 
 					delete(routes, id)
-
-					// trigger another reconcile
-					r.QueueReconcile()
 				} else {
 					return fmt.Errorf("error updating resource: %w", err)
 				}
@@ -131,5 +132,9 @@ func (ctrl *RouteMergeController) Run(ctx context.Context, r controller.Runtime,
 				}
 			}
 		}
+
+		if conflictsDetected > 0 {
+			return fmt.Errorf("%d conflict(s) detected", conflictsDetected)
+		}
 	}
 }