[release-0.18] 🐛 Use leader elector with client timeout

pkg/leaderelection/leader_election.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"time"
 
 	"k8s.io/apimachinery/pkg/util/uuid"
 	coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1"
@@ -49,6 +50,12 @@ type Options struct {
 	// LeaderElectionID determines the name of the resource that leader election
 	// will use for holding the leader lock.
 	LeaderElectionID string
+
+	// RenewDeadline is the renew deadline for this leader election client.
+	// Must be set to ensure the resource lock has an appropriate client timeout.
+	// Without that, a single slow response from the API server can result
+	// in losing leadership.
+	RenewDeadline time.Duration
 }
 
 // NewResourceLock creates a new resource lock for use in a leader election loop.
@@ -88,6 +95,20 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
 
 	// Construct clients for leader election
 	rest.AddUserAgent(config, "leader-election")
+
+	if options.RenewDeadline != 0 {
+		return resourcelock.NewFromKubeconfig(options.LeaderElectionResourceLock,
+			options.LeaderElectionNamespace,
+			options.LeaderElectionID,
+			resourcelock.ResourceLockConfig{
+				Identity:      id,
+				EventRecorder: recorderProvider.GetEventRecorderFor(id),
+			},
+			config,
+			options.RenewDeadline,
+		)
+	}
+
 	corev1Client, err := corev1client.NewForConfig(config)
 	if err != nil {
 		return nil, err
@@ -97,7 +118,6 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
 	if err != nil {
 		return nil, err
 	}
-
 	return resourcelock.New(options.LeaderElectionResourceLock,
 		options.LeaderElectionNamespace,
 		options.LeaderElectionID,
@@ -106,7 +126,8 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
 		resourcelock.ResourceLockConfig{
 			Identity:      id,
 			EventRecorder: recorderProvider.GetEventRecorderFor(id),
-		})
+		},
+	)
 }
 
 func getInClusterNamespace() (string, error) {

pkg/manager/manager.go

@@ -389,6 +389,7 @@ func New(config *rest.Config, options Options) (Manager, error) {
 			LeaderElectionResourceLock: options.LeaderElectionResourceLock,
 			LeaderElectionID:           options.LeaderElectionID,
 			LeaderElectionNamespace:    options.LeaderElectionNamespace,
+			RenewDeadline:              *options.RenewDeadline,
 		})
 		if err != nil {
 			return nil, err

pkg/manager/manager_test.go

@@ -316,6 +316,28 @@ var _ = Describe("manger.Manager", func() {
 				<-m2done
 			})
 
+			It("should default RenewDeadline for leader election config", func() {
+				var rl resourcelock.Interface
+				m1, err := New(cfg, Options{
+					LeaderElection:          true,
+					LeaderElectionNamespace: "default",
+					LeaderElectionID:        "test-leader-election-id",
+					newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) {
+						if options.RenewDeadline != 10*time.Second {
+							return nil, fmt.Errorf("expected RenewDeadline to be 10s, got %v", options.RenewDeadline)
+						}
+						var err error
+						rl, err = leaderelection.NewResourceLock(config, recorderProvider, options)
+						return rl, err
+					},
+					HealthProbeBindAddress: "0",
+					Metrics:                metricsserver.Options{BindAddress: "0"},
+					PprofBindAddress:       "0",
+				})
+				Expect(err).ToNot(HaveOccurred())
+				Expect(m1).ToNot(BeNil())
+			})
+
 			It("should default ID to controller-runtime if ID is not set", func() {
 				var rl resourcelock.Interface
 				m1, err := New(cfg, Options{