Merge pull request #959 from p0lyn0mial/etcd-client-retry

etcd-client starts retrying transient errors from the etcd cluster

staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go

@@ -137,8 +137,13 @@ func newETCD3Client(c storagebackend.TransportConfig) (*clientv3.Client, error)
 	}
 	dialOptions := []grpc.DialOption{
 		grpc.WithBlock(), // block until the underlying connection is up
-		grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor),
-		grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor),
+		// use chained interceptors so that the default (retry and backoff) interceptors are added.
+		// otherwise they will be overwritten by the metric interceptor.
+		//
+		// these optional interceptors will be placed after the default ones.
+		// which seems to be what we want as the metrics will be collected on each attempt (retry)
+		grpc.WithChainUnaryInterceptor(grpcprom.UnaryClientInterceptor),
+		grpc.WithChainStreamInterceptor(grpcprom.StreamClientInterceptor),
 	}
 	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
 		tracingOpts := []otelgrpc.Option{