Allow managers not to expose a remote API port

integration/cluster.go

@@ -73,17 +73,20 @@ func (c *testCluster) RandomManager() *testNode {
 	return managers[idx]
 }
 
-// AddManager adds node with Manager role(both agent and manager).
-func (c *testCluster) AddManager() error {
+// AddManager adds a node with the Manager role. The node will function as both
+// an agent and a manager. If lateBind is set, the manager is started before a
+// remote API port is bound. This setting only applies to the first manager.
+func (c *testCluster) AddManager(lateBind bool) error {
 	// first node
 	var n *testNode
 	if len(c.nodes) == 0 {
-		node, err := newTestNode("", "")
+		node, err := newTestNode("", "", lateBind)
 		if err != nil {
 			return err
 		}
 		n = node
 	} else {
+		lateBind = false
 		joinAddr, err := c.RandomManager().node.RemoteAPIAddr()
 		if err != nil {
 			return err
@@ -95,7 +98,7 @@ func (c *testCluster) AddManager() error {
 		if len(clusterInfo.Clusters) == 0 {
 			return fmt.Errorf("joining manager: there is no cluster created in storage")
 		}
-		node, err := newTestNode(joinAddr, clusterInfo.Clusters[0].RootCA.JoinTokens.Manager)
+		node, err := newTestNode(joinAddr, clusterInfo.Clusters[0].RootCA.JoinTokens.Manager, false)
 		if err != nil {
 			return err
 		}
@@ -119,6 +122,20 @@ func (c *testCluster) AddManager() error {
 
 	c.nodes[n.node.NodeID()] = n
 	c.nodesOrder[n.node.NodeID()] = c.counter
+
+	if lateBind {
+		// Verify that the control API works
+		clusterInfo, err := c.api.ListClusters(context.Background(), &api.ListClustersRequest{})
+		if err != nil {
+			return err
+		}
+		if len(clusterInfo.Clusters) == 0 {
+			return fmt.Errorf("joining manager: there is no cluster created in storage")
+		}
+
+		return n.node.BindRemote(context.Background(), "127.0.0.1:0", "")
+	}
+
 	return nil
 }
 
@@ -140,7 +157,7 @@ func (c *testCluster) AddAgent() error {
 	if len(clusterInfo.Clusters) == 0 {
 		return fmt.Errorf("joining agent: there is no cluster created in storage")
 	}
-	node, err := newTestNode(joinAddr, clusterInfo.Clusters[0].RootCA.JoinTokens.Worker)
+	node, err := newTestNode(joinAddr, clusterInfo.Clusters[0].RootCA.JoinTokens.Worker, false)
 	if err != nil {
 		return err
 	}

integration/integration_test.go

@@ -125,7 +125,7 @@ func pollServiceReady(t *testing.T, c *testCluster, sid string) {
 func newCluster(t *testing.T, numWorker, numManager int) *testCluster {
 	cl := newTestCluster()
 	for i := 0; i < numManager; i++ {
-		require.NoError(t, cl.AddManager(), "manager number %d", i+1)
+		require.NoError(t, cl.AddManager(false), "manager number %d", i+1)
 	}
 	for i := 0; i < numWorker; i++ {
 		require.NoError(t, cl.AddAgent(), "agent number %d", i+1)
@@ -145,6 +145,28 @@ func TestClusterCreate(t *testing.T) {
 	}()
 }
 
+func TestServiceCreateLateBind(t *testing.T) {
+	t.Parallel()
+
+	numWorker, numManager := 3, 3
+
+	cl := newTestCluster()
+	for i := 0; i < numManager; i++ {
+		require.NoError(t, cl.AddManager(true), "manager number %d", i+1)
+	}
+	for i := 0; i < numWorker; i++ {
+		require.NoError(t, cl.AddAgent(), "agent number %d", i+1)
+	}
+
+	defer func() {
+		require.NoError(t, cl.Stop())
+	}()
+
+	sid, err := cl.CreateService("test_service", 60)
+	require.NoError(t, err)
+	pollServiceReady(t, cl, sid)
+}
+
 func TestServiceCreate(t *testing.T) {
 	t.Parallel()
 

integration/node.go

@@ -24,23 +24,25 @@ type testNode struct {
 
 // newNode creates new node with specific role(manager or agent) and joins to
 // existing cluster. if joinAddr is empty string, then new cluster will be initialized.
-// It uses TestExecutor as executor.
-func newTestNode(joinAddr, joinToken string) (*testNode, error) {
+// It uses TestExecutor as executor. If lateBind is set, the remote API port is not
+// bound.
+func newTestNode(joinAddr, joinToken string, lateBind bool) (*testNode, error) {
 	tmpDir, err := ioutil.TempDir("", "swarmkit-integration-")
 	if err != nil {
 		return nil, err
 	}
 
-	rAddr := "127.0.0.1:0"
 	cAddr := filepath.Join(tmpDir, "control.sock")
 	cfg := &node.Config{
-		ListenRemoteAPI:  rAddr,
 		ListenControlAPI: cAddr,
 		JoinAddr:         joinAddr,
 		StateDir:         tmpDir,
 		Executor:         &TestExecutor{},
 		JoinToken:        joinToken,
 	}
+	if !lateBind {
+		cfg.ListenRemoteAPI = "127.0.0.1:0"
+	}
 	node, err := node.New(cfg)
 	if err != nil {
 		return nil, err
@@ -122,6 +124,5 @@ func (n *testNode) ControlClient(ctx context.Context) (api.ControlClient, error)
 }
 
 func (n *testNode) IsManager() bool {
-	_, err := n.node.RemoteAPIAddr()
-	return err == nil
+	return n.node.Manager() != nil
 }