Merge pull request #3456 from hashicorp/gossip-fix

Adds gossip keys to network segment memberlist configs.

agent/agent.go

@@ -31,6 +31,7 @@ import (
 	"github.com/hashicorp/consul/types"
 	"github.com/hashicorp/consul/watch"
 	"github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/memberlist"
 	"github.com/hashicorp/raft"
 	"github.com/hashicorp/serf/serf"
 	"github.com/shirou/gopsutil/host"
@@ -772,7 +773,8 @@ func (a *Agent) consulConfig() (*consul.Config, error) {
 	// Setup the loggers
 	base.LogOutput = a.LogOutput
 
-	// This will set up the LAN keyring, as well as the WAN for servers.
+	// This will set up the LAN keyring, as well as the WAN and any segments
+	// for servers.
 	if err := a.setupKeyrings(base); err != nil {
 		return nil, fmt.Errorf("Failed to configure keyring: %v", err)
 	}
@@ -946,8 +948,8 @@ func (a *Agent) setupNodeID(config *Config) error {
 	return nil
 }
 
-// setupKeyrings is used to initialize and load keyrings during agent startup
-func (a *Agent) setupKeyrings(config *consul.Config) error {
+// setupBaseKeyrings configures the LAN and WAN keyrings.
+func (a *Agent) setupBaseKeyrings(config *consul.Config) error {
 	// If the keyring file is disabled then just poke the provided key
 	// into the in-memory keyring.
 	if a.config.DisableKeyringFile {
@@ -1006,6 +1008,34 @@ LOAD:
 	return nil
 }
 
+// setupKeyrings is used to initialize and load keyrings during agent startup.
+func (a *Agent) setupKeyrings(config *consul.Config) error {
+	// First set up the LAN and WAN keyrings.
+	if err := a.setupBaseKeyrings(config); err != nil {
+		return err
+	}
+
+	// If there's no LAN keyring then there's nothing else to set up for
+	// any segments.
+	lanKeyring := config.SerfLANConfig.MemberlistConfig.Keyring
+	if lanKeyring == nil {
+		return nil
+	}
+
+	// Copy the initial state of the LAN keyring into each segment config.
+	// Segments don't have their own keyring file, they rely on the LAN
+	// holding the state so things can't get out of sync.
+	k, pk := lanKeyring.GetKeys(), lanKeyring.GetPrimaryKey()
+	for _, segment := range config.Segments {
+		keyring, err := memberlist.NewKeyring(k, pk)
+		if err != nil {
+			return err
+		}
+		segment.SerfConfig.MemberlistConfig.Keyring = keyring
+	}
+	return nil
+}
+
 // registerEndpoint registers a handler for the consul RPC server
 // under a unique name while making it accessible under the provided
 // name. This allows overwriting handlers for the golang net/rpc

agent/consul/internal_endpoint.go

@@ -149,12 +149,12 @@ func (m *Internal) executeKeyringOp(
 
 	if wan {
 		mgr := m.srv.KeyManagerWAN()
-		m.executeKeyringOpMgr(mgr, args, reply, wan)
+		m.executeKeyringOpMgr(mgr, args, reply, wan, "")
 	} else {
 		segments := m.srv.LANSegments()
-		for _, segment := range segments {
+		for name, segment := range segments {
 			mgr := segment.KeyManager()
-			m.executeKeyringOpMgr(mgr, args, reply, wan)
+			m.executeKeyringOpMgr(mgr, args, reply, wan, name)
 		}
 	}
 }
@@ -166,7 +166,8 @@ func (m *Internal) executeKeyringOpMgr(
 	mgr *serf.KeyManager,
 	args *structs.KeyringRequest,
 	reply *structs.KeyringResponses,
-	wan bool) {
+	wan bool,
+	segment string) {
 	var serfResp *serf.KeyResponse
 	var err error
 
@@ -190,6 +191,7 @@ func (m *Internal) executeKeyringOpMgr(
 	reply.Responses = append(reply.Responses, &structs.KeyringResponse{
 		WAN:        wan,
 		Datacenter: m.srv.config.Datacenter,
+		Segment:    segment,
 		Messages:   serfResp.Messages,
 		Keys:       serfResp.Keys,
 		NumNodes:   serfResp.NumNodes,

agent/structs/structs.go

@@ -887,6 +887,7 @@ func (r *KeyringRequest) RequestDatacenter() string {
 type KeyringResponse struct {
 	WAN        bool
 	Datacenter string
+	Segment    string
 	Messages   map[string]string `json:",omitempty"`
 	Keys       map[string]int
 	NumNodes   int

api/operator_keyring.go

@@ -13,6 +13,9 @@ type KeyringResponse struct {
 	// The datacenter name this request corresponds to
 	Datacenter string
 
+	// Segment has the network segment this request corresponds to.
+	Segment string
+
 	// A map of the encryption keys to the number of nodes they're installed on
 	Keys map[string]int
 

command/keyring.go

@@ -129,6 +129,9 @@ func (c *KeyringCommand) Run(args []string) int {
 func (c *KeyringCommand) handleList(responses []*consulapi.KeyringResponse) {
 	for _, response := range responses {
 		pool := response.Datacenter + " (LAN)"
+		if response.Segment != "" {
+			pool += fmt.Sprintf(" [%s]", response.Segment)
+		}
 		if response.WAN {
 			pool = "WAN"
 		}

website/source/api/operator/keyring.html.md

@@ -55,6 +55,7 @@ $ curl \
   {
     "WAN": true,
     "Datacenter": "dc1",
+    "Segment": "",
     "Keys": {
       "0eK8RjnsGC/+I1fJErQsBA==": 1,
       "G/3/L4yOw3e5T7NTvuRi9g==": 1,
@@ -65,6 +66,7 @@ $ curl \
   {
     "WAN": false,
     "Datacenter": "dc1",
+    "Segment": "",
     "Keys": {
       "0eK8RjnsGC/+I1fJErQsBA==": 1,
       "G/3/L4yOw3e5T7NTvuRi9g==": 1,
@@ -80,6 +82,8 @@ $ curl \
 
 - `Datacenter` is the datacenter the block refers to.
 
+- `Segment` is the network segment the block refers to.
+
 - `Keys` is a map of each gossip key to the number of nodes it's currently
   installed on.
 

website/source/docs/commands/keyring.html.markdown.erb

@@ -73,11 +73,14 @@ dc2 (LAN):
 
 dc1 (LAN):
   a1i101sMY8rxB+0eAKD/gw== [2/2]
+
+dc1 (LAN) [alpha]:
+  a1i101sMY8rxB+0eAKD/gw== [2/2]
 ```
 
-As you can see, the output above is divided first by gossip pool, and then by
-encryption key. The indicator to the right of each key displays the number of
-nodes the key is installed on over the total number of nodes in the pool.
+As you can see, the output above is divided first by gossip pool, including any network
+segments, and then by encryption key. The indicator to the right of each key displays
+the number of nodes the key is installed on over the total number of nodes in the pool.
 
 ## Errors