Implement flag --set-member-localaddr

Which sets the LocalAddr to an IP address from --initial-advertise-peer-urls.

Also adds e2e test that requires this flag to succeed.

Signed-off-by: Edwin Xie <edwin.xie@broadcom.com>

client/pkg/transport/listener.go

@@ -194,7 +194,7 @@ type TLSInfo struct {
 	// If true, ClientConfig() will return an error for a cert with non empty CN.
 	EmptyCN bool
 
-	// LocalAddr is the local IP address to use when communicating peer.
+	// LocalAddr is the local IP address to use when communicating with a peer.
 	LocalAddr string
 }
 

server/config/config.go

@@ -205,7 +205,7 @@ type ServerConfig struct {
 	// V2Deprecation defines a phase of v2store deprecation process.
 	V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
 
-	// LocalAddr is the local IP address to use when communicating peer.
+	// LocalAddr is the local IP address to use when communicating with a peer.
 	LocalAddr string `json:"local-address"`
 }
 

server/embed/config.go

@@ -22,6 +22,7 @@ import (
 	"math"
 	"net"
 	"net/http"
+	"net/netip"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -220,6 +221,12 @@ type Config struct {
 	ClientAutoTLS                                          bool
 	PeerTLSInfo                                            transport.TLSInfo
 	PeerAutoTLS                                            bool
+
+	// SetMemberLocalAddr specifies if true, that etcd should use the first
+	// specified and non-loopback host in AdvertisePeerUrls as the LocalAddr when
+	// communicating with a peer.
+	SetMemberLocalAddr bool `json:"set-member-localaddr"`
+
 	// SelfSignedCertValidity specifies the validity period of the client and peer certificates
 	// that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS,
 	// the unit is year, and the default is 1
@@ -327,9 +334,6 @@ type Config struct {
 	// AuthTokenTTL in seconds of the simple token
 	AuthTokenTTL uint `json:"auth-token-ttl"`
 
-	// PeerLocalAddr is the local IP address to use when communicating peer.
-	PeerLocalAddr string `json:"peer-local-addr"`
-
 	ExperimentalInitialCorruptCheck     bool          `json:"experimental-initial-corrupt-check"`
 	ExperimentalCorruptCheckTime        time.Duration `json:"experimental-corrupt-check-time"`
 	ExperimentalCompactHashCheckEnabled bool          `json:"experimental-compact-hash-check-enabled"`
@@ -614,6 +618,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
 		"initial-advertise-peer-urls",
 		"List of this member's peer URLs to advertise to the rest of the cluster.",
 	)
+	fs.BoolVar(&cfg.SetMemberLocalAddr, "set-member-localaddr", false, "Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.")
+
 	fs.Var(
 		flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""),
 		"advertise-client-urls",
@@ -665,8 +671,6 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
 	fs.StringVar(&cfg.PeerTLSInfo.ClientKeyFile, "peer-client-key-file", "", "Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required.")
 	fs.BoolVar(&cfg.PeerTLSInfo.ClientCertAuth, "peer-client-cert-auth", false, "Enable peer client cert authentication.")
 	fs.StringVar(&cfg.PeerTLSInfo.TrustedCAFile, "peer-trusted-ca-file", "", "Path to the peer server TLS trusted CA file.")
-	fs.StringVar(&cfg.PeerTLSInfo.LocalAddr, "peer-local-addr", "", "peer-local-addr is the local IP address to use when communicating peer.")
-
 	fs.BoolVar(&cfg.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates")
 	fs.UintVar(&cfg.SelfSignedCertValidity, "self-signed-cert-validity", 1, "The validity period of the client and peer certificates, unit is year")
 	fs.StringVar(&cfg.PeerTLSInfo.CRLFile, "peer-crl-file", "", "Path to the peer certificate revocation list file.")
@@ -1141,6 +1145,26 @@ func (cfg *Config) InitialClusterFromName(name string) (ret string) {
 	return ret[1:]
 }
 
+// InferLocalAddr tries to determine the LocalAddr used when communicating with
+// an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host
+// from AdvertisePeerUrls by searching for the first URL with a specified
+// non-loopback address. Otherwise, it defaults to empty string and the
+// LocalAddr used will be the default for the Golang HTTP client.
+func (cfg *Config) InferLocalAddr() string {
+	if !cfg.SetMemberLocalAddr {
+		return ""
+	}
+	for _, peerUrl := range cfg.AdvertisePeerUrls {
+		if addr, err := netip.ParseAddr(peerUrl.Hostname()); err == nil {
+			if addr.IsLoopback() || addr.IsUnspecified() {
+				continue
+			}
+			return addr.String()
+		}
+	}
+	return ""
+}
+
 func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
 func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
 

server/embed/config_test.go

@@ -152,6 +152,102 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) {
 	}
 }
 
+func TestInferLocalAddr(t *testing.T) {
+	tests := []struct {
+		name               string
+		advertisePeerUrls  []string
+		setMemberLocalAddr bool
+		expectedLocalAddr  string
+	}{
+		{
+			"defaults, SetMemberLocalAddr=false ",
+			[]string{DefaultInitialAdvertisePeerURLs},
+			false,
+			"",
+		},
+		{
+			"IPv4 address, SetMemberLocalAddr=false ",
+			[]string{"https://192.168.100.110:2380"},
+			false,
+			"",
+		},
+		{
+			"defaults, SetMemberLocalAddr=true",
+			[]string{DefaultInitialAdvertisePeerURLs},
+			true,
+			"",
+		},
+		{
+			"IPv4 unspecified address, SetMemberLocalAddr=true",
+			[]string{"https://0.0.0.0:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv6 unspecified address, SetMemberLocalAddr=true",
+			[]string{"https://[::]:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv4 loopback address, SetMemberLocalAddr=true",
+			[]string{"https://127.0.0.1:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv6 loopback address, SetMemberLocalAddr=true",
+			[]string{"https://[::1]:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv4 address, SetMemberLocalAddr=true",
+			[]string{"https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"Hostname and IPv4 address, SetMemberLocalAddr=true",
+			[]string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv4 and IPv6 address, SetMemberLocalAddr=true",
+			// IPv4 addresses will always sort before IPv6 ones anyways
+			[]string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv6 address, SetMemberLocalAddr=true",
+			[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"2001:db8:85a3::8a2e:370:7334",
+		},
+	}
+
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := NewConfig()
+			cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerUrls)
+			cfg.SetMemberLocalAddr = tt.setMemberLocalAddr
+
+			if err := cfg.Validate(); err != nil {
+				t.Errorf("#%d: failed to validate test Config: %v", i, err)
+				return
+			}
+
+			inferredLocalAddr := cfg.InferLocalAddr()
+			if inferredLocalAddr != tt.expectedLocalAddr {
+				t.Errorf("#%d: LocalAddr = %s, want = %s", i, inferredLocalAddr, tt.expectedLocalAddr)
+			}
+		})
+	}
+
+}
+
 func (s *securityConfig) equals(t *transport.TLSInfo) bool {
 	return s.CertFile == t.CertFile &&
 		s.CertAuth == t.ClientCertAuth &&

server/embed/etcd.go

@@ -227,6 +227,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
 		ExperimentalMaxLearners:                       cfg.ExperimentalMaxLearners,
 		V2Deprecation:                                 cfg.V2DeprecationEffective(),
+		LocalAddr:                                     cfg.InferLocalAddr(),
 	}
 
 	if srvcfg.ExperimentalEnableDistributedTracing {
@@ -245,6 +246,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		)
 	}
 
+	srvcfg.PeerTLSInfo.LocalAddr = srvcfg.LocalAddr
+
 	print(e.cfg.logger, *cfg, srvcfg, memberInitialized)
 
 	if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
@@ -336,6 +339,8 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized
 		zap.Strings("advertise-client-urls", ec.getAdvertiseClientUrls()),
 		zap.Strings("listen-client-urls", ec.getListenClientUrls()),
 		zap.Strings("listen-metrics-urls", ec.getMetricsURLs()),
+		zap.Bool("set-member-localaddr", ec.SetMemberLocalAddr),
+		zap.String("local-addr", sc.LocalAddr),
 		zap.Strings("cors", cors),
 		zap.Strings("host-whitelist", hss),
 		zap.String("initial-cluster", sc.InitialPeerURLsMap.String()),

server/etcdmain/help.go

@@ -107,6 +107,8 @@ Member:
 Clustering:
   --initial-advertise-peer-urls 'http://localhost:2380'
     List of this member's peer URLs to advertise to the rest of the cluster.
+  --set-member-localaddr 'false'
+    Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.
   --initial-cluster 'default=http://localhost:2380'
     Initial cluster configuration for bootstrapping.
   --initial-cluster-state 'new'
@@ -221,9 +223,6 @@ Security:
     Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3.
   --tls-max-version ''
     Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty will be auto-populated by Go).
-  --peer-local-addr ''
-    LocalAddr is the local IP address to use when communicating peer.
-
 
 Auth:
   --auth-token 'simple'

tests/e2e/etcd_config_test.go

@@ -17,6 +17,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"strings"
 	"testing"
@@ -274,6 +275,107 @@ func TestEtcdPeerNameAuth(t *testing.T) {
 	}
 }
 
+// TestEtcdPeerLocalAddr checks that the inter peer auth works with when
+// the member LocalAddr is set.
+func TestEtcdPeerLocalAddr(t *testing.T) {
+	e2e.SkipInShortMode(t)
+
+	nodeIP, err := getLocalIP()
+	t.Log("Using node IP", nodeIP)
+	require.NoError(t, err)
+
+	peers, tmpdirs := make([]string, 3), make([]string, 3)
+
+	for i := range peers {
+		peerIP := nodeIP
+		if i == 0 {
+			peerIP = "127.0.0.1"
+		}
+		peers[i] = fmt.Sprintf("e%d=https://%s:%d", i, peerIP, e2e.EtcdProcessBasePort+i)
+		tmpdirs[i] = t.TempDir()
+	}
+	procs := make([]*expect.ExpectProcess, len(peers))
+	defer func() {
+		for i := range procs {
+			if procs[i] != nil {
+				procs[i].Stop()
+				procs[i].Close()
+			}
+			os.RemoveAll(tmpdirs[i])
+		}
+	}()
+
+	tempDir := t.TempDir()
+	caFile, certFiles, keyFiles, err := generateCertsForIPs(tempDir, []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP(nodeIP)})
+	require.NoError(t, err)
+
+	defer func() {
+		os.RemoveAll(tempDir)
+	}()
+
+	// node 0 (127.0.0.1) does not set localaddr, while nodes 1 and nodes 2 (both use host's IP) do.
+	// The other two nodes will reject connections from node 0 warning that node 0's certificate is valid only for
+	// 127.0.0.1, not the host IP, since node 0 will try to connect to the other peers with the host IP
+	// as the client address.
+	// Node 0 will not reject connections from the other nodes since they will
+	// use the host's IP to connect (due to --set-member-localaddr)
+	for i := range procs {
+		peerIP := nodeIP
+		if i == 0 {
+			peerIP = "127.0.0.1"
+		}
+		ic := strings.Join(peers, ",")
+		commonArgs := []string{
+			e2e.BinPath.Etcd,
+			"--name", fmt.Sprintf("e%d", i),
+			"--listen-client-urls", "http://0.0.0.0:0",
+			"--data-dir", tmpdirs[i],
+			"--advertise-client-urls", "http://0.0.0.0:0",
+			"--initial-advertise-peer-urls", fmt.Sprintf("https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i),
+			"--listen-peer-urls", fmt.Sprintf("https://%s:%d,https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i, peerIP, e2e.EtcdProcessBasePort+len(peers)+i),
+			"--initial-cluster", ic,
+		}
+
+		var args []string
+		if i == 0 {
+			args = []string{
+				"--peer-cert-file", certFiles[0],
+				"--peer-key-file", keyFiles[0],
+				"--peer-trusted-ca-file", caFile,
+				"--peer-client-cert-auth",
+			}
+		} else {
+			args = []string{
+				"--peer-cert-file", certFiles[1],
+				"--peer-key-file", keyFiles[1],
+				"--peer-trusted-ca-file", caFile,
+				"--peer-client-cert-auth",
+				"--set-member-localaddr",
+			}
+		}
+
+		commonArgs = append(commonArgs, args...)
+
+		p, err := e2e.SpawnCmd(commonArgs, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		procs[i] = p
+	}
+
+	for i, p := range procs {
+		var expect []string
+		if i == 0 {
+			expect = e2e.EtcdServerReadyLines
+		} else {
+			expect = []string{"x509: certificate is valid for 127.0.0.1, not "}
+		}
+		if err := e2e.WaitReadyExpectProc(context.TODO(), p, expect); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
 func TestGrpcproxyAndCommonName(t *testing.T) {
 	e2e.SkipInShortMode(t)