From cef3a131a3a2f13fae744754df4b31130489dd4c Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Thu, 13 Jun 2019 15:45:01 -0400 Subject: [PATCH 1/7] Add tagged addresses into the data model for services Currently these are not used anywhere but they will be eventually. --- agent/agent_endpoint.go | 10 +++++ agent/config/builder.go | 21 +++++++++++ agent/config/config.go | 30 +++++++++------ agent/config/runtime_test.go | 34 ++++++++++++++++- agent/structs/service_definition.go | 9 +++++ agent/structs/structs.go | 25 ++++++++++++ agent/structs/structs_filtering_test.go | 33 ++++++++++++++++ agent/structs/structs_test.go | 39 ++++++++++++++++++- api/agent.go | 20 +++++----- api/agent_test.go | 23 +++++++++++ api/catalog.go | 20 ++++++++++ command/services/register/register.go | 44 +++++++++++++++------- command/services/register/register_test.go | 35 +++++++++++++++++ website/source/docs/agent/services.html.md | 12 +++++- 14 files changed, 317 insertions(+), 38 deletions(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 229969df04b0..7fdf3e23b59c 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -166,6 +166,15 @@ func buildAgentService(s *structs.NodeService, proxies map[string]*local.Managed } weights.Warning = s.Weights.Warning } + + var taggedAddrs map[string]api.ServiceAddress + if len(s.TaggedAddresses) > 0 { + taggedAddrs = make(map[string]api.ServiceAddress) + for k, v := range s.TaggedAddresses { + taggedAddrs[k] = v.ToAPIServiceAddress() + } + } + as := api.AgentService{ Kind: api.ServiceKind(s.Kind), ID: s.ID, @@ -174,6 +183,7 @@ func buildAgentService(s *structs.NodeService, proxies map[string]*local.Managed Meta: s.Meta, Port: s.Port, Address: s.Address, + TaggedAddresses: taggedAddrs, EnableTagOverride: s.EnableTagOverride, CreateIndex: s.CreateIndex, ModifyIndex: s.ModifyIndex, diff --git a/agent/config/builder.go b/agent/config/builder.go index dafc25a9f521..4e633ff2a902 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1177,6 +1177,26 @@ func (b *Builder) checkVal(v *CheckDefinition) *structs.CheckDefinition { } } +func (b *Builder) svcTaggedAddresses(v map[string]ServiceAddress) map[string]structs.ServiceAddress { + if len(v) <= 0 { + return nil + } + + svcAddrs := make(map[string]structs.ServiceAddress) + for addrName, addrConf := range v { + addr := structs.ServiceAddress{} + if addrConf.Address != nil { + addr.Address = *addrConf.Address + } + if addrConf.Port != nil { + addr.Port = *addrConf.Port + } + + svcAddrs[addrName] = addr + } + return svcAddrs +} + func (b *Builder) serviceVal(v *ServiceDefinition) *structs.ServiceDefinition { if v == nil { return nil @@ -1215,6 +1235,7 @@ func (b *Builder) serviceVal(v *ServiceDefinition) *structs.ServiceDefinition { Name: b.stringVal(v.Name), Tags: v.Tags, Address: b.stringVal(v.Address), + TaggedAddresses: b.svcTaggedAddresses(v.TaggedAddresses), Meta: meta, Port: b.intVal(v.Port), Token: b.stringVal(v.Token), diff --git a/agent/config/config.go b/agent/config/config.go index fdafdb1636f1..4cf66e51ae87 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -360,19 +360,25 @@ type ServiceWeights struct { Warning *int `json:"warning,omitempty" hcl:"warning" mapstructure:"warning"` } +type ServiceAddress struct { + Address *string `json:"address,omitempty" hcl:"address" mapstructure:"address"` + Port *int `json:"port,omitempty" hcl:"port" mapstructure:"port"` +} + type ServiceDefinition struct { - Kind *string `json:"kind,omitempty" hcl:"kind" mapstructure:"kind"` - ID *string `json:"id,omitempty" hcl:"id" mapstructure:"id"` - Name *string `json:"name,omitempty" hcl:"name" mapstructure:"name"` - Tags []string `json:"tags,omitempty" hcl:"tags" mapstructure:"tags"` - Address *string `json:"address,omitempty" hcl:"address" mapstructure:"address"` - Meta map[string]string `json:"meta,omitempty" hcl:"meta" mapstructure:"meta"` - Port *int `json:"port,omitempty" hcl:"port" mapstructure:"port"` - Check *CheckDefinition `json:"check,omitempty" hcl:"check" mapstructure:"check"` - Checks []CheckDefinition `json:"checks,omitempty" hcl:"checks" mapstructure:"checks"` - Token *string `json:"token,omitempty" hcl:"token" mapstructure:"token"` - Weights *ServiceWeights `json:"weights,omitempty" hcl:"weights" mapstructure:"weights"` - EnableTagOverride *bool `json:"enable_tag_override,omitempty" hcl:"enable_tag_override" mapstructure:"enable_tag_override"` + Kind *string `json:"kind,omitempty" hcl:"kind" mapstructure:"kind"` + ID *string `json:"id,omitempty" hcl:"id" mapstructure:"id"` + Name *string `json:"name,omitempty" hcl:"name" mapstructure:"name"` + Tags []string `json:"tags,omitempty" hcl:"tags" mapstructure:"tags"` + Address *string `json:"address,omitempty" hcl:"address" mapstructure:"address"` + TaggedAddresses map[string]ServiceAddress `json:"tagged_addresses,omitempty" hcl:"tagged_addresses" mapstructure:"tagged_addresses"` + Meta map[string]string `json:"meta,omitempty" hcl:"meta" mapstructure:"meta"` + Port *int `json:"port,omitempty" hcl:"port" mapstructure:"port"` + Check *CheckDefinition `json:"check,omitempty" hcl:"check" mapstructure:"check"` + Checks []CheckDefinition `json:"checks,omitempty" hcl:"checks" mapstructure:"checks"` + Token *string `json:"token,omitempty" hcl:"token" mapstructure:"token"` + Weights *ServiceWeights `json:"weights,omitempty" hcl:"weights" mapstructure:"weights"` + EnableTagOverride *bool `json:"enable_tag_override,omitempty" hcl:"enable_tag_override" mapstructure:"enable_tag_override"` // DEPRECATED (ProxyDestination) - remove this when removing ProxyDestination ProxyDestination *string `json:"proxy_destination,omitempty" hcl:"proxy_destination" mapstructure:"proxy_destination"` Proxy *ServiceProxy `json:"proxy,omitempty" hcl:"proxy" mapstructure:"proxy"` diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 70e6eb3d9637..9cff4dfd7f54 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -3251,6 +3251,16 @@ func TestFullConfig(t *testing.T) { "meta": { "mymeta": "data" }, + "tagged_addresses": { + "lan": { + "address": "2d79888a", + "port": 2143 + }, + "wan": { + "address": "d4db85e2", + "port": 6109 + } + }, "tags": ["nkwshvM5", "NTDWn3ek"], "address": "cOlSOhbp", "token": "msy7iWER", @@ -3820,6 +3830,16 @@ func TestFullConfig(t *testing.T) { meta = { mymeta = "data" } + tagged_addresses = { + lan = { + address = "2d79888a" + port = 2143 + } + wan = { + address = "d4db85e2" + port = 6109 + } + } tags = ["nkwshvM5", "NTDWn3ek"] address = "cOlSOhbp" token = "msy7iWER" @@ -4603,8 +4623,18 @@ func TestFullConfig(t *testing.T) { }, }, { - ID: "dLOXpSCI", - Name: "o1ynPkp0", + ID: "dLOXpSCI", + Name: "o1ynPkp0", + TaggedAddresses: map[string]structs.ServiceAddress{ + "lan": structs.ServiceAddress{ + Address: "2d79888a", + Port: 2143, + }, + "wan": structs.ServiceAddress{ + Address: "d4db85e2", + Port: 6109, + }, + }, Tags: []string{"nkwshvM5", "NTDWn3ek"}, Address: "cOlSOhbp", Token: "msy7iWER", diff --git a/agent/structs/service_definition.go b/agent/structs/service_definition.go index 93a53ff96a9d..a9439da67483 100644 --- a/agent/structs/service_definition.go +++ b/agent/structs/service_definition.go @@ -19,6 +19,7 @@ type ServiceDefinition struct { Name string Tags []string Address string + TaggedAddresses map[string]ServiceAddress Meta map[string]string Port int Check CheckType @@ -76,6 +77,14 @@ func (s *ServiceDefinition) NodeService() *NodeService { if ns.ID == "" && ns.Service != "" { ns.ID = ns.Service } + if len(s.TaggedAddresses) > 0 { + taggedAddrs := make(map[string]ServiceAddress) + for k, v := range s.TaggedAddresses { + taggedAddrs[k] = v + } + + ns.TaggedAddresses = taggedAddrs + } return ns } diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 720e614acebb..62cdfe5ae60a 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -591,6 +591,7 @@ type ServiceNode struct { ServiceName string ServiceTags []string ServiceAddress string + ServiceTaggedAddresses map[string]ServiceAddress `json:",omitempty"` ServiceWeights Weights ServiceMeta map[string]string ServicePort int @@ -613,6 +614,14 @@ func (s *ServiceNode) PartialClone() *ServiceNode { nsmeta[k] = v } + var svcTaggedAddrs map[string]ServiceAddress + if len(s.ServiceTaggedAddresses) > 0 { + svcTaggedAddrs = make(map[string]ServiceAddress) + for k, v := range s.ServiceTaggedAddresses { + svcTaggedAddrs[k] = v + } + } + return &ServiceNode{ // Skip ID, see above. Node: s.Node, @@ -623,6 +632,7 @@ func (s *ServiceNode) PartialClone() *ServiceNode { ServiceName: s.ServiceName, ServiceTags: tags, ServiceAddress: s.ServiceAddress, + ServiceTaggedAddresses: svcTaggedAddrs, ServicePort: s.ServicePort, ServiceMeta: nsmeta, ServiceWeights: s.ServiceWeights, @@ -646,6 +656,7 @@ func (s *ServiceNode) ToNodeService() *NodeService { Service: s.ServiceName, Tags: s.ServiceTags, Address: s.ServiceAddress, + TaggedAddresses: s.ServiceTaggedAddresses, Port: s.ServicePort, Meta: s.ServiceMeta, Weights: &s.ServiceWeights, @@ -683,6 +694,16 @@ const ( ServiceKindConnectProxy ServiceKind = "connect-proxy" ) +// Type to hold a address and port of a service +type ServiceAddress struct { + Address string + Port int +} + +func (a ServiceAddress) ToAPIServiceAddress() api.ServiceAddress { + return api.ServiceAddress{Address: a.Address, Port: a.Port} +} + // NodeService is a service provided by a node type NodeService struct { // Kind is the kind of service this is. Different kinds of services may @@ -694,6 +715,7 @@ type NodeService struct { Service string Tags []string Address string + TaggedAddresses map[string]ServiceAddress `json:",omitempty"` Meta map[string]string Port int Weights *Weights @@ -844,6 +866,7 @@ func (s *NodeService) IsSame(other *NodeService) bool { !reflect.DeepEqual(s.Tags, other.Tags) || s.Address != other.Address || s.Port != other.Port || + !reflect.DeepEqual(s.TaggedAddresses, other.TaggedAddresses) || !reflect.DeepEqual(s.Weights, other.Weights) || !reflect.DeepEqual(s.Meta, other.Meta) || s.EnableTagOverride != other.EnableTagOverride || @@ -876,6 +899,7 @@ func (s *ServiceNode) IsSameService(other *ServiceNode) bool { s.ServiceName != other.ServiceName || !reflect.DeepEqual(s.ServiceTags, other.ServiceTags) || s.ServiceAddress != other.ServiceAddress || + !reflect.DeepEqual(s.ServiceTaggedAddresses, other.ServiceTaggedAddresses) || s.ServicePort != other.ServicePort || !reflect.DeepEqual(s.ServiceMeta, other.ServiceMeta) || !reflect.DeepEqual(s.ServiceWeights, other.ServiceWeights) || @@ -915,6 +939,7 @@ func (s *NodeService) ToServiceNode(node string) *ServiceNode { ServiceName: s.Service, ServiceTags: s.Tags, ServiceAddress: s.Address, + ServiceTaggedAddresses: s.TaggedAddresses, ServicePort: s.Port, ServiceMeta: s.Meta, ServiceWeights: theWeights, diff --git a/agent/structs/structs_filtering_test.go b/agent/structs/structs_filtering_test.go index 5227d6efa9de..446f4b3669ef 100644 --- a/agent/structs/structs_filtering_test.go +++ b/agent/structs/structs_filtering_test.go @@ -162,6 +162,19 @@ var expectedFieldConfigNode bexpr.FieldConfigurations = bexpr.FieldConfiguration }, } +var expectedFieldConfigMapStringServiceAddress bexpr.FieldConfigurations = bexpr.FieldConfigurations{ + "Address": &bexpr.FieldConfiguration{ + StructFieldName: "Address", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, + "Port": &bexpr.FieldConfiguration{ + StructFieldName: "Port", + CoerceFn: bexpr.CoerceInt, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, + }, +} + var expectedFieldConfigNodeService bexpr.FieldConfigurations = bexpr.FieldConfigurations{ "Kind": &bexpr.FieldConfiguration{ StructFieldName: "Kind", @@ -188,6 +201,16 @@ var expectedFieldConfigNodeService bexpr.FieldConfigurations = bexpr.FieldConfig CoerceFn: bexpr.CoerceString, SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, }, + "TaggedAddresses": &bexpr.FieldConfiguration{ + StructFieldName: "TaggedAddresses", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchIsEmpty, bexpr.MatchIsNotEmpty, bexpr.MatchIn, bexpr.MatchNotIn}, + SubFields: bexpr.FieldConfigurations{ + bexpr.FieldNameAny: &bexpr.FieldConfiguration{ + SubFields: expectedFieldConfigMapStringServiceAddress, + }, + }, + }, "Meta": &bexpr.FieldConfiguration{ StructFieldName: "Meta", CoerceFn: bexpr.CoerceString, @@ -276,6 +299,16 @@ var expectedFieldConfigServiceNode bexpr.FieldConfigurations = bexpr.FieldConfig CoerceFn: bexpr.CoerceString, SupportedOperations: []bexpr.MatchOperator{bexpr.MatchEqual, bexpr.MatchNotEqual}, }, + "ServiceTaggedAddresses": &bexpr.FieldConfiguration{ + StructFieldName: "ServiceTaggedAddresses", + CoerceFn: bexpr.CoerceString, + SupportedOperations: []bexpr.MatchOperator{bexpr.MatchIsEmpty, bexpr.MatchIsNotEmpty, bexpr.MatchIn, bexpr.MatchNotIn}, + SubFields: bexpr.FieldConfigurations{ + bexpr.FieldNameAny: &bexpr.FieldConfiguration{ + SubFields: expectedFieldConfigMapStringServiceAddress, + }, + }, + }, "ServiceMeta": &bexpr.FieldConfiguration{ StructFieldName: "ServiceMeta", CoerceFn: bexpr.CoerceString, diff --git a/agent/structs/structs_test.go b/agent/structs/structs_test.go index e407f758c185..fdee9443d328 100644 --- a/agent/structs/structs_test.go +++ b/agent/structs/structs_test.go @@ -144,7 +144,17 @@ func testServiceNode(t *testing.T) *ServiceNode { ServiceName: "dogs", ServiceTags: []string{"prod", "v1"}, ServiceAddress: "127.0.0.2", - ServicePort: 8080, + ServiceTaggedAddresses: map[string]ServiceAddress{ + "lan": ServiceAddress{ + Address: "127.0.0.2", + Port: 8080, + }, + "wan": ServiceAddress{ + Address: "198.18.0.1", + Port: 80, + }, + }, + ServicePort: 8080, ServiceMeta: map[string]string{ "service": "metadata", }, @@ -241,6 +251,7 @@ func TestStructs_ServiceNode_IsSameService(t *testing.T) { serviceProxyDestination := sn.ServiceProxyDestination serviceProxy := sn.ServiceProxy serviceConnect := sn.ServiceConnect + serviceTaggedAddresses := sn.ServiceTaggedAddresses n := sn.ToNodeService().ToServiceNode(node) other := sn.ToNodeService().ToServiceNode(node) @@ -275,6 +286,7 @@ func TestStructs_ServiceNode_IsSameService(t *testing.T) { check(func() { other.ServiceWeights = Weights{Passing: 42, Warning: 41} }, func() { other.ServiceWeights = serviceWeights }) check(func() { other.ServiceProxy = ConnectProxyConfig{} }, func() { other.ServiceProxy = serviceProxy }) check(func() { other.ServiceConnect = ServiceConnect{} }, func() { other.ServiceConnect = serviceConnect }) + check(func() { other.ServiceTaggedAddresses = nil }, func() { other.ServiceTaggedAddresses = serviceTaggedAddresses }) } func TestStructs_ServiceNode_PartialClone(t *testing.T) { @@ -321,6 +333,10 @@ func TestStructs_ServiceNode_PartialClone(t *testing.T) { if reflect.DeepEqual(sn, clone) { t.Fatalf("clone wasn't independent of the original for Meta") } + + // ensure that the tagged addresses were copied and not just a pointer to the map + sn.ServiceTaggedAddresses["foo"] = ServiceAddress{Address: "consul.is.awesome", Port: 443} + require.NotEqual(t, sn, clone) } func TestStructs_ServiceNode_Conversions(t *testing.T) { @@ -472,6 +488,16 @@ func TestStructs_NodeService_IsSame(t *testing.T) { Service: "theservice", Tags: []string{"foo", "bar"}, Address: "127.0.0.1", + TaggedAddresses: map[string]ServiceAddress{ + "lan": ServiceAddress{ + Address: "127.0.0.1", + Port: 3456, + }, + "wan": ServiceAddress{ + Address: "198.18.0.1", + Port: 1234, + }, + }, Meta: map[string]string{ "meta1": "value1", "meta2": "value2", @@ -497,6 +523,16 @@ func TestStructs_NodeService_IsSame(t *testing.T) { Address: "127.0.0.1", Port: 1234, EnableTagOverride: true, + TaggedAddresses: map[string]ServiceAddress{ + "wan": ServiceAddress{ + Address: "198.18.0.1", + Port: 1234, + }, + "lan": ServiceAddress{ + Address: "127.0.0.1", + Port: 3456, + }, + }, Meta: map[string]string{ // We don't care about order "meta2": "value2", @@ -559,6 +595,7 @@ func TestStructs_NodeService_IsSame(t *testing.T) { if !otherServiceNode.IsSameService(otherServiceNodeCopy2) { t.Fatalf("copy should be the same, but was\n %#v\nVS\n %#v", otherServiceNode, otherServiceNodeCopy2) } + check(func() { other.TaggedAddresses["lan"] = ServiceAddress{Address: "127.0.0.1", Port: 9999} }, func() { other.TaggedAddresses["lan"] = ServiceAddress{Address: "127.0.0.1", Port: 3456} }) } func TestStructs_HealthCheck_IsSame(t *testing.T) { diff --git a/api/agent.go b/api/agent.go index 04043ba842f9..de5f4da4fd6a 100644 --- a/api/agent.go +++ b/api/agent.go @@ -82,6 +82,7 @@ type AgentService struct { Meta map[string]string Port int Address string + TaggedAddresses map[string]ServiceAddress Weights AgentWeights EnableTagOverride bool CreateIndex uint64 `json:",omitempty" bexpr:"-"` @@ -157,15 +158,16 @@ type MembersOpts struct { // AgentServiceRegistration is used to register a new service type AgentServiceRegistration struct { - Kind ServiceKind `json:",omitempty"` - ID string `json:",omitempty"` - Name string `json:",omitempty"` - Tags []string `json:",omitempty"` - Port int `json:",omitempty"` - Address string `json:",omitempty"` - EnableTagOverride bool `json:",omitempty"` - Meta map[string]string `json:",omitempty"` - Weights *AgentWeights `json:",omitempty"` + Kind ServiceKind `json:",omitempty"` + ID string `json:",omitempty"` + Name string `json:",omitempty"` + Tags []string `json:",omitempty"` + Port int `json:",omitempty"` + Address string `json:",omitempty"` + TaggedAddresses map[string]ServiceAddress `json:",omitempty"` + EnableTagOverride bool `json:",omitempty"` + Meta map[string]string `json:",omitempty"` + Weights *AgentWeights `json:",omitempty"` Check *AgentServiceCheck Checks AgentServiceChecks // DEPRECATED (ProxyDestination) - remove this field diff --git a/api/agent_test.go b/api/agent_test.go index f37cd15e61c8..071885439f65 100644 --- a/api/agent_test.go +++ b/api/agent_test.go @@ -179,6 +179,12 @@ func TestAPI_AgentServices(t *testing.T) { Name: "foo", ID: "foo", Tags: []string{"bar", "baz"}, + TaggedAddresses: map[string]ServiceAddress{ + "lan": ServiceAddress{ + Address: "198.18.0.1", + Port: 80, + }, + }, Port: 8000, Check: &AgentServiceCheck{ TTL: "15s", @@ -605,6 +611,16 @@ func TestAPI_AgentServiceAddress(t *testing.T) { reg2 := &AgentServiceRegistration{ Name: "foo2", Port: 8000, + TaggedAddresses: map[string]ServiceAddress{ + "lan": ServiceAddress{ + Address: "192.168.0.43", + Port: 8000, + }, + "wan": ServiceAddress{ + Address: "198.18.0.1", + Port: 80, + }, + }, } if err := agent.ServiceRegister(reg1); err != nil { t.Fatalf("err: %v", err) @@ -631,6 +647,13 @@ func TestAPI_AgentServiceAddress(t *testing.T) { if services["foo2"].Address != "" { t.Fatalf("missing Address field in service foo2: %v", services) } + require.NotNil(t, services["foo2"].TaggedAddresses) + require.Contains(t, services["foo2"].TaggedAddresses, "lan") + require.Contains(t, services["foo2"].TaggedAddresses, "wan") + require.Equal(t, services["foo2"].TaggedAddresses["lan"].Address, "192.168.0.43") + require.Equal(t, services["foo2"].TaggedAddresses["lan"].Port, 8000) + require.Equal(t, services["foo2"].TaggedAddresses["wan"].Address, "198.18.0.1") + require.Equal(t, services["foo2"].TaggedAddresses["wan"].Port, 80) if err := agent.ServiceDeregister("foo"); err != nil { t.Fatalf("err: %v", err) diff --git a/api/catalog.go b/api/catalog.go index c175c3fff53b..62c0af9557e7 100644 --- a/api/catalog.go +++ b/api/catalog.go @@ -1,5 +1,10 @@ package api +import ( + "net" + "strconv" +) + type Weights struct { Passing int Warning int @@ -16,6 +21,11 @@ type Node struct { ModifyIndex uint64 } +type ServiceAddress struct { + Address string + Port int +} + type CatalogService struct { ID string Node string @@ -26,6 +36,7 @@ type CatalogService struct { ServiceID string ServiceName string ServiceAddress string + ServiceTaggedAddresses map[string]ServiceAddress ServiceTags []string ServiceMeta map[string]string ServicePort int @@ -242,3 +253,12 @@ func (c *Catalog) Node(node string, q *QueryOptions) (*CatalogNode, *QueryMeta, } return out, qm, nil } + +func ParseServiceAddr(addrPort string) (ServiceAddress, error) { + port := 0 + host, portStr, err := net.SplitHostPort(addrPort) + if err == nil { + port, err = strconv.Atoi(portStr) + } + return ServiceAddress{Address: host, Port: port}, err +} diff --git a/command/services/register/register.go b/command/services/register/register.go index 323bb3db9516..b4af715413d1 100644 --- a/command/services/register/register.go +++ b/command/services/register/register.go @@ -23,12 +23,13 @@ type cmd struct { help string // flags - flagId string - flagName string - flagAddress string - flagPort int - flagTags []string - flagMeta map[string]string + flagId string + flagName string + flagAddress string + flagPort int + flagTags []string + flagMeta map[string]string + flagTaggedAddresses map[string]string } func (c *cmd) init() { @@ -43,11 +44,14 @@ func (c *cmd) init() { c.flags.IntVar(&c.flagPort, "port", 0, "Port of the service to register for arg-based registration.") c.flags.Var((*flags.FlagMapValue)(&c.flagMeta), "meta", - "Metadata to set on the intention, formatted as key=value. This flag "+ + "Metadata to set on the service, formatted as key=value. This flag "+ "may be specified multiple times to set multiple meta fields.") c.flags.Var((*flags.AppendSliceValue)(&c.flagTags), "tag", "Tag to add to the service. This flag can be specified multiple "+ "times to set multiple tags.") + c.flags.Var((*flags.FlagMapValue)(&c.flagTaggedAddresses), "tagged-address", + "Tagged address to set on the service, formatted as key=value. This flag "+ + "may be specified multiple times to set multiple addresses.") c.http = &flags.HTTPFlags{} flags.Merge(c.flags, c.http.ClientFlags()) @@ -60,13 +64,27 @@ func (c *cmd) Run(args []string) int { return 1 } + var taggedAddrs map[string]api.ServiceAddress + if len(c.flagTaggedAddresses) > 0 { + taggedAddrs = make(map[string]api.ServiceAddress) + for k, v := range c.flagTaggedAddresses { + addr, err := api.ParseServiceAddr(v) + if err != nil { + c.UI.Error(fmt.Sprintf("Invalid Tagged Address: %v", err)) + return 1 + } + taggedAddrs[k] = addr + } + } + svcs := []*api.AgentServiceRegistration{&api.AgentServiceRegistration{ - ID: c.flagId, - Name: c.flagName, - Address: c.flagAddress, - Port: c.flagPort, - Tags: c.flagTags, - Meta: c.flagMeta, + ID: c.flagId, + Name: c.flagName, + Address: c.flagAddress, + Port: c.flagPort, + Tags: c.flagTags, + Meta: c.flagMeta, + TaggedAddresses: taggedAddrs, }} // Check for arg validation diff --git a/command/services/register/register_test.go b/command/services/register/register_test.go index 4b1eb5b8311e..fa2d95daae14 100644 --- a/command/services/register/register_test.go +++ b/command/services/register/register_test.go @@ -118,6 +118,41 @@ func TestCommand_Flags(t *testing.T) { require.NotNil(svc) } +func TestCommand_Flags_TaggedAddresses(t *testing.T) { + t.Parallel() + + require := require.New(t) + a := agent.NewTestAgent(t, t.Name(), ``) + defer a.Shutdown() + client := a.Client() + + ui := cli.NewMockUi() + c := New(ui) + + args := []string{ + "-http-addr=" + a.HTTPAddr(), + "-name", "web", + "-tagged-address", "lan=127.0.0.1:1234", + "-tagged-address", "v6=[2001:db8::12]:1234", + } + + require.Equal(0, c.Run(args), ui.ErrorWriter.String()) + + svcs, err := client.Agent().Services() + require.NoError(err) + require.Len(svcs, 1) + + svc := svcs["web"] + require.NotNil(svc) + require.Len(svc.TaggedAddresses, 2) + require.Contains(svc.TaggedAddresses, "lan") + require.Contains(svc.TaggedAddresses, "v6") + require.Equal(svc.TaggedAddresses["lan"].Address, "127.0.0.1") + require.Equal(svc.TaggedAddresses["lan"].Port, 1234) + require.Equal(svc.TaggedAddresses["v6"].Address, "2001:db8::12") + require.Equal(svc.TaggedAddresses["v6"].Port, 1234) +} + func testFile(t *testing.T, suffix string) *os.File { f := testutil.TempFile(t, "register-test-file") if err := f.Close(); err != nil { diff --git a/website/source/docs/agent/services.html.md b/website/source/docs/agent/services.html.md index ce85d92de3e1..e2545b93c3ff 100644 --- a/website/source/docs/agent/services.html.md +++ b/website/source/docs/agent/services.html.md @@ -37,6 +37,16 @@ example shows all possible fields, but note that only a few are required. "meta": { "meta": "for my service" }, + "tagged_addresses": { + "lan": { + "address": "192.168.0.55", + "port": 8000, + }, + "wan": { + "address": "198.18.0.23", + "port": 80 + } + }, "port": 8000, "enable_tag_override": false, "checks": [ @@ -280,7 +290,7 @@ For historical reasons Consul's API uses `CamelCased` parameter names in responses, however it's configuration file uses `snake_case` for both HCL and JSON representations. For this reason the registration _HTTP APIs_ accept both name styles for service definition parameters although APIs will return the -listings using `CamelCase`. +listings using `CamelCase`. Note though that **all config file formats require `snake_case` fields**. We always document service definition examples using From e5c5c52d16342bf48e3a4ca797301e9e65a2018a Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 14 Jun 2019 11:17:06 -0400 Subject: [PATCH 2/7] Enable WAN address translation for tagged service addresses. --- agent/dns.go | 10 +++++----- agent/translate_addr.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/agent/dns.go b/agent/dns.go index a4bffa18ecec..5fa8df162f7b 100644 --- a/agent/dns.go +++ b/agent/dns.go @@ -1354,8 +1354,8 @@ func (d *DNSServer) serviceNodeRecords(cfg *dnsConfig, dc string, nodes structs. // Start with the translated address but use the service address, // if specified. addr := d.agent.TranslateAddress(dc, node.Node.Address, node.Node.TaggedAddresses) - if node.Service.Address != "" { - addr = node.Service.Address + if svcAddr := d.agent.TranslateServiceAddress(dc, node.Service.Address, node.Service.TaggedAddresses); svcAddr != "" { + addr = svcAddr } // If the service address is a CNAME for the service we are looking @@ -1488,7 +1488,7 @@ func (d *DNSServer) serviceSRVRecords(cfg *dnsConfig, dc string, nodes structs.C }, Priority: 1, Weight: uint16(weight), - Port: uint16(node.Service.Port), + Port: uint16(d.agent.TranslateServicePort(dc, node.Service.Port, node.Service.TaggedAddresses)), Target: fmt.Sprintf("%s.node.%s.%s", node.Node.Node, dc, d.domain), } resp.Answer = append(resp.Answer, srvRec) @@ -1496,8 +1496,8 @@ func (d *DNSServer) serviceSRVRecords(cfg *dnsConfig, dc string, nodes structs.C // Start with the translated address but use the service address, // if specified. addr := d.agent.TranslateAddress(dc, node.Node.Address, node.Node.TaggedAddresses) - if node.Service.Address != "" { - addr = node.Service.Address + if svcAddr := d.agent.TranslateServiceAddress(dc, node.Service.Address, node.Service.TaggedAddresses); svcAddr != "" { + addr = svcAddr } // Add the extra record diff --git a/agent/translate_addr.go b/agent/translate_addr.go index 138f7aadcbc5..082229e33071 100644 --- a/agent/translate_addr.go +++ b/agent/translate_addr.go @@ -6,6 +6,30 @@ import ( "github.com/hashicorp/consul/agent/structs" ) +// TranslateServicePort is used to provide the final, translated port for a service, +// depending on how the agent and the other node are configured. The dc +// parameter is the dc the datacenter this node is from. +func (a *Agent) TranslateServicePort(dc string, port int, taggedAddresses map[string]structs.ServiceAddress) int { + if a.config.TranslateWANAddrs && (a.config.Datacenter != dc) { + if wanAddr, ok := taggedAddresses["wan"]; ok && wanAddr.Port != 0 { + return wanAddr.Port + } + } + return port +} + +// TranslateServiceAddress is used to provide the final, translated address for a node, +// depending on how the agent and the other node are configured. The dc +// parameter is the dc the datacenter this node is from. +func (a *Agent) TranslateServiceAddress(dc string, addr string, taggedAddresses map[string]structs.ServiceAddress) string { + if a.config.TranslateWANAddrs && (a.config.Datacenter != dc) { + if wanAddr, ok := taggedAddresses["wan"]; ok && wanAddr.Address != "" { + return wanAddr.Address + } + } + return addr +} + // TranslateAddress is used to provide the final, translated address for a node, // depending on how the agent and the other node are configured. The dc // parameter is the dc the datacenter this node is from. @@ -45,6 +69,8 @@ func (a *Agent) TranslateAddresses(dc string, subj interface{}) { case structs.CheckServiceNodes: for _, entry := range v { entry.Node.Address = a.TranslateAddress(dc, entry.Node.Address, entry.Node.TaggedAddresses) + entry.Service.Address = a.TranslateServiceAddress(dc, entry.Service.Address, entry.Service.TaggedAddresses) + entry.Service.Port = a.TranslateServicePort(dc, entry.Service.Port, entry.Service.TaggedAddresses) } case *structs.Node: v.Address = a.TranslateAddress(dc, v.Address, v.TaggedAddresses) @@ -55,6 +81,8 @@ func (a *Agent) TranslateAddresses(dc string, subj interface{}) { case structs.ServiceNodes: for _, entry := range v { entry.Address = a.TranslateAddress(dc, entry.Address, entry.TaggedAddresses) + entry.ServiceAddress = a.TranslateServiceAddress(dc, entry.ServiceAddress, entry.ServiceTaggedAddresses) + entry.ServicePort = a.TranslateServicePort(dc, entry.ServicePort, entry.ServiceTaggedAddresses) } default: panic(fmt.Errorf("Unhandled type passed to address translator: %#v", subj)) From b59c26e4af42b501d1359a0192d0a1fe381d03e8 Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 14 Jun 2019 11:58:54 -0400 Subject: [PATCH 3/7] Update the API docs --- website/source/api/agent/service.html.md | 67 ++++++++++++++++++++++++ website/source/api/catalog.html.md | 41 ++++++++++++++- website/source/api/health.html.md | 13 +++++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/website/source/api/agent/service.html.md b/website/source/api/agent/service.html.md index d7d93d8960cc..32eb7d8557f3 100644 --- a/website/source/api/agent/service.html.md +++ b/website/source/api/agent/service.html.md @@ -58,6 +58,16 @@ $ curl \ "ID": "redis", "Service": "redis", "Tags": [], + "TaggedAddresses": { + "lan": { + "address": "127.0.0.1", + "port": 8000 + }, + "wan": { + "address": "198.18.0.53", + "port": 80 + } + }, "Meta": { "redis_version": "4.0" }, @@ -99,6 +109,9 @@ following selectors and filter operations being supported: | `Proxy.Upstreams.LocalBindAddress` | Equal, Not Equal | | `Proxy.Upstreams.LocalBindPort` | Equal, Not Equal | | `Service` | Equal, Not Equal | +| `TaggedAddresses` | In, Not In, Is Empty, Is Not Empty | +| `TaggedAddresses..Address` | Equal, Not Equal | +| `TaggedAddresses..Port` | Equal, Not Equal | | `Tags` | In, Not In, Is Empty, Is Not Empty | | `Weights.Passing` | Equal, Not Equal | | `Weights.Warning` | Equal, Not Equal | @@ -157,6 +170,16 @@ $ curl \ "Meta": null, "Port": 18080, "Address": "", + "TaggedAddresses": { + "lan": { + "address": "127.0.0.1", + "port": 8000 + }, + "wan": { + "address": "198.18.0.53", + "port": 80 + } + }, "Weights": { "Passing": 1, "Warning": 1 @@ -270,6 +293,16 @@ curl localhost:8500/v1/agent/health/service/name/web "rails" ], "Address": "", + "TaggedAddresses": { + "lan": { + "address": "127.0.0.1", + "port": 8000 + }, + "wan": { + "address": "198.18.0.53", + "port": 80 + } + }, "Meta": null, "Port": 80, "EnableTagOverride": false, @@ -290,6 +323,16 @@ curl localhost:8500/v1/agent/health/service/name/web "rails" ], "Address": "", + "TaggedAddresses": { + "lan": { + "address": "127.0.0.1", + "port": 8000 + }, + "wan": { + "address": "198.18.0.53", + "port": 80 + } + }, "Meta": null, "Port": 80, "EnableTagOverride": false, @@ -332,6 +375,16 @@ curl localhost:8500/v1/agent/health/service/id/web2 "rails" ], "Address": "", + "TaggedAddresses": { + "lan": { + "address": "127.0.0.1", + "port": 8000 + }, + "wan": { + "address": "198.18.0.53", + "port": 80 + } + }, "Meta": null, "Port": 80, "EnableTagOverride": false, @@ -370,6 +423,16 @@ curl localhost:8500/v1/agent/health/service/id/web1 "rails" ], "Address": "", + "TaggedAddresses": { + "lan": { + "address": "127.0.0.1", + "port": 8000 + }, + "wan": { + "address": "198.18.0.53", + "port": 80 + } + }, "Meta": null, "Port": 80, "EnableTagOverride": false, @@ -443,6 +506,10 @@ service definition keys for compatibility with the config file format. provided, the agent's address is used as the address for the service during DNS queries. +- `TaggedAddresses` `(map: nil)` - Specifies a map of explicit LAN + and WAN addresses for the service instance. Both the address and port can be + specified within the map values. + - `Meta` `(map: nil)` - Specifies arbitrary KV metadata linked to the service instance. diff --git a/website/source/api/catalog.html.md b/website/source/api/catalog.html.md index e619144c53d7..1bc7ef0e017c 100644 --- a/website/source/api/catalog.html.md +++ b/website/source/api/catalog.html.md @@ -56,7 +56,7 @@ The table below shows this endpoint's support for Only one service with a given `ID` may be present per node. The service `Tags`, `Address`, `Meta`, and `Port` fields are all optional. For more information about these fields and the implications of setting them, - see the [Service - Agent API](https://www.consul.io/api/agent/service.html) page + see the [Service - Agent API](/api/agent/service.html) page as registering services differs between using this or the Services Agent endpoint. - `Check` `(Check: nil)` - Specifies to register a check. The register API @@ -112,6 +112,16 @@ and vice versa. A catalog entry can have either, neither, or both. "v1" ], "Address": "127.0.0.1", + "TaggedAddresses": { + "lan": { + "address": "127.0.0.1", + "port": 8000 + }, + "wan": { + "address": "198.18.0.1", + "port": 80 + } + }, "Meta": { "redis_version": "4.0" }, @@ -475,6 +485,16 @@ $ curl \ "ServiceMeta": { "foobar_meta_value": "baz" }, + "ServiceTaggedAddresses": { + "lan": { + "address": "172.17.0.3", + "port": 5000 + }, + "wan": { + "address": "198.18.0.1", + "port": 512 + } + }, "ServiceTags": [ "tacos" ], @@ -529,6 +549,9 @@ $ curl \ - `ServiceTags` is a list of tags for the service +- `ServiceTaggedAddresses` is the map of explicit LAN and WAN addresses for the + service instance. This includes both the address as well as the port. + - `ServiceKind` is the kind of service, usually "". See the Agent registration API for more information. @@ -576,6 +599,9 @@ following selectors and filter operations being supported: | `ServiceProxy.Upstreams.DestinationType` | Equal, Not Equal | | `ServiceProxy.Upstreams.LocalBindAddress` | Equal, Not Equal | | `ServiceProxy.Upstreams.LocalBindPort` | Equal, Not Equal | +| `ServiceTaggedAddresses` | In, Not In, Is Empty, Is Not Empty | +| `ServiceTaggedAddresses..Address` | Equal, Not Equal | +| `ServiceTaggedAddresses..Port` | Equal, Not Equal | | `ServiceTags` | In, Not In, Is Empty, Is Not Empty | | `ServiceWeights.Passing` | Equal, Not Equal | | `ServiceWeights.Warning` | Equal, Not Equal | @@ -662,6 +688,16 @@ $ curl \ "redis": { "ID": "redis", "Service": "redis", + "TaggedAddresses": { + "lan": { + "address": "10.1.10.12", + "port": 8000, + }, + "wan": { + "address": "198.18.1.2", + "port": 80 + } + }, "Tags": [ "v1" ], @@ -701,6 +737,9 @@ top level Node object. The following selectors and filter operations are support | `Proxy.Upstreams.LocalBindAddress` | Equal, Not Equal | | `Proxy.Upstreams.LocalBindPort` | Equal, Not Equal | | `Service` | Equal, Not Equal | +| `TaggedAddresses` | In, Not In, Is Empty, Is Not Empty | +| `TaggedAddresses..Address` | Equal, Not Equal | +| `TaggedAddresses..Port` | Equal, Not Equal | | `Tags` | In, Not In, Is Empty, Is Not Empty | | `Weights.Passing` | Equal, Not Equal | | `Weights.Warning` | Equal, Not Equal | diff --git a/website/source/api/health.html.md b/website/source/api/health.html.md index 93990449e7d5..ac057469ef66 100644 --- a/website/source/api/health.html.md +++ b/website/source/api/health.html.md @@ -265,6 +265,16 @@ $ curl \ "Service": "redis", "Tags": ["primary"], "Address": "10.1.10.12", + "TaggedAddresses": { + "lan": { + "address": "10.1.10.12", + "port": 8000 + }, + "wan": { + "address": "198.18.1.2", + "port": 80 + } + }, "Meta": { "redis_version": "4.0" }, @@ -347,6 +357,9 @@ following selectors and filter operations being supported: | `Service.Proxy.Upstreams.LocalBindAddress` | Equal, Not Equal | | `Service.Proxy.Upstreams.LocalBindPort` | Equal, Not Equal | | `Service.Service` | Equal, Not Equal | +| `Service.TaggedAddresses` | In, Not In, Is Empty, Is Not Empty | +| `Service.TaggedAddresses..Address` | Equal, Not Equal | +| `Service.TaggedAddresses..Port` | Equal, Not Equal | | `Service.Tags` | In, Not In, Is Empty, Is Not Empty | | `Service.Weights.Passing` | Equal, Not Equal | | `Service.Weights.Warning` | Equal, Not Equal | From 35fc5e975f917538e142f544faa8554917d60780 Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 14 Jun 2019 13:35:42 -0400 Subject: [PATCH 4/7] Ensure that we can use snake cased svc tagged_addresses --- agent/agent_endpoint.go | 3 +++ agent/agent_endpoint_test.go | 20 ++++++++++++++++++++ agent/config/runtime_test.go | 23 +++++++++++++++++++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/agent/agent_endpoint.go b/agent/agent_endpoint.go index 7fdf3e23b59c..30cc271e92c5 100644 --- a/agent/agent_endpoint.go +++ b/agent/agent_endpoint.go @@ -897,6 +897,9 @@ func (s *HTTPServer) AgentRegisterService(resp http.ResponseWriter, req *http.Re // DON'T Recurse into these opaque config maps or we might mangle user's // keys. Note empty canonical is a special sentinel to prevent recursion. "Meta": "", + + "tagged_addresses": "TaggedAddresses", + // upstreams is an array but this prevents recursion into config field of // any item in the array. "Proxy.Config": "", diff --git a/agent/agent_endpoint_test.go b/agent/agent_endpoint_test.go index 1aa8db7084d7..8ab009f6a169 100644 --- a/agent/agent_endpoint_test.go +++ b/agent/agent_endpoint_test.go @@ -2503,6 +2503,16 @@ func TestAgent_RegisterService_TranslateKeys(t *testing.T) { "name":"test", "port":8000, "enable_tag_override": true, + "tagged_addresses": { + "lan": { + "address": "1.2.3.4", + "port": 5353 + }, + "wan": { + "address": "2.3.4.5", + "port": 53 + } + }, "meta": { "some": "meta", "enable_tag_override": "meta is 'opaque' so should not get translated" @@ -2595,6 +2605,16 @@ func TestAgent_RegisterService_TranslateKeys(t *testing.T) { svc := &structs.NodeService{ ID: "test", Service: "test", + TaggedAddresses: map[string]structs.ServiceAddress{ + "lan": { + Address: "1.2.3.4", + Port: 5353, + }, + "wan": { + Address: "2.3.4.5", + Port: 53, + }, + }, Meta: map[string]string{ "some": "meta", "enable_tag_override": "meta is 'opaque' so should not get translated", diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 9cff4dfd7f54..b008cf6bc9b9 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -2194,6 +2194,12 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { "service": { "name": "a", "port": 80, + "tagged_addresses": { + "wan": { + "address": "198.18.3.4", + "port": 443 + } + }, "enable_tag_override": true, "check": { "id": "x", @@ -2210,6 +2216,12 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { name = "a" port = 80 enable_tag_override = true + tagged_addresses = { + wan = { + address = "198.18.3.4" + port = 443 + } + } check = { id = "x" name = "y" @@ -2222,8 +2234,14 @@ func TestConfigFlagsAndEdgecases(t *testing.T) { patch: func(rt *RuntimeConfig) { rt.Services = []*structs.ServiceDefinition{ &structs.ServiceDefinition{ - Name: "a", - Port: 80, + Name: "a", + Port: 80, + TaggedAddresses: map[string]structs.ServiceAddress{ + "wan": structs.ServiceAddress{ + Address: "198.18.3.4", + Port: 443, + }, + }, EnableTagOverride: true, Checks: []*structs.CheckType{ &structs.CheckType{ @@ -5317,6 +5335,7 @@ func TestSanitize(t *testing.T) { "Port": 0, "Proxy": null, "ProxyDestination": "", + "TaggedAddresses": {}, "Tags": [], "Token": "hidden", "Weights": { From 53335810cedba2c535c5460e4ba74be77bc1af68 Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 14 Jun 2019 13:54:08 -0400 Subject: [PATCH 5/7] Add tests to ensure that the TaggedAddresses of a service are omitted when empty/nil --- agent/structs/structs_test.go | 73 +++++++++++++++++++++++++++++++++++ api/agent.go | 2 +- api/agent_test.go | 37 ++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/agent/structs/structs_test.go b/agent/structs/structs_test.go index fdee9443d328..8ca852f815c1 100644 --- a/agent/structs/structs_test.go +++ b/agent/structs/structs_test.go @@ -1,6 +1,7 @@ package structs import ( + "encoding/json" "fmt" "reflect" "strings" @@ -1082,3 +1083,75 @@ func TestSpecificServiceRequest_CacheInfo(t *testing.T) { }) } } + +func TestNodeService_JSON_OmitTaggedAdddresses(t *testing.T) { + t.Parallel() + cases := []struct { + name string + ns NodeService + }{ + { + "nil", + NodeService{ + TaggedAddresses: nil, + }, + }, + { + "empty", + NodeService{ + TaggedAddresses: make(map[string]ServiceAddress), + }, + }, + } + + for _, tc := range cases { + name := tc.name + ns := tc.ns + t.Run(name, func(t *testing.T) { + t.Parallel() + data, err := json.Marshal(ns) + require.NoError(t, err) + var raw map[string]interface{} + err = json.Unmarshal(data, &raw) + require.NoError(t, err) + require.NotContains(t, raw, "TaggedAddresses") + require.NotContains(t, raw, "tagged_addresses") + }) + } +} + +func TestServiceNode_JSON_OmitServiceTaggedAdddresses(t *testing.T) { + t.Parallel() + cases := []struct { + name string + sn ServiceNode + }{ + { + "nil", + ServiceNode{ + ServiceTaggedAddresses: nil, + }, + }, + { + "empty", + ServiceNode{ + ServiceTaggedAddresses: make(map[string]ServiceAddress), + }, + }, + } + + for _, tc := range cases { + name := tc.name + sn := tc.sn + t.Run(name, func(t *testing.T) { + t.Parallel() + data, err := json.Marshal(sn) + require.NoError(t, err) + var raw map[string]interface{} + err = json.Unmarshal(data, &raw) + require.NoError(t, err) + require.NotContains(t, raw, "ServiceTaggedAddresses") + require.NotContains(t, raw, "service_tagged_addresses") + }) + } +} diff --git a/api/agent.go b/api/agent.go index de5f4da4fd6a..a1e2a2f57037 100644 --- a/api/agent.go +++ b/api/agent.go @@ -82,7 +82,7 @@ type AgentService struct { Meta map[string]string Port int Address string - TaggedAddresses map[string]ServiceAddress + TaggedAddresses map[string]ServiceAddress `json:",omitempty"` Weights AgentWeights EnableTagOverride bool CreateIndex uint64 `json:",omitempty" bexpr:"-"` diff --git a/api/agent_test.go b/api/agent_test.go index 071885439f65..d449755fd4e3 100644 --- a/api/agent_test.go +++ b/api/agent_test.go @@ -1,6 +1,7 @@ package api import ( + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -1685,3 +1686,39 @@ func TestAPI_AgentHealthService(t *testing.T) { require.Nil(t, err) requireServiceHealthName(t, testServiceName, HealthPassing, true) } + +func TestAgentService_JSON_OmitTaggedAdddresses(t *testing.T) { + t.Parallel() + cases := []struct { + name string + as AgentService + }{ + { + "nil", + AgentService{ + TaggedAddresses: nil, + }, + }, + { + "empty", + AgentService{ + TaggedAddresses: make(map[string]ServiceAddress), + }, + }, + } + + for _, tc := range cases { + name := tc.name + as := tc.as + t.Run(name, func(t *testing.T) { + t.Parallel() + data, err := json.Marshal(as) + require.NoError(t, err) + var raw map[string]interface{} + err = json.Unmarshal(data, &raw) + require.NoError(t, err) + require.NotContains(t, raw, "TaggedAddresses") + require.NotContains(t, raw, "tagged_addresses") + }) + } +} From 7d688786c569c88251d286c4757623c986bf4bd2 Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 14 Jun 2019 14:50:31 -0400 Subject: [PATCH 6/7] Add tests for HTTP endpoints that do WAN translation. --- agent/catalog_endpoint.go | 4 +- agent/catalog_endpoint_test.go | 126 +++++++++++++------------- agent/health_endpoint_test.go | 73 ++++++++------- agent/prepared_query_endpoint_test.go | 40 ++++---- agent/translate_addr.go | 8 ++ 5 files changed, 130 insertions(+), 121 deletions(-) diff --git a/agent/catalog_endpoint.go b/agent/catalog_endpoint.go index 7381cdf6420c..6fd649d1cf93 100644 --- a/agent/catalog_endpoint.go +++ b/agent/catalog_endpoint.go @@ -284,8 +284,8 @@ RETRY_ONCE: goto RETRY_ONCE } out.ConsistencyLevel = args.QueryOptions.ConsistencyLevel() - if out.NodeServices != nil && out.NodeServices.Node != nil { - s.agent.TranslateAddresses(args.Datacenter, out.NodeServices.Node) + if out.NodeServices != nil { + s.agent.TranslateAddresses(args.Datacenter, out.NodeServices) } // TODO: The NodeServices object in IndexedNodeServices is a pointer to diff --git a/agent/catalog_endpoint_test.go b/agent/catalog_endpoint_test.go index c045969914ae..e071e6f0f586 100644 --- a/agent/catalog_endpoint_test.go +++ b/agent/catalog_endpoint_test.go @@ -769,13 +769,10 @@ func TestCatalogServiceNodes_WanTranslation(t *testing.T) { // Wait for the WAN join. addr := fmt.Sprintf("127.0.0.1:%d", a1.Config.SerfPortWAN) - if _, err := a2.srv.agent.JoinWAN([]string{addr}); err != nil { - t.Fatalf("err: %v", err) - } + _, err := a2.srv.agent.JoinWAN([]string{addr}) + require.NoError(t, err) retry.Run(t, func(r *retry.R) { - if got, want := len(a1.WANMembers()), 2; got < want { - r.Fatalf("got %d WAN members want at least %d", got, want) - } + require.Len(r, a1.WANMembers(), 2) }) // Register a node with DC2. @@ -789,51 +786,51 @@ func TestCatalogServiceNodes_WanTranslation(t *testing.T) { }, Service: &structs.NodeService{ Service: "http_wan_translation_test", + Address: "127.0.0.1", + Port: 8080, + TaggedAddresses: map[string]structs.ServiceAddress{ + "wan": structs.ServiceAddress{ + Address: "1.2.3.4", + Port: 80, + }, + }, }, } var out struct{} - if err := a2.RPC("Catalog.Register", args, &out); err != nil { - t.Fatalf("err: %v", err) - } + require.NoError(t, a2.RPC("Catalog.Register", args, &out)) } // Query for the node in DC2 from DC1. req, _ := http.NewRequest("GET", "/v1/catalog/service/http_wan_translation_test?dc=dc2", nil) resp1 := httptest.NewRecorder() obj1, err1 := a1.srv.CatalogServiceNodes(resp1, req) - if err1 != nil { - t.Fatalf("err: %v", err1) - } - assertIndex(t, resp1) + require.NoError(t, err1) + require.NoError(t, checkIndex(resp1)) // Expect that DC1 gives us a WAN address (since the node is in DC2). - nodes1 := obj1.(structs.ServiceNodes) - if len(nodes1) != 1 { - t.Fatalf("bad: %v", obj1) - } + nodes1, ok := obj1.(structs.ServiceNodes) + require.True(t, ok, "obj1 is not a structs.ServiceNodes") + require.Len(t, nodes1, 1) node1 := nodes1[0] - if node1.Address != "127.0.0.2" { - t.Fatalf("bad: %v", node1) - } + require.Equal(t, node1.Address, "127.0.0.2") + require.Equal(t, node1.ServiceAddress, "1.2.3.4") + require.Equal(t, node1.ServicePort, 80) // Query DC2 from DC2. resp2 := httptest.NewRecorder() obj2, err2 := a2.srv.CatalogServiceNodes(resp2, req) - if err2 != nil { - t.Fatalf("err: %v", err2) - } - assertIndex(t, resp2) + require.NoError(t, err2) + require.NoError(t, checkIndex(resp2)) // Expect that DC2 gives us a local address (since the node is in DC2). - nodes2 := obj2.(structs.ServiceNodes) - if len(nodes2) != 1 { - t.Fatalf("bad: %v", obj2) - } + nodes2, ok := obj2.(structs.ServiceNodes) + require.True(t, ok, "obj2 is not a structs.ServiceNodes") + require.Len(t, nodes2, 1) node2 := nodes2[0] - if node2.Address != "127.0.0.1" { - t.Fatalf("bad: %v", node2) - } + require.Equal(t, node2.Address, "127.0.0.1") + require.Equal(t, node2.ServiceAddress, "127.0.0.1") + require.Equal(t, node2.ServicePort, 8080) } func TestCatalogServiceNodes_DistanceSort(t *testing.T) { @@ -1150,13 +1147,10 @@ func TestCatalogNodeServices_WanTranslation(t *testing.T) { // Wait for the WAN join. addr := fmt.Sprintf("127.0.0.1:%d", a1.Config.SerfPortWAN) - if _, err := a2.srv.agent.JoinWAN([]string{addr}); err != nil { - t.Fatalf("err: %v", err) - } + _, err := a2.srv.agent.JoinWAN([]string{addr}) + require.NoError(t, err) retry.Run(t, func(r *retry.R) { - if got, want := len(a1.WANMembers()), 2; got < want { - r.Fatalf("got %d WAN members want at least %d", got, want) - } + require.Len(r, a1.WANMembers(), 2) }) // Register a node with DC2. @@ -1170,49 +1164,53 @@ func TestCatalogNodeServices_WanTranslation(t *testing.T) { }, Service: &structs.NodeService{ Service: "http_wan_translation_test", + Address: "127.0.0.1", + Port: 8080, + TaggedAddresses: map[string]structs.ServiceAddress{ + "wan": structs.ServiceAddress{ + Address: "1.2.3.4", + Port: 80, + }, + }, }, } var out struct{} - if err := a2.RPC("Catalog.Register", args, &out); err != nil { - t.Fatalf("err: %v", err) - } + require.NoError(t, a2.RPC("Catalog.Register", args, &out)) } // Query for the node in DC2 from DC1. req, _ := http.NewRequest("GET", "/v1/catalog/node/foo?dc=dc2", nil) resp1 := httptest.NewRecorder() obj1, err1 := a1.srv.CatalogNodeServices(resp1, req) - if err1 != nil { - t.Fatalf("err: %v", err1) - } - assertIndex(t, resp1) + require.NoError(t, err1) + require.NoError(t, checkIndex(resp1)) // Expect that DC1 gives us a WAN address (since the node is in DC2). - services1 := obj1.(*structs.NodeServices) - if len(services1.Services) != 1 { - t.Fatalf("bad: %v", obj1) - } - service1 := services1.Node - if service1.Address != "127.0.0.2" { - t.Fatalf("bad: %v", service1) - } + service1, ok := obj1.(*structs.NodeServices) + require.True(t, ok, "obj1 is not a *structs.NodeServices") + require.NotNil(t, service1.Node) + require.Equal(t, service1.Node.Address, "127.0.0.2") + require.Len(t, service1.Services, 1) + ns1, ok := service1.Services["http_wan_translation_test"] + require.True(t, ok, "Missing service http_wan_translation_test") + require.Equal(t, "1.2.3.4", ns1.Address) + require.Equal(t, 80, ns1.Port) // Query DC2 from DC2. resp2 := httptest.NewRecorder() obj2, err2 := a2.srv.CatalogNodeServices(resp2, req) - if err2 != nil { - t.Fatalf("err: %v", err2) - } - assertIndex(t, resp2) + require.NoError(t, err2) + require.NoError(t, checkIndex(resp2)) // Expect that DC2 gives us a private address (since the node is in DC2). - services2 := obj2.(*structs.NodeServices) - if len(services2.Services) != 1 { - t.Fatalf("bad: %v", obj2) - } - service2 := services2.Node - if service2.Address != "127.0.0.1" { - t.Fatalf("bad: %v", service2) - } + service2 := obj2.(*structs.NodeServices) + require.True(t, ok, "obj2 is not a *structs.NodeServices") + require.NotNil(t, service2.Node) + require.Equal(t, service2.Node.Address, "127.0.0.1") + require.Len(t, service2.Services, 1) + ns2, ok := service2.Services["http_wan_translation_test"] + require.True(t, ok, "Missing service http_wan_translation_test") + require.Equal(t, ns2.Address, "127.0.0.1") + require.Equal(t, ns2.Port, 8080) } diff --git a/agent/health_endpoint_test.go b/agent/health_endpoint_test.go index 9cd61537a39f..eb4e82c6ee23 100644 --- a/agent/health_endpoint_test.go +++ b/agent/health_endpoint_test.go @@ -977,7 +977,6 @@ func TestHealthServiceNodes_WanTranslation(t *testing.T) { acl_datacenter = "" `) defer a1.Shutdown() - testrpc.WaitForLeader(t, a1.RPC, "dc1") a2 := NewTestAgent(t, t.Name(), ` datacenter = "dc2" @@ -985,17 +984,13 @@ func TestHealthServiceNodes_WanTranslation(t *testing.T) { acl_datacenter = "" `) defer a2.Shutdown() - testrpc.WaitForLeader(t, a2.RPC, "dc2") // Wait for the WAN join. addr := fmt.Sprintf("127.0.0.1:%d", a1.Config.SerfPortWAN) - if _, err := a2.JoinWAN([]string{addr}); err != nil { - t.Fatalf("err: %v", err) - } + _, err := a2.srv.agent.JoinWAN([]string{addr}) + require.NoError(t, err) retry.Run(t, func(r *retry.R) { - if got, want := len(a1.WANMembers()), 2; got < want { - r.Fatalf("got %d WAN members want at least %d", got, want) - } + require.Len(r, a1.WANMembers(), 2) }) // Register a node with DC2. @@ -1009,51 +1004,55 @@ func TestHealthServiceNodes_WanTranslation(t *testing.T) { }, Service: &structs.NodeService{ Service: "http_wan_translation_test", + Address: "127.0.0.1", + Port: 8080, + TaggedAddresses: map[string]structs.ServiceAddress{ + "wan": structs.ServiceAddress{ + Address: "1.2.3.4", + Port: 80, + }, + }, }, } var out struct{} - if err := a2.RPC("Catalog.Register", args, &out); err != nil { - t.Fatalf("err: %v", err) - } + require.NoError(t, a2.RPC("Catalog.Register", args, &out)) } // Query for a service in DC2 from DC1. req, _ := http.NewRequest("GET", "/v1/health/service/http_wan_translation_test?dc=dc2", nil) resp1 := httptest.NewRecorder() obj1, err1 := a1.srv.HealthServiceNodes(resp1, req) - if err1 != nil { - t.Fatalf("err: %v", err1) - } - assertIndex(t, resp1) + require.NoError(t, err1) + require.NoError(t, checkIndex(resp1)) // Expect that DC1 gives us a WAN address (since the node is in DC2). - nodes1 := obj1.(structs.CheckServiceNodes) - if len(nodes1) != 1 { - t.Fatalf("bad: %v", obj1) - } - node1 := nodes1[0].Node - if node1.Address != "127.0.0.2" { - t.Fatalf("bad: %v", node1) - } + nodes1, ok := obj1.(structs.CheckServiceNodes) + require.True(t, ok, "obj1 is not a structs.CheckServiceNodes") + require.Len(t, nodes1, 1) + node1 := nodes1[0] + require.NotNil(t, node1.Node) + require.Equal(t, node1.Node.Address, "127.0.0.2") + require.NotNil(t, node1.Service) + require.Equal(t, node1.Service.Address, "1.2.3.4") + require.Equal(t, node1.Service.Port, 80) // Query DC2 from DC2. resp2 := httptest.NewRecorder() obj2, err2 := a2.srv.HealthServiceNodes(resp2, req) - if err2 != nil { - t.Fatalf("err: %v", err2) - } - assertIndex(t, resp2) - - // Expect that DC2 gives us a private address (since the node is in DC2). - nodes2 := obj2.(structs.CheckServiceNodes) - if len(nodes2) != 1 { - t.Fatalf("bad: %v", obj2) - } - node2 := nodes2[0].Node - if node2.Address != "127.0.0.1" { - t.Fatalf("bad: %v", node2) - } + require.NoError(t, err2) + require.NoError(t, checkIndex(resp2)) + + // Expect that DC2 gives us a local address (since the node is in DC2). + nodes2, ok := obj2.(structs.CheckServiceNodes) + require.True(t, ok, "obj2 is not a structs.ServiceNodes") + require.Len(t, nodes2, 1) + node2 := nodes2[0] + require.NotNil(t, node2.Node) + require.Equal(t, node2.Node.Address, "127.0.0.1") + require.NotNil(t, node2.Service) + require.Equal(t, node2.Service.Address, "127.0.0.1") + require.Equal(t, node2.Service.Port, 8080) } func TestHealthConnectServiceNodes(t *testing.T) { diff --git a/agent/prepared_query_endpoint_test.go b/agent/prepared_query_endpoint_test.go index b1b8b0eb9778..9df24be25288 100644 --- a/agent/prepared_query_endpoint_test.go +++ b/agent/prepared_query_endpoint_test.go @@ -514,37 +514,41 @@ func TestPreparedQuery_Execute(t *testing.T) { "wan": "127.0.0.2", }, } + nodesResponse[0].Service = &structs.NodeService{ + Service: "foo", + Address: "10.0.1.1", + Port: 8080, + TaggedAddresses: map[string]structs.ServiceAddress{ + "wan": structs.ServiceAddress{ + Address: "198.18.0.1", + Port: 80, + }, + }, + } reply.Nodes = nodesResponse reply.Datacenter = "dc2" return nil }, } - if err := a.registerEndpoint("PreparedQuery", &m); err != nil { - t.Fatalf("err: %v", err) - } + require.NoError(t, a.registerEndpoint("PreparedQuery", &m)) body := bytes.NewBuffer(nil) req, _ := http.NewRequest("GET", "/v1/query/my-id/execute?dc=dc2", body) resp := httptest.NewRecorder() obj, err := a.srv.PreparedQuerySpecific(resp, req) - if err != nil { - t.Fatalf("err: %v", err) - } - if resp.Code != 200 { - t.Fatalf("bad code: %d", resp.Code) - } + require.NoError(t, err) + require.Equal(t, 200, resp.Code) r, ok := obj.(structs.PreparedQueryExecuteResponse) - if !ok { - t.Fatalf("unexpected: %T", obj) - } - if r.Nodes == nil || len(r.Nodes) != 1 { - t.Fatalf("bad: %v", r) - } + require.True(t, ok, "unexpected: %T", obj) + require.NotNil(t, r.Nodes) + require.Len(t, r.Nodes, 1) node := r.Nodes[0] - if node.Node.Address != "127.0.0.2" { - t.Fatalf("bad: %v", node.Node) - } + require.NotNil(t, node.Node) + require.Equal(t, "127.0.0.2", node.Node.Address) + require.NotNil(t, node.Service) + require.Equal(t, "198.18.0.1", node.Service.Address) + require.Equal(t, 80, node.Service.Port) }) // Ensure WAN translation doesn't occur for the local DC. diff --git a/agent/translate_addr.go b/agent/translate_addr.go index 082229e33071..a0ba41972638 100644 --- a/agent/translate_addr.go +++ b/agent/translate_addr.go @@ -84,6 +84,14 @@ func (a *Agent) TranslateAddresses(dc string, subj interface{}) { entry.ServiceAddress = a.TranslateServiceAddress(dc, entry.ServiceAddress, entry.ServiceTaggedAddresses) entry.ServicePort = a.TranslateServicePort(dc, entry.ServicePort, entry.ServiceTaggedAddresses) } + case *structs.NodeServices: + if v.Node != nil { + v.Node.Address = a.TranslateAddress(dc, v.Node.Address, v.Node.TaggedAddresses) + } + for _, entry := range v.Services { + entry.Address = a.TranslateServiceAddress(dc, entry.Address, entry.TaggedAddresses) + entry.Port = a.TranslateServicePort(dc, entry.Port, entry.TaggedAddresses) + } default: panic(fmt.Errorf("Unhandled type passed to address translator: %#v", subj)) } From 0dc2c7e5b2490865f9a812b4a3397be4f66ad830 Mon Sep 17 00:00:00 2001 From: Matt Keeler Date: Fri, 14 Jun 2019 15:35:38 -0400 Subject: [PATCH 7/7] Implement tests for DNS wan service address translation --- agent/dns_test.go | 302 ++++++++++++++++++++++++---------------------- 1 file changed, 161 insertions(+), 141 deletions(-) diff --git a/agent/dns_test.go b/agent/dns_test.go index fbb02d672a1f..543fc57e5f3c 100644 --- a/agent/dns_test.go +++ b/agent/dns_test.go @@ -2345,7 +2345,7 @@ func TestDNS_ServiceLookup_ServiceAddressIPV6(t *testing.T) { } } -func TestDNS_ServiceLookup_WanAddress(t *testing.T) { +func TestDNS_ServiceLookup_WanTranslation(t *testing.T) { t.Parallel() a1 := NewTestAgent(t, t.Name(), ` datacenter = "dc1" @@ -2363,38 +2363,11 @@ func TestDNS_ServiceLookup_WanAddress(t *testing.T) { // Join WAN cluster addr := fmt.Sprintf("127.0.0.1:%d", a1.Config.SerfPortWAN) - if _, err := a2.JoinWAN([]string{addr}); err != nil { - t.Fatalf("err: %v", err) - } + _, err := a2.JoinWAN([]string{addr}) + require.NoError(t, err) retry.Run(t, func(r *retry.R) { - if got, want := len(a1.WANMembers()), 2; got < want { - r.Fatalf("got %d WAN members want at least %d", got, want) - } - if got, want := len(a2.WANMembers()), 2; got < want { - r.Fatalf("got %d WAN members want at least %d", got, want) - } - }) - - // Register a remote node with a service. This is in a retry since we - // need the datacenter to have a route which takes a little more time - // beyond the join, and we don't have direct access to the router here. - retry.Run(t, func(r *retry.R) { - args := &structs.RegisterRequest{ - Datacenter: "dc2", - Node: "foo", - Address: "127.0.0.1", - TaggedAddresses: map[string]string{ - "wan": "127.0.0.2", - }, - Service: &structs.NodeService{ - Service: "db", - }, - } - - var out struct{} - if err := a2.RPC("Catalog.Register", args, &out); err != nil { - r.Fatalf("err: %v", err) - } + require.Len(t, a1.WANMembers(), 2) + require.Len(t, a2.WANMembers(), 2) }) // Register an equivalent prepared query. @@ -2410,126 +2383,173 @@ func TestDNS_ServiceLookup_WanAddress(t *testing.T) { }, }, } - if err := a2.RPC("PreparedQuery.Apply", args, &id); err != nil { - t.Fatalf("err: %v", err) - } + require.NoError(t, a2.RPC("PreparedQuery.Apply", args, &id)) } - // Look up the SRV record via service and prepared query. - questions := []string{ - "db.service.dc2.consul.", - id + ".query.dc2.consul.", - } - for _, question := range questions { - m := new(dns.Msg) - m.SetQuestion(question, dns.TypeSRV) + type testCase struct { + nodeTaggedAddresses map[string]string + serviceAddress string + serviceTaggedAddresses map[string]structs.ServiceAddress - c := new(dns.Client) - - addr := a1.config.DNSAddrs[0] - in, _, err := c.Exchange(m, addr.String()) - if err != nil { - t.Fatalf("err: %v", err) - } - - if len(in.Answer) != 1 { - t.Fatalf("Bad: %#v", in) - } - - aRec, ok := in.Extra[0].(*dns.A) - if !ok { - t.Fatalf("Bad: %#v", in.Extra[0]) - } - if aRec.Hdr.Name != "7f000002.addr.dc2.consul." { - t.Fatalf("Bad: %#v", in.Extra[0]) - } - if aRec.A.String() != "127.0.0.2" { - t.Fatalf("Bad: %#v", in.Extra[0]) - } - } - - // Also check the A record directly - for _, question := range questions { - m := new(dns.Msg) - m.SetQuestion(question, dns.TypeA) - - c := new(dns.Client) - addr := a1.config.DNSAddrs[0] - in, _, err := c.Exchange(m, addr.String()) - if err != nil { - t.Fatalf("err: %v", err) - } + dnsAddr string - if len(in.Answer) != 1 { - t.Fatalf("Bad: %#v", in) - } - - aRec, ok := in.Answer[0].(*dns.A) - if !ok { - t.Fatalf("Bad: %#v", in.Answer[0]) - } - if aRec.Hdr.Name != question { - t.Fatalf("Bad: %#v", in.Answer[0]) - } - if aRec.A.String() != "127.0.0.2" { - t.Fatalf("Bad: %#v", in.Answer[0]) - } + expectedPort uint16 + expectedAddress string + expectedARRName string } - // Now query from the same DC and make sure we get the local address - for _, question := range questions { - m := new(dns.Msg) - m.SetQuestion(question, dns.TypeSRV) - - c := new(dns.Client) - addr := a2.Config.DNSAddrs[0] - in, _, err := c.Exchange(m, addr.String()) - if err != nil { - t.Fatalf("err: %v", err) - } - - if len(in.Answer) != 1 { - t.Fatalf("Bad: %#v", in) - } - - aRec, ok := in.Extra[0].(*dns.A) - if !ok { - t.Fatalf("Bad: %#v", in.Extra[0]) - } - if aRec.Hdr.Name != "foo.node.dc2.consul." { - t.Fatalf("Bad: %#v", in.Extra[0]) - } - if aRec.A.String() != "127.0.0.1" { - t.Fatalf("Bad: %#v", in.Extra[0]) - } + cases := map[string]testCase{ + "node-addr-from-dc1": testCase{ + dnsAddr: a1.config.DNSAddrs[0].String(), + expectedPort: 8080, + expectedAddress: "127.0.0.1", + expectedARRName: "foo.node.dc2.consul.", + }, + "node-wan-from-dc1": testCase{ + dnsAddr: a1.config.DNSAddrs[0].String(), + nodeTaggedAddresses: map[string]string{ + "wan": "127.0.0.2", + }, + expectedPort: 8080, + expectedAddress: "127.0.0.2", + expectedARRName: "7f000002.addr.dc2.consul.", + }, + "service-addr-from-dc1": testCase{ + dnsAddr: a1.config.DNSAddrs[0].String(), + nodeTaggedAddresses: map[string]string{ + "wan": "127.0.0.2", + }, + serviceAddress: "10.0.1.1", + expectedPort: 8080, + expectedAddress: "10.0.1.1", + expectedARRName: "0a000101.addr.dc2.consul.", + }, + "service-wan-from-dc1": testCase{ + dnsAddr: a1.config.DNSAddrs[0].String(), + nodeTaggedAddresses: map[string]string{ + "wan": "127.0.0.2", + }, + serviceAddress: "10.0.1.1", + serviceTaggedAddresses: map[string]structs.ServiceAddress{ + "wan": structs.ServiceAddress{ + Address: "198.18.0.1", + Port: 80, + }, + }, + expectedPort: 80, + expectedAddress: "198.18.0.1", + expectedARRName: "c6120001.addr.dc2.consul.", + }, + "node-addr-from-dc2": testCase{ + dnsAddr: a2.config.DNSAddrs[0].String(), + expectedPort: 8080, + expectedAddress: "127.0.0.1", + expectedARRName: "foo.node.dc2.consul.", + }, + "node-wan-from-dc2": testCase{ + dnsAddr: a2.config.DNSAddrs[0].String(), + nodeTaggedAddresses: map[string]string{ + "wan": "127.0.0.2", + }, + expectedPort: 8080, + expectedAddress: "127.0.0.1", + expectedARRName: "foo.node.dc2.consul.", + }, + "service-addr-from-dc2": testCase{ + dnsAddr: a2.config.DNSAddrs[0].String(), + nodeTaggedAddresses: map[string]string{ + "wan": "127.0.0.2", + }, + serviceAddress: "10.0.1.1", + expectedPort: 8080, + expectedAddress: "10.0.1.1", + expectedARRName: "0a000101.addr.dc2.consul.", + }, + "service-wan-from-dc2": testCase{ + dnsAddr: a2.config.DNSAddrs[0].String(), + nodeTaggedAddresses: map[string]string{ + "wan": "127.0.0.2", + }, + serviceAddress: "10.0.1.1", + serviceTaggedAddresses: map[string]structs.ServiceAddress{ + "wan": structs.ServiceAddress{ + Address: "198.18.0.1", + Port: 80, + }, + }, + expectedPort: 8080, + expectedAddress: "10.0.1.1", + expectedARRName: "0a000101.addr.dc2.consul.", + }, } - // Also check the A record directly from DC2 - for _, question := range questions { - m := new(dns.Msg) - m.SetQuestion(question, dns.TypeA) + for name, tc := range cases { + name := name + tc := tc + t.Run(name, func(t *testing.T) { + // Register a remote node with a service. This is in a retry since we + // need the datacenter to have a route which takes a little more time + // beyond the join, and we don't have direct access to the router here. + retry.Run(t, func(r *retry.R) { + args := &structs.RegisterRequest{ + Datacenter: "dc2", + Node: "foo", + Address: "127.0.0.1", + TaggedAddresses: tc.nodeTaggedAddresses, + Service: &structs.NodeService{ + Service: "db", + Address: tc.serviceAddress, + Port: 8080, + TaggedAddresses: tc.serviceTaggedAddresses, + }, + } - c := new(dns.Client) - addr := a2.Config.DNSAddrs[0] - in, _, err := c.Exchange(m, addr.String()) - if err != nil { - t.Fatalf("err: %v", err) - } + var out struct{} + require.NoError(t, a2.RPC("Catalog.Register", args, &out)) + }) - if len(in.Answer) != 1 { - t.Fatalf("Bad: %#v", in) - } + // Look up the SRV record via service and prepared query. + questions := []string{ + "db.service.dc2.consul.", + id + ".query.dc2.consul.", + } + for _, question := range questions { + m := new(dns.Msg) + m.SetQuestion(question, dns.TypeSRV) + + c := new(dns.Client) + + addr := tc.dnsAddr + in, _, err := c.Exchange(m, addr) + require.NoError(t, err) + require.Len(t, in.Answer, 1) + srvRec, ok := in.Answer[0].(*dns.SRV) + require.True(t, ok, "Bad: %#v", in.Answer[0]) + require.Equal(t, tc.expectedPort, srvRec.Port) + + aRec, ok := in.Extra[0].(*dns.A) + require.True(t, ok, "Bad: %#v", in.Extra[0]) + require.Equal(t, tc.expectedARRName, aRec.Hdr.Name) + require.Equal(t, tc.expectedAddress, aRec.A.String()) + } - aRec, ok := in.Answer[0].(*dns.A) - if !ok { - t.Fatalf("Bad: %#v", in.Answer[0]) - } - if aRec.Hdr.Name != question { - t.Fatalf("Bad: %#v", in.Answer[0]) - } - if aRec.A.String() != "127.0.0.1" { - t.Fatalf("Bad: %#v", in.Answer[0]) - } + // Also check the A record directly + for _, question := range questions { + m := new(dns.Msg) + m.SetQuestion(question, dns.TypeA) + + c := new(dns.Client) + addr := tc.dnsAddr + in, _, err := c.Exchange(m, addr) + require.NoError(t, err) + require.Len(t, in.Answer, 1) + + aRec, ok := in.Answer[0].(*dns.A) + require.True(t, ok, "Bad: %#v", in.Answer[0]) + require.Equal(t, question, aRec.Hdr.Name) + require.Equal(t, tc.expectedAddress, aRec.A.String()) + } + }) } }