Skip to content

Commit

Permalink
feat(dns): Use generated vips instead of rebuilding it
Browse files Browse the repository at this point in the history
This avoids computing the same thing twice. Now that allocator
persists the correct configuration we can just reuse this

Signed-off-by: Charly Molter <charly.molter@konghq.com>
  • Loading branch information
lahabana committed Aug 9, 2021
1 parent 7d7785c commit 952b21e
Show file tree
Hide file tree
Showing 7 changed files with 142 additions and 465 deletions.
5 changes: 1 addition & 4 deletions pkg/xds/context/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp"
mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
"github.com/kumahq/kuma/pkg/core/xds"
"github.com/kumahq/kuma/pkg/dns/resolver"
"github.com/kumahq/kuma/pkg/envoy/admin"
"github.com/kumahq/kuma/pkg/tls"
)
Expand All @@ -27,7 +26,6 @@ type ControlPlaneContext struct {
SdsTlsCert []byte
AdminProxyKeyPair *tls.KeyPair
CLACache xds.CLACache
DNSResolver resolver.DNSResolver
}

func (c Context) SDSLocation() string {
Expand All @@ -41,7 +39,7 @@ type MeshContext struct {
Hash string
}

func BuildControlPlaneContext(config kuma_cp.Config, claCache xds.CLACache, dnsResolver resolver.DNSResolver) (*ControlPlaneContext, error) {
func BuildControlPlaneContext(config kuma_cp.Config, claCache xds.CLACache) (*ControlPlaneContext, error) {
var sdsCert []byte
if config.DpServer.TlsCertFile != "" {
c, err := ioutil.ReadFile(config.DpServer.TlsCertFile)
Expand All @@ -60,6 +58,5 @@ func BuildControlPlaneContext(config kuma_cp.Config, claCache xds.CLACache, dnsR
SdsTlsCert: sdsCert,
AdminProxyKeyPair: &adminKeyPair,
CLACache: claCache,
DNSResolver: dnsResolver,
}, nil
}
11 changes: 1 addition & 10 deletions pkg/xds/generator/dns_generator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ import (
mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
model "github.com/kumahq/kuma/pkg/core/xds"
"github.com/kumahq/kuma/pkg/dns/resolver"
"github.com/kumahq/kuma/pkg/dns/vips"
. "github.com/kumahq/kuma/pkg/test/matchers"
test_model "github.com/kumahq/kuma/pkg/test/resources/model"
util_proto "github.com/kumahq/kuma/pkg/util/proto"
Expand All @@ -32,19 +30,12 @@ var _ = Describe("DNSGenerator", func() {
func(given testCase) {
// setup
gen := &generator.DNSGenerator{}

dnsResolver := resolver.NewDNSResolver("mesh")
dnsResolver.SetVIPs(vips.List{
vips.NewServiceEntry("backend_test-ns_svc_8080"): "240.0.0.0",
vips.NewServiceEntry("httpbin"): "240.0.0.1",
})
ctx := xds_context.Context{
ConnectionInfo: xds_context.ConnectionInfo{
Authority: "kuma-system:5677",
},
ControlPlane: &xds_context.ControlPlaneContext{
SdsTlsCert: []byte("12345"),
DNSResolver: dnsResolver,
SdsTlsCert: []byte("12345"),
},
Mesh: xds_context.MeshContext{
Resource: &mesh_core.MeshResource{
Expand Down
2 changes: 1 addition & 1 deletion pkg/xds/server/components.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func RegisterXDS(rt core_runtime.Runtime) error {
if err != nil {
return err
}
envoyCpCtx, err := xds_context.BuildControlPlaneContext(rt.Config(), claCache, rt.DNSResolver())
envoyCpCtx, err := xds_context.BuildControlPlaneContext(rt.Config(), claCache)
if err != nil {
return err
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/xds/sync/components.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ var (

func defaultDataplaneProxyBuilder(rt core_runtime.Runtime, metadataTracker DataplaneMetadataTracker, apiVersion envoy.APIVersion) *DataplaneProxyBuilder {
return &DataplaneProxyBuilder{
ConfigManager: rt.ConfigManager(),
CachingResManager: rt.ReadOnlyResourceManager(),
NonCachingResManager: rt.ResourceManager(),
LookupIP: rt.LookupIP(),
Expand All @@ -30,6 +31,7 @@ func defaultDataplaneProxyBuilder(rt core_runtime.Runtime, metadataTracker Datap
RateLimitMatcher: ratelimits.RateLimitMatcher{ResourceManager: rt.ReadOnlyResourceManager()},
Zone: rt.Config().Multizone.Zone.Name,
apiVersion: apiVersion,
TopLevelDomain: rt.Config().DNSServer.Domain,
}
}

Expand Down
27 changes: 17 additions & 10 deletions pkg/xds/sync/dataplane_proxy_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
"github.com/kumahq/kuma/pkg/core"
config_manager "github.com/kumahq/kuma/pkg/core/config/manager"
"github.com/kumahq/kuma/pkg/core/datasource"
"github.com/kumahq/kuma/pkg/core/dns/lookup"
"github.com/kumahq/kuma/pkg/core/faultinjections"
Expand All @@ -16,7 +17,7 @@ import (
core_model "github.com/kumahq/kuma/pkg/core/resources/model"
core_store "github.com/kumahq/kuma/pkg/core/resources/store"
"github.com/kumahq/kuma/pkg/core/xds"
"github.com/kumahq/kuma/pkg/dns/resolver"
"github.com/kumahq/kuma/pkg/dns/vips"
xds_context "github.com/kumahq/kuma/pkg/xds/context"
"github.com/kumahq/kuma/pkg/xds/envoy"
xds_topology "github.com/kumahq/kuma/pkg/xds/topology"
Expand All @@ -35,8 +36,10 @@ type DataplaneProxyBuilder struct {
FaultInjectionMatcher faultinjections.FaultInjectionMatcher
RateLimitMatcher ratelimits.RateLimitMatcher

Zone string
apiVersion envoy.APIVersion
Zone string
apiVersion envoy.APIVersion
ConfigManager config_manager.ConfigManager
TopLevelDomain string
}

func (p *DataplaneProxyBuilder) build(key core_model.ResourceKey, envoyContext *xds_context.Context) (*xds.Proxy, error) {
Expand All @@ -47,7 +50,7 @@ func (p *DataplaneProxyBuilder) build(key core_model.ResourceKey, envoyContext *
return nil, err
}

routing, destinations, err := p.resolveRouting(ctx, &envoyContext.Mesh, envoyContext.ControlPlane.DNSResolver, dp)
routing, destinations, err := p.resolveRouting(ctx, &envoyContext.Mesh, dp)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -88,7 +91,6 @@ func (p *DataplaneProxyBuilder) resolveDataplane(ctx context.Context, key core_m
func (p *DataplaneProxyBuilder) resolveRouting(
ctx context.Context,
meshContext *xds_context.MeshContext,
dnsResolver resolver.DNSResolver,
dataplane *core_mesh.DataplaneResource,
) (*xds.Routing, xds.DestinationMap, error) {
externalServices := &core_mesh.ExternalServiceResourceList{}
Expand All @@ -109,16 +111,21 @@ func (p *DataplaneProxyBuilder) resolveRouting(
var domains []xds.VIPDomains
outbounds := dataplane.Spec.Networking.Outbound
if dataplane.Spec.Networking.GetTransparentProxying() != nil {
pers := vips.NewPersistence(p.CachingResManager, p.ConfigManager)
virtualOutboundView, err := pers.GetByMesh(dataplane.Meta.GetMesh())
if err != nil {
return nil, nil, err
}
// resolve all the domains
domains, outbounds = xds_topology.VIPOutbounds(core_model.MetaToResourceKey(dataplane.Meta), meshContext.Dataplanes.Items, zoneIngresses.Items, dnsResolver.GetVIPs(), dnsResolver.GetDomain(), matchedExternalServices)
domains, outbounds = xds_topology.VIPOutbounds(virtualOutboundView, p.TopLevelDomain)

// Update the outbound of the dataplane with the vips
vips := map[string]bool{}
// Update the outbound of the dataplane with the generatedVips
generatedVips := map[string]bool{}
for _, ob := range outbounds {
vips[ob.Address] = true
generatedVips[ob.Address] = true
}
for _, outbound := range dataplane.Spec.Networking.GetOutbound() {
if vips[outbound.Address] { // Useful while we still have resources with computed vip outbounds
if generatedVips[outbound.Address] { // Useful while we still have resources with computed vip outbounds
continue
}
outbounds = append(outbounds, outbound)
Expand Down
172 changes: 37 additions & 135 deletions pkg/xds/topology/dns.go
Original file line number Diff line number Diff line change
@@ -1,163 +1,65 @@
package topology

import (
"sort"
"strconv"
"strings"

"github.com/asaskevich/govalidator"
"github.com/pkg/errors"

mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
"github.com/kumahq/kuma/pkg/core/resources/model"
"github.com/kumahq/kuma/pkg/core/xds"
"github.com/kumahq/kuma/pkg/dns/vips"
)

const VIPListenPort = uint32(80)

func VIPOutbounds(
resourceKey model.ResourceKey,
dataplanes []*core_mesh.DataplaneResource,
zoneIngresses []*core_mesh.ZoneIngressResource,
vipList vips.List,
virtualOutboundView *vips.VirtualOutboundView,
tldomain string,
externalServices []*core_mesh.ExternalServiceResource,
) ([]xds.VIPDomains, []*mesh_proto.Dataplane_Networking_Outbound) {
type vipEntry struct {
ip string
port uint32
entryType vips.EntryType
}
serviceVIPMap := map[string][]vipEntry{}
services := []string{}
for _, dataplane := range dataplanes {
// backwards compatibility
if dataplane.Spec.IsIngress() {
for _, service := range dataplane.Spec.GetNetworking().GetIngress().GetAvailableServices() {
if service.Mesh == resourceKey.Mesh {
// Only add outbounds for services in the same mesh
inService := service.Tags[mesh_proto.ServiceTag]
if _, found := serviceVIPMap[inService]; !found {
vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
if err == nil {
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, vips.Service})
services = append(services, inService)
}
}
}
}
} else {
for _, inbound := range dataplane.Spec.GetNetworking().GetInbound() {
inService := inbound.GetTags()[mesh_proto.ServiceTag]
if _, found := serviceVIPMap[inService]; !found {
vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
if err == nil {
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, vips.Service})
services = append(services, inService)
}
}
}
var vipDomains []xds.VIPDomains
var outbounds []*mesh_proto.Dataplane_Networking_Outbound
for _, key := range virtualOutboundView.Keys() {
voutbound := virtualOutboundView.Get(key)
if voutbound.Address == "" {
continue
}
}

for _, zi := range zoneIngresses {
for _, service := range zi.Spec.GetAvailableServices() {
if service.Mesh == resourceKey.Mesh {
// Only add outbounds for services in the same mesh
inService := service.Tags[mesh_proto.ServiceTag]
if _, found := serviceVIPMap[inService]; !found {
vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
if err == nil {
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, vips.Service})
services = append(services, inService)
domain := xds.VIPDomains{Address: voutbound.Address}
switch key.Type {
case vips.Host, vips.FullyQualifiedDomain:
for _, ob := range voutbound.Outbounds {
if govalidator.IsDNSName(key.Name) {
domain.Domains = []string{key.Name}
if ob.Port != 0 {
outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
Address: voutbound.Address,
Port: ob.Port,
Tags: ob.TagSet,
})
}
// TODO remove the `vips.Host` on the next major version it's there for backward compatibility
if key.Type == vips.Host {
outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
Address: voutbound.Address,
Port: VIPListenPort,
Tags: ob.TagSet,
})
}
}
}
}
}

externalServicesByServiceName := map[string]*core_mesh.ExternalServiceResource{}
for _, externalService := range externalServices {
inService := externalService.Spec.Tags[mesh_proto.ServiceTag]
externalServicesByServiceName[inService] = externalService
host := externalService.Spec.GetHost()
if _, found := serviceVIPMap[inService]; !found {
vip1, err := ForwardLookup(vipList, vips.NewHostEntry(host))
if err == nil {
port := externalService.Spec.GetPort()
var p32 uint32
if p64, err := strconv.ParseUint(port, 10, 32); err != nil {
p32 = VIPListenPort
} else {
p32 = uint32(p64)
}
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip1, p32, vips.Host})
services = append(services, inService)
}
vip2, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
if err == nil {
port := externalService.Spec.GetPort()
var p32 uint32
if p64, err := strconv.ParseUint(port, 10, 32); err != nil {
p32 = VIPListenPort
} else {
p32 = uint32(p64)
}
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip2, p32, vips.Service})
services = append(services, inService)
case vips.Service:
service := voutbound.Outbounds[0].TagSet[mesh_proto.ServiceTag]
domain.Domains = []string{service + "." + tldomain}
cleanedDomain := strings.ReplaceAll(service, "_", ".") + "." + tldomain
if cleanedDomain != domain.Domains[0] {
domain.Domains = append(domain.Domains, cleanedDomain)
}
}
}

sort.Strings(services)
var vipDomains []xds.VIPDomains
var outbounds []*mesh_proto.Dataplane_Networking_Outbound
for _, service := range services {
entries := serviceVIPMap[service]
for _, entry := range entries {
outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
Address: entry.ip,
Tags: map[string]string{mesh_proto.ServiceTag: service},
Port: entry.port,
Address: voutbound.Address,
Port: VIPListenPort,
Tags: voutbound.Outbounds[0].TagSet,
})
vip := xds.VIPDomains{
Address: entry.ip,
}
switch entry.entryType {
case vips.Service:
// add regular .mesh domain
vip.Domains = []string{service + "." + tldomain}
cleanedDomain := strings.ReplaceAll(service, "_", ".") + "." + tldomain
if cleanedDomain != vip.Domains[0] {
vip.Domains = append(vip.Domains, cleanedDomain)
}
// todo (lobkovilya): backwards compatibility, could be deleted in the next major release Kuma 1.2.x
if entry.port != VIPListenPort {
outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
Address: entry.ip,
Tags: map[string]string{mesh_proto.ServiceTag: service},
Port: VIPListenPort,
})
}
case vips.Host:
host := externalServicesByServiceName[service].Spec.GetHost()
if govalidator.IsDNSName(host) {
vip.Domains = append(vip.Domains, host)
}
}
vipDomains = append(vipDomains, vip)
}
vipDomains = append(vipDomains, domain)
}

return vipDomains, outbounds
}

func ForwardLookup(vips vips.List, entry vips.Entry) (string, error) {
ip, found := vips[entry]
if !found {
return "", errors.Errorf("entry name [%s] not found", entry.Name)
}
return ip, nil
}
Loading

0 comments on commit 952b21e

Please sign in to comment.