Skip to content

Commit

Permalink
feat(*): added integraton test and code review changes
Browse files Browse the repository at this point in the history
Signed-off-by: Łukasz Dziedziak <lukidzi@gmail.com>
  • Loading branch information
lukidzi committed Jun 10, 2022
1 parent 56f41dc commit 7ab78fd
Show file tree
Hide file tree
Showing 5 changed files with 182 additions and 111 deletions.
1 change: 0 additions & 1 deletion pkg/config/dp-server/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ type DpServerAuthConfig struct {
// If empty, autoconfigured based on the environment - "serviceAccountToken" on Kubernetes, "dpToken" on Universal.
Type string `yaml:"type" envconfig:"kuma_dp_server_auth_type"`
// UseTokenPath define if should use config for ads with path to token that can be reloaded.
// Default value: false
UseTokenPath bool `yaml:"useTokenPath" envconfig:"kuma_dp_server_auth_use_token_path"`
}

Expand Down
172 changes: 90 additions & 82 deletions pkg/xds/bootstrap/template_v3.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package bootstrap

import (
"fmt"
"net"
"strconv"

Expand Down Expand Up @@ -96,7 +95,9 @@ func genConfig(parameters configParameters, useTokenPath bool) (*envoy_bootstrap
ApiType: envoy_core_v3.ApiConfigSource_GRPC,
TransportApiVersion: envoy_core_v3.ApiVersion_V3,
SetNodeOnFirstMessageOnly: true,
GrpcServices: getGrpcServices(parameters, useTokenPath),
GrpcServices: []*envoy_core_v3.GrpcService{
buildGrpcService(parameters, useTokenPath),
},
},
},
StaticResources: &envoy_bootstrap_v3.Bootstrap_StaticResources{
Expand All @@ -119,79 +120,7 @@ func genConfig(parameters configParameters, useTokenPath bool) (*envoy_bootstrap
},
},
},
Clusters: []*envoy_cluster_v3.Cluster{
{
// TODO does timeout and keepAlive make sense on this as it uses unix domain sockets?
Name: "access_log_sink",
ConnectTimeout: util_proto.Duration(parameters.XdsConnectTimeout),
Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{},
LbPolicy: envoy_cluster_v3.Cluster_ROUND_ROBIN,
UpstreamConnectionOptions: &envoy_cluster_v3.UpstreamConnectionOptions{
TcpKeepalive: &envoy_core_v3.TcpKeepalive{
KeepaliveProbes: util_proto.UInt32(3),
KeepaliveTime: util_proto.UInt32(10),
KeepaliveInterval: util_proto.UInt32(10),
},
},
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC},
LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
ClusterName: "access_log_sink",
Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
{
LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
{
HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
Endpoint: &envoy_config_endpoint_v3.Endpoint{
Address: &envoy_core_v3.Address{
Address: &envoy_core_v3.Address_Pipe{Pipe: &envoy_core_v3.Pipe{Path: parameters.AccessLogPipe}},
},
},
},
},
},
},
},
},
},
{
Name: "ads_cluster",
ConnectTimeout: util_proto.Duration(parameters.XdsConnectTimeout),
Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{},
LbPolicy: envoy_cluster_v3.Cluster_ROUND_ROBIN,
UpstreamConnectionOptions: &envoy_cluster_v3.UpstreamConnectionOptions{
TcpKeepalive: &envoy_core_v3.TcpKeepalive{
KeepaliveProbes: util_proto.UInt32(3),
KeepaliveTime: util_proto.UInt32(10),
KeepaliveInterval: util_proto.UInt32(10),
},
},
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: clusterTypeFromHost(parameters.XdsHost)},
DnsLookupFamily: dnsLookupFamilyFromXdsHost(parameters.XdsHost, net.LookupIP),
LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
ClusterName: "ads_cluster",
Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
{
LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
{
HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
Endpoint: &envoy_config_endpoint_v3.Endpoint{
Address: &envoy_core_v3.Address{
Address: &envoy_core_v3.Address_SocketAddress{
SocketAddress: &envoy_core_v3.SocketAddress{
Address: parameters.XdsHost,
PortSpecifier: &envoy_core_v3.SocketAddress_PortValue{PortValue: parameters.XdsPort},
},
},
},
},
},
},
},
},
},
},
},
},
Clusters: buildStaticClusters(parameters, useTokenPath),
},
}
for _, r := range res.StaticResources.Clusters {
Expand Down Expand Up @@ -226,7 +155,9 @@ func genConfig(parameters configParameters, useTokenPath bool) (*envoy_bootstrap
ApiType: envoy_core_v3.ApiConfigSource_GRPC,
TransportApiVersion: envoy_core_v3.ApiVersion_V3,
SetNodeOnFirstMessageOnly: true,
GrpcServices: getGrpcServices(parameters, useTokenPath),
GrpcServices: []*envoy_core_v3.GrpcService{
buildGrpcService(parameters, useTokenPath),
},
}
}

Expand Down Expand Up @@ -314,13 +245,12 @@ func clusterTypeFromHost(host string) envoy_cluster_v3.Cluster_DiscoveryType {
return envoy_cluster_v3.Cluster_STRICT_DNS
}

func getGrpcServices(params configParameters, useTokenPath bool) []*envoy_core_v3.GrpcService {
var grpcSerivces []*envoy_core_v3.GrpcService
func buildGrpcService(params configParameters, useTokenPath bool) *envoy_core_v3.GrpcService {
if useTokenPath && params.DataplaneTokenPath != "" {
googleGrpcService := &envoy_core_v3.GrpcService{
TargetSpecifier: &envoy_core_v3.GrpcService_GoogleGrpc_{
GoogleGrpc: &envoy_core_v3.GrpcService_GoogleGrpc{
TargetUri: fmt.Sprintf("%s:%d", params.XdsHost, params.XdsPort),
TargetUri: net.JoinHostPort(params.XdsHost, strconv.FormatUint(uint64(params.XdsPort), 10)),
StatPrefix: "ads",
CredentialsFactoryName: "envoy.grpc_credentials.file_based_metadata",
CallCredentials: []*envoy_core_v3.GrpcService_GoogleGrpc_CallCredentials{
Expand Down Expand Up @@ -355,7 +285,7 @@ func getGrpcServices(params configParameters, useTokenPath bool) []*envoy_core_v
},
}
}
grpcSerivces = append(grpcSerivces, googleGrpcService)
return googleGrpcService
} else {
envoyGrpcSerivce := &envoy_core_v3.GrpcService{
TargetSpecifier: &envoy_core_v3.GrpcService_EnvoyGrpc_{
Expand All @@ -364,7 +294,85 @@ func getGrpcServices(params configParameters, useTokenPath bool) []*envoy_core_v
},
},
}
grpcSerivces = append(grpcSerivces, envoyGrpcSerivce)
return envoyGrpcSerivce
}
}

func buildStaticClusters(parameters configParameters, useTokenPath bool) []*envoy_cluster_v3.Cluster {
clusters := []*envoy_cluster_v3.Cluster{
{
// TODO does timeout and keepAlive make sense on this as it uses unix domain sockets?
Name: "access_log_sink",
ConnectTimeout: util_proto.Duration(parameters.XdsConnectTimeout),
Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{},
LbPolicy: envoy_cluster_v3.Cluster_ROUND_ROBIN,
UpstreamConnectionOptions: &envoy_cluster_v3.UpstreamConnectionOptions{
TcpKeepalive: &envoy_core_v3.TcpKeepalive{
KeepaliveProbes: util_proto.UInt32(3),
KeepaliveTime: util_proto.UInt32(10),
KeepaliveInterval: util_proto.UInt32(10),
},
},
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC},
LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
ClusterName: "access_log_sink",
Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
{
LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
{
HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
Endpoint: &envoy_config_endpoint_v3.Endpoint{
Address: &envoy_core_v3.Address{
Address: &envoy_core_v3.Address_Pipe{Pipe: &envoy_core_v3.Pipe{Path: parameters.AccessLogPipe}},
},
},
},
},
},
},
},
},
},
}
if parameters.DataplaneTokenPath == "" || !useTokenPath {
clusters = append(clusters, &envoy_cluster_v3.Cluster{
Name: "ads_cluster",
ConnectTimeout: util_proto.Duration(parameters.XdsConnectTimeout),
Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{},
LbPolicy: envoy_cluster_v3.Cluster_ROUND_ROBIN,
UpstreamConnectionOptions: &envoy_cluster_v3.UpstreamConnectionOptions{
TcpKeepalive: &envoy_core_v3.TcpKeepalive{
KeepaliveProbes: util_proto.UInt32(3),
KeepaliveTime: util_proto.UInt32(10),
KeepaliveInterval: util_proto.UInt32(10),
},
},
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: clusterTypeFromHost(parameters.XdsHost)},
DnsLookupFamily: dnsLookupFamilyFromXdsHost(parameters.XdsHost, net.LookupIP),
LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
ClusterName: "ads_cluster",
Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
{
LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
{
HostIdentifier: &envoy_config_endpoint_v3.LbEndpoint_Endpoint{
Endpoint: &envoy_config_endpoint_v3.Endpoint{
Address: &envoy_core_v3.Address{
Address: &envoy_core_v3.Address_SocketAddress{
SocketAddress: &envoy_core_v3.SocketAddress{
Address: parameters.XdsHost,
PortSpecifier: &envoy_core_v3.SocketAddress_PortValue{PortValue: parameters.XdsPort},
},
},
},
},
},
},
},
},
},
},
})
}
return grpcSerivces
return clusters
}
Original file line number Diff line number Diff line change
Expand Up @@ -96,34 +96,6 @@ staticResources:
keepaliveInterval: 10
keepaliveProbes: 3
keepaliveTime: 10
- connectTimeout: 1s
http2ProtocolOptions: {}
loadAssignment:
clusterName: ads_cluster
endpoints:
- lbEndpoints:
- endpoint:
address:
socketAddress:
address: localhost
portValue: 5678
name: ads_cluster
transportSocket:
name: envoy.transport_sockets.tls
typedConfig:
'@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
commonTlsContext:
tlsParams:
tlsMinimumProtocolVersion: TLSv1_2
validationContextSdsSecretConfig:
name: cp_validation_ctx
sni: localhost
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 10
keepaliveProbes: 3
keepaliveTime: 10
secrets:
- name: cp_validation_ctx
validationContext:
Expand Down
16 changes: 16 additions & 0 deletions test/e2e/projectedsatoken/e2e_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package projectedsatoken_test

import (
"testing"

. "github.com/onsi/ginkgo/v2"

"github.com/kumahq/kuma/pkg/test"
"github.com/kumahq/kuma/test/e2e/projectedsatoken"
)

func TestE2E(t *testing.T) {
test.RunSpecs(t, "E2E Projected SAT")
}

var _ = Describe("Test Projected Service Account Token on Universal", projectedsatoken.ProjectedServiceAccountToken)
76 changes: 76 additions & 0 deletions test/e2e/projectedsatoken/psat_universal.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package projectedsatoken

import (
"strings"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"github.com/kumahq/kuma/pkg/config/core"
. "github.com/kumahq/kuma/test/framework"
)

func ProjectedServiceAccountToken() {
var universal Cluster

BeforeEach(func() {
clusters, err := NewUniversalClusters([]string{Kuma1}, Silent)
Expect(err).ToNot(HaveOccurred())

universal = clusters.GetCluster(Kuma1)
Expect(NewClusterSetup().
Install(Kuma(core.Standalone,
WithEnv("KUMA_DP_SERVER_AUTH_USE_TOKEN_PATH", "true"),
)).
Install(DemoClientUniversal("demo-client", "default")).
Setup(universal)).To(Succeed())
})

E2EAfterEach(func() {
Expect(universal.DismissCluster()).To(Succeed())
})

It("should connect to restarted control plane with new token without dp restart", func() {
// given
uniCluster := universal.(*UniversalCluster)
kumaCP := universal.(*UniversalCluster).GetApp(AppModeCP)

// then should have dataplane in control plane
Eventually(func() bool {
stdout, _, err := uniCluster.Exec("", "", "kuma-cp", "kumactl", "get", "dataplanes")
if err != nil {
return false
}
return strings.Contains(stdout, "demo-client")
}, "60s", "1s").Should(BeTrue())

// when restart control-plane
Expect(kumaCP.ReStart()).Should(Succeed())

// then should not have demo-client in dataplanes
Eventually(func() bool {
stdout, _, err := uniCluster.Exec("", "", "kuma-cp", "kumactl", "get", "dataplanes")
if err != nil {
return true
}
return strings.Contains(stdout, "demo-client")
}, "60s", "1s").Should(BeFalse())

// when new token generated
token, err := universal.GetKuma().GenerateDpToken("default", "demo-client")
Expect(err).ToNot(HaveOccurred())

// and set token
_, _, err = universal.Exec("", "", "demo-client", "printf", "\""+token+"\"", ">", "/kuma/token-demo-client")
Expect(err).ToNot(HaveOccurred())

// then there should be dataplane for demo-client
Eventually(func() bool {
stdout, _, err := uniCluster.Exec("", "", "kuma-cp", "kumactl", "get", "dataplanes")
if err != nil {
return false
}
return strings.Contains(stdout, "demo-client")
}, "60s", "1s").Should(BeTrue())
})
}

0 comments on commit 7ab78fd

Please sign in to comment.