Added jaeger support for VirtualGateway Envoy

Update jaegar tracing Envoy config to v3 API (#468)

Bump Envoy image version to v1.17.2.0-prod (#471)

VirtualGateway and VirtualNode should use same EnvoyTemplate variables
Restructured code such that VirtulalNode and VirtualGateway uses same sidecarbuilder class to get Env variables
Tested with Jaeger endpoint

reverted unnecesaary changes

pkg/inject/envoy.go

@@ -2,42 +2,17 @@ package inject
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
+
 	appmesh "github.com/aws/aws-app-mesh-controller-for-k8s/apis/appmesh/v1beta2"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
-	"strconv"
-	"strings"
 )
 
-const envoyTracingConfigVolumeName = "envoy-tracing-config"
 const envoyContainerName = "envoy"
 
-type EnvoyTemplateVariables struct {
-	AWSRegion                    string
-	MeshName                     string
-	VirtualNodeName              string
-	Preview                      string
-	EnableSDS                    bool
-	SdsUdsPath                   string
-	LogLevel                     string
-	AdminAccessPort              int32
-	AdminAccessLogFile           string
-	PreStopDelay                 string
-	SidecarImage                 string
-	EnvoyTracingConfigVolumeName string
-	EnableXrayTracing            bool
-	XrayDaemonPort               int32
-	EnableJaegerTracing          bool
-	EnableDatadogTracing         bool
-	DatadogTracerPort            int32
-	DatadogTracerAddress         string
-	EnableStatsTags              bool
-	EnableStatsD                 bool
-	StatsDPort                   int32
-	StatsDAddress                string
-}
-
 type envoyMutatorConfig struct {
 	accountID                  string
 	awsRegion                  string
@@ -58,6 +33,8 @@ type envoyMutatorConfig struct {
 	enableXrayTracing          bool
 	xrayDaemonPort             int32
 	enableJaegerTracing        bool
+	jaegerPort                 string
+	jaegerAddress              string
 	enableDatadogTracing       bool
 	datadogTracerPort          int32
 	datadogTracerAddress       string
@@ -130,28 +107,29 @@ func (m *envoyMutator) buildTemplateVariables(pod *corev1.Pod) EnvoyTemplateVari
 	}
 
 	return EnvoyTemplateVariables{
-		AWSRegion:                    m.mutatorConfig.awsRegion,
-		MeshName:                     meshName,
-		VirtualNodeName:              virtualNodeName,
-		Preview:                      preview,
-		EnableSDS:                    sdsEnabled,
-		SdsUdsPath:                   m.mutatorConfig.sdsUdsPath,
-		LogLevel:                     m.mutatorConfig.logLevel,
-		AdminAccessPort:              m.mutatorConfig.adminAccessPort,
-		AdminAccessLogFile:           m.mutatorConfig.adminAccessLogFile,
-		PreStopDelay:                 m.mutatorConfig.preStopDelay,
-		SidecarImage:                 m.mutatorConfig.sidecarImage,
-		EnvoyTracingConfigVolumeName: envoyTracingConfigVolumeName,
-		EnableXrayTracing:            m.mutatorConfig.enableXrayTracing,
-		XrayDaemonPort:               m.mutatorConfig.xrayDaemonPort,
-		EnableJaegerTracing:          m.mutatorConfig.enableJaegerTracing,
-		EnableDatadogTracing:         m.mutatorConfig.enableDatadogTracing,
-		DatadogTracerPort:            m.mutatorConfig.datadogTracerPort,
-		DatadogTracerAddress:         m.mutatorConfig.datadogTracerAddress,
-		EnableStatsTags:              m.mutatorConfig.enableStatsTags,
-		EnableStatsD:                 m.mutatorConfig.enableStatsD,
-		StatsDPort:                   m.mutatorConfig.statsDPort,
-		StatsDAddress:                m.mutatorConfig.statsDAddress,
+		AWSRegion:                m.mutatorConfig.awsRegion,
+		MeshName:                 meshName,
+		VirtualGatewayOrNodeName: virtualNodeName,
+		Preview:                  preview,
+		EnableSDS:                sdsEnabled,
+		SdsUdsPath:               m.mutatorConfig.sdsUdsPath,
+		LogLevel:                 m.mutatorConfig.logLevel,
+		AdminAccessPort:          m.mutatorConfig.adminAccessPort,
+		AdminAccessLogFile:       m.mutatorConfig.adminAccessLogFile,
+		PreStopDelay:             m.mutatorConfig.preStopDelay,
+		SidecarImage:             m.mutatorConfig.sidecarImage,
+		EnableXrayTracing:        m.mutatorConfig.enableXrayTracing,
+		XrayDaemonPort:           m.mutatorConfig.xrayDaemonPort,
+		EnableJaegerTracing:      m.mutatorConfig.enableJaegerTracing,
+		JaegerPort:               m.mutatorConfig.jaegerPort,
+		JaegerAddress:            m.mutatorConfig.jaegerAddress,
+		EnableDatadogTracing:     m.mutatorConfig.enableDatadogTracing,
+		DatadogTracerPort:        m.mutatorConfig.datadogTracerPort,
+		DatadogTracerAddress:     m.mutatorConfig.datadogTracerAddress,
+		EnableStatsTags:          m.mutatorConfig.enableStatsTags,
+		EnableStatsD:             m.mutatorConfig.enableStatsD,
+		StatsDPort:               m.mutatorConfig.statsDPort,
+		StatsDAddress:            m.mutatorConfig.statsDAddress,
 	}
 }
 
@@ -225,13 +203,3 @@ func (m *envoyMutator) getCustomEnv(pod *corev1.Pod) (map[string]string, error)
 	}
 	return customEnv, nil
 }
-
-// containsEnvoyTracingConfigVolume checks whether pod already contains "envoy-tracing-config" volume
-func containsEnvoyTracingConfigVolume(pod *corev1.Pod) bool {
-	for _, volume := range pod.Spec.Volumes {
-		if volume.Name == envoyTracingConfigVolumeName {
-			return true
-		}
-	}
-	return false
-}

pkg/inject/envoy_test.go

@@ -2,6 +2,8 @@ package inject
 
 import (
 	"errors"
+	"testing"
+
 	appmesh "github.com/aws/aws-app-mesh-controller-for-k8s/apis/appmesh/v1beta2"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/google/go-cmp/cmp"
@@ -10,7 +12,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"testing"
 )
 
 func Test_envoyMutator_mutate(t *testing.T) {
@@ -461,6 +462,8 @@ func Test_envoyMutator_mutate(t *testing.T) {
 					sidecarCPURequests:         cpuRequests.String(),
 					sidecarMemoryRequests:      memoryRequests.String(),
 					enableJaegerTracing:        true,
+					jaegerPort:                 "8000",
+					jaegerAddress:              "localhost",
 				},
 			},
 			args: args{
@@ -529,18 +532,20 @@ func Test_envoyMutator_mutate(t *testing.T) {
 									Value: "9901",
 								},
 								{
-									Name:  "ENVOY_TRACING_CFG_FILE",
-									Value: "/tmp/envoy/envoyconf.yaml",
+									Name:  "ENABLE_ENVOY_JAEGER_TRACING",
+									Value: "1",
 								},
 								{
-									Name:  "AWS_REGION",
-									Value: "us-west-2",
+									Name:  "JAEGER_TRACER_PORT",
+									Value: "8000",
+								},
+								{
+									Name:  "JAEGER_TRACER_ADDRESS",
+									Value: "localhost",
 								},
-							},
-							VolumeMounts: []corev1.VolumeMount{
 								{
-									Name:      "envoy-tracing-config",
-									MountPath: "/tmp/envoy",
+									Name:  "AWS_REGION",
+									Value: "us-west-2",
 								},
 							},
 							Resources: corev1.ResourceRequirements{

pkg/inject/inject.go

@@ -129,6 +129,8 @@ func (m *SidecarInjector) injectAppMeshPatches(ms *appmesh.Mesh, vn *appmesh.Vir
 				enableXrayTracing:          m.config.EnableXrayTracing,
 				xrayDaemonPort:             m.config.XrayDaemonPort,
 				enableJaegerTracing:        m.config.EnableJaegerTracing,
+				jaegerPort:                 m.config.JaegerPort,
+				jaegerAddress:              m.config.JaegerAddress,
 				enableDatadogTracing:       m.config.EnableDatadogTracing,
 				datadogTracerPort:          m.config.DatadogPort,
 				datadogTracerAddress:       m.config.DatadogAddress,
@@ -169,6 +171,16 @@ func (m *SidecarInjector) injectAppMeshPatches(ms *appmesh.Mesh, vn *appmesh.Vir
 			readinessProbePeriod:       m.config.ReadinessProbePeriod,
 			enableXrayTracing:          m.config.EnableXrayTracing,
 			xrayDaemonPort:             m.config.XrayDaemonPort,
+			enableJaegerTracing:        m.config.EnableJaegerTracing,
+			jaegerPort:                 m.config.JaegerPort,
+			jaegerAddress:              m.config.JaegerAddress,
+			enableDatadogTracing:       m.config.EnableDatadogTracing,
+			datadogTracerPort:          m.config.DatadogPort,
+			datadogTracerAddress:       m.config.DatadogAddress,
+			enableStatsTags:            m.config.EnableStatsTags,
+			enableStatsD:               m.config.EnableStatsD,
+			statsDPort:                 m.config.StatsDPort,
+			statsDAddress:              m.config.StatsDAddress,
 		}, ms, vg),
 			newXrayMutator(xrayMutatorConfig{
 				awsRegion:             m.awsRegion,
@@ -179,6 +191,10 @@ func (m *SidecarInjector) injectAppMeshPatches(ms *appmesh.Mesh, vn *appmesh.Vir
 				xRayImage:             m.config.XRayImage,
 				xRayDaemonPort:        m.config.XrayDaemonPort,
 			}, m.config.EnableXrayTracing),
+			newJaegerMutator(jaegerMutatorConfig{
+				jaegerAddress: m.config.JaegerAddress,
+				jaegerPort:    m.config.JaegerPort,
+			}, m.config.EnableJaegerTracing),
 		}
 	}
 

pkg/inject/inject_test.go

@@ -22,7 +22,7 @@ func getConfig(fp func(Config) Config) Config {
 		IgnoredIPs:                  "169.254.169.254",
 		LogLevel:                    "debug",
 		Preview:                     false,
-		SidecarImage:                "840364872350.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-envoy:v1.16.1.1-prod",
+		SidecarImage:                "840364872350.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-envoy:v1.17.2.0-prod",
 		InitImage:                   "840364872350.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-proxy-route-manager:v3-prod",
 		SidecarMemoryRequests:       "32Mi",
 		SidecarCpuRequests:          "10m",

pkg/inject/jaeger.go

@@ -10,7 +10,7 @@ tracing:
  http:
   name: envoy.tracers.zipkin
   typed_config:
-   "@type": type.googleapis.com/envoy.config.trace.v2.ZipkinConfig
+   "@type": type.googleapis.com/envoy.config.trace.v3.ZipkinConfig
    collector_cluster: jaeger
    collector_endpoint: "/api/v2/spans"
    collector_endpoint_version: HTTP_JSON

pkg/inject/jaeger_test.go

@@ -110,7 +110,7 @@ tracing:
  http:
   name: envoy.tracers.zipkin
   typed_config:
-   "@type": type.googleapis.com/envoy.config.trace.v2.ZipkinConfig
+   "@type": type.googleapis.com/envoy.config.trace.v3.ZipkinConfig
    collector_cluster: jaeger
    collector_endpoint: "/api/v2/spans"
    collector_endpoint_version: HTTP_JSON

pkg/inject/sidecar_builder.go

@@ -2,43 +2,48 @@ package inject
 
 import (
 	"fmt"
+	"strconv"
+
 	"github.com/aws/aws-sdk-go/aws"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
-	"strconv"
 )
 
-func buildEnvoySidecar(vars EnvoyTemplateVariables, env map[string]string) corev1.Container {
-
-	envoy := corev1.Container{
-		Name:  "envoy",
-		Image: vars.SidecarImage,
-		SecurityContext: &corev1.SecurityContext{
-			RunAsUser: aws.Int64(1337),
-		},
-		Ports: []corev1.ContainerPort{
-			{
-				Name:          "stats",
-				ContainerPort: vars.AdminAccessPort,
-				Protocol:      "TCP",
-			},
-		},
-		Lifecycle: &corev1.Lifecycle{
-			PostStart: nil,
-			PreStop: &corev1.Handler{
-				Exec: &corev1.ExecAction{Command: []string{
-					"sh", "-c", fmt.Sprintf("sleep %s", vars.PreStopDelay),
-				}},
-			},
-		},
-	}
-
-	vn := fmt.Sprintf("mesh/%s/virtualNode/%s", vars.MeshName, vars.VirtualNodeName)
+const envoyTracingConfigVolumeName = "envoy-tracing-config"
+
+// Envoy template variables used by envoys in pod and the envoy in VirtualGateway
+//as we use the same envoy image
+type EnvoyTemplateVariables struct {
+	AWSRegion                string
+	MeshName                 string
+	VirtualGatewayOrNodeName string
+	Preview                  string
+	EnableSDS                bool
+	SdsUdsPath               string
+	LogLevel                 string
+	AdminAccessPort          int32
+	AdminAccessLogFile       string
+	PreStopDelay             string
+	SidecarImage             string
+	EnableXrayTracing        bool
+	XrayDaemonPort           int32
+	EnableJaegerTracing      bool
+	JaegerPort               string
+	JaegerAddress            string
+	EnableDatadogTracing     bool
+	DatadogTracerPort        int32
+	DatadogTracerAddress     string
+	EnableStatsTags          bool
+	EnableStatsD             bool
+	StatsDPort               int32
+	StatsDAddress            string
+}
 
+func updateEnvMapForEnvoy(vars EnvoyTemplateVariables, env map[string]string, vname string) {
 	// add all the controller managed env to the map so
 	// 1) we remove duplicates
 	// 2) we don't allow overriding controller managed env with pod annotations
-	env["APPMESH_VIRTUAL_NODE_NAME"] = vn
+	env["APPMESH_VIRTUAL_NODE_NAME"] = vname
 	env["AWS_REGION"] = vars.AWSRegion
 
 	// Set the value to 1 to connect to the App Mesh Preview Channel endpoint.
@@ -109,19 +114,39 @@ func buildEnvoySidecar(vars EnvoyTemplateVariables, env map[string]string) corev
 	}
 
 	if vars.EnableJaegerTracing {
-		// Specify a file path in the Envoy container file system.
-		// See https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/trace/v2/http_tracer.proto
-		env["ENVOY_TRACING_CFG_FILE"] = "/tmp/envoy/envoyconf.yaml"
+		env["ENABLE_ENVOY_JAEGER_TRACING"] = "1"
+		env["JAEGER_TRACER_PORT"] = vars.JaegerPort
+		env["JAEGER_TRACER_ADDRESS"] = vars.JaegerAddress
+	}
+}
+
+func buildEnvoySidecar(vars EnvoyTemplateVariables, env map[string]string) corev1.Container {
 
-		vol_mount := []corev1.VolumeMount{
+	envoy := corev1.Container{
+		Name:  "envoy",
+		Image: vars.SidecarImage,
+		SecurityContext: &corev1.SecurityContext{
+			RunAsUser: aws.Int64(1337),
+		},
+		Ports: []corev1.ContainerPort{
 			{
-				Name:      vars.EnvoyTracingConfigVolumeName,
-				MountPath: "/tmp/envoy",
+				Name:          "stats",
+				ContainerPort: vars.AdminAccessPort,
+				Protocol:      "TCP",
 			},
-		}
-		envoy.VolumeMounts = vol_mount
+		},
+		Lifecycle: &corev1.Lifecycle{
+			PostStart: nil,
+			PreStop: &corev1.Handler{
+				Exec: &corev1.ExecAction{Command: []string{
+					"sh", "-c", fmt.Sprintf("sleep %s", vars.PreStopDelay),
+				}},
+			},
+		},
 	}
 
+	vname := fmt.Sprintf("mesh/%s/virtualNode/%s", vars.MeshName, vars.VirtualGatewayOrNodeName)
+	updateEnvMapForEnvoy(vars, env, vname)
 	envoy.Env = getEnvoyEnv(env)
 	return envoy
 
@@ -263,3 +288,13 @@ func envVar(envName, envVal string) corev1.EnvVar {
 		Value: envVal,
 	}
 }
+
+// containsEnvoyTracingConfigVolume checks whether pod already contains "envoy-tracing-config" volume
+func containsEnvoyTracingConfigVolume(pod *corev1.Pod) bool {
+	for _, volume := range pod.Spec.Volumes {
+		if volume.Name == envoyTracingConfigVolumeName {
+			return true
+		}
+	}
+	return false
+}