feat(mads): add possibility to run MADS on TLS

UPGRADE.md

@@ -33,7 +33,8 @@ However, Zone Ingress Token is now deprecated and will be removed in the future.
 
 ### Kuma-cp
 
-By default the minimum TLS version allowed on servers is TLSv1.2. If you require using TLS < 1.2 you can set `KUMA_GENERAL_TLS_MIN_VERSION`. 
+- By default, the minimum TLS version allowed on servers is TLSv1.2. If you require using TLS < 1.2 you can set `KUMA_GENERAL_TLS_MIN_VERSION`.
+- `KUMA_MONITORING_ASSIGNMENT_SERVER_GRPC_PORT` was removed after a long deprecation period use `KUMA_MONITORING_ASSIGNMENT_SERVER_PORT` instead.
 
 ## Upgrade to `1.8.x`
 

pkg/config/app/kuma-cp/kuma-cp.defaults.yaml

@@ -98,6 +98,16 @@ monitoringAssignmentServer:
   assignmentRefreshInterval: 1s # ENV: KUMA_MONITORING_ASSIGNMENT_SERVER_ASSIGNMENT_REFRESH_INTERVAL
   # The default timeout for a single fetch-based discovery request, if not specified
   defaultFetchTimeout: 30s # ENV: KUMA_MONITORING_ASSIGNMENT_SERVER_DEFAULT_FETCH_TIMEOUT
+  # Path to TLS certificate file
+  tlsCertFile: "" # ENV: KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_CERT_FILE
+  # Path to TLS key file
+  tlsKeyFile: "" # ENV: KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_KEY_FILE
+  # TlsMinVersion the minimum version of TLS used across all the Kuma Servers.
+  tlsMinVersion: "TLSv1_2" # ENV: KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_MIN_VERSION
+  # TlsMaxVersion the maximum version of TLS used across all the Kuma Servers.
+  tlsMaxVersion: # ENV: KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_MAX_VERSION
+  # TlsCipherSuites the list of cipher suites to be used across all the Kuma Servers.
+  tlsCipherSuites: [] # ENV: KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_CIPHER_SUITES
 
 # Envoy XDS server configuration
 xdsServer:

pkg/config/app/kuma-dp/config.go

@@ -70,7 +70,7 @@ type ControlPlane struct {
 	Retry CpRetry `yaml:"retry,omitempty"`
 	// CaCert defines Certificate Authority that will be used to verify connection to the Control Plane. It takes precedence over CaCertFile.
 	CaCert string `yaml:"caCert" envconfig:"kuma_control_plane_ca_cert"`
-	// CaCertFile defines a file for Certificate Authority that will be used to verifiy connection to the Control Plane.
+	// CaCertFile defines a file for Certificate Authority that will be used to verify connection to the Control Plane.
 	CaCertFile string `yaml:"caCertFile" envconfig:"kuma_control_plane_ca_cert_file"`
 }
 

pkg/config/loader_test.go

@@ -134,13 +134,17 @@ var _ = Describe("Config loader", func() {
 			Expect(cfg.ApiServer.Authn.Tokens.BootstrapAdminToken).To(BeFalse())
 			Expect(cfg.ApiServer.CorsAllowedDomains).To(Equal([]string{"https://kuma", "https://someapi"}))
 
-			// nolint: staticcheck
-			Expect(cfg.MonitoringAssignmentServer.GrpcPort).To(Equal(uint32(3333)))
 			Expect(cfg.MonitoringAssignmentServer.Port).To(Equal(uint32(2222)))
 			Expect(cfg.MonitoringAssignmentServer.AssignmentRefreshInterval).To(Equal(12 * time.Second))
 			Expect(cfg.MonitoringAssignmentServer.DefaultFetchTimeout).To(Equal(45 * time.Second))
 			Expect(cfg.MonitoringAssignmentServer.ApiVersions).To(HaveLen(1))
 			Expect(cfg.MonitoringAssignmentServer.ApiVersions).To(ContainElements("v1"))
+			Expect(cfg.MonitoringAssignmentServer.TlsEnabled).To(Equal(true))
+			Expect(cfg.MonitoringAssignmentServer.TlsCertFile).To(Equal("/cert"))
+			Expect(cfg.MonitoringAssignmentServer.TlsKeyFile).To(Equal("/key"))
+			Expect(cfg.MonitoringAssignmentServer.TlsMinVersion).To(Equal("TLSv1_3"))
+			Expect(cfg.MonitoringAssignmentServer.TlsMaxVersion).To(Equal("TLSv1_3"))
+			Expect(cfg.MonitoringAssignmentServer.TlsCipherSuites).To(Equal([]string{"TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_AES_256_GCM_SHA384"}))
 
 			Expect(cfg.Runtime.Kubernetes.ControlPlaneServiceName).To(Equal("custom-control-plane"))
 			Expect(cfg.Runtime.Kubernetes.ServiceAccountName).To(Equal("custom-sa"))
@@ -348,11 +352,16 @@ apiServer:
     - https://kuma
     - https://someapi
 monitoringAssignmentServer:
-  grpcPort: 3333
   port: 2222
   defaultFetchTimeout: 45s
   apiVersions: [v1]
   assignmentRefreshInterval: 12s
+  tlsEnabled: true
+  tlsCertFile: "/cert"
+  tlsKeyFile: "/key"
+  tlsMinVersion: TLSv1_3
+  tlsMaxVersion: TLSv1_3
+  tlsCipherSuites: ["TLS_RSA_WITH_AES_128_CBC_SHA", "TLS_AES_256_GCM_SHA384"]
 runtime:
   universal:
     dataplaneCleanupAge: 1h
@@ -581,11 +590,16 @@ proxy:
 				"KUMA_API_SERVER_AUTHN_TYPE":                                                               "custom-authn",
 				"KUMA_API_SERVER_AUTHN_LOCALHOST_IS_ADMIN":                                                 "false",
 				"KUMA_API_SERVER_AUTHN_TOKENS_BOOTSTRAP_ADMIN_TOKEN":                                       "false",
-				"KUMA_MONITORING_ASSIGNMENT_SERVER_GRPC_PORT":                                              "3333",
 				"KUMA_MONITORING_ASSIGNMENT_SERVER_PORT":                                                   "2222",
 				"KUMA_MONITORING_ASSIGNMENT_SERVER_DEFAULT_FETCH_TIMEOUT":                                  "45s",
 				"KUMA_MONITORING_ASSIGNMENT_SERVER_API_VERSIONS":                                           "v1",
 				"KUMA_MONITORING_ASSIGNMENT_SERVER_ASSIGNMENT_REFRESH_INTERVAL":                            "12s",
+				"KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_ENABLED":                                            "true",
+				"KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_CERT_FILE":                                          "/cert",
+				"KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_KEY_FILE":                                           "/key",
+				"KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_MIN_VERSION":                                        "TLSv1_3",
+				"KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_MAX_VERSION":                                        "TLSv1_3",
+				"KUMA_MONITORING_ASSIGNMENT_SERVER_TLS_CIPHER_SUITES":                                      "TLS_RSA_WITH_AES_128_CBC_SHA,TLS_AES_256_GCM_SHA384",
 				"KUMA_REPORTS_ENABLED":                                                                     "false",
 				"KUMA_RUNTIME_KUBERNETES_CONTROL_PLANE_SERVICE_NAME":                                       "custom-control-plane",
 				"KUMA_RUNTIME_KUBERNETES_SERVICE_ACCOUNT_NAME":                                             "custom-sa",

pkg/config/mads/config.go

@@ -7,27 +7,23 @@ import (
 	"go.uber.org/multierr"
 
 	"github.com/kumahq/kuma/pkg/config"
-	"github.com/kumahq/kuma/pkg/core"
+	config_types "github.com/kumahq/kuma/pkg/config/types"
 	"github.com/kumahq/kuma/pkg/mads"
 )
 
-var log = core.Log.WithName("mads-config")
-
 func DefaultMonitoringAssignmentServerConfig() *MonitoringAssignmentServerConfig {
 	return &MonitoringAssignmentServerConfig{
 		Port:                      5676,
 		DefaultFetchTimeout:       30 * time.Second,
 		ApiVersions:               []mads.ApiVersion{mads.API_V1},
 		AssignmentRefreshInterval: 1 * time.Second,
+		TlsMinVersion:             "TLSv1_2",
+		TlsCipherSuites:           []string{},
 	}
 }
 
 // Monitoring Assignment Discovery Service (MADS) server configuration.
 type MonitoringAssignmentServerConfig struct {
-	// GrpcPort is the port of the gRPC server that serves Monitoring Assignment Discovery Service (MADS).
-	//
-	// Deprecated: GrpcPort has been replaced with Port to multiplex both HTTP and gRPC
-	GrpcPort uint32 `yaml:"grpcPort" envconfig:"kuma_monitoring_assignment_server_grpc_port"`
 	// Port of the server that serves Monitoring Assignment Discovery Service (MADS)
 	// over both grpc and http.
 	Port uint32 `yaml:"port" envconfig:"kuma_monitoring_assignment_server_port"`
@@ -37,6 +33,18 @@ type MonitoringAssignmentServerConfig struct {
 	ApiVersions []string `yaml:"apiVersions" envconfig:"kuma_monitoring_assignment_server_api_versions"`
 	// Interval for re-generating monitoring assignments for clients connected to the Control Plane.
 	AssignmentRefreshInterval time.Duration `yaml:"assignmentRefreshInterval" envconfig:"kuma_monitoring_assignment_server_assignment_refresh_interval"`
+	// TlsEnabled whether tls is enabled or not
+	TlsEnabled bool `yaml:"tlsEnabled" envconfig:"kuma_monitoring_assignment_server_tls_enabled"`
+	// TlsCertFile defines a path to a file with PEM-encoded TLS cert. If empty, autoconfigured from general.tlsCertFile
+	TlsCertFile string `yaml:"tlsCertFile" envconfig:"kuma_monitoring_assignment_server_tls_cert_file"`
+	// TlsKeyFile defines a path to a file with PEM-encoded TLS key. If empty, autoconfigured from general.tlsKeyFile
+	TlsKeyFile string `yaml:"tlsKeyFile" envconfig:"kuma_monitoring_assignment_server_tls_key_file"`
+	// TlsMinVersion defines the minimum TLS version to be used
+	TlsMinVersion string `yaml:"tlsMinVersion" envconfig:"kuma_monitoring_assignment_server_tls_min_version"`
+	// TlsMaxVersion defines the maximum TLS version to be used
+	TlsMaxVersion string `yaml:"tlsMaxVersion" envconfig:"kuma_monitoring_assignment_server_tls_max_version"`
+	// TlsCipherSuites defines the list of ciphers to use
+	TlsCipherSuites []string `yaml:"tlsCipherSuites" envconfig:"kuma_monitoring_assignment_server_tls_cipher_suites"`
 }
 
 var _ config.Config = &MonitoringAssignmentServerConfig{}
@@ -45,13 +53,6 @@ func (c *MonitoringAssignmentServerConfig) Sanitize() {
 }
 
 func (c *MonitoringAssignmentServerConfig) Validate() (errs error) {
-	if c.GrpcPort != 0 {
-		log.Info(".GrpcPort is deprecated. Please use .Port instead")
-		if c.Port == 0 {
-			c.Port = c.GrpcPort
-		}
-	}
-
 	if 65535 < c.Port {
 		errs = multierr.Append(errs, errors.Errorf(".Port must be in the range [0, 65535]"))
 	}
@@ -69,6 +70,21 @@ func (c *MonitoringAssignmentServerConfig) Validate() (errs error) {
 	if c.AssignmentRefreshInterval <= 0 {
 		return errors.New(".AssignmentRefreshInterval must be positive")
 	}
+	if c.TlsCertFile == "" && c.TlsKeyFile != "" {
+		errs = multierr.Append(errs, errors.New(".TlsCertFile cannot be empty if TlsKeyFile has been set"))
+	}
+	if c.TlsKeyFile == "" && c.TlsCertFile != "" {
+		errs = multierr.Append(errs, errors.New(".TlsKeyFile cannot be empty if TlsCertFile has been set"))
+	}
+	if _, err := config_types.TLSVersion(c.TlsMinVersion); err != nil {
+		errs = multierr.Append(errs, errors.New(".TlsMinVersion"+err.Error()))
+	}
+	if _, err := config_types.TLSVersion(c.TlsMaxVersion); err != nil {
+		errs = multierr.Append(errs, errors.New(".TlsMaxVersion"+err.Error()))
+	}
+	if _, err := config_types.TLSCiphers(c.TlsCipherSuites); err != nil {
+		errs = multierr.Append(errs, errors.New(".TlsCipherSuites"+err.Error()))
+	}
 	return
 }
 

pkg/core/bootstrap/autoconfig.go

@@ -72,15 +72,27 @@ func autoconfigureServersTLS(cfg *kuma_cp.Config) {
 		cfg.ApiServer.HTTPS.TlsCertFile = cfg.General.TlsCertFile
 		cfg.ApiServer.HTTPS.TlsKeyFile = cfg.General.TlsKeyFile
 	}
+	if cfg.MonitoringAssignmentServer.TlsCertFile == "" {
+		cfg.MonitoringAssignmentServer.TlsCertFile = cfg.General.TlsCertFile
+		cfg.MonitoringAssignmentServer.TlsKeyFile = cfg.General.TlsKeyFile
+	}
 	if cfg.General.TlsMinVersion != "" {
 		cfg.Multizone.Global.KDS.TlsMinVersion = cfg.General.TlsMinVersion
 		cfg.DpServer.TlsMinVersion = cfg.General.TlsMinVersion
 		cfg.ApiServer.HTTPS.TlsMinVersion = cfg.General.TlsMinVersion
+		cfg.MonitoringAssignmentServer.TlsMinVersion = cfg.General.TlsMinVersion
+	}
+	if cfg.General.TlsMaxVersion != "" {
+		cfg.Multizone.Global.KDS.TlsMaxVersion = cfg.General.TlsMaxVersion
+		cfg.DpServer.TlsMaxVersion = cfg.General.TlsMaxVersion
+		cfg.ApiServer.HTTPS.TlsMaxVersion = cfg.General.TlsMaxVersion
+		cfg.MonitoringAssignmentServer.TlsMaxVersion = cfg.General.TlsMaxVersion
 	}
 	if len(cfg.General.TlsCipherSuites) > 0 {
 		cfg.Multizone.Global.KDS.TlsCipherSuites = cfg.General.TlsCipherSuites
 		cfg.DpServer.TlsCipherSuites = cfg.General.TlsCipherSuites
 		cfg.ApiServer.HTTPS.TlsCipherSuites = cfg.General.TlsCipherSuites
+		cfg.MonitoringAssignmentServer.TlsCipherSuites = cfg.General.TlsCipherSuites
 	}
 }
 

pkg/mads/server/server.go

@@ -1,13 +1,15 @@
 package server
 
 import (
+	"crypto/tls"
 	"fmt"
 	"net"
 	"net/http"
 	"strings"
 	"time"
 
 	"github.com/emicklei/go-restful/v3"
+	"github.com/pkg/errors"
 	http_prometheus "github.com/slok/go-http-metrics/metrics/prometheus"
 	"github.com/slok/go-http-metrics/middleware"
 	"github.com/soheilhy/cmux"
@@ -17,6 +19,7 @@ import (
 	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
 	config_core "github.com/kumahq/kuma/pkg/config/core"
 	mads_config "github.com/kumahq/kuma/pkg/config/mads"
+	config_types "github.com/kumahq/kuma/pkg/config/types"
 	"github.com/kumahq/kuma/pkg/core"
 	core_runtime "github.com/kumahq/kuma/pkg/core/runtime"
 	"github.com/kumahq/kuma/pkg/core/runtime/component"
@@ -113,13 +116,29 @@ func (s *muxServer) Start(stop <-chan struct{}) error {
 		}
 	}()
 
+	if s.config.TlsEnabled {
+		cert, err := tls.LoadX509KeyPair(s.config.TlsCertFile, s.config.TlsKeyFile)
+		if err != nil {
+			return errors.Wrap(err, "failed to load TLS certificate")
+		}
+		tlsConfig := &tls.Config{Certificates: []tls.Certificate{cert}}
+		if tlsConfig.MinVersion, err = config_types.TLSVersion(s.config.TlsMinVersion); err != nil {
+			return err
+		}
+		if tlsConfig.MaxVersion, err = config_types.TLSVersion(s.config.TlsMaxVersion); err != nil {
+			return err
+		}
+		if tlsConfig.CipherSuites, err = config_types.TLSCiphers(s.config.TlsCipherSuites); err != nil {
+			return err
+		}
+		l = tls.NewListener(l, tlsConfig)
+	}
 	m := cmux.New(l)
 
 	grpcL := m.MatchWithWriters(
 		cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc"),
+		cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc+proto"),
 	)
-	httpL := m.Match(cmux.HTTP1Fast())
-
 	grpcS := s.createGRPCServer()
 	errChanGrpc := make(chan error)
 	go func() {
@@ -135,6 +154,7 @@ func (s *muxServer) Start(stop <-chan struct{}) error {
 		}
 	}()
 
+	httpL := m.Match(cmux.Any())
 	httpS := &http.Server{
 		Handler: s.createHttpServicesHandler(),
 	}