add support for serving metrics with self-signed cert

Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
kubeovn · Jul 17, 2024 · 8bf5036 · 8bf5036
1 parent 1f58f75
commit 8bf5036
Show file tree

Hide file tree

Showing 13 changed files with 199 additions and 41 deletions.
diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml
@@ -116,13 +116,18 @@ spec:
           - --keep-vm-ip={{- .Values.func.ENABLE_KEEP_VM_IP }}
           - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
           - --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }}
+          - --secure-serving={{- .Values.func.SECURE_SERVING }}
           env:
             - name: ENABLE_SSL
               value: "{{ .Values.networking.ENABLE_SSL }}"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
             - name: KUBE_NAMESPACE
               valueFrom:
                 fieldRef:

diff --git a/charts/kube-ovn/templates/monitor-deploy.yaml b/charts/kube-ovn/templates/monitor-deploy.yaml
@@ -44,6 +44,7 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           command: ["/kube-ovn/start-ovn-monitor.sh"]
           args:
+          - --secure-serving={{- .Values.func.SECURE_SERVING }}
           - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
           - --logtostderr=false
           - --alsologtostderr=true
@@ -58,6 +59,14 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
             - name: POD_IPS
               valueFrom:
                 fieldRef:

diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml
@@ -82,6 +82,7 @@ spec:
           - --kubelet-dir={{ .Values.kubelet_conf.KUBELET_DIR }}
           - --enable-tproxy={{ .Values.func.ENABLE_TPROXY }}
           - --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }}
+          - --secure-serving={{- .Values.func.SECURE_SERVING }}
         securityContext:
           runAsUser: 0
           privileged: true
@@ -96,6 +97,14 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: spec.nodeName
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: POD_IPS
             valueFrom:
               fieldRef:

diff --git a/charts/kube-ovn/values.yaml b/charts/kube-ovn/values.yaml
@@ -68,6 +68,7 @@ func:
   CHECK_GATEWAY: true
   LOGICAL_GATEWAY: false
   ENABLE_BIND_LOCAL_IP: true
+  SECURE_SERVING: false
   U2O_INTERCONNECTION: false
   ENABLE_TPROXY: false
   ENABLE_IC: false

diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go
@@ -22,11 +22,15 @@ import (
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/controller"
+	"github.com/kubeovn/kube-ovn/pkg/tls"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
-const ovnLeaderResource = "kube-ovn-controller"
+const (
+	svcName           = "kube-ovn-controller"
+	ovnLeaderResource = "kube-ovn-controller"
+)
 
 func CmdMain() {
 	defer klog.Flush()
@@ -79,14 +83,22 @@ func CmdMain() {
 				}
 			}
 		}
-		// conform to Gosec G114
-		// https://github.com/securego/gosec#available-rules
-		server := &http.Server{
-			Addr:              fmt.Sprintf("%s:%d", addr, config.PprofPort),
-			ReadHeaderTimeout: 3 * time.Second,
-			Handler:           mux,
+
+		addr = util.JoinHostPort(addr, config.PprofPort)
+		if !config.SecureServing {
+			server := &http.Server{
+				Addr:              addr,
+				ReadHeaderTimeout: 3 * time.Second,
+				Handler:           mux,
+			}
+			util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
+		} else {
+			ch, err := tls.SecureServing(addr, svcName, mux)
+			if err != nil {
+				util.LogFatalAndExit(err, "failed to serve on %s", addr)
+			}
+			<-ch
 		}
-		util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
 	}()
 
 	//	ctx, cancel := context.WithCancel(context.Background())

diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go
@@ -18,10 +18,13 @@ import (
 	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
 	"github.com/kubeovn/kube-ovn/pkg/daemon"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
+	"github.com/kubeovn/kube-ovn/pkg/tls"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
+const svcName = "kube-ovn-cni"
+
 func CmdMain() {
 	defer klog.Flush()
 
@@ -114,14 +117,21 @@ func CmdMain() {
 		}()
 	}
 
-	// conform to Gosec G114
-	// https://github.com/securego/gosec#available-rules
-	server := &http.Server{
-		Addr:              fmt.Sprintf("%s:%d", addr, config.PprofPort),
-		ReadHeaderTimeout: 3 * time.Second,
-		Handler:           mux,
+	addr = util.JoinHostPort(addr, config.PprofPort)
+	if !config.SecureServing {
+		server := &http.Server{
+			Addr:              addr,
+			ReadHeaderTimeout: 3 * time.Second,
+			Handler:           mux,
+		}
+		util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
+	} else {
+		ch, err := tls.SecureServing(addr, svcName, mux)
+		if err != nil {
+			util.LogFatalAndExit(err, "failed to serve on %s", addr)
+		}
+		<-ch
 	}
-	util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and serve on %s", server.Addr)
 }
 
 func mvCNIConf(configDir, configFile, confName string) error {

diff --git a/cmd/ovn_monitor/ovn_monitor.go b/cmd/ovn_monitor/ovn_monitor.go
@@ -12,10 +12,13 @@ import (
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	ovn "github.com/kubeovn/kube-ovn/pkg/ovnmonitor"
+	"github.com/kubeovn/kube-ovn/pkg/tls"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
+const svcName = "kube-ovn-monitor"
+
 func CmdMain() {
 	defer klog.Flush()
 
@@ -25,21 +28,6 @@ func CmdMain() {
 		util.LogFatalAndExit(err, "failed to parse config")
 	}
 
-	exporter := ovn.NewExporter(config)
-	if err = exporter.StartConnection(); err != nil {
-		klog.Errorf("%s failed to connect db socket properly: %s", ovn.GetExporterName(), err)
-		go exporter.TryClientConnection()
-	}
-	exporter.StartOvnMetrics()
-	mux := http.NewServeMux()
-	if config.EnableMetrics {
-		mux.Handle(config.MetricsPath, promhttp.Handler())
-		klog.Infoln("Listening on", config.ListenAddress)
-	}
-
-	// conform to Gosec G114
-	// https://github.com/securego/gosec#available-rules
-
 	addr := config.ListenAddress
 	if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
 		podIpsEnv := os.Getenv("POD_IPS")
@@ -54,10 +42,30 @@ func CmdMain() {
 		}
 	}
 
-	server := &http.Server{
-		Addr:              addr,
-		ReadHeaderTimeout: 3 * time.Second,
-		Handler:           mux,
+	exporter := ovn.NewExporter(config)
+	if err = exporter.StartConnection(); err != nil {
+		klog.Errorf("%s failed to connect db socket properly: %s", ovn.GetExporterName(), err)
+		go exporter.TryClientConnection()
+	}
+	exporter.StartOvnMetrics()
+	mux := http.NewServeMux()
+	if config.EnableMetrics {
+		mux.Handle(config.MetricsPath, promhttp.Handler())
+		klog.Infoln("Listening on", addr)
+	}
+
+	if !config.SecureServing {
+		server := &http.Server{
+			Addr:              addr,
+			ReadHeaderTimeout: 3 * time.Second,
+			Handler:           mux,
+		}
+		util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", addr)
+	} else {
+		ch, err := tls.SecureServing(addr, svcName, mux)
+		if err != nil {
+			util.LogFatalAndExit(err, "failed to serve on %s", addr)
+		}
+		<-ch
 	}
-	util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", config.ListenAddress)
 }
diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -38,6 +38,7 @@ ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-true}
 ENABLE_TPROXY=${ENABLE_TPROXY:-false}
 OVS_VSCTL_CONCURRENCY=${OVS_VSCTL_CONCURRENCY:-100}
 ENABLE_COMPACT=${ENABLE_COMPACT:-false}
+SECURE_SERVING=${SECURE_SERVING:-false}
 
 # debug
 DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
@@ -4058,13 +4059,18 @@ spec:
           - --enable-lb-svc=$ENABLE_LB_SVC
           - --keep-vm-ip=$ENABLE_KEEP_VM_IP
           - --node-local-dns-ip=$NODE_LOCAL_DNS_IP
+          - --secure-serving=${SECURE_SERVING}
           env:
             - name: ENABLE_SSL
               value: "$ENABLE_SSL"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
             - name: KUBE_NAMESPACE
               valueFrom:
                 fieldRef:
@@ -4199,6 +4205,7 @@ spec:
           - --kubelet-dir=$KUBELET_DIR
           - --enable-tproxy=$ENABLE_TPROXY
           - --ovs-vsctl-concurrency=$OVS_VSCTL_CONCURRENCY
+          - --secure-serving=${SECURE_SERVING}
         securityContext:
           runAsUser: 0
           privileged: true
@@ -4217,6 +4224,14 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: status.podIPs
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: ENABLE_BIND_LOCAL_IP
             value: "$ENABLE_BIND_LOCAL_IP"
           - name: DBUS_SYSTEM_BUS_ADDRESS
@@ -4482,6 +4497,7 @@ spec:
           imagePullPolicy: $IMAGE_PULL_POLICY
           command: ["/kube-ovn/start-ovn-monitor.sh"]
           args:
+          - --secure-serving=${SECURE_SERVING}
           - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
           - --logtostderr=false
           - --alsologtostderr=true
@@ -4496,6 +4512,14 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
             - name: POD_IPS
               valueFrom:
                 fieldRef:

diff --git a/go.mod b/go.mod
@@ -44,6 +44,7 @@ require (
 	gopkg.in/k8snetworkplumbingwg/multus-cni.v4 v4.0.2
 	k8s.io/api v0.30.2
 	k8s.io/apimachinery v0.30.2
+	k8s.io/apiserver v0.30.2
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kubectl v0.30.2
@@ -241,7 +242,6 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.30.2 // indirect
-	k8s.io/apiserver v0.30.2 // indirect
 	k8s.io/cli-runtime v0.30.2 // indirect
 	k8s.io/cloud-provider v0.30.2 // indirect
 	k8s.io/cluster-bootstrap v0.30.2 // indirect

diff --git a/pkg/controller/config.go b/pkg/controller/config.go
@@ -21,7 +21,6 @@ import (
 
 // Configuration is the controller conf
 type Configuration struct {
-	BindAddress            string
 	OvnNbAddr              string
 	OvnSbAddr              string
 	OvnTimeout             int
@@ -68,8 +67,9 @@ type Configuration struct {
 	PodNicType   string
 
 	WorkerNum       int
-	PprofPort       int
+	PprofPort       int32
 	EnablePprof     bool
+	SecureServing   bool
 	NodePgProbeTime int
 
 	NetworkType             string
@@ -143,7 +143,8 @@ func ParseFlags() (*Configuration, error) {
 
 		argWorkerNum       = pflag.Int("worker-num", 3, "The parallelism of each worker")
 		argEnablePprof     = pflag.Bool("enable-pprof", false, "Enable pprof")
-		argPprofPort       = pflag.Int("pprof-port", 10660, "The port to get profiling data")
+		argPprofPort       = pflag.Int32("pprof-port", 10660, "The port to get profiling data")
+		argSecureServing   = pflag.Bool("secure-serving", false, "Enable secure serving")
 		argNodePgProbeTime = pflag.Int("nodepg-probe-time", 1, "The probe interval for node port-group, the unit is minute")
 
 		argNetworkType             = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type")
@@ -226,6 +227,7 @@ func ParseFlags() (*Configuration, error) {
 		WorkerNum:                      *argWorkerNum,
 		EnablePprof:                    *argEnablePprof,
 		PprofPort:                      *argPprofPort,
+		SecureServing:                  *argSecureServing,
 		NetworkType:                    *argNetworkType,
 		DefaultVlanID:                  *argDefaultVlanID,
 		LsDnatModDlDst:                 *argLsDnatModDlDst,

diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go
@@ -49,7 +49,8 @@ type Configuration struct {
 	EncapChecksum             bool
 	EnablePprof               bool
 	MacLearningFallback       bool
-	PprofPort                 int
+	PprofPort                 int32
+	SecureServing             bool
 	NetworkType               string
 	CniConfDir                string
 	CniConfFile               string
@@ -86,7 +87,8 @@ func ParseFlags() *Configuration {
 		argNodeSwitch            = pflag.String("node-switch", "join", "The name of node gateway switch which help node to access pod network")
 		argEncapChecksum         = pflag.Bool("encap-checksum", true, "Enable checksum")
 		argEnablePprof           = pflag.Bool("enable-pprof", false, "Enable pprof")
-		argPprofPort             = pflag.Int("pprof-port", 10665, "The port to get profiling data")
+		argPprofPort             = pflag.Int32("pprof-port", 10665, "The port to get profiling data")
+		argSecureServing         = pflag.Bool("secure-serving", false, "Enable secure serving")
 		argMacLearningFallback   = pflag.Bool("mac-learning-fallback", false, "Fallback to the legacy MAC learning mode")
 
 		argsNetworkType              = pflag.String("network-type", util.NetworkTypeGeneve, "Tunnel encapsulation protocol in overlay networks")
@@ -138,6 +140,7 @@ func ParseFlags() *Configuration {
 		OvsSocket:                 *argOvsSocket,
 		KubeConfigFile:            *argKubeConfigFile,
 		EnablePprof:               *argEnablePprof,
+		SecureServing:             *argSecureServing,
 		PprofPort:                 *argPprofPort,
 		MacLearningFallback:       *argMacLearningFallback,
 		NodeName:                  strings.ToLower(*argNodeName),