metrics: add support for secure serving (kubeovn#4297)

* add support for serving metrics with self-signed cert * add support for serving metrics with authn/authz --------- Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
zhangzujian · Jul 19, 2024 · 1078173 · 1078173
1 parent b577211
commit 1078173
Show file tree

Hide file tree

Showing 20 changed files with 451 additions and 104 deletions.
diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml
@@ -116,6 +116,7 @@ spec:
           - --keep-vm-ip={{- .Values.func.ENABLE_KEEP_VM_IP }}
           - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
           - --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }}
+          - --secure-serving={{- .Values.func.SECURE_SERVING }}
           securityContext:
             runAsUser: 0
             privileged: false
@@ -129,6 +130,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
             - name: KUBE_NAMESPACE
               valueFrom:
                 fieldRef:
@@ -139,6 +144,10 @@ spec:
                   fieldPath: spec.nodeName
             - name: OVN_DB_IPS
               value: "{{ .Values.MASTER_NODES }}"
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
             - name: POD_IPS
               valueFrom:
                 fieldRef:

diff --git a/charts/kube-ovn/templates/monitor-deploy.yaml b/charts/kube-ovn/templates/monitor-deploy.yaml
@@ -44,6 +44,7 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           command: ["/kube-ovn/start-ovn-monitor.sh"]
           args:
+          - --secure-serving={{- .Values.func.SECURE_SERVING }}
           - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log
           - --logtostderr=false
           - --alsologtostderr=true
@@ -58,6 +59,18 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
             - name: POD_IPS
               valueFrom:
                 fieldRef:

diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml
@@ -240,7 +240,18 @@ rules:
       - get
       - list
       - watch
-
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -263,3 +274,15 @@ rules:
       - daemonsets
     verbs:
       - get
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
diff --git a/charts/kube-ovn/templates/ovn-CRB.yaml b/charts/kube-ovn/templates/ovn-CRB.yaml
@@ -38,7 +38,20 @@ subjects:
   - kind: ServiceAccount
     name: kube-ovn-cni
     namespace: kube-system
-
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-cni
+    namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,3 +65,17 @@ subjects:
   - kind: ServiceAccount
     name: kube-ovn-app
     namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-app
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-app
+    namespace: kube-system
diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml
@@ -80,6 +80,7 @@ spec:
           - --kubelet-dir={{ .Values.kubelet_conf.KUBELET_DIR }}
           - --enable-tproxy={{ .Values.func.ENABLE_TPROXY }}
           - --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }}
+          - --secure-serving={{- .Values.func.SECURE_SERVING }}
         securityContext:
           runAsUser: 0
           privileged: false
@@ -104,6 +105,14 @@ spec:
             value: "{{- .Values.performance.MODULES }}"
           - name: RPMS
             value: "{{- .Values.performance.RPMS }}"
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
           - name: POD_IPS
             valueFrom:
               fieldRef:

diff --git a/charts/kube-ovn/values.yaml b/charts/kube-ovn/values.yaml
@@ -64,6 +64,7 @@ func:
   LS_DNAT_MOD_DL_DST: true
   LS_CT_SKIP_DST_LPORT_IPS: true
   ENABLE_BIND_LOCAL_IP: true
+  SECURE_SERVING: false
   U2O_INTERCONNECTION: false
   ENABLE_TPROXY: false
   ENABLE_IC: false

diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go
@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -23,11 +22,15 @@ import (
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/controller"
+	"github.com/kubeovn/kube-ovn/pkg/server"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
-const ovnLeaderResource = "kube-ovn-controller"
+const (
+	svcName           = "kube-ovn-controller"
+	ovnLeaderResource = "kube-ovn-controller"
+)
 
 func CmdMain() {
 	defer klog.Flush()
@@ -68,27 +71,21 @@ func CmdMain() {
 			mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
 		}
 
-		addr := "0.0.0.0"
-		if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
-			podIpsEnv := os.Getenv("POD_IPS")
-			podIps := strings.Split(podIpsEnv, ",")
-			// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
-			// so not support bind local ip when in dual mode
-			if len(podIps) == 1 {
-				addr = podIps[0]
-				if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
-					addr = fmt.Sprintf("[%s]", podIps[0])
-				}
+		addr := util.JoinHostPort(util.GetDefaultListenAddr(), config.PprofPort)
+		if !config.SecureServing {
+			server := &http.Server{
+				Addr:              addr,
+				ReadHeaderTimeout: 3 * time.Second,
+				Handler:           mux,
 			}
+			util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
+		} else {
+			ch, err := server.SecureServing(addr, svcName, mux)
+			if err != nil {
+				util.LogFatalAndExit(err, "failed to serve on %s", addr)
+			}
+			<-ch
 		}
-		// conform to Gosec G114
-		// https://github.com/securego/gosec#available-rules
-		server := &http.Server{
-			Addr:              fmt.Sprintf("%s:%d", addr, config.PprofPort),
-			ReadHeaderTimeout: 3 * time.Second,
-			Handler:           mux,
-		}
-		util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
 	}()
 
 	//	ctx, cancel := context.WithCancel(context.Background())

diff --git a/cmd/controller_health_check/controller_health_check.go b/cmd/controller_health_check/controller_health_check.go
@@ -1,29 +1,17 @@
 package controller_health_check
 
 import (
-	"fmt"
 	"net"
 	"os"
-	"strings"
 	"time"
 
-	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 )
 
 func CmdMain() {
 	addr := "127.0.0.1:10660"
 	if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
-		podIpsEnv := os.Getenv("POD_IPS")
-		podIps := strings.Split(podIpsEnv, ",")
-		// when pod in dual mode, golang can't support bind v4 and v6 address in the same time,
-		// so not support bind local ip when in dual mode
-		if len(podIps) == 1 {
-			addr = fmt.Sprintf("%s:10660", podIps[0])
-			if util.CheckProtocol(podIps[0]) == kubeovnv1.ProtocolIPv6 {
-				addr = fmt.Sprintf("[%s]:10660", podIps[0])
-			}
-		}
+		addr = util.JoinHostPort(os.Getenv("POD_IP"), 10660)
 	}
 
 	conn, err := net.DialTimeout("tcp", addr, 3*time.Second)

diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go
@@ -18,10 +18,13 @@ import (
 	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
 	"github.com/kubeovn/kube-ovn/pkg/daemon"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
+	"github.com/kubeovn/kube-ovn/pkg/server"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
+const svcName = "kube-ovn-cni"
+
 func CmdMain() {
 	defer klog.Flush()
 
@@ -96,31 +99,37 @@ func CmdMain() {
 	}
 
 	addr := util.GetDefaultListenAddr()
-
 	if config.EnableVerboseConnCheck {
 		go func() {
-			connListenaddr := fmt.Sprintf("%s:%d", addr, config.TCPConnCheckPort)
+			connListenaddr := util.JoinHostPort(addr, config.TCPConnCheckPort)
 			if err := util.TCPConnectivityListen(connListenaddr); err != nil {
 				util.LogFatalAndExit(err, "failed to start TCP listen on addr %s ", addr)
 			}
 		}()
 
 		go func() {
-			connListenaddr := fmt.Sprintf("%s:%d", addr, config.UDPConnCheckPort)
+			connListenaddr := util.JoinHostPort(addr, config.UDPConnCheckPort)
 			if err := util.UDPConnectivityListen(connListenaddr); err != nil {
 				util.LogFatalAndExit(err, "failed to start UDP listen on addr %s ", addr)
 			}
 		}()
 	}
 
-	// conform to Gosec G114
-	// https://github.com/securego/gosec#available-rules
-	server := &http.Server{
-		Addr:              fmt.Sprintf("%s:%d", addr, config.PprofPort),
-		ReadHeaderTimeout: 3 * time.Second,
-		Handler:           mux,
+	listenAddr := util.JoinHostPort(addr, config.PprofPort)
+	if !config.SecureServing {
+		server := &http.Server{
+			Addr:              listenAddr,
+			ReadHeaderTimeout: 3 * time.Second,
+			Handler:           mux,
+		}
+		util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
+	} else {
+		ch, err := server.SecureServing(listenAddr, svcName, mux)
+		if err != nil {
+			util.LogFatalAndExit(err, "failed to serve on %s", listenAddr)
+		}
+		<-ch
 	}
-	util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and serve on %s", server.Addr)
 }
 
 func mvCNIConf(configDir, configFile, confName string) error {