refactor metrics

charts/kube-ovn/templates/ovn-CR.yaml

@@ -163,7 +163,18 @@ rules:
       - get
       - list
       - watch
-
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

charts/kube-ovn/templates/ovn-CRB.yaml

@@ -10,7 +10,20 @@ subjects:
   - kind: ServiceAccount
     name: ovn
     namespace: {{ .Values.namespace }}
-
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: ovn
+  namespace: {{ .Values.namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: ovn
+    namespace: {{ .Values.namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -43,15 +56,15 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: kube-ovn-cni
-  namespace: kube-system
+  namespace: {{ .Values.namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: extension-apiserver-authentication-reader
 subjects:
   - kind: ServiceAccount
     name: kube-ovn-cni
-    namespace: kube-system
+    namespace: {{ .Values.namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -70,12 +83,12 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: kube-ovn-app
-  namespace: kube-system
+  namespace: {{ .Values.namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: extension-apiserver-authentication-reader
 subjects:
   - kind: ServiceAccount
     name: kube-ovn-app
-    namespace: kube-system
+    namespace: {{ .Values.namespace }}

cmd/controller/controller.go

@@ -3,12 +3,12 @@ package controller
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"net/http/pprof"
 	"os"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	v1 "k8s.io/api/authorization/v1"
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -18,36 +18,25 @@ import (
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/controller"
-	"github.com/kubeovn/kube-ovn/pkg/server"
+	"github.com/kubeovn/kube-ovn/pkg/metrics"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
-const (
-	svcName           = "kube-ovn-controller"
-	ovnLeaderResource = "kube-ovn-controller"
-)
+const ovnLeaderResource = "kube-ovn-controller"
 
 func CmdMain() {
 	defer klog.Flush()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go func() {
-		stopCh := signals.SetupSignalHandler().Done()
-		<-stopCh
-		cancel()
-	}()
+	ctx := signals.SetupSignalHandler()
 
 	klog.Infof(versions.String())
 
-	controller.InitClientGoMetrics()
-	controller.InitWorkQueueMetrics()
-	util.InitKlogMetrics()
 	config, err := controller.ParseFlags()
 	if err != nil {
 		util.LogFatalAndExit(err, "failed to parse config")
@@ -59,36 +48,47 @@ func CmdMain() {
 	utilruntime.Must(kubeovnv1.AddToScheme(scheme.Scheme))
 
 	go func() {
-		mux := http.NewServeMux()
-		if config.EnableMetrics {
-			mux.Handle("/metrics", promhttp.Handler())
-		}
 		if config.EnablePprof {
+			mux := http.NewServeMux()
 			mux.HandleFunc("/debug/pprof/", pprof.Index)
 			mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
 			mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
 			mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
 			mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-		}
 
-		addr := util.JoinHostPort(util.GetDefaultListenAddr(), config.PprofPort)
-		if !config.SecureServing {
-			server := &http.Server{
-				Addr:              addr,
-				ReadHeaderTimeout: 3 * time.Second,
-				Handler:           mux,
-			}
-			util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
-		} else {
-			ch, err := server.SecureServing(addr, svcName, mux)
+			listerner, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(config.PprofPort)})
 			if err != nil {
-				util.LogFatalAndExit(err, "failed to serve on %s", addr)
+				util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort("127.0.0.1", config.PprofPort))
 			}
-			<-ch
+			svr := manager.Server{
+				Name: "pprof",
+				Server: &http.Server{
+					Handler:           mux,
+					MaxHeaderBytes:    1 << 20,
+					IdleTimeout:       90 * time.Second,
+					ReadHeaderTimeout: 32 * time.Second,
+				},
+				Listener: listerner,
+			}
+			go func() {
+				if err = svr.Start(ctx); err != nil {
+					util.LogFatalAndExit(err, "failed to run pprof server")
+				}
+			}()
+		}
+
+		if !config.EnableMetrics {
+			return
+		}
+		metrics.InitKlogMetrics()
+		metrics.InitClientGoMetrics()
+		addr := util.JoinHostPort(util.GetDefaultListenAddr(), config.PprofPort)
+		if err := metrics.Run(ctx, config.KubeRestConfig, addr, config.SecureServing); err != nil {
+			util.LogFatalAndExit(err, "failed to run metrics server")
 		}
+		<-ctx.Done()
 	}()
 
-	//	ctx, cancel := context.WithCancel(context.Background())
 	recorder := record.NewBroadcaster().NewRecorder(scheme.Scheme, apiv1.EventSource{
 		Component: ovnLeaderResource,
 		Host:      os.Getenv(util.HostnameEnv),

cmd/daemon/cniserver.go

@@ -2,34 +2,33 @@ package daemon
 
 import (
 	"fmt"
+	"net"
 	"net/http"
 	"net/http/pprof"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/klog/v2"
-	"k8s.io/sample-controller/pkg/signals"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
 	"github.com/kubeovn/kube-ovn/pkg/daemon"
+	"github.com/kubeovn/kube-ovn/pkg/metrics"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
-	"github.com/kubeovn/kube-ovn/pkg/server"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
-const svcName = "kube-ovn-cni"
-
 func CmdMain() {
 	defer klog.Flush()
 
 	daemon.InitMetrics()
-	util.InitKlogMetrics()
+	metrics.InitKlogMetrics()
 
 	config := daemon.ParseFlags()
 	klog.Infof(versions.String())
@@ -62,7 +61,8 @@ func CmdMain() {
 		util.LogFatalAndExit(err, "failed to do the OS initialization")
 	}
 
-	stopCh := signals.SetupSignalHandler().Done()
+	ctx := signals.SetupSignalHandler()
+	stopCh := ctx.Done()
 	podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0,
 		kubeinformers.WithTweakListOptions(func(listOption *v1.ListOptions) {
 			listOption.FieldSelector = fmt.Sprintf("spec.nodeName=%s", config.NodeName)
@@ -87,18 +87,6 @@ func CmdMain() {
 		util.LogFatalAndExit(err, "failed to mv cni config file")
 	}
 
-	mux := http.NewServeMux()
-	if config.EnableMetrics {
-		mux.Handle("/metrics", promhttp.Handler())
-	}
-	if config.EnablePprof {
-		mux.HandleFunc("/debug/pprof/", pprof.Index)
-		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
-		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-	}
-
 	addr := util.GetDefaultListenAddr()
 	if config.EnableVerboseConnCheck {
 		go func() {
@@ -116,21 +104,40 @@ func CmdMain() {
 		}()
 	}
 
-	listenAddr := util.JoinHostPort(addr, config.PprofPort)
-	if !config.SecureServing {
-		server := &http.Server{
-			Addr:              listenAddr,
-			ReadHeaderTimeout: 3 * time.Second,
-			Handler:           mux,
-		}
-		util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", server.Addr)
-	} else {
-		ch, err := server.SecureServing(listenAddr, svcName, mux)
+	if config.EnablePprof {
+		mux := http.NewServeMux()
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
+		listerner, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: int(config.PprofPort)})
 		if err != nil {
-			util.LogFatalAndExit(err, "failed to serve on %s", listenAddr)
+			util.LogFatalAndExit(err, "failed to listen on %s", util.JoinHostPort("127.0.0.1", config.PprofPort))
+		}
+		svr := manager.Server{
+			Name: "pprof",
+			Server: &http.Server{
+				Handler:           mux,
+				MaxHeaderBytes:    1 << 20,
+				IdleTimeout:       90 * time.Second,
+				ReadHeaderTimeout: 32 * time.Second,
+			},
+			Listener: listerner,
 		}
-		<-ch
+		go func() {
+			if err = svr.Start(ctx); err != nil {
+				util.LogFatalAndExit(err, "failed to run pprof server")
+			}
+		}()
+	}
+
+	listenAddr := util.JoinHostPort(addr, config.PprofPort)
+	if err = metrics.Run(ctx, nil, listenAddr, config.SecureServing); err != nil {
+		util.LogFatalAndExit(err, "failed to run metrics server")
 	}
+	<-stopCh
 }
 
 func mvCNIConf(configDir, configFile, confName string) error {

cmd/ovn_monitor/ovn_monitor.go

@@ -1,22 +1,18 @@
 package ovn_monitor
 
 import (
-	"net/http"
 	"os"
 	"strings"
-	"time"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
+	"github.com/kubeovn/kube-ovn/pkg/metrics"
 	ovn "github.com/kubeovn/kube-ovn/pkg/ovnmonitor"
-	"github.com/kubeovn/kube-ovn/pkg/server"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
 )
 
-const svcName = "kube-ovn-monitor"
-
 const port = 10661
 
 func CmdMain() {
@@ -41,24 +37,10 @@ func CmdMain() {
 		go exporter.TryClientConnection()
 	}
 	exporter.StartOvnMetrics()
-	mux := http.NewServeMux()
-	if config.EnableMetrics {
-		mux.Handle(config.MetricsPath, promhttp.Handler())
-		klog.Infoln("Listening on", addr)
-	}
 
-	if !config.SecureServing {
-		server := &http.Server{
-			Addr:              addr,
-			ReadHeaderTimeout: 3 * time.Second,
-			Handler:           mux,
-		}
-		util.LogFatalAndExit(server.ListenAndServe(), "failed to listen and server on %s", addr)
-	} else {
-		ch, err := server.SecureServing(addr, svcName, mux)
-		if err != nil {
-			util.LogFatalAndExit(err, "failed to serve on %s", addr)
-		}
-		<-ch
+	ctx := signals.SetupSignalHandler()
+	if err = metrics.Run(ctx, nil, addr, config.SecureServing); err != nil {
+		util.LogFatalAndExit(err, "failed to run metrics server")
 	}
+	<-ctx.Done()
 }