add support for serving metrics with authn/authz

Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
kubeovn · Jul 17, 2024 · a3d41a8 · a3d41a8
1 parent bf76ed3
commit a3d41a8
Show file tree

Hide file tree

Showing 4 changed files with 137 additions and 6 deletions.
diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml
@@ -248,7 +248,18 @@ rules:
       - get
       - list
       - watch
-
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -271,3 +282,15 @@ rules:
       - daemonsets
     verbs:
       - get
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
diff --git a/charts/kube-ovn/templates/ovn-CRB.yaml b/charts/kube-ovn/templates/ovn-CRB.yaml
@@ -38,7 +38,20 @@ subjects:
   - kind: ServiceAccount
     name: kube-ovn-cni
     namespace: {{ .Values.namespace }}
-
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-cni
+    namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -52,3 +65,17 @@ subjects:
   - kind: ServiceAccount
     name: kube-ovn-app
     namespace: {{ .Values.namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-app
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-app
+    namespace: kube-system
diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -3170,6 +3170,18 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -3183,6 +3195,20 @@ subjects:
   - kind: ServiceAccount
     name: kube-ovn-cni
     namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-cni
+    namespace: kube-system
 EOF
 
 cat <<EOF > kube-ovn-app-sa.yaml
@@ -3214,6 +3240,18 @@ rules:
       - daemonsets
     verbs:
       - get
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -3227,6 +3265,20 @@ subjects:
   - kind: ServiceAccount
     name: kube-ovn-app
     namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-app
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: kube-ovn-app
+    namespace: kube-system
 EOF
 
 kubectl apply -f kube-ovn-crd.yaml

diff --git a/pkg/server/server.go b/pkg/server/server.go
@@ -8,9 +8,14 @@ import (
 	"strconv"
 	"strings"
 
+	"k8s.io/apiserver/pkg/endpoints/filters"
+	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/options"
+	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
+
+	"github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme"
 )
 
 func SecureServing(addr, svcName string, handler http.Handler) (<-chan struct{}, error) {
@@ -31,9 +36,15 @@ func SecureServing(addr, svcName string, handler http.Handler) (<-chan struct{},
 		}
 	}
 
-	opt := options.NewSecureServingOptions()
+	var clientConfig *rest.Config
+	opt := options.NewSecureServingOptions().WithLoopback()
+	authnOpt := options.NewDelegatingAuthenticationOptions()
+	authzOpt := options.NewDelegatingAuthorizationOptions()
 	opt.ServerCert.PairName = svcName
 	opt.ServerCert.CertDirectory = ""
+	authnOpt.RemoteKubeConfigFileOptional = true
+	authzOpt.RemoteKubeConfigFileOptional = true
+
 	if host != "" {
 		ip := net.ParseIP(host)
 		if ip == nil {
@@ -55,14 +66,32 @@ func SecureServing(addr, svcName string, handler http.Handler) (<-chan struct{},
 		return nil, fmt.Errorf("failed to genarate self signed certificates: %v", err)
 	}
 
-	var c *server.SecureServingInfo
-	if err = opt.ApplyTo(&c); err != nil {
+	var serving *server.SecureServingInfo
+	var authn server.AuthenticationInfo
+	var authz server.AuthorizationInfo
+	if err = opt.ApplyTo(&serving, &clientConfig); err != nil {
 		klog.Error(err)
 		return nil, fmt.Errorf("failed to apply secure serving options to secure serving info: %v", err)
 	}
+	if err = authnOpt.ApplyTo(&authn, serving, nil); err != nil {
+		klog.Error(err)
+		return nil, fmt.Errorf("failed to apply authn options to authn info: %v", err)
+	}
+	if err = authzOpt.ApplyTo(&authz); err != nil {
+		klog.Error(err)
+		return nil, fmt.Errorf("failed to apply authz options to authz info: %v", err)
+	}
+
+	handler = filters.WithAuthorization(handler, authz.Authorizer, scheme.Codecs)
+	handler = filters.WithAuthentication(handler, authn.Authenticator, filters.Unauthorized(scheme.Codecs), nil, nil)
+
+	requestInfoResolver := &request.RequestInfoFactory{}
+	handler = filters.WithRequestInfo(handler, requestInfoResolver)
+	handler = filters.WithCacheControl(handler)
+	server.AuthorizeClientBearerToken(clientConfig, &authn, &authz)
 
 	stopCh := make(chan struct{}, 1)
-	_, listenerStoppedCh, err := c.Serve(handler, 0, stopCh)
+	_, listenerStoppedCh, err := serving.Serve(handler, 0, stopCh)
 	if err != nil {
 		klog.Error(err)
 		return nil, fmt.Errorf("failed to serve on %s: %v", addr, err)