Add RBAC auth and CA controller to Theia manager

build/charts/theia/README.md

@@ -66,6 +66,7 @@ Kubernetes: `>= 1.16.0-0`
 | sparkOperator.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-spark-operator","tag":"v1beta2-1.3.3-3.1.1"}` | Container image used by Spark Operator. |
 | sparkOperator.name | string | `"policy-recommendation"` | Name of Spark Operator. |
 | theiaManager.apiServer.apiPort | int | `11347` | The port for the Theia Manager APIServer to serve on. |
+| theiaManager.apiServer.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "theia-manager-tls" must be provided with the following keys: ca.crt, tls.crt, tls.key. |
 | theiaManager.apiServer.tlsCipherSuites | string | `""` | Comma-separated list of cipher suites that will be used by the Theia Manager APIservers. If empty, the default Go Cipher Suites will be used. |
 | theiaManager.apiServer.tlsMinVersion | string | `""` | TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. |
 | theiaManager.enable | bool | `false` | Determine whether to install Theia Manager. |

build/charts/theia/conf/theia-manager.conf

@@ -3,6 +3,13 @@ apiServer:
   # The port for the theia-manager APIServer to serve on.
   apiPort: {{ .Values.theiaManager.apiServer.apiPort }}
 
+  # Indicates whether to use auto-generated self-signed TLS certificate.
+  # If false, a Secret named "theia-manager-tls" must be provided with the following keys:
+  #   ca.crt: <CA certificate>
+  #   tls.crt: <TLS certificate>
+  #   tls.key: <TLS private key>
+  selfSignedCert: {{ .Values.theiaManager.apiServer.selfSignedCert }}
+
   # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used.
   # https://golang.org/pkg/crypto/tls/#pkg-constants
   # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always

build/charts/theia/templates/theia-cli/clusterrole.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.theiaManager.enable }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: theia-cli
+  labels:
+    app: theia-cli
+rules:
+  - apiGroups:
+      - intelligence.theia.antrea.io
+    resources:
+      - networkpolicyrecommendations
+    verbs:
+      - get
+      - list
+{{- end }}

build/charts/theia/templates/theia-cli/clusterrolebinding.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.theiaManager.enable }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: theia-cli
+  name: theia-cli
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: theia-cli
+subjects:
+  - kind: ServiceAccount
+    name: theia-cli
+    namespace: {{ .Release.Namespace }}
+{{- end }}

build/charts/theia/templates/theia-cli/secret.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.theiaManager.enable }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: theia-cli-account-token
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    kubernetes.io/service-account.name: theia-cli
+type: kubernetes.io/service-account-token
+{{- end }}

build/charts/theia/templates/theia-cli/serviceaccount.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.theiaManager.enable }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: theia-cli
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: theia-cli
+{{- end }}

build/charts/theia/templates/theia-manager/clusterrole.yaml

@@ -6,6 +6,33 @@ metadata:
     app: theia-manager
   name: theia-manager-role
 rules:
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    resourceNames:
+      - theia-ca
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
   # This is the content of built-in role kube-system/extension-apiserver-authentication-reader.
   # But it doesn't have list/watch permission before K8s v1.17.0 so the extension apiserver (antrea-agent) will
   # have permission issue after bumping up apiserver library to a version that supports dynamic authentication.

build/charts/theia/templates/theia-manager/deployment.yaml

@@ -47,6 +47,8 @@ spec:
             - mountPath: /etc/theia-manager
               name: theia-manager-config
               readOnly: true
+            - mountPath: /var/run/theia/theia-manager-tls
+              name: theia-manager-tls
             - mountPath: /var/log/antrea/theia-manager
               name: host-var-log-antrea-theia-manager
       nodeSelector:
@@ -57,6 +59,12 @@ spec:
         - name: theia-manager-config
           configMap:
             name: theia-manager-configmap
+        # Make it optional as we only read it when selfSignedCert=false.
+        - name: theia-manager-tls
+          secret:
+            secretName: theia-manager-tls
+            defaultMode: 0400
+            optional: true
         - name: host-var-log-antrea-theia-manager
           hostPath:
             path: /var/log/antrea/theia-manager

build/charts/theia/values.yaml

@@ -224,6 +224,10 @@ theiaManager:
   apiServer:
     # -- The port for the Theia Manager APIServer to serve on.
     apiPort: 11347
+    # -- Indicates whether to use auto-generated self-signed TLS certificates. If
+    # false, a Secret named "theia-manager-tls" must be provided with the
+    # following keys: ca.crt, tls.crt, tls.key.
+    selfSignedCert: true
     # -- Comma-separated list of cipher suites that will be used by the Theia Manager
     # APIservers. If empty, the default Go Cipher Suites will be used.
     tlsCipherSuites: ""

cmd/theia-manager/options.go

@@ -81,4 +81,11 @@ func (o *Options) setDefaults() {
 	if o.config.APIServer.APIPort == 0 {
 		o.config.APIServer.APIPort = apis.TheiaManagerAPIPort
 	}
+	if o.config.APIServer.SelfSignedCert == nil {
+		o.config.APIServer.SelfSignedCert = ptrBool(true)
+	}
+}
+
+func ptrBool(value bool) *bool {
+	return &value
 }

cmd/theia-manager/theia-manager.go

@@ -15,39 +15,106 @@
 package main
 
 import (
+	"context"
 	"fmt"
+	"net"
+	"os"
+	"path"
 	"time"
 
 	"antrea.io/antrea/pkg/log"
 	"antrea.io/antrea/pkg/signals"
 	"antrea.io/antrea/pkg/util/cipher"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 
 	"antrea.io/theia/pkg/apiserver"
+	"antrea.io/theia/pkg/apiserver/certificate"
 	crdclientset "antrea.io/theia/pkg/client/clientset/versioned"
 	crdinformers "antrea.io/theia/pkg/client/informers/externalversions"
 	"antrea.io/theia/pkg/controller/networkpolicyrecommendation"
+	"antrea.io/theia/pkg/querier"
 )
 
 // informerDefaultResync is the default resync period if a handler doesn't specify one.
 // Use the same default value as kube-controller-manager:
 // https://github.com/kubernetes/kubernetes/blob/release-1.17/pkg/controller/apis/config/v1alpha1/defaults.go#L120
 const informerDefaultResync = 12 * time.Hour
 
+func createAPIServerConfig(
+	client clientset.Interface,
+	selfSignedCert bool,
+	bindPort int,
+	cipherSuites []uint16,
+	tlsMinVersion uint16,
+	nprq querier.NPRecommendationQuerier) (*apiserver.Config, error) {
+	secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
+	authentication := genericoptions.NewDelegatingAuthenticationOptions()
+	authorization := genericoptions.NewDelegatingAuthorizationOptions()
+
+	caCertController, err := certificate.ApplyServerCert(selfSignedCert, client, secureServing, apiserver.DefaultCAConfig())
+	if err != nil {
+		return nil, fmt.Errorf("error applying server cert: %v", err)
+	}
+
+	secureServing.BindAddress = net.IPv4zero
+	secureServing.BindPort = bindPort
+
+	authentication.WithRequestTimeout(apiserver.AuthenticationTimeout)
+
+	serverConfig := genericapiserver.NewConfig(apiserver.Codecs)
+	if err := secureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil {
+		return nil, err
+	}
+	if err := authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil {
+		return nil, err
+	}
+	if err := authorization.ApplyTo(&serverConfig.Authorization); err != nil {
+		return nil, err
+	}
+
+	if err := os.MkdirAll(path.Dir(apiserver.TokenPath), os.ModeDir); err != nil {
+		return nil, fmt.Errorf("error when creating dirs of token file: %v", err)
+	}
+	if err := os.WriteFile(apiserver.TokenPath, []byte(serverConfig.LoopbackClientConfig.BearerToken), 0600); err != nil {
+		return nil, fmt.Errorf("error when writing loopback access token to file: %v", err)
+	}
+
+	serverConfig.SecureServing.CipherSuites = cipherSuites
+	serverConfig.SecureServing.MinTLSVersion = tlsMinVersion
+
+	return apiserver.NewConfig(
+		serverConfig,
+		client,
+		caCertController,
+		nprq), nil
+}
+
 func run(o *Options) error {
 	klog.InfoS("Theia manager starting...")
 	// Set up signal capture: the first SIGTERM / SIGINT signal is handled gracefully and will
 	// cause the stopCh channel to be closed; if another signal is received before the program
 	// exits, we will force exit.
 	stopCh := signals.RegisterSignalHandlers()
+	// Generate a context for functions which require one (instead of stopCh).
+	// We cancel the context when the function returns, which in the normal case will be when
+	// stopCh is closed.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	log.StartLogFileNumberMonitor(stopCh)
 
 	kubeConfig, err := rest.InClusterConfig()
 	if err != nil {
 		return fmt.Errorf("error when generating KubeConfig: %v", err)
 	}
+	client, err := clientset.NewForConfig(kubeConfig)
+	if err != nil {
+		return fmt.Errorf("error when generating k8s client: %v", err)
+	}
 	crdClient, err := crdclientset.NewForConfig(kubeConfig)
 	if err != nil {
 		return fmt.Errorf("error when generating CRD client: %v", err)
@@ -60,18 +127,25 @@ func run(o *Options) error {
 	if err != nil {
 		return fmt.Errorf("error when generating Cipher Suite list: %v", err)
 	}
-	apiServer, err := apiserver.New(
-		npRecoController,
+
+	apiServerConfig, err := createAPIServerConfig(
+		client,
+		*o.config.APIServer.SelfSignedCert,
 		o.config.APIServer.APIPort,
 		cipherSuites,
-		cipher.TLSVersionMap[o.config.APIServer.TLSMinVersion])
+		cipher.TLSVersionMap[o.config.APIServer.TLSMinVersion],
+		npRecoController)
+	if err != nil {
+		return fmt.Errorf("error creating API server config: %v", err)
+	}
+	apiServer, err := apiServerConfig.New()
 	if err != nil {
 		return fmt.Errorf("error when creating API server: %v", err)
 	}
 
 	crdInformerFactory.Start(stopCh)
 	go npRecoController.Run(stopCh)
-	go apiServer.Run(stopCh)
+	go apiServer.Run(ctx)
 
 	<-stopCh
 	klog.InfoS("Stopping theia manager")