Add API server cert controller

This change adds certificate controller to Theia manager. The public key
of API server TLS in case of self-signed, or CA cert in case of user
provided TLS, will be exposed to clients via configmap "theia-ca" in
flow-visibility namespace. This will allow cURL or client requests to be
made in "secure" fashion if the ca cert is added to trust chain.

The configmap will be updated when user provided TLS bundle is changed,
or the self-signed cert is rotated upon expiration.

Signed-off-by: Shawn Wang <wshaoquan@vmware.com>

build/charts/theia/README.md

@@ -66,6 +66,7 @@ Kubernetes: `>= 1.16.0-0`
 | sparkOperator.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-spark-operator","tag":"v1beta2-1.3.3-3.1.1"}` | Container image used by Spark Operator. |
 | sparkOperator.name | string | `"policy-recommendation"` | Name of Spark Operator. |
 | theiaManager.apiServer.apiPort | int | `11347` | The port for the Theia Manager APIServer to serve on. |
+| theiaManager.apiServer.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "theia-manager-tls" must be provided with the following keys: ca.crt, tls.crt, tls.key. |
 | theiaManager.apiServer.tlsCipherSuites | string | `""` | Comma-separated list of cipher suites that will be used by the Theia Manager APIservers. If empty, the default Go Cipher Suites will be used. |
 | theiaManager.apiServer.tlsMinVersion | string | `""` | TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. |
 | theiaManager.enable | bool | `false` | Determine whether to install Theia Manager. |

build/charts/theia/conf/theia-manager.conf

@@ -3,6 +3,13 @@ apiServer:
   # The port for the theia-manager APIServer to serve on.
   apiPort: {{ .Values.theiaManager.apiServer.apiPort }}
 
+  # Indicates whether to use auto-generated self-signed TLS certificate.
+  # If false, a Secret named "theia-manager-tls" must be provided with the following keys:
+  #   ca.crt: <CA certificate>
+  #   tls.crt: <TLS certificate>
+  #   tls.key: <TLS private key>
+  selfSignedCert: {{ .Values.theiaManager.apiServer.selfSignedCert }}
+
   # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used.
   # https://golang.org/pkg/crypto/tls/#pkg-constants
   # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always

build/charts/theia/templates/theia-cli/clusterrole.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.theiaManager.enable }}
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -12,3 +13,4 @@ rules:
     verbs:
       - get
       - list
+{{- end }}

build/charts/theia/templates/theia-cli/clusterrolebinding.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.theiaManager.enable }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -12,3 +13,4 @@ subjects:
   - kind: ServiceAccount
     name: theia-cli
     namespace: {{ .Release.Namespace }}
+{{- end }}

build/charts/theia/templates/theia-cli/secret.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.theiaManager.enable }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -6,3 +7,4 @@ metadata:
   annotations:
     kubernetes.io/service-account.name: theia-cli
 type: kubernetes.io/service-account-token
+{{- end }}

build/charts/theia/templates/theia-cli/serviceaccount.yaml

@@ -1,7 +1,9 @@
+{{- if .Values.theiaManager.enable }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: theia-cli
   namespace: {{ .Release.Namespace }}
   labels:
     app: theia-cli
+{{- end }}

build/charts/theia/templates/theia-manager/clusterrole.yaml

@@ -18,6 +18,21 @@ rules:
       - subjectaccessreviews
     verbs:
       - create
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    resourceNames:
+      - theia-ca
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
   # This is the content of built-in role kube-system/extension-apiserver-authentication-reader.
   # But it doesn't have list/watch permission before K8s v1.17.0 so the extension apiserver (antrea-agent) will
   # have permission issue after bumping up apiserver library to a version that supports dynamic authentication.

build/charts/theia/templates/theia-manager/deployment.yaml

@@ -47,6 +47,8 @@ spec:
             - mountPath: /etc/theia-manager
               name: theia-manager-config
               readOnly: true
+            - mountPath: /var/run/theia/theia-manager-tls
+              name: theia-manager-tls
             - mountPath: /var/log/antrea/theia-manager
               name: host-var-log-antrea-theia-manager
       nodeSelector:
@@ -57,6 +59,12 @@ spec:
         - name: theia-manager-config
           configMap:
             name: theia-manager-configmap
+        # Make it optional as we only read it when selfSignedCert=false.
+        - name: theia-manager-tls
+          secret:
+            secretName: theia-manager-tls
+            defaultMode: 0400
+            optional: true
         - name: host-var-log-antrea-theia-manager
           hostPath:
             path: /var/log/antrea/theia-manager

build/charts/theia/values.yaml

@@ -224,6 +224,10 @@ theiaManager:
   apiServer:
     # -- The port for the Theia Manager APIServer to serve on.
     apiPort: 11347
+    # -- Indicates whether to use auto-generated self-signed TLS certificates. If
+    # false, a Secret named "theia-manager-tls" must be provided with the
+    # following keys: ca.crt, tls.crt, tls.key.
+    selfSignedCert: true
     # -- Comma-separated list of cipher suites that will be used by the Theia Manager
     # APIservers. If empty, the default Go Cipher Suites will be used.
     tlsCipherSuites: ""

build/yamls/flow-visibility.yml

@@ -18,14 +18,6 @@ metadata:
   name: grafana
   namespace: flow-visibility
 ---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  labels:
-    app: theia-cli
-  name: theia-cli
-  namespace: flow-visibility
----
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -44,21 +36,6 @@ rules:
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app: theia-cli
-  name: theia-cli
-rules:
-- apiGroups:
-  - intelligence.theia.antrea.io
-  resources:
-  - networkpolicyrecommendations
-  verbs:
-  - get
-  - list
----
-apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   labels:
@@ -74,21 +51,6 @@ subjects:
   name: grafana
   namespace: flow-visibility
 ---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    app: theia-cli
-  name: theia-cli
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: theia-cli
-subjects:
-- kind: ServiceAccount
-  name: theia-cli
-  namespace: flow-visibility
----
 apiVersion: v1
 data:
   0-3-0_0-2-0.sql: |
@@ -5898,15 +5860,6 @@ stringData:
 type: Opaque
 ---
 apiVersion: v1
-kind: Secret
-metadata:
-  annotations:
-    kubernetes.io/service-account.name: theia-cli
-  name: theia-cli-account-token
-  namespace: flow-visibility
-type: kubernetes.io/service-account-token
----
-apiVersion: v1
 kind: Service
 metadata:
   name: grafana

cmd/theia-manager/options.go

@@ -81,4 +81,11 @@ func (o *Options) setDefaults() {
 	if o.config.APIServer.APIPort == 0 {
 		o.config.APIServer.APIPort = apis.TheiaManagerAPIPort
 	}
+	if o.config.APIServer.SelfSignedCert == nil {
+		o.config.APIServer.SelfSignedCert = ptrBool(true)
+	}
+}
+
+func ptrBool(value bool) *bool {
+	return &value
 }

cmd/theia-manager/theia-manager.go

@@ -15,39 +15,106 @@
 package main
 
 import (
+	"context"
 	"fmt"
+	"net"
+	"os"
+	"path"
 	"time"
 
 	"antrea.io/antrea/pkg/log"
 	"antrea.io/antrea/pkg/signals"
 	"antrea.io/antrea/pkg/util/cipher"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
+	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 
 	"antrea.io/theia/pkg/apiserver"
+	"antrea.io/theia/pkg/apiserver/certificate"
 	crdclientset "antrea.io/theia/pkg/client/clientset/versioned"
 	crdinformers "antrea.io/theia/pkg/client/informers/externalversions"
 	"antrea.io/theia/pkg/controller/networkpolicyrecommendation"
+	"antrea.io/theia/pkg/querier"
 )
 
 // informerDefaultResync is the default resync period if a handler doesn't specify one.
 // Use the same default value as kube-controller-manager:
 // https://github.com/kubernetes/kubernetes/blob/release-1.17/pkg/controller/apis/config/v1alpha1/defaults.go#L120
 const informerDefaultResync = 12 * time.Hour
 
+func createAPIServerConfig(
+	client clientset.Interface,
+	selfSignedCert bool,
+	bindPort int,
+	cipherSuites []uint16,
+	tlsMinVersion uint16,
+	nprq querier.NPRecommendationQuerier) (*apiserver.Config, error) {
+	secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
+	authentication := genericoptions.NewDelegatingAuthenticationOptions()
+	authorization := genericoptions.NewDelegatingAuthorizationOptions()
+
+	caCertController, err := certificate.ApplyServerCert(selfSignedCert, client, secureServing, apiserver.DefaultCAConfig())
+	if err != nil {
+		return nil, fmt.Errorf("error applying server cert: %v", err)
+	}
+
+	secureServing.BindAddress = net.IPv4zero
+	secureServing.BindPort = bindPort
+
+	authentication.WithRequestTimeout(apiserver.AuthenticationTimeout)
+
+	serverConfig := genericapiserver.NewConfig(apiserver.Codecs)
+	if err := secureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil {
+		return nil, err
+	}
+	if err := authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil {
+		return nil, err
+	}
+	if err := authorization.ApplyTo(&serverConfig.Authorization); err != nil {
+		return nil, err
+	}
+
+	if err := os.MkdirAll(path.Dir(apiserver.TokenPath), os.ModeDir); err != nil {
+		return nil, fmt.Errorf("error when creating dirs of token file: %v", err)
+	}
+	if err := os.WriteFile(apiserver.TokenPath, []byte(serverConfig.LoopbackClientConfig.BearerToken), 0600); err != nil {
+		return nil, fmt.Errorf("error when writing loopback access token to file: %v", err)
+	}
+
+	serverConfig.SecureServing.CipherSuites = cipherSuites
+	serverConfig.SecureServing.MinTLSVersion = tlsMinVersion
+
+	return apiserver.NewConfig(
+		serverConfig,
+		client,
+		caCertController,
+		nprq), nil
+}
+
 func run(o *Options) error {
 	klog.InfoS("Theia manager starting...")
 	// Set up signal capture: the first SIGTERM / SIGINT signal is handled gracefully and will
 	// cause the stopCh channel to be closed; if another signal is received before the program
 	// exits, we will force exit.
 	stopCh := signals.RegisterSignalHandlers()
+	// Generate a context for functions which require one (instead of stopCh).
+	// We cancel the context when the function returns, which in the normal case will be when
+	// stopCh is closed.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	log.StartLogFileNumberMonitor(stopCh)
 
 	kubeConfig, err := rest.InClusterConfig()
 	if err != nil {
 		return fmt.Errorf("error when generating KubeConfig: %v", err)
 	}
+	client, err := clientset.NewForConfig(kubeConfig)
+	if err != nil {
+		return fmt.Errorf("error when generating k8s client: %v", err)
+	}
 	crdClient, err := crdclientset.NewForConfig(kubeConfig)
 	if err != nil {
 		return fmt.Errorf("error when generating CRD client: %v", err)
@@ -60,18 +127,25 @@ func run(o *Options) error {
 	if err != nil {
 		return fmt.Errorf("error when generating Cipher Suite list: %v", err)
 	}
-	apiServer, err := apiserver.New(
-		npRecoController,
+
+	apiServerConfig, err := createAPIServerConfig(
+		client,
+		*o.config.APIServer.SelfSignedCert,
 		o.config.APIServer.APIPort,
 		cipherSuites,
-		cipher.TLSVersionMap[o.config.APIServer.TLSMinVersion])
+		cipher.TLSVersionMap[o.config.APIServer.TLSMinVersion],
+		npRecoController)
+	if err != nil {
+		return fmt.Errorf("error creating API server config: %v", err)
+	}
+	apiServer, err := apiServerConfig.New()
 	if err != nil {
 		return fmt.Errorf("error when creating API server: %v", err)
 	}
 
 	crdInformerFactory.Start(stopCh)
 	go npRecoController.Run(stopCh)
-	go apiServer.Run(stopCh)
+	go apiServer.Run(ctx)
 
 	<-stopCh
 	klog.InfoS("Stopping theia manager")