From c13b90db5f111fac426bab7a1bf90c2e92d0b556 Mon Sep 17 00:00:00 2001 From: Shawn Wang Date: Mon, 19 Sep 2022 22:47:53 -0700 Subject: [PATCH 1/2] Add RBAC auth for Theia Manager This change adds k8s auth delegation to theia manager, and adds template of cli service account / cluster role to allow access for specified API groups and resources. A SA toekn secret is also added that can be used for CLI to auth with Theia manager. Signed-off-by: Shawn Wang --- .../templates/theia-cli/clusterrole.yaml | 14 ++++++ .../theia-cli/clusterrolebinding.yaml | 14 ++++++ .../theia/templates/theia-cli/secret.yaml | 8 ++++ .../templates/theia-cli/serviceaccount.yaml | 7 +++ .../templates/theia-manager/clusterrole.yaml | 12 +++++ build/yamls/flow-visibility.yml | 47 +++++++++++++++++++ pkg/apiserver/apiserver.go | 8 ++-- 7 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 build/charts/theia/templates/theia-cli/clusterrole.yaml create mode 100644 build/charts/theia/templates/theia-cli/clusterrolebinding.yaml create mode 100644 build/charts/theia/templates/theia-cli/secret.yaml create mode 100644 build/charts/theia/templates/theia-cli/serviceaccount.yaml diff --git a/build/charts/theia/templates/theia-cli/clusterrole.yaml b/build/charts/theia/templates/theia-cli/clusterrole.yaml new file mode 100644 index 00000000..c6f9f145 --- /dev/null +++ b/build/charts/theia/templates/theia-cli/clusterrole.yaml @@ -0,0 +1,14 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: theia-cli + labels: + app: theia-cli +rules: + - apiGroups: + - intelligence.theia.antrea.io + resources: + - networkpolicyrecommendations + verbs: + - get + - list diff --git a/build/charts/theia/templates/theia-cli/clusterrolebinding.yaml b/build/charts/theia/templates/theia-cli/clusterrolebinding.yaml new file mode 100644 index 00000000..fbfa355a --- /dev/null +++ b/build/charts/theia/templates/theia-cli/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: theia-cli + name: theia-cli +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: theia-cli +subjects: + - kind: ServiceAccount + name: theia-cli + namespace: {{ .Release.Namespace }} diff --git a/build/charts/theia/templates/theia-cli/secret.yaml b/build/charts/theia/templates/theia-cli/secret.yaml new file mode 100644 index 00000000..21aac560 --- /dev/null +++ b/build/charts/theia/templates/theia-cli/secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: theia-cli-account-token + namespace: {{ .Release.Namespace }} + annotations: + kubernetes.io/service-account.name: theia-cli +type: kubernetes.io/service-account-token diff --git a/build/charts/theia/templates/theia-cli/serviceaccount.yaml b/build/charts/theia/templates/theia-cli/serviceaccount.yaml new file mode 100644 index 00000000..a865f1bf --- /dev/null +++ b/build/charts/theia/templates/theia-cli/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: theia-cli + namespace: {{ .Release.Namespace }} + labels: + app: theia-cli diff --git a/build/charts/theia/templates/theia-manager/clusterrole.yaml b/build/charts/theia/templates/theia-manager/clusterrole.yaml index ee9b57fa..b685cf69 100644 --- a/build/charts/theia/templates/theia-manager/clusterrole.yaml +++ b/build/charts/theia/templates/theia-manager/clusterrole.yaml @@ -6,6 +6,18 @@ metadata: app: theia-manager name: theia-manager-role rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create # This is the content of built-in role kube-system/extension-apiserver-authentication-reader. # But it doesn't have list/watch permission before K8s v1.17.0 so the extension apiserver (antrea-agent) will # have permission issue after bumping up apiserver library to a version that supports dynamic authentication. diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml index 5b2db121..22b5b358 100644 --- a/build/yamls/flow-visibility.yml +++ b/build/yamls/flow-visibility.yml @@ -18,6 +18,14 @@ metadata: name: grafana namespace: flow-visibility --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: theia-cli + name: theia-cli + namespace: flow-visibility +--- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -36,6 +44,21 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: theia-cli + name: theia-cli +rules: +- apiGroups: + - intelligence.theia.antrea.io + resources: + - networkpolicyrecommendations + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -51,6 +74,21 @@ subjects: name: grafana namespace: flow-visibility --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: theia-cli + name: theia-cli +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: theia-cli +subjects: +- kind: ServiceAccount + name: theia-cli + namespace: flow-visibility +--- apiVersion: v1 data: 0-3-0_0-2-0.sql: | @@ -5860,6 +5898,15 @@ stringData: type: Opaque --- apiVersion: v1 +kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: theia-cli + name: theia-cli-account-token + namespace: flow-visibility +type: kubernetes.io/service-account-token +--- +apiVersion: v1 kind: Service metadata: name: grafana diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 91ac01a1..997c11d5 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -24,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apiserver/pkg/authorization/authorizerfactory" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" @@ -74,6 +73,7 @@ func (s *theiaManagerAPIServer) Run(stopCh <-chan struct{}) error { func newConfig(bindPort int) (*genericapiserver.CompletedConfig, error) { secureServing := genericoptions.NewSecureServingOptions().WithLoopback() authentication := genericoptions.NewDelegatingAuthenticationOptions() + authorization := genericoptions.NewDelegatingAuthorizationOptions() // Set the PairName but leave certificate directory blank to generate in-memory by default. secureServing.ServerCert.CertDirectory = "" @@ -93,10 +93,10 @@ func newConfig(bindPort int) (*genericapiserver.CompletedConfig, error) { if err := authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil { return nil, err } + if err := authorization.ApplyTo(&serverConfig.Authorization); err != nil { + return nil, err + } - // TODO (wshaoquan): use native k8s role-based auth - serverConfig.Authorization = genericapiserver.AuthorizationInfo{ - Authorizer: authorizerfactory.NewAlwaysAllowAuthorizer()} if err := os.MkdirAll(path.Dir(TokenPath), os.ModeDir); err != nil { return nil, fmt.Errorf("error when creating dirs of token file: %v", err) } From dc4b28fb8769f5c547719121b49b700298b5f73f Mon Sep 17 00:00:00 2001 From: Shawn Wang Date: Tue, 20 Sep 2022 14:01:09 -0700 Subject: [PATCH 2/2] Add API server cert controller This change adds certificate controller to Theia manager. The public key of API server TLS in case of self-signed, or CA cert in case of user provided TLS, will be exposed to clients via configmap "theia-ca" in flow-visibility namespace. This will allow cURL or client requests to be made in "secure" fashion if the ca cert is added to trust chain. The configmap will be updated when user provided TLS bundle is changed, or the self-signed cert is rotated upon expiration. Signed-off-by: Shawn Wang --- build/charts/theia/README.md | 1 + build/charts/theia/conf/theia-manager.conf | 7 + .../templates/theia-cli/clusterrole.yaml | 2 + .../theia-cli/clusterrolebinding.yaml | 2 + .../theia/templates/theia-cli/secret.yaml | 2 + .../templates/theia-cli/serviceaccount.yaml | 2 + .../templates/theia-manager/clusterrole.yaml | 15 ++ .../templates/theia-manager/deployment.yaml | 8 + build/charts/theia/values.yaml | 4 + build/yamls/flow-visibility.yml | 47 ---- cmd/theia-manager/options.go | 7 + cmd/theia-manager/theia-manager.go | 82 ++++++- pkg/apiserver/apiserver.go | 124 +++++----- .../certificate/cacert_controller.go | 205 ++++++++++++++++ pkg/apiserver/certificate/certificate.go | 188 +++++++++++++++ pkg/apiserver/certificate/certificate_test.go | 222 ++++++++++++++++++ pkg/apiserver/certificate/config.go | 45 ++++ pkg/config/theiamanager/config.go | 7 + pkg/util/env/env.go | 49 ++++ pkg/util/env/env_test.go | 65 +++++ 20 files changed, 975 insertions(+), 109 deletions(-) create mode 100644 pkg/apiserver/certificate/cacert_controller.go create mode 100644 pkg/apiserver/certificate/certificate.go create mode 100644 pkg/apiserver/certificate/certificate_test.go create mode 100644 pkg/apiserver/certificate/config.go create mode 100644 pkg/util/env/env.go create mode 100644 pkg/util/env/env_test.go diff --git a/build/charts/theia/README.md b/build/charts/theia/README.md index bfd75e75..4b55158f 100644 --- a/build/charts/theia/README.md +++ b/build/charts/theia/README.md @@ -66,6 +66,7 @@ Kubernetes: `>= 1.16.0-0` | sparkOperator.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-spark-operator","tag":"v1beta2-1.3.3-3.1.1"}` | Container image used by Spark Operator. | | sparkOperator.name | string | `"policy-recommendation"` | Name of Spark Operator. | | theiaManager.apiServer.apiPort | int | `11347` | The port for the Theia Manager APIServer to serve on. | +| theiaManager.apiServer.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "theia-manager-tls" must be provided with the following keys: ca.crt, tls.crt, tls.key. | | theiaManager.apiServer.tlsCipherSuites | string | `""` | Comma-separated list of cipher suites that will be used by the Theia Manager APIservers. If empty, the default Go Cipher Suites will be used. | | theiaManager.apiServer.tlsMinVersion | string | `""` | TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. | | theiaManager.enable | bool | `false` | Determine whether to install Theia Manager. | diff --git a/build/charts/theia/conf/theia-manager.conf b/build/charts/theia/conf/theia-manager.conf index a040ce86..a258f54d 100644 --- a/build/charts/theia/conf/theia-manager.conf +++ b/build/charts/theia/conf/theia-manager.conf @@ -3,6 +3,13 @@ apiServer: # The port for the theia-manager APIServer to serve on. apiPort: {{ .Values.theiaManager.apiServer.apiPort }} + # Indicates whether to use auto-generated self-signed TLS certificate. + # If false, a Secret named "theia-manager-tls" must be provided with the following keys: + # ca.crt: + # tls.crt: + # tls.key: + selfSignedCert: {{ .Values.theiaManager.apiServer.selfSignedCert }} + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. # https://golang.org/pkg/crypto/tls/#pkg-constants # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always diff --git a/build/charts/theia/templates/theia-cli/clusterrole.yaml b/build/charts/theia/templates/theia-cli/clusterrole.yaml index c6f9f145..000870a6 100644 --- a/build/charts/theia/templates/theia-cli/clusterrole.yaml +++ b/build/charts/theia/templates/theia-cli/clusterrole.yaml @@ -1,3 +1,4 @@ +{{- if .Values.theiaManager.enable }} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -12,3 +13,4 @@ rules: verbs: - get - list +{{- end }} diff --git a/build/charts/theia/templates/theia-cli/clusterrolebinding.yaml b/build/charts/theia/templates/theia-cli/clusterrolebinding.yaml index fbfa355a..e849ea06 100644 --- a/build/charts/theia/templates/theia-cli/clusterrolebinding.yaml +++ b/build/charts/theia/templates/theia-cli/clusterrolebinding.yaml @@ -1,3 +1,4 @@ +{{- if .Values.theiaManager.enable }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -12,3 +13,4 @@ subjects: - kind: ServiceAccount name: theia-cli namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/build/charts/theia/templates/theia-cli/secret.yaml b/build/charts/theia/templates/theia-cli/secret.yaml index 21aac560..2ff4a376 100644 --- a/build/charts/theia/templates/theia-cli/secret.yaml +++ b/build/charts/theia/templates/theia-cli/secret.yaml @@ -1,3 +1,4 @@ +{{- if .Values.theiaManager.enable }} apiVersion: v1 kind: Secret metadata: @@ -6,3 +7,4 @@ metadata: annotations: kubernetes.io/service-account.name: theia-cli type: kubernetes.io/service-account-token +{{- end }} diff --git a/build/charts/theia/templates/theia-cli/serviceaccount.yaml b/build/charts/theia/templates/theia-cli/serviceaccount.yaml index a865f1bf..ab995531 100644 --- a/build/charts/theia/templates/theia-cli/serviceaccount.yaml +++ b/build/charts/theia/templates/theia-cli/serviceaccount.yaml @@ -1,3 +1,4 @@ +{{- if .Values.theiaManager.enable }} apiVersion: v1 kind: ServiceAccount metadata: @@ -5,3 +6,4 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: theia-cli +{{- end }} diff --git a/build/charts/theia/templates/theia-manager/clusterrole.yaml b/build/charts/theia/templates/theia-manager/clusterrole.yaml index b685cf69..651bc60d 100644 --- a/build/charts/theia/templates/theia-manager/clusterrole.yaml +++ b/build/charts/theia/templates/theia-manager/clusterrole.yaml @@ -18,6 +18,21 @@ rules: - subjectaccessreviews verbs: - create + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - theia-ca + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create # This is the content of built-in role kube-system/extension-apiserver-authentication-reader. # But it doesn't have list/watch permission before K8s v1.17.0 so the extension apiserver (antrea-agent) will # have permission issue after bumping up apiserver library to a version that supports dynamic authentication. diff --git a/build/charts/theia/templates/theia-manager/deployment.yaml b/build/charts/theia/templates/theia-manager/deployment.yaml index f1933850..45f07283 100644 --- a/build/charts/theia/templates/theia-manager/deployment.yaml +++ b/build/charts/theia/templates/theia-manager/deployment.yaml @@ -47,6 +47,8 @@ spec: - mountPath: /etc/theia-manager name: theia-manager-config readOnly: true + - mountPath: /var/run/theia/theia-manager-tls + name: theia-manager-tls - mountPath: /var/log/antrea/theia-manager name: host-var-log-antrea-theia-manager nodeSelector: @@ -57,6 +59,12 @@ spec: - name: theia-manager-config configMap: name: theia-manager-configmap + # Make it optional as we only read it when selfSignedCert=false. + - name: theia-manager-tls + secret: + secretName: theia-manager-tls + defaultMode: 0400 + optional: true - name: host-var-log-antrea-theia-manager hostPath: path: /var/log/antrea/theia-manager diff --git a/build/charts/theia/values.yaml b/build/charts/theia/values.yaml index 33c8d7df..6b380484 100644 --- a/build/charts/theia/values.yaml +++ b/build/charts/theia/values.yaml @@ -224,6 +224,10 @@ theiaManager: apiServer: # -- The port for the Theia Manager APIServer to serve on. apiPort: 11347 + # -- Indicates whether to use auto-generated self-signed TLS certificates. If + # false, a Secret named "theia-manager-tls" must be provided with the + # following keys: ca.crt, tls.crt, tls.key. + selfSignedCert: true # -- Comma-separated list of cipher suites that will be used by the Theia Manager # APIservers. If empty, the default Go Cipher Suites will be used. tlsCipherSuites: "" diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml index 22b5b358..5b2db121 100644 --- a/build/yamls/flow-visibility.yml +++ b/build/yamls/flow-visibility.yml @@ -18,14 +18,6 @@ metadata: name: grafana namespace: flow-visibility --- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: theia-cli - name: theia-cli - namespace: flow-visibility ---- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -44,21 +36,6 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app: theia-cli - name: theia-cli -rules: -- apiGroups: - - intelligence.theia.antrea.io - resources: - - networkpolicyrecommendations - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -74,21 +51,6 @@ subjects: name: grafana namespace: flow-visibility --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app: theia-cli - name: theia-cli -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: theia-cli -subjects: -- kind: ServiceAccount - name: theia-cli - namespace: flow-visibility ---- apiVersion: v1 data: 0-3-0_0-2-0.sql: | @@ -5898,15 +5860,6 @@ stringData: type: Opaque --- apiVersion: v1 -kind: Secret -metadata: - annotations: - kubernetes.io/service-account.name: theia-cli - name: theia-cli-account-token - namespace: flow-visibility -type: kubernetes.io/service-account-token ---- -apiVersion: v1 kind: Service metadata: name: grafana diff --git a/cmd/theia-manager/options.go b/cmd/theia-manager/options.go index cc383ca4..43362550 100644 --- a/cmd/theia-manager/options.go +++ b/cmd/theia-manager/options.go @@ -81,4 +81,11 @@ func (o *Options) setDefaults() { if o.config.APIServer.APIPort == 0 { o.config.APIServer.APIPort = apis.TheiaManagerAPIPort } + if o.config.APIServer.SelfSignedCert == nil { + o.config.APIServer.SelfSignedCert = ptrBool(true) + } +} + +func ptrBool(value bool) *bool { + return &value } diff --git a/cmd/theia-manager/theia-manager.go b/cmd/theia-manager/theia-manager.go index cfe847eb..c6568819 100644 --- a/cmd/theia-manager/theia-manager.go +++ b/cmd/theia-manager/theia-manager.go @@ -15,19 +15,28 @@ package main import ( + "context" "fmt" + "net" + "os" + "path" "time" "antrea.io/antrea/pkg/log" "antrea.io/antrea/pkg/signals" "antrea.io/antrea/pkg/util/cipher" + genericapiserver "k8s.io/apiserver/pkg/server" + genericoptions "k8s.io/apiserver/pkg/server/options" + clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/klog/v2" "antrea.io/theia/pkg/apiserver" + "antrea.io/theia/pkg/apiserver/certificate" crdclientset "antrea.io/theia/pkg/client/clientset/versioned" crdinformers "antrea.io/theia/pkg/client/informers/externalversions" "antrea.io/theia/pkg/controller/networkpolicyrecommendation" + "antrea.io/theia/pkg/querier" ) // informerDefaultResync is the default resync period if a handler doesn't specify one. @@ -35,12 +44,66 @@ import ( // https://github.com/kubernetes/kubernetes/blob/release-1.17/pkg/controller/apis/config/v1alpha1/defaults.go#L120 const informerDefaultResync = 12 * time.Hour +func createAPIServerConfig( + client clientset.Interface, + selfSignedCert bool, + bindPort int, + cipherSuites []uint16, + tlsMinVersion uint16, + nprq querier.NPRecommendationQuerier) (*apiserver.Config, error) { + secureServing := genericoptions.NewSecureServingOptions().WithLoopback() + authentication := genericoptions.NewDelegatingAuthenticationOptions() + authorization := genericoptions.NewDelegatingAuthorizationOptions() + + caCertController, err := certificate.ApplyServerCert(selfSignedCert, client, secureServing, apiserver.DefaultCAConfig()) + if err != nil { + return nil, fmt.Errorf("error applying server cert: %v", err) + } + + secureServing.BindAddress = net.IPv4zero + secureServing.BindPort = bindPort + + authentication.WithRequestTimeout(apiserver.AuthenticationTimeout) + + serverConfig := genericapiserver.NewConfig(apiserver.Codecs) + if err := secureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil { + return nil, err + } + if err := authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil { + return nil, err + } + if err := authorization.ApplyTo(&serverConfig.Authorization); err != nil { + return nil, err + } + + if err := os.MkdirAll(path.Dir(apiserver.TokenPath), os.ModeDir); err != nil { + return nil, fmt.Errorf("error when creating dirs of token file: %v", err) + } + if err := os.WriteFile(apiserver.TokenPath, []byte(serverConfig.LoopbackClientConfig.BearerToken), 0600); err != nil { + return nil, fmt.Errorf("error when writing loopback access token to file: %v", err) + } + + serverConfig.SecureServing.CipherSuites = cipherSuites + serverConfig.SecureServing.MinTLSVersion = tlsMinVersion + + return apiserver.NewConfig( + serverConfig, + client, + caCertController, + nprq), nil +} + func run(o *Options) error { klog.InfoS("Theia manager starting...") // Set up signal capture: the first SIGTERM / SIGINT signal is handled gracefully and will // cause the stopCh channel to be closed; if another signal is received before the program // exits, we will force exit. stopCh := signals.RegisterSignalHandlers() + // Generate a context for functions which require one (instead of stopCh). + // We cancel the context when the function returns, which in the normal case will be when + // stopCh is closed. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() log.StartLogFileNumberMonitor(stopCh) @@ -48,6 +111,10 @@ func run(o *Options) error { if err != nil { return fmt.Errorf("error when generating KubeConfig: %v", err) } + client, err := clientset.NewForConfig(kubeConfig) + if err != nil { + return fmt.Errorf("error when generating k8s client: %v", err) + } crdClient, err := crdclientset.NewForConfig(kubeConfig) if err != nil { return fmt.Errorf("error when generating CRD client: %v", err) @@ -60,18 +127,25 @@ func run(o *Options) error { if err != nil { return fmt.Errorf("error when generating Cipher Suite list: %v", err) } - apiServer, err := apiserver.New( - npRecoController, + + apiServerConfig, err := createAPIServerConfig( + client, + *o.config.APIServer.SelfSignedCert, o.config.APIServer.APIPort, cipherSuites, - cipher.TLSVersionMap[o.config.APIServer.TLSMinVersion]) + cipher.TLSVersionMap[o.config.APIServer.TLSMinVersion], + npRecoController) + if err != nil { + return fmt.Errorf("error creating API server config: %v", err) + } + apiServer, err := apiServerConfig.New() if err != nil { return fmt.Errorf("error when creating API server: %v", err) } crdInformerFactory.Start(stopCh) go npRecoController.Run(stopCh) - go apiServer.Run(stopCh) + go apiServer.Run(ctx) <-stopCh klog.InfoS("Stopping theia manager") diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 997c11d5..d5837581 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -1,4 +1,4 @@ -// Copyright 2020 Antre2 Authors +// Copyright 2022 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,10 +15,8 @@ package apiserver import ( - "fmt" - "net" - "os" - "path" + "context" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -26,21 +24,25 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" - genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" intelligenceinstall "antrea.io/theia/pkg/apis/intelligence/install" intelligence "antrea.io/theia/pkg/apis/intelligence/v1alpha1" + "antrea.io/theia/pkg/apiserver/certificate" "antrea.io/theia/pkg/apiserver/registry/intelligence/networkpolicyrecommendation" "antrea.io/theia/pkg/querier" ) const ( - Name = "theia-manager-api" - // authenticationTimeout specifies a time limit for requests made by the authorization webhook client + CertDir = "/var/run/theia/theia-manager-tls" + SelfSignedCertDir = "/var/run/theia/theia-manager-self-signed" + Name = "theia-manager-api" + // AuthenticationTimeout specifies a time limit for requests made by the authorization webhook client // The default value (10 seconds) is not long enough as defined in // https://pkg.go.dev/k8s.io/apiserver@v0.21.0/pkg/server/options#NewDelegatingAuthenticationOptions // A value of zero means no timeout. - authenticationTimeout = 0 + AuthenticationTimeout = 0 ) var ( @@ -48,7 +50,7 @@ var ( scheme = runtime.NewScheme() // Codecs provides methods for retrieving codecs and serializers for specific // versions and content types. - codecs = serializer.NewCodecFactory(scheme) + Codecs = serializer.NewCodecFactory(scheme) // ParameterCodec defines methods for serializing and deserializing url values // to versioned API objects and back. parameterCodec = runtime.NewParameterCodec(scheme) @@ -61,56 +63,52 @@ func init() { metav1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) } -type theiaManagerAPIServer struct { - GenericAPIServer *genericapiserver.GenericAPIServer - NPRecommendationQuerier querier.NPRecommendationQuerier +// ExtraConfig holds custom apiserver config. +type ExtraConfig struct { + k8sClient kubernetes.Interface + caCertController *certificate.CACertController + npRecommendationQuerier querier.NPRecommendationQuerier } -func (s *theiaManagerAPIServer) Run(stopCh <-chan struct{}) error { - return s.GenericAPIServer.PrepareRun().Run(stopCh) +// Config defines the config for Theia manager apiserver. +type Config struct { + genericConfig *genericapiserver.Config + extraConfig ExtraConfig } -func newConfig(bindPort int) (*genericapiserver.CompletedConfig, error) { - secureServing := genericoptions.NewSecureServingOptions().WithLoopback() - authentication := genericoptions.NewDelegatingAuthenticationOptions() - authorization := genericoptions.NewDelegatingAuthorizationOptions() - - // Set the PairName but leave certificate directory blank to generate in-memory by default. - secureServing.ServerCert.CertDirectory = "" - secureServing.ServerCert.PairName = Name - secureServing.BindAddress = net.IPv4zero - secureServing.BindPort = bindPort - - authentication.WithRequestTimeout(authenticationTimeout) +type TheiaManagerAPIServer struct { + GenericAPIServer *genericapiserver.GenericAPIServer + caCertController *certificate.CACertController + NPRecommendationQuerier querier.NPRecommendationQuerier +} - if err := secureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1"), net.IPv6loopback}); err != nil { - return nil, fmt.Errorf("error creating self-signed certificates: %v", err) - } - serverConfig := genericapiserver.NewConfig(codecs) - if err := secureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil { - return nil, err - } - if err := authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil { - return nil, err - } - if err := authorization.ApplyTo(&serverConfig.Authorization); err != nil { - return nil, err +func (s *TheiaManagerAPIServer) Run(ctx context.Context) error { + // Make sure CACertController runs once to publish the CA cert before starting APIServer. + if err := s.caCertController.RunOnce(ctx); err != nil { + klog.Warningf("caCertController RunOnce failed: %v", err) } + go s.caCertController.Run(ctx, 1) + return s.GenericAPIServer.PrepareRun().Run(ctx.Done()) +} - if err := os.MkdirAll(path.Dir(TokenPath), os.ModeDir); err != nil { - return nil, fmt.Errorf("error when creating dirs of token file: %v", err) +func NewConfig( + genericConfig *genericapiserver.Config, + k8sClient kubernetes.Interface, + caCertController *certificate.CACertController, + npRecommendationQuerier querier.NPRecommendationQuerier) *Config { + return &Config{ + genericConfig: genericConfig, + extraConfig: ExtraConfig{ + k8sClient: k8sClient, + caCertController: caCertController, + npRecommendationQuerier: npRecommendationQuerier, + }, } - if err := os.WriteFile(TokenPath, []byte(serverConfig.LoopbackClientConfig.BearerToken), 0600); err != nil { - return nil, fmt.Errorf("error when writing loopback access token to file: %v", err) - } - - completedServerCfg := serverConfig.Complete(nil) - return &completedServerCfg, nil } -func installAPIGroup(s *theiaManagerAPIServer) error { +func installAPIGroup(s *TheiaManagerAPIServer) error { npRecommendationStorage := networkpolicyrecommendation.NewREST(s.NPRecommendationQuerier) - intelligenceGroup := genericapiserver.NewDefaultAPIGroupInfo(intelligence.GroupName, scheme, parameterCodec, codecs) + intelligenceGroup := genericapiserver.NewDefaultAPIGroupInfo(intelligence.GroupName, scheme, parameterCodec, Codecs) v1alpha1Storage := map[string]rest.Storage{} v1alpha1Storage["networkpolicyrecommendations"] = npRecommendationStorage intelligenceGroup.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1Storage @@ -125,20 +123,30 @@ func installAPIGroup(s *theiaManagerAPIServer) error { return nil } -func New(nprq querier.NPRecommendationQuerier, bindPort int, cipherSuites []uint16, tlsMinVersion uint16) (*theiaManagerAPIServer, error) { - cfg, err := newConfig(bindPort) +func (c Config) New() (*TheiaManagerAPIServer, error) { + completedServerCfg := c.genericConfig.Complete(nil) + s, err := completedServerCfg.New(Name, genericapiserver.NewEmptyDelegate()) if err != nil { return nil, err } - s, err := cfg.New(Name, genericapiserver.NewEmptyDelegate()) - if err != nil { - return nil, err - } - s.SecureServingInfo.CipherSuites = cipherSuites - s.SecureServingInfo.MinTLSVersion = tlsMinVersion - apiServer := &theiaManagerAPIServer{GenericAPIServer: s, NPRecommendationQuerier: nprq} + apiServer := &TheiaManagerAPIServer{ + GenericAPIServer: s, + caCertController: c.extraConfig.caCertController, + NPRecommendationQuerier: c.extraConfig.npRecommendationQuerier} if err := installAPIGroup(apiServer); err != nil { return nil, err } return apiServer, nil } + +func DefaultCAConfig() *certificate.CAConfig { + return &certificate.CAConfig{ + CAConfigMapName: certificate.TheiaCAConfigMapName, + CertDir: CertDir, + SelfSignedCertDir: SelfSignedCertDir, + CertReadyTimeout: 2 * time.Minute, + MaxRotateDuration: time.Hour * (24 * 365), + ServiceName: certificate.TheiaServiceName, + PairName: Name, + } +} diff --git a/pkg/apiserver/certificate/cacert_controller.go b/pkg/apiserver/certificate/cacert_controller.go new file mode 100644 index 00000000..23b22cf3 --- /dev/null +++ b/pkg/apiserver/certificate/cacert_controller.go @@ -0,0 +1,205 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificate + +import ( + "context" + "fmt" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "antrea.io/theia/pkg/util/env" +) + +const ( + CAConfigMapKey = "ca.crt" +) + +// CACertController is responsible for taking the CA certificate from the +// caContentProvider and publishing it to the ConfigMap and the APIServices. +type CACertController struct { + mutex sync.RWMutex + + // caContentProvider provides the very latest content of the ca bundle. + caContentProvider dynamiccertificates.CAContentProvider + // queue only ever has one item, but it has nice error handling backoff/retry semantics + queue workqueue.RateLimitingInterface + + client kubernetes.Interface + caConfig *CAConfig +} + +var _ dynamiccertificates.Listener = &CACertController{} + +func GetCAConfigMapNamespace() string { + return env.GetTheiaNamespace() +} + +func newCACertController(caContentProvider dynamiccertificates.CAContentProvider, + client kubernetes.Interface, + caConfig *CAConfig, +) *CACertController { + c := &CACertController{ + caContentProvider: caContentProvider, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "CACertController"), + client: client, + caConfig: caConfig, + } + if notifier, ok := caContentProvider.(dynamiccertificates.Notifier); ok { + notifier.AddListener(c) + } + return c +} + +func (c *CACertController) UpdateCertificate(ctx context.Context) error { + if controller, ok := c.caContentProvider.(dynamiccertificates.ControllerRunner); ok { + if err := controller.RunOnce(ctx); err != nil { + klog.Warningf("Updating of CA content failed: %v", err) + c.Enqueue() + return err + } + } + + return nil +} + +// getCertificate exposes the certificate for testing. +func (c *CACertController) getCertificate() []byte { + return c.caContentProvider.CurrentCABundleContent() +} + +// Enqueue will be called after CACertController is registered as a listener of CA cert change. +func (c *CACertController) Enqueue() { + // The key can be anything as we only have single item. + c.queue.Add("key") +} + +func (c *CACertController) syncCACert() error { + caCert := c.caContentProvider.CurrentCABundleContent() + + if err := c.syncConfigMap(caCert); err != nil { + return err + } + + return nil +} + +// syncConfigMap updates the ConfigMap that holds the CA bundle, which will be read by API clients. +func (c *CACertController) syncConfigMap(caCert []byte) error { + klog.InfoS("Syncing CA certificate with ConfigMap") + // Use the Theia manager Pod Namespace for the CA cert ConfigMap. + caConfigMapNamespace := GetCAConfigMapNamespace() + caConfigMap, err := c.client.CoreV1().ConfigMaps(caConfigMapNamespace).Get(context.TODO(), c.caConfig.CAConfigMapName, metav1.GetOptions{}) + exists := true + if err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("error getting ConfigMap %s: %v", c.caConfig.CAConfigMapName, err) + } + exists = false + caConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.caConfig.CAConfigMapName, + Namespace: caConfigMapNamespace, + Labels: map[string]string{ + "app": "theia", + }, + }, + } + } + if caConfigMap.Data != nil && caConfigMap.Data[CAConfigMapKey] == string(caCert) { + return nil + } + caConfigMap.Data = map[string]string{ + CAConfigMapKey: string(caCert), + } + if exists { + if _, err := c.client.CoreV1().ConfigMaps(caConfigMapNamespace).Update(context.TODO(), caConfigMap, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("error updating ConfigMap %s: %v", c.caConfig.CAConfigMapName, err) + } + } else { + if _, err := c.client.CoreV1().ConfigMaps(caConfigMapNamespace).Create(context.TODO(), caConfigMap, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("error creating ConfigMap %s: %v", c.caConfig.CAConfigMapName, err) + } + } + return nil +} + +// RunOnce runs a single sync step to ensure that we have a valid starting configuration. +func (c *CACertController) RunOnce(ctx context.Context) error { + if controller, ok := c.caContentProvider.(dynamiccertificates.ControllerRunner); ok { + if err := controller.RunOnce(ctx); err != nil { + klog.Warningf("Initial population of CA content failed: %v", err) + c.Enqueue() + return err + } + } + if err := c.syncCACert(); err != nil { + klog.Warningf("Initial sync of CA content failed: %v", err) + c.Enqueue() + return err + } + return nil +} + +// Run starts the CACertController and blocks until the context is canceled. +func (c *CACertController) Run(ctx context.Context, workers int) { + defer c.queue.ShutDown() + + klog.InfoS("Starting CACertController") + defer klog.InfoS("Shutting down CACertController") + + if controller, ok := c.caContentProvider.(dynamiccertificates.ControllerRunner); ok { + // doesn't matter what workers say, only start one. + go controller.Run(ctx, 1) + } + + // doesn't matter what workers say, only start one. + go wait.Until(c.runWorker, time.Second, ctx.Done()) + + <-ctx.Done() +} + +func (c *CACertController) runWorker() { + for c.processNextWorkItem() { + } +} + +func (c *CACertController) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + + err := c.syncCACert() + if err == nil { + c.queue.Forget(key) + return true + } + + klog.ErrorS(err, "Error when syncing CA cert, requeuing") + c.queue.AddRateLimited(key) + + return true +} diff --git a/pkg/apiserver/certificate/certificate.go b/pkg/apiserver/certificate/certificate.go new file mode 100644 index 00000000..5c545fd8 --- /dev/null +++ b/pkg/apiserver/certificate/certificate.go @@ -0,0 +1,188 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificate + +import ( + "context" + "fmt" + "net" + "os" + "path" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/apiserver/pkg/server/options" + "k8s.io/client-go/kubernetes" + certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" + "k8s.io/klog/v2" + + "antrea.io/theia/pkg/util/env" +) + +const ( + // The names of the files that should contain the CA certificate and the TLS key pair. + CACertFile = "ca.crt" + TLSCertFile = "tls.crt" + TLSKeyFile = "tls.key" +) + +// GetTheiaServerNames returns the DNS names that the TLS certificate will be signed with. +func GetTheiaServerNames(serviceName string) []string { + namespace := env.GetTheiaNamespace() + theiaServerName := serviceName + "." + namespace + ".svc" + // TODO: Add the whole FQDN "theia-manager..svc." as an + // alternate DNS name when other clients need to access it directly with that name. + return []string{theiaServerName} +} + +func ApplyServerCert(selfSignedCert bool, + client kubernetes.Interface, + secureServing *options.SecureServingOptionsWithLoopback, + caConfig *CAConfig) (*CACertController, error) { + var err error + var caContentProvider dynamiccertificates.CAContentProvider + if selfSignedCert { + caContentProvider, err = generateSelfSignedCertificate(secureServing, caConfig) + if err != nil { + return nil, fmt.Errorf("error creating self-signed CA certificate: %v", err) + } + } else { + caCertPath := path.Join(caConfig.CertDir, CACertFile) + tlsCertPath := path.Join(caConfig.CertDir, TLSCertFile) + tlsKeyPath := path.Join(caConfig.CertDir, TLSKeyFile) + // The secret may be created after the Pod is created, for example, when cert-manager is used the secret + // is created asynchronously. It waits for a while before it's considered to be failed. + if err = wait.PollImmediate(2*time.Second, caConfig.CertReadyTimeout, func() (bool, error) { + for _, path := range []string{caCertPath, tlsCertPath, tlsKeyPath} { + f, err := os.Open(path) + if err != nil { + klog.Warningf("Couldn't read %s when applying server certificate, retrying", path) + return false, nil + } + f.Close() + } + return true, nil + }); err != nil { + return nil, fmt.Errorf("error reading TLS certificate and/or key. Please make sure the TLS CA (%s), cert (%s), and key (%s) files are present in \"%s\", when selfSignedCert is set to false", CACertFile, TLSCertFile, TLSKeyFile, caConfig.CertDir) + } + // Since 1.17.0 (https://github.com/kubernetes/kubernetes/commit/3f5fbfbfac281f40c11de2f57d58cc332affc37b), + // apiserver reloads certificate cert and key file from disk every minute, allowing serving tls config to be updated. + secureServing.ServerCert.CertKey.CertFile = tlsCertPath + secureServing.ServerCert.CertKey.KeyFile = tlsKeyPath + + caContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("user-provided CA cert", caCertPath) + if err != nil { + return nil, fmt.Errorf("error reading user-provided CA certificate: %v", err) + } + } + + caCertController := newCACertController(caContentProvider, client, caConfig) + + if selfSignedCert { + go rotateSelfSignedCertificates(caCertController, secureServing, caConfig.MaxRotateDuration) + } + + return caCertController, nil +} + +// generateSelfSignedCertificate generates a new self signed certificate. +func generateSelfSignedCertificate(secureServing *options.SecureServingOptionsWithLoopback, caConfig *CAConfig) (dynamiccertificates.CAContentProvider, error) { + var err error + var caContentProvider dynamiccertificates.CAContentProvider + + // Set the PairName and CertDirectory to generate the certificate files. + secureServing.ServerCert.CertDirectory = caConfig.SelfSignedCertDir + secureServing.ServerCert.PairName = caConfig.PairName + + if err := secureServing.MaybeDefaultWithSelfSignedCerts(caConfig.ServiceName, GetTheiaServerNames(caConfig.ServiceName), []net.IP{net.ParseIP("127.0.0.1"), net.IPv6loopback}); err != nil { + return nil, fmt.Errorf("error creating self-signed certificates: %v", err) + } + + caContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("self-signed cert", secureServing.ServerCert.CertKey.CertFile) + if err != nil { + return nil, fmt.Errorf("error reading self-signed CA certificate: %v", err) + } + + return caContentProvider, nil +} + +// Used to determine which is sooner, the provided maxRotateDuration or the expiration date +// of the cert. Used to allow for unit testing with a far shorter rotation period. +// Also can be used to pass a user provided rotation window. +func nextRotationDuration(secureServing *options.SecureServingOptionsWithLoopback, + maxRotateDuration time.Duration) (time.Duration, error) { + + x509Cert, err := certutil.CertsFromFile(secureServing.ServerCert.CertKey.CertFile) + if err != nil { + return time.Duration(0), fmt.Errorf("error parsing generated certificate: %v", err) + } + + // Attempt to rotate the certificate at the half-way point of expiration. + // Unless the halfway point is longer than maxRotateDuration + duration := x509Cert[0].NotAfter.Sub(time.Now()) / 2 + + waitDuration := duration + if maxRotateDuration < waitDuration { + waitDuration = maxRotateDuration + } + + return waitDuration, nil +} + +// rotateSelfSignedCertificates calculates the rotation duration for the current certificate. +// Then once the duration is complete, generates a new self-signed certificate and repeats the process. +func rotateSelfSignedCertificates(c *CACertController, secureServing *options.SecureServingOptionsWithLoopback, + maxRotateDuration time.Duration) { + for { + rotationDuration, err := nextRotationDuration(secureServing, maxRotateDuration) + if err != nil { + klog.ErrorS(err, "Error when reading expiration date of cert") + return + } + + klog.InfoS("Certificate will be rotated at", "time", time.Now().Add(rotationDuration)) + + time.Sleep(rotationDuration) + + klog.InfoS("Rotating self signed certificate") + + err = generateNewServingCertificate(secureServing, c.caConfig) + if err != nil { + klog.ErrorS(err, "Error when generating new cert") + return + } + c.UpdateCertificate(context.TODO()) + } +} + +func generateNewServingCertificate(secureServing *options.SecureServingOptionsWithLoopback, caConfig *CAConfig) error { + cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures(caConfig.ServiceName, []net.IP{net.ParseIP("127.0.0.1"), net.IPv6loopback}, GetTheiaServerNames(caConfig.ServiceName), secureServing.ServerCert.FixtureDirectory) + if err != nil { + return fmt.Errorf("unable to generate self signed cert: %v", err) + } + + if err := certutil.WriteCert(secureServing.ServerCert.CertKey.CertFile, cert); err != nil { + return err + } + if err := keyutil.WriteKey(secureServing.ServerCert.CertKey.KeyFile, key); err != nil { + return err + } + klog.InfoS("Generated self-signed certificate", "cert", secureServing.ServerCert.CertKey.CertFile, + "key", secureServing.ServerCert.CertKey.KeyFile) + + return nil +} diff --git a/pkg/apiserver/certificate/certificate_test.go b/pkg/apiserver/certificate/certificate_test.go new file mode 100644 index 00000000..e4b7a02f --- /dev/null +++ b/pkg/apiserver/certificate/certificate_test.go @@ -0,0 +1,222 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificate + +import ( + "bytes" + "os" + "path" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/wait" + genericoptions "k8s.io/apiserver/pkg/server/options" + fakeclientset "k8s.io/client-go/kubernetes/fake" + certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" +) + +const ( + fakeTLSCert = `-----BEGIN CERTIFICATE----- +MIICBDCCAW2gAwIBAgIJAPgVBh+4xbGoMA0GCSqGSIb3DQEBCwUAMBsxGTAXBgNV +BAMMEHdlYmhvb2tfdGVzdHNfY2EwIBcNMTcwNzI4MjMxNTI4WhgPMjI5MTA1MTMy +MzE1MjhaMB8xHTAbBgNVBAMMFHdlYmhvb2tfdGVzdHNfY2xpZW50MIGfMA0GCSqG +SIb3DQEBAQUAA4GNADCBiQKBgQDkGXXSm6Yun5o3Jlmx45rItcQ2pmnoDk4eZfl0 +rmPa674s2pfYo3KywkXQ1Fp3BC8GUgzPLSfJ8xXya9Lg1Wo8sHrDln0iRg5HXxGu +uFNhRBvj2S0sIff0ZG/IatB9I6WXVOUYuQj6+A0CdULNj1vBqH9+7uWbLZ6lrD4b +a44x/wIDAQABo0owSDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAU +BggrBgEFBQcDAgYIKwYBBQUHAwEwDwYDVR0RBAgwBocEfwAAATANBgkqhkiG9w0B +AQsFAAOBgQCpN27uh/LjUVCaBK7Noko25iih/JSSoWzlvc8CaipvSPofNWyGx3Vu +OdcSwNGYX/pp4ZoAzFij/Y5u0vKTVLkWXATeTMVmlPvhmpYjj9gPkCSY6j/SiKlY +kGy0xr+0M5UQkMBcfIh9oAp9um1fZHVWAJAGP/ikZgkcUey0LmBn8w== +-----END CERTIFICATE-----` + fakeTLSKey = `-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQDkGXXSm6Yun5o3Jlmx45rItcQ2pmnoDk4eZfl0rmPa674s2pfY +o3KywkXQ1Fp3BC8GUgzPLSfJ8xXya9Lg1Wo8sHrDln0iRg5HXxGuuFNhRBvj2S0s +Iff0ZG/IatB9I6WXVOUYuQj6+A0CdULNj1vBqH9+7uWbLZ6lrD4ba44x/wIDAQAB +AoGAZbWwowvCq1GBq4vPPRI3h739Uz0bRl1ymf1woYXNguXRtCB4yyH+2BTmmrrF +6AIWkePuUEdbUaKyK5nGu3iOWM+/i6NP3kopQANtbAYJ2ray3kwvFlhqyn1bxX4n +gl/Cbdw1If4zrDrB66y8mYDsjzK7n/gFaDNcY4GArjvOXKkCQQD9Lgv+WD73y4RP +yS+cRarlEeLLWVsX/pg2oEBLM50jsdUnrLSW071MjBgP37oOXzqynF9SoDbP2Y5C +x+aGux9LAkEA5qPlQPv0cv8Wc3qTI+LixZ/86PPHKWnOnwaHm3b9vQjZAkuVQg3n +Wgg9YDmPM87t3UFH7ZbDihUreUxwr9ZjnQJAZ9Z95shMsxbOYmbSVxafu6m1Sc+R +M+sghK7/D5jQpzYlhUspGf8n0YBX0hLhXUmjamQGGH5LXL4Owcb4/mM6twJAEVio +SF/qva9jv+GrKVrKFXT374lOJFY53Qn/rvifEtWUhLCslCA5kzLlctRBafMZPrfH +Mh5RrJP1BhVysDbenQJASGcc+DiF7rB6K++ZGyC11E2AP29DcZ0pgPESSV7npOGg ++NqPRZNVCSZOiVmNuejZqmwKhZNGZnBFx1Y+ChAAgw== +-----END RSA PRIVATE KEY-----` + fakeCACert = `-----BEGIN CERTIFICATE----- +MIICBDCCAW2gAwIBAgIJAPgVBh+4xbGoMA0GCSqGSIb3DQEBCwUAMBsxGTAXBgNV +BAMMEHdlYmhvb2tfdGVzdHNfY2EwIBcNMTcwNzI4MjMxNTI4WhgPMjI5MTA1MTMy +MzE1MjhaMB8xHTAbBgNVBAMMFHdlYmhvb2tfdGVzdHNfY2xpZW50MIGfMA0GCSqG +SIb3DQEBAQUAA4GNADCBiQKBgQDkGXXSm6Yun5o3Jlmx45rItcQ2pmnoDk4eZfl0 +rmPa674s2pfYo3KywkXQ1Fp3BC8GUgzPLSfJ8xXya9Lg1Wo8sHrDln0iRg5HXxGu +uFNhRBvj2S0sIff0ZG/IatB9I6WXVOUYuQj6+A0CdULNj1vBqH9+7uWbLZ6lrD4b +a44x/wIDAQABo0owSDAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAU +BggrBgEFBQcDAgYIKwYBBQUHAwEwDwYDVR0RBAgwBocEfwAAATANBgkqhkiG9w0B +AQsFAAOBgQCpN27uh/LjUVCaBK7Noko25iih/JSSoWzlvc8CaipvSPofNWyGx3Vu +OdcSwNGYX/pp4ZoAzFij/Y5u0vKTVLkWXATeTMVmlPvhmpYjj9gPkCSY6j/SiKlY +kGy0xr+0M5UQkMBcfIh9oAp9um1fZHVWAJAGP/ikZgkcUey0LmBn8w== +-----END CERTIFICATE-----` +) + +func TestApplyServerCert(t *testing.T) { + tests := []struct { + name string + selfSignedCert bool + tlsCert []byte + tlsKey []byte + caCert []byte + wantErr bool + wantCertKey bool + wantGeneratedCert bool + wantCACert []byte + testRotate bool + }{ + { + name: "self-signed", + selfSignedCert: true, + tlsCert: nil, + tlsKey: nil, + caCert: nil, + wantErr: false, + wantCertKey: false, + wantGeneratedCert: true, + wantCACert: nil, + testRotate: false, + }, + { + name: "user-provided", + selfSignedCert: false, + tlsCert: []byte(fakeTLSCert), + tlsKey: []byte(fakeTLSKey), + caCert: []byte(fakeCACert), + wantErr: false, + wantCertKey: true, + wantGeneratedCert: false, + wantCACert: []byte(fakeCACert), + testRotate: false, + }, + { + name: "user-provided-missing-tls-crt", + selfSignedCert: false, + tlsCert: nil, + tlsKey: []byte(fakeTLSKey), + caCert: []byte(fakeCACert), + wantErr: true, + testRotate: false, + }, + { + name: "user-provided-missing-tls-key", + selfSignedCert: false, + tlsCert: []byte(fakeTLSCert), + tlsKey: nil, + caCert: []byte(fakeCACert), + wantErr: true, + testRotate: false, + }, + { + name: "user-provided-missing-ca-crt", + selfSignedCert: false, + tlsCert: []byte(fakeTLSCert), + tlsKey: []byte(fakeTLSKey), + caCert: nil, + wantErr: true, + testRotate: false, + }, + { + name: "self-signed-rotate", + selfSignedCert: true, + tlsCert: nil, + tlsKey: nil, + caCert: nil, + wantErr: false, + wantCertKey: false, + wantGeneratedCert: true, + wantCACert: nil, + testRotate: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + caConfig := &CAConfig{ + ServiceName: TheiaServiceName, + PairName: "theia-manager-api", + } + var err error + caConfig.CertDir, err = os.MkdirTemp("", "theia-tls-test") + if err != nil { + t.Fatalf("Unable to create temporary directory: %v", err) + } + defer os.RemoveAll(caConfig.CertDir) + caConfig.SelfSignedCertDir, err = os.MkdirTemp("", "theia-self-signed") + if err != nil { + t.Fatalf("Unable to create temporary directory: %v", err) + } + defer os.RemoveAll(caConfig.SelfSignedCertDir) + caConfig.CertReadyTimeout = 100 * time.Millisecond + secureServing := genericoptions.NewSecureServingOptions().WithLoopback() + if tt.tlsCert != nil { + certutil.WriteCert(path.Join(caConfig.CertDir, TLSCertFile), tt.tlsCert) + } + if tt.tlsKey != nil { + keyutil.WriteKey(path.Join(caConfig.CertDir, TLSKeyFile), tt.tlsKey) + } + if tt.caCert != nil { + certutil.WriteCert(path.Join(caConfig.CertDir, CACertFile), tt.caCert) + } + + if tt.testRotate { + caConfig.MaxRotateDuration = 2 * time.Second + } + + clientset := fakeclientset.NewSimpleClientset() + got, err := ApplyServerCert(tt.selfSignedCert, clientset, secureServing, caConfig) + + if err != nil || tt.wantErr { + if (err != nil) != tt.wantErr { + t.Errorf("ApplyServerCert() error = %v, wantErr %v", err, tt.wantErr) + } + return + } + + if tt.selfSignedCert && tt.testRotate { + oldCertKeyContent := got.getCertificate() + err := wait.Poll(time.Second, 8*time.Second, func() (bool, error) { + newCertKeyContent := got.getCertificate() + equal := bytes.Equal(oldCertKeyContent, newCertKeyContent) + return !equal, nil + }) + + assert.Nil(t, err, "CA cert not updated") + } + + if tt.wantCertKey { + assert.Equal(t, genericoptions.CertKey{CertFile: caConfig.CertDir + "/tls.crt", KeyFile: caConfig.CertDir + "/tls.key"}, secureServing.ServerCert.CertKey, "CertKey doesn't match") + } + if tt.wantGeneratedCert { + assert.Equal(t, genericoptions.CertKey{CertFile: caConfig.SelfSignedCertDir + "/theia-manager-api.crt", KeyFile: caConfig.SelfSignedCertDir + "/theia-manager-api.key"}, secureServing.ServerCert.CertKey, "SelfSigned certs not generated") + } else { + assert.NotEqual(t, genericoptions.CertKey{CertFile: caConfig.SelfSignedCertDir + "/theia-manager-api.crt", KeyFile: caConfig.SelfSignedCertDir + "/theia-manager-api.key"}, secureServing.ServerCert.CertKey, "SelfSigned certs generated erroneously") + } + if tt.wantCACert != nil { + assert.Equal(t, tt.wantCACert, got.caContentProvider.CurrentCABundleContent(), "CA cert doesn't match") + } else { + assert.NotEmpty(t, got.caContentProvider.CurrentCABundleContent(), "CA cert is empty") + } + }) + } +} diff --git a/pkg/apiserver/certificate/config.go b/pkg/apiserver/certificate/config.go new file mode 100644 index 00000000..d8160489 --- /dev/null +++ b/pkg/apiserver/certificate/config.go @@ -0,0 +1,45 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificate + +import "time" + +const ( + TheiaCAConfigMapName = "theia-ca" + TheiaServiceName = "theia-manager" +) + +type CAConfig struct { + // Name of the ConfigMap that will hold the CA certificate that signs the TLS + // certificate of theia manager. + CAConfigMapName string + + // CertDir is the directory that the TLS Secret should be mounted to. Declaring it as a variable for testing. + CertDir string + + // SelfSignedCertDir is the dir self-signed certificates are created in. + SelfSignedCertDir string + + // CertReadyTimeout is the timeout we will wait for the TLS Secret being ready. Declaring it as a variable for testing. + CertReadyTimeout time.Duration + + // MaxRotateDuration is the max duration for rotating self-signed certificate generated. + // In most cases we will rotate the certificate when we reach half the expiration time of the certificate (see nextRotationDuration). + // MaxRotateDuration ensures that if a self-signed certificate has a really long expiration (N years), we still attempt to rotate it + // within a reasonable time, in this case one year. maxRotateDuration is also used to force certificate rotation in unit tests. + MaxRotateDuration time.Duration + ServiceName string + PairName string +} diff --git a/pkg/config/theiamanager/config.go b/pkg/config/theiamanager/config.go index 323569ba..994702b2 100644 --- a/pkg/config/theiamanager/config.go +++ b/pkg/config/theiamanager/config.go @@ -23,6 +23,13 @@ type APIServerConfig struct { // APIPort is the port for the theia-manager APIServer to serve on. // Defaults to 11347. APIPort int `yaml:"apiPort,omitempty"` + // Indicates whether to use auto-generated self-signed TLS certificate. + // If false, a Secret named "theia-manager-tls" must be provided with the following keys: + // ca.crt: + // tls.crt: + // tls.key: + // Defaults to true. + SelfSignedCert *bool `yaml:"selfSignedCert,omitempty"` // Cipher suites to use. TLSCipherSuites string `yaml:"tlsCipherSuites,omitempty"` // TLS min version. diff --git a/pkg/util/env/env.go b/pkg/util/env/env.go new file mode 100644 index 00000000..8f429c71 --- /dev/null +++ b/pkg/util/env/env.go @@ -0,0 +1,49 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package env + +import ( + "os" + + "k8s.io/klog/v2" +) + +const ( + podNamespaceEnvKey = "POD_NAMESPACE" + + defaultTheiaNamespace = "flow-visibility" +) + +// GetPodNamespace returns Namespace of the Pod where the code executes. +func GetPodNamespace() string { + podNamespace := os.Getenv(podNamespaceEnvKey) + if podNamespace == "" { + klog.Warningf("Environment variable %s not found", podNamespaceEnvKey) + } + return podNamespace +} + +// GetTheiaNamespace tries to determine the Namespace in which Theia is running by looking at the +// POD_NAMESPACE environment variable. If this environment variable is not set (e.g. because the +// Theia component is not run as a Pod), "flow-visibility" is returned. +func GetTheiaNamespace() string { + namespace := GetPodNamespace() + if namespace == "" { + klog.Warningf("Failed to get Pod Namespace from environment. Using \"%s\" as the "+ + "Theia Service Namespace", defaultTheiaNamespace) + namespace = defaultTheiaNamespace + } + return namespace +} diff --git a/pkg/util/env/env_test.go b/pkg/util/env/env_test.go new file mode 100644 index 00000000..9de6b08c --- /dev/null +++ b/pkg/util/env/env_test.go @@ -0,0 +1,65 @@ +// Copyright 2022 Antrea Authors + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package env + +import ( + "os" + "testing" +) + +func TestGetPodNamespace(t *testing.T) { + testTable := map[string]string{ + "test-namespace": "test-namespace", + "": "", + } + + for k, v := range testTable { + comparePodNamespace(k, v, t) + } +} + +func comparePodNamespace(k, v string, t *testing.T) { + if k != "" { + _ = os.Setenv(podNamespaceEnvKey, k) + defer os.Unsetenv(podNamespaceEnvKey) + } + podNamespace := GetPodNamespace() + if podNamespace != v { + t.Errorf("Failed to retrieve pod namespace, want: %s, get: %s", v, podNamespace) + } +} + +func TestGetTheiaNamespace(t *testing.T) { + testTable := map[string]string{ + "test-namespace": "test-namespace", + "": "flow-visibility", + } + + for k, v := range testTable { + compareTheiaNamespace(k, v, t) + } +} + +func compareTheiaNamespace(k, v string, t *testing.T) { + if k != "" { + _ = os.Setenv(podNamespaceEnvKey, k) + defer os.Unsetenv(podNamespaceEnvKey) + } + podNamespace := GetTheiaNamespace() + if podNamespace != v { + t.Errorf("Failed to retrieve pod namespace, want: %s, get: %s", v, podNamespace) + } +}