Auto generate and sign certificates for TLS enabled cluster

charts/tidb-cluster/templates/_helpers.tpl

@@ -37,6 +37,13 @@ config-file: |-
     {{- if .Values.pd.config }}
 {{ .Values.pd.config | indent 2 }}
     {{- end -}}
+    {{- if .Values.enableTLSCluster }}
+  [security]
+  cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  cert-path = "/var/lib/pd-tls/cert"
+  key-path = "/var/lib/pd-tls/key"
+    {{- end -}}
+
 {{- end -}}
 
 {{- define "pd-configmap.data-digest" -}}
@@ -53,6 +60,13 @@ config-file: |-
     {{- if .Values.tikv.config }}
 {{ .Values.tikv.config | indent 2 }}
     {{- end -}}
+    {{- if .Values.enableTLSCluster }}
+  [security]
+  ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  cert-path = "/var/lib/tikv-tls/cert"
+  key-path = "/var/lib/tikv-tls/key"
+    {{- end -}}
+
 {{- end -}}
 
 {{- define "tikv-configmap.data-digest" -}}
@@ -73,6 +87,20 @@ config-file: |-
     {{- if .Values.tidb.config }}
 {{ .Values.tidb.config | indent 2 }}
     {{- end -}}
+    {{- if or .Values.enableTLSCluster .Values.enableTLSClient }}
+  [security]
+    {{- end -}}
+    {{- if .Values.enableTLSCluster }}
+  cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  cluster-ssl-cert = "/var/lib/tidb-tls/cert"
+  cluster-ssl-key = "/var/lib/tidb-tls/key"
+    {{- end -}}
+    {{- if .Values.tidb.enableTLSClient }}
+  ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  ssl-cert = "/var/lib/tidb-server-tls/cert"
+  ssl-key = "/var/lib/tidb-server-tls/key"
+    {{- end -}}
+
 {{- end -}}
 
 {{- define "tidb-configmap.data-digest" -}}

charts/tidb-cluster/templates/config/_drainer-config.tpl

@@ -11,7 +11,7 @@ detect-interval = {{ .Values.binlog.drainer.detectInterval | default 10 }}
 data-dir = "/data"
 
 # a comma separated list of PD endpoints
-pd-urls = "http://{{ template "cluster.name" . }}-pd:2379"
+pd-urls = "{{ if .Values.enableTLSCluster }}https{{ else }}http{{ end }}://{{ template "cluster.name" . }}-pd:2379"
 
 # Use the specified compressor to compress payload between pump and drainer
 compressor = ""

charts/tidb-cluster/templates/config/_prometheus-config.tpl

@@ -19,6 +19,13 @@ scrape_configs:
         - {{ .Release.Namespace }}
     tls_config:
       insecure_skip_verify: true
+    {{- if .Values.enableTLSCluster }}
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      cert_file: /var/lib/pd-client-tls/cert
+      key_file: /var/lib/pd-client-tls/key
+
+    scheme: https
+    {{- end }}
     relabel_configs:
     - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
       action: keep
@@ -44,11 +51,63 @@ scrape_configs:
     - source_labels: [__meta_kubernetes_pod_ip]
       action: replace
       target_label: kubernetes_pod_ip
+    {{- if .Values.enableTLSCluster }}
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: drop
+      regex: .*\-tikv\-\d*$
+    {{- end }}
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: replace
+      target_label: instance
+    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+      action: replace
+      target_label: cluster
+  {{- if .Values.enableTLSCluster }}
+  - job_name: 'tidb-cluster-tikv'
+    scrape_interval: 15s
+    honor_labels: true
+    kubernetes_sd_configs:
+    - role: pod
+      namespaces:
+        names:
+        - {{ .Release.Namespace }}
+    tls_config:
+      insecure_skip_verify: true
+    scheme: http
+    relabel_configs:
+    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+      action: keep
+      regex: {{ .Release.Name }}
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+      action: keep
+      regex: true
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+      action: replace
+      target_label: __metrics_path__
+      regex: (.+)
+    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+      action: replace
+      regex: ([^:]+)(?::\d+)?;(\d+)
+      replacement: $1:$2
+      target_label: __address__
+    - source_labels: [__meta_kubernetes_namespace]
+      action: replace
+      target_label: kubernetes_namespace
+    - source_labels: [__meta_kubernetes_pod_node_name]
+      action: replace
+      target_label: kubernetes_node
+    - source_labels: [__meta_kubernetes_pod_ip]
+      action: replace
+      target_label: kubernetes_pod_ip
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: keep
+      regex: .*\-tikv\-\d*$
     - source_labels: [__meta_kubernetes_pod_name]
       action: replace
       target_label: instance
     - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
       action: replace
       target_label: cluster
+  {{- end }}
 rule_files:
   - '/prometheus-rules/rules/*.rules.yml'

charts/tidb-cluster/templates/config/_pump-config.tpl

@@ -17,7 +17,7 @@ data-dir = "/data"
 heartbeat-interval = {{ .Values.binlog.pump.heartbeatInterval | default 2 }}
 
 # a comma separated list of PD endpoints
-pd-urls = "http://{{ template "cluster.name" . }}-pd:2379"
+pd-urls = "{{ if .Values.enableTLSCluster }}https{{ else }}http{{ end }}://{{ template "cluster.name" . }}-pd:2379"
 
 #[security]
 # Path of file that contains list of trusted SSL CAs for connection with cluster components.

charts/tidb-cluster/templates/discovery-deployment.yaml

@@ -45,4 +45,3 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
-

charts/tidb-cluster/templates/discovery-rbac.yaml

@@ -14,6 +14,9 @@ rules:
   resources: ["tidbclusters"]
   resourceNames: [{{ template "cluster.name" . }}]
   verbs: ["get"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1beta1

charts/tidb-cluster/templates/monitor-deployment.yaml

@@ -113,6 +113,11 @@ spec:
           - name: prometheus-rules
             mountPath: /prometheus-rules
             readOnly: false
+          {{- if .Values.enableTLSCluster }}
+          - name: tls-pd-client
+            mountPath: /var/lib/pd-client-tls
+            readOnly: true
+          {{- end }}
       {{- if .Values.monitor.grafana.create }}
       - name: reloader
         image: {{ .Values.monitor.reloader.image }}
@@ -208,6 +213,12 @@ spec:
       - emptyDir: {}
         name: grafana-dashboard
       {{- end }}
+      {{- if .Values.enableTLSCluster }}
+      - name: tls-pd-client
+        secret:
+          defaultMode: 420
+          secretName: {{ .Release.Name }}-pd-client
+      {{- end }}
     {{- if .Values.monitor.tolerations }}
       tolerations:
 {{ toYaml .Values.monitor.tolerations | indent 6 }}

charts/tidb-cluster/templates/scripts/_start_drainer.sh.tpl

@@ -24,9 +24,11 @@ while true; do
     fi
 done
 
+SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }}
+
 /drainer \
 -L={{ .Values.binlog.drainer.logLevel | default "info" }} \
--pd-urls=http://{{ template "cluster.name" . }}-pd:2379 \
+-pd-urls=${SCHEME}://{{ template "cluster.name" . }}-pd:2379 \
 -addr=`echo ${HOSTNAME}`.{{ template "cluster.name" . }}-drainer:8249 \
 -config=/etc/drainer/drainer.toml \
 -disable-detect={{ .Values.binlog.drainer.disableDetect | default false }} \

charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl

@@ -58,12 +58,14 @@ while true; do
     fi
 done
 
+SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }}
+
 ARGS="--data-dir=/var/lib/pd \
 --name=${POD_NAME} \
---peer-urls=http://0.0.0.0:2380 \
---advertise-peer-urls=http://${domain}:2380 \
---client-urls=http://0.0.0.0:2379 \
---advertise-client-urls=http://${domain}:2379 \
+--peer-urls=${SCHEME}://0.0.0.0:2380 \
+--advertise-peer-urls=${SCHEME}://${domain}:2380 \
+--client-urls=${SCHEME}://0.0.0.0:2379 \
+--advertise-client-urls=${SCHEME}://${domain}:2379 \
 --config=/etc/pd/pd.toml \
 "
 

charts/tidb-cluster/templates/scripts/_start_pump.sh.tpl

@@ -1,6 +1,9 @@
 set -euo pipefail
+
+SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }}
+
 /pump \
--pd-urls=http://{{ template "cluster.name" . }}-pd:2379 \
+-pd-urls=${SCHEME}://{{ template "cluster.name" . }}-pd:2379 \
 -L={{ .Values.binlog.pump.logLevel | default "info" }} \
 -advertise-addr=`echo ${HOSTNAME}`.{{ template "cluster.name" . }}-pump:8250 \
 -config=/etc/pump/pump.toml \

charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl

@@ -28,9 +28,11 @@ then
 	tail -f /dev/null
 fi
 
+SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }}
+
 # Use HOSTNAME if POD_NAME is unset for backward compatibility.
 POD_NAME=${POD_NAME:-$HOSTNAME}
-ARGS="--pd=${CLUSTER_NAME}-pd:2379 \
+ARGS="--pd=${SCHEME}://${CLUSTER_NAME}-pd:2379 \
 --advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20160 \
 --addr=0.0.0.0:20160 \
 --status-addr=0.0.0.0:20180 \

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -20,6 +20,7 @@ metadata:
 spec:
   pvReclaimPolicy: {{ .Values.pvReclaimPolicy }}
   timezone: {{ .Values.timezone | default "UTC" }}
+  enableTLSCluster: {{ .Values.enableTLSCluster | default false }}
   services:
 {{ toYaml .Values.services | indent 4 }}
   schedulerName: {{ .Values.schedulerName | default "default-scheduler" }}
@@ -70,6 +71,7 @@ spec:
   {{- end }}
     hostNetwork: {{ .Values.tikv.hostNetwork }}
   tidb:
+    enableTLSClient: {{ .Values.tidb.enableTLSClient | default false }}
     replicas: {{ .Values.tidb.replicas }}
     image: {{ .Values.tidb.image }}
     imagePullPolicy: {{ .Values.tidb.imagePullPolicy | default "IfNotPresent" }}

charts/tidb-cluster/values.yaml

@@ -47,6 +47,11 @@ discovery:
 # if the ConfigMap was not changed.
 enableConfigMapRollout: true
 
+# Whether enable TLS connections between server nodes.
+# When enabled, PD/TiDB/TiKV will use TLS encrypted connections to transfer data between each node,
+# certificates will be generated automatically (if not already present).
+enableTLSCluster: false
+
 pd:
   # Please refer to https://github.com/pingcap/pd/blob/master/conf/config.toml for the default
   # pd configurations (change to the tags of your pd version),
@@ -245,6 +250,7 @@ tidb:
   config: |
     [log]
     level = "info"
+
   # # Here are some parameters you MUST customize (Please configure in the above 'tidb.config' section):
   # [performance]
   #   # Normally it should be tuned to `tidb.resources.limits.cpu`, for example: 16000m -> 16
@@ -319,6 +325,12 @@ tidb:
     # the start argument to specify the plugin id (name "-" version) that needs to be loaded, e.g. 'conn_limit-1'.
     list: ["whitelist-1"]
 
+  # Whether enable TLS connection between TiDB server and MySQL client.
+  # When enabled, TiDB will accept TLS encrypted connections from MySQL client, certificates will be generated
+  # automatically.
+  # Note: TLS connection is not forced on the server side, plain connections are also accepted after enableing.
+  enableTLSClient: false
+
 # mysqlClient is used to set password for TiDB
 # it must has Python MySQL client installed
 mysqlClient:

charts/tidb-operator/templates/controller-manager-rbac.yaml

@@ -35,7 +35,7 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "delete"]
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["get", "list", "watch"]
+  verbs: ["create", "get", "list", "watch"]
 - apiGroups: [""]
   resources: ["persistentvolumeclaims"]
   verbs: ["get", "list", "watch", "create", "update", "delete"]
@@ -63,6 +63,12 @@ rules:
 - apiGroups: [""]
   resources: ["persistentvolumes"]
   verbs: ["get", "list", "watch", "patch","update"]
+- apiGroups: ["certificates.k8s.io"]
+  resources: ["certificatesigningrequests"]
+  verbs: ["create", "get", "watch", "delete"]
+- apiGroups: ["certificates.k8s.io"]
+  resources: ["certificatesigningrequests/approval", "certificatesigningrequests/status"]
+  verbs: ["update"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1beta1
@@ -108,7 +114,7 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "delete"]
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["get", "list", "watch"]
+  verbs: ["create", "get", "list", "watch"]
 - apiGroups: [""]
   resources: ["persistentvolumeclaims"]
   verbs: ["get", "list", "watch", "create", "update", "delete"]

cmd/controller-manager/main.go

@@ -136,6 +136,10 @@ func main() {
 	}
 
 	tcController := tidbcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory, autoFailover, pdFailoverPeriod, tikvFailoverPeriod, tidbFailoverPeriod)
+	secControl := controller.NewRealSecretControl(kubeCli, kubeInformerFactory.Core().V1().Secrets().Lister())
+	certController := controller.NewRealCertControl(kubeCli,
+		kubeInformerFactory.Certificates().V1beta1().CertificateSigningRequests().Lister(),
+		secControl)
 	backupController := backup.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
 	restoreController := restore.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
 	bsController := backupschedule.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
@@ -145,6 +149,9 @@ func main() {
 	go kubeInformerFactory.Start(controllerCtx.Done())
 
 	onStarted := func(ctx context.Context) {
+		if err := generateClientCert(ns, "tidb-operator", certController); err != nil {
+			return
+		}
 		go wait.Forever(func() { backupController.Run(workers, ctx.Done()) }, waitDuration)
 		go wait.Forever(func() { restoreController.Run(workers, ctx.Done()) }, waitDuration)
 		go wait.Forever(func() { bsController.Run(workers, ctx.Done()) }, waitDuration)
@@ -170,3 +177,25 @@ func main() {
 
 	glog.Fatal(http.ListenAndServe(":6060", nil))
 }
+
+func generateClientCert(ns string, commonName string, certController controller.CertControlInterface) error {
+	secretName := "tidb-operator-pd-client"
+	if certController.CheckSecret(ns, secretName) {
+		return nil
+	}
+
+	hostList := []string{
+		commonName,
+	}
+
+	certOpts := &controller.TiDBClusterCertOptions{
+		Namespace:  ns,
+		Instance:   commonName,
+		CommonName: commonName,
+		HostList:   hostList,
+		Component:  "tidb-operator",
+		Suffix:     "pd-client",
+	}
+
+	return certController.Create(certOpts)
+}

cmd/discovery/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/pingcap/tidb-operator/version"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/util/logs"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 )
 
@@ -46,9 +47,13 @@ func main() {
 	if err != nil {
 		glog.Fatalf("failed to create Clientset: %v", err)
 	}
+	kubeCli, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		glog.Fatalf("failed to get kubernetes Clientset: %v", err)
+	}
 
 	go wait.Forever(func() {
-		server.StartServer(cli, port)
+		server.StartServer(cli, kubeCli, port)
 	}, 5*time.Second)
 	glog.Fatal(http.ListenAndServe(":6060", nil))
 }