Auto generate and sign certificates for TLS enabled cluster

charts/tidb-cluster/templates/_helpers.tpl

@@ -27,6 +27,10 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- default .Release.Name .Values.clusterName }}
 {{- end -}}
 
+{{- define "cluster.scheme" -}}
+{{ if .Values.enableTLSCluster }}https{{ else }}http{{ end }}
+{{- end -}}
+
 {{/*
 Encapsulate PD configmap data for consistent digest calculation
 */}}
@@ -40,8 +44,8 @@ config-file: |-
     {{- if .Values.enableTLSCluster }}
   [security]
   cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-  cert-path = "/var/lib/pd-tls/pd.crt"
-  key-path = "/var/lib/pd-tls/pd.key"
+  cert-path = "/var/lib/pd-tls/cert"
+  key-path = "/var/lib/pd-tls/key"
     {{- end -}}
 
 {{- end -}}
@@ -63,8 +67,8 @@ config-file: |-
     {{- if .Values.enableTLSCluster }}
   [security]
   ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-  cert-path = "/var/lib/tikv-tls/tikv.crt"
-  key-path = "/var/lib/tikv-tls/tikv.key"
+  cert-path = "/var/lib/tikv-tls/cert"
+  key-path = "/var/lib/tikv-tls/key"
     {{- end -}}
 
 {{- end -}}
@@ -92,13 +96,13 @@ config-file: |-
     {{- end -}}
     {{- if .Values.enableTLSCluster }}
   cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-  cluster-ssl-cert = "/var/lib/tidb-tls/tidb.crt"
-  cluster-ssl-key = "/var/lib/tidb-tls/tidb.key"
+  cluster-ssl-cert = "/var/lib/tidb-tls/cert"
+  cluster-ssl-key = "/var/lib/tidb-tls/key"
     {{- end -}}
     {{- if .Values.tidb.enableTLSClient }}
   ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-  ssl-cert = "/var/lib/tidb-tls/tidb.crt"
-  ssl-key = "/var/lib/tidb-tls/tidb.key"
+  ssl-cert = "/var/lib/tidb-server-tls/cert"
+  ssl-key = "/var/lib/tidb-server-tls/key"
     {{- end -}}
 
 {{- end -}}

charts/tidb-cluster/templates/config/_drainer-config.tpl

@@ -11,7 +11,7 @@ detect-interval = {{ .Values.binlog.drainer.detectInterval | default 10 }}
 data-dir = "/data"
 
 # a comma separated list of PD endpoints
-pd-urls = "http://{{ template "cluster.name" . }}-pd:2379"
+pd-urls = "{{ template "cluster.scheme" . }}://{{ template "cluster.name" . }}-pd:2379"
 
 # Use the specified compressor to compress payload between pump and drainer
 compressor = ""

charts/tidb-cluster/templates/config/_prometheus-config.tpl

@@ -21,6 +21,13 @@ scrape_configs:
     {{- end }}
     tls_config:
       insecure_skip_verify: true
+    {{- if .Values.enableTLSCluster }}
+      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      cert_file: /var/lib/pd-client-tls/cert
+      key_file: /var/lib/pd-client-tls/key
+
+    scheme: https
+    {{- end }}
     relabel_configs:
     - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
       action: keep
@@ -46,11 +53,67 @@ scrape_configs:
     - source_labels: [__meta_kubernetes_pod_ip]
       action: replace
       target_label: kubernetes_pod_ip
+    {{- if .Values.enableTLSCluster }}
+    # This is a workaround of https://github.com/tikv/tikv/issues/5340 and should
+    # be removed after TiKV fix this issue
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: drop
+      regex: .*\-tikv\-\d*$
+    {{- end }}
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: replace
+      target_label: instance
+    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+      action: replace
+      target_label: cluster
+  {{- if .Values.enableTLSCluster }}
+  # This is a workaround of https://github.com/tikv/tikv/issues/5340 and should
+  # be removed after TiKV fix this issue
+  - job_name: 'tidb-cluster-tikv'
+    scrape_interval: 15s
+    honor_labels: true
+    kubernetes_sd_configs:
+    - role: pod
+      namespaces:
+        names:
+        - {{ .Release.Namespace }}
+    tls_config:
+      insecure_skip_verify: true
+    scheme: http
+    relabel_configs:
+    - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+      action: keep
+      regex: {{ .Release.Name }}
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+      action: keep
+      regex: true
+    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+      action: replace
+      target_label: __metrics_path__
+      regex: (.+)
+    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+      action: replace
+      regex: ([^:]+)(?::\d+)?;(\d+)
+      replacement: $1:$2
+      target_label: __address__
+    - source_labels: [__meta_kubernetes_namespace]
+      action: replace
+      target_label: kubernetes_namespace
+    - source_labels: [__meta_kubernetes_pod_node_name]
+      action: replace
+      target_label: kubernetes_node
+    - source_labels: [__meta_kubernetes_pod_ip]
+      action: replace
+      target_label: kubernetes_pod_ip
+    - source_labels: [__meta_kubernetes_pod_name]
+      action: keep
+      regex: .*\-tikv\-\d*$
     - source_labels: [__meta_kubernetes_pod_name]
       action: replace
       target_label: instance
     - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
       action: replace
       target_label: cluster
+  {{- end }}
 rule_files:
   - '/prometheus-rules/rules/*.rules.yml'

charts/tidb-cluster/templates/config/_pump-config.tpl

@@ -17,7 +17,7 @@ data-dir = "/data"
 heartbeat-interval = {{ .Values.binlog.pump.heartbeatInterval | default 2 }}
 
 # a comma separated list of PD endpoints
-pd-urls = "http://{{ template "cluster.name" . }}-pd:2379"
+pd-urls = "{{ template "cluster.scheme" . }}://{{ template "cluster.name" . }}-pd:2379"
 
 #[security]
 # Path of file that contains list of trusted SSL CAs for connection with cluster components.

charts/tidb-cluster/templates/discovery-deployment.yaml

@@ -49,14 +49,3 @@ spec:
           - name: TZ
             value: {{ .Values.timezone | default "UTC" }}
         {{- end }}
-{{- if .Values.enableTLSCluster }}
-        volumeMounts:
-        - mountPath: /var/lib/tls
-          name: tls
-          readOnly: true
-      volumes:
-      - name: tls
-        secret:
-          defaultMode: 420
-          secretName: client-tls
-{{- end -}}

charts/tidb-cluster/templates/discovery-rbac.yaml

@@ -14,6 +14,9 @@ rules:
   resources: ["tidbclusters"]
   resourceNames: [{{ template "cluster.name" . }}]
   verbs: ["get"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1beta1

charts/tidb-cluster/templates/monitor-deployment.yaml

@@ -134,6 +134,11 @@ spec:
           - name: prometheus-rules
             mountPath: /prometheus-rules
             readOnly: false
+          {{- if .Values.enableTLSCluster }}
+          - name: tls-pd-client
+            mountPath: /var/lib/pd-client-tls
+            readOnly: true
+          {{- end }}
       {{- if .Values.monitor.grafana.create }}
       - name: reloader
         image: {{ .Values.monitor.reloader.image }}
@@ -236,6 +241,12 @@ spec:
         name: prometheus-rules
       - emptyDir: {}
         name: grafana-dashboard
+      {{- if .Values.enableTLSCluster }}
+      - name: tls-pd-client
+        secret:
+          defaultMode: 420
+          secretName: {{ .Release.Name }}-pd-client
+      {{- end }}
     {{- if .Values.monitor.tolerations }}
       tolerations:
 {{ toYaml .Values.monitor.tolerations | indent 6 }}

charts/tidb-cluster/templates/scripts/_start_drainer.sh.tpl

@@ -26,7 +26,7 @@ done
 
 /drainer \
 -L={{ .Values.binlog.drainer.logLevel | default "info" }} \
--pd-urls=http://{{ template "cluster.name" . }}-pd:2379 \
+-pd-urls={{ template "cluster.scheme" . }}://{{ template "cluster.name" . }}-pd:2379 \
 -addr=`echo ${HOSTNAME}`.{{ template "cluster.name" . }}-drainer:8249 \
 -config=/etc/drainer/drainer.toml \
 -disable-detect={{ .Values.binlog.drainer.disableDetect | default false }} \

charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl

@@ -58,14 +58,12 @@ while true; do
     fi
 done
 
-SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }}
-
 ARGS="--data-dir=/var/lib/pd \
 --name=${POD_NAME} \
---peer-urls=${SCHEME}://0.0.0.0:2380 \
---advertise-peer-urls=${SCHEME}://${domain}:2380 \
---client-urls=${SCHEME}://0.0.0.0:2379 \
---advertise-client-urls=${SCHEME}://${domain}:2379 \
+--peer-urls={{ template "cluster.scheme" . }}://0.0.0.0:2380 \
+--advertise-peer-urls={{ template "cluster.scheme" . }}://${domain}:2380 \
+--client-urls={{ template "cluster.scheme" . }}://0.0.0.0:2379 \
+--advertise-client-urls={{ template "cluster.scheme" . }}://${domain}:2379 \
 --config=/etc/pd/pd.toml \
 "
 

charts/tidb-cluster/templates/scripts/_start_pump.sh.tpl

@@ -1,6 +1,7 @@
 set -euo pipefail
+
 /pump \
--pd-urls=http://{{ template "cluster.name" . }}-pd:2379 \
+-pd-urls={{ template "cluster.scheme" . }}://{{ template "cluster.name" . }}-pd:2379 \
 -L={{ .Values.binlog.pump.logLevel | default "info" }} \
 -advertise-addr=`echo ${HOSTNAME}`.{{ template "cluster.name" . }}-pump:8250 \
 -config=/etc/pump/pump.toml \

charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl

@@ -28,11 +28,9 @@ then
 	tail -f /dev/null
 fi
 
-SCHEME={{ if .Values.enableTLSCluster }}"https"{{ else }}"http"{{ end }}
-
 # Use HOSTNAME if POD_NAME is unset for backward compatibility.
 POD_NAME=${POD_NAME:-$HOSTNAME}
-ARGS="--pd=${SCHEME}://${CLUSTER_NAME}-pd:2379 \
+ARGS="--pd={{ template "cluster.scheme" . }}://${CLUSTER_NAME}-pd:2379 \
 --advertise-addr=${POD_NAME}.${HEADLESS_SERVICE_NAME}.${NAMESPACE}.svc:20160 \
 --addr=0.0.0.0:20160 \
 --status-addr=0.0.0.0:20180 \

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -22,7 +22,6 @@ spec:
   enablePVReclaim: {{ .Values.enablePVReclaim }}
   timezone: {{ .Values.timezone | default "UTC" }}
   enableTLSCluster: {{ .Values.enableTLSCluster | default false }}
-  enableTLSClient: {{ .Values.enableTLSClient | default false }}
   services:
 {{ toYaml .Values.services | indent 4 }}
   schedulerName: {{ .Values.schedulerName | default "default-scheduler" }}
@@ -89,6 +88,7 @@ spec:
   {{- end }}
     maxFailoverCount: {{ .Values.tikv.maxFailoverCount | default 3 }}
   tidb:
+    enableTLSClient: {{ .Values.tidb.enableTLSClient | default false }}
     replicas: {{ .Values.tidb.replicas }}
     image: {{ .Values.tidb.image }}
     imagePullPolicy: {{ .Values.tidb.imagePullPolicy | default "IfNotPresent" }}

charts/tidb-operator/templates/controller-manager-rbac.yaml

@@ -35,7 +35,7 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "delete"]
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["get", "list", "watch"]
+  verbs: ["create", "get", "list", "watch"]
 - apiGroups: [""]
   resources: ["persistentvolumeclaims"]
   verbs: ["get", "list", "watch", "create", "update", "delete"]
@@ -63,6 +63,12 @@ rules:
 - apiGroups: [""]
   resources: ["persistentvolumes"]
   verbs: ["get", "list", "watch", "patch","update"]
+- apiGroups: ["certificates.k8s.io"]
+  resources: ["certificatesigningrequests"]
+  verbs: ["create", "get", "list", "watch", "delete"]
+- apiGroups: ["certificates.k8s.io"]
+  resources: ["certificatesigningrequests/approval", "certificatesigningrequests/status"]
+  verbs: ["update"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1beta1
@@ -108,7 +114,7 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "delete"]
 - apiGroups: [""]
   resources: ["secrets"]
-  verbs: ["get", "list", "watch"]
+  verbs: ["create", "get", "list", "watch"]
 - apiGroups: [""]
   resources: ["persistentvolumeclaims"]
   verbs: ["get", "list", "watch", "create", "update", "delete"]

cmd/discovery/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/pingcap/tidb-operator/pkg/discovery/server"
 	"github.com/pingcap/tidb-operator/pkg/version"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/component-base/logs"
 	glog "k8s.io/klog"
@@ -46,9 +47,13 @@ func main() {
 	if err != nil {
 		glog.Fatalf("failed to create Clientset: %v", err)
 	}
+	kubeCli, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		glog.Fatalf("failed to get kubernetes Clientset: %v", err)
+	}
 
 	go wait.Forever(func() {
-		server.StartServer(cli, port)
+		server.StartServer(cli, kubeCli, port)
 	}, 5*time.Second)
 	glog.Fatal(http.ListenAndServe(":6060", nil))
 }

go.mod

@@ -63,6 +63,7 @@ require (
 	github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect

go.sum

@@ -30,7 +30,6 @@ github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5
 github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
 github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e h1:eb0Pzkt15Bm7f2FFYv7sjY7NPFi3cPkS3tv1CcrFBWA=
 github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
-github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc=
@@ -740,6 +739,7 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e h1:JgcxKXxCjrA2tyDP/aNU9K0Ck5Czfk6C7e2tMw7+bSI=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

manifests/crd.yaml

@@ -77,7 +77,7 @@ spec:
             enablePVReclaim:
               type: boolean
             enableTLSCluster:
-              description: Enable TLS connection between TiDB server compoments
+              description: Enable TLS connection between TiDB server components
               type: boolean
             pd:
               description: PDSpec contains details of PD members

pkg/apis/pingcap/v1alpha1/types.go

@@ -101,7 +101,7 @@ type TidbClusterSpec struct {
 	PVReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"`
 	EnablePVReclaim bool                                 `json:"enablePVReclaim,omitempty"`
 	Timezone        string                               `json:"timezone,omitempty"`
-	// Enable TLS connection between TiDB server compoments
+	// Enable TLS connection between TiDB server components
 	EnableTLSCluster bool `json:"enableTLSCluster,omitempty"`
 }