From 7076bec3398b111e02ea06fb53f9bf9533db161a Mon Sep 17 00:00:00 2001 From: weekface Date: Wed, 18 Mar 2020 17:17:04 +0800 Subject: [PATCH 1/2] TLS for Pump and Drainer --- charts/tidb-drainer/templates/_helpers.tpl | 8 +-- .../templates/drainer-statefulset.yaml | 4 +- charts/tidb-drainer/values.yaml | 19 +++++-- pkg/manager/member/pump_member_manager.go | 49 ++----------------- pkg/manager/member/template.go | 2 +- 5 files changed, 24 insertions(+), 58 deletions(-) diff --git a/charts/tidb-drainer/templates/_helpers.tpl b/charts/tidb-drainer/templates/_helpers.tpl index 49c13cabec..ee0ff83794 100644 --- a/charts/tidb-drainer/templates/_helpers.tpl +++ b/charts/tidb-drainer/templates/_helpers.tpl @@ -7,7 +7,7 @@ {{- end -}} {{- define "drainer.tlsSecretName" -}} -{{ .Values.clusterName }}-drainer +{{ .Values.clusterName }}-drainer-cluster-secret {{- end -}} {{/* @@ -18,9 +18,9 @@ config-file: |- {{- if .Values.config }} {{ .Values.config | indent 2 }} {{- end -}} - {{- if .Values.enableTLSCluster }} + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} [security] - ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + ssl-ca = "/var/lib/drainer-tls/ca.crt" ssl-cert = "/var/lib/drainer-tls/tls.crt" ssl-key = "/var/lib/drainer-tls/tls.key" {{- end -}} @@ -31,7 +31,7 @@ config-file: |- {{- end -}} {{- define "cluster.scheme" -}} -{{ if .Values.enableTLSCluster }}https{{ else }}http{{ end }} +{{ if and .Values.tlsCluster .Values.tlsCluster.enabled }}https{{ else }}http{{ end }} {{- end -}} {{- define "helm-toolkit.utils.template" -}} diff --git a/charts/tidb-drainer/templates/drainer-statefulset.yaml b/charts/tidb-drainer/templates/drainer-statefulset.yaml index 19383b25ad..e0bb02776a 100644 --- a/charts/tidb-drainer/templates/drainer-statefulset.yaml +++ b/charts/tidb-drainer/templates/drainer-statefulset.yaml @@ -46,7 +46,7 @@ spec: mountPath: /data - name: config mountPath: /etc/drainer - {{- if .Values.enableTLSCluster }} + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} - name: drainer-tls mountPath: /var/lib/drainer-tls readOnly: true @@ -65,7 +65,7 @@ spec: items: - key: config-file path: drainer.toml - {{- if .Values.enableTLSCluster }} + {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }} - name: drainer-tls secret: secretName: {{ include "drainer.tlsSecretName" . }} diff --git a/charts/tidb-drainer/values.yaml b/charts/tidb-drainer/values.yaml index 4c35d88daf..0f701b1ffc 100644 --- a/charts/tidb-drainer/values.yaml +++ b/charts/tidb-drainer/values.yaml @@ -29,13 +29,22 @@ disableDetect: false # if drainer donesn't have checkpoint, use initial commitTS to initial checkpoint initialCommitTs: 0 -# Whether enable TLS connections between server nodes. -# When enabled, DRAINER will use TLS encrypted connections to transfer data with PUMP node, -# certificates will be generated by script "hack/create-cert.sh" manually -enableTLSCluster: false +# Whether enable the TLS connection between TiDB server components +tlsCluster: + # The steps to enable this feature: + # 1. Generate Drainer certificate. + # There are multiple ways to generate these certificates: + # - user-provided certificates: https://pingcap.com/docs/stable/how-to/secure/generate-self-signed-certificates/ + # - use the K8s built-in certificate signing system signed certificates: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/ + # - or use cert-manager signed certificates: https://cert-manager.io/ + # 2. Create one secret object for Drainer which contains the certificates created above. + # The name of this Secret must be: -drainer-cluster-secret. + # For Drainer: kubectl create secret generic -drainer-cluster-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + # 3. Then create the Drainer cluster with `tlsCluster.enabled` set to `true`. + enabled: false # Refer to https://github.com/pingcap/tidb-binlog/blob/master/cmd/drainer/drainer.toml -# [security] section will be generated automatically if enableTLSCluster is set to true so users do not need to configure it. +# [security] section will be generated automatically if tlsCluster.enabled is set to true so users do not need to configure it. config: | detect-interval = 10 compressor = "" diff --git a/pkg/manager/member/pump_member_manager.go b/pkg/manager/member/pump_member_manager.go index 83902915d7..5899c4e878 100644 --- a/pkg/manager/member/pump_member_manager.go +++ b/pkg/manager/member/pump_member_manager.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/pkg/manager" + "github.com/pingcap/tidb-operator/pkg/util" apps "k8s.io/api/apps/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -99,13 +100,6 @@ func (pmm *pumpMemberManager) syncPumpStatefulSetForTidbCluster(tc *v1alpha1.Tid return err } - if tc.IsTLSClusterEnabled() { - err := pmm.syncPumpStatefulsetCerts(tc) - if err != nil { - return err - } - } - newPumpSet, err := getNewPumpStatefulSet(tc, cm) if err != nil { return err @@ -260,7 +254,7 @@ func getNewPumpConfigMap(tc *v1alpha1.TidbCluster) (*corev1.ConfigMap, error) { confTextStr = strings.Join([]string{ confTextStr, "[security]", - fmt.Sprintf("ssl-ca = \"%s\"", serviceAccountCAPath), + fmt.Sprintf("ssl-ca = \"%s\"", path.Join(pumpCertPath, corev1.ServiceAccountRootCAKey)), fmt.Sprintf("ssl-cert = \"%s\"", path.Join(pumpCertPath, corev1.TLSCertKey)), fmt.Sprintf("ssl-key = \"%s\"", path.Join(pumpCertPath, corev1.TLSPrivateKeyKey))}, "\n") } @@ -379,7 +373,7 @@ func getNewPumpStatefulSet(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*app volumes = append(volumes, corev1.Volume{ Name: pumpCertVolumeMount, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: controller.PumpMemberName(tc.Name), + SecretName: util.ClusterTLSSecretName(tc.Name, label.PumpLabelVal), }, }, }) @@ -479,43 +473,6 @@ func getPumpLogLevel(tc *v1alpha1.TidbCluster) string { return logLevel } -// syncPumpStatefulsetCerts creates the cert pair for Pump if not exist, the cert -// pair is used to communicate with other TiDB components, like TiDB and Drainer -func (pmm *pumpMemberManager) syncPumpStatefulsetCerts(tc *v1alpha1.TidbCluster) error { - ns := tc.GetNamespace() - tcName := tc.GetName() - svcName := controller.PumpMemberName(tcName) - peerName := controller.PumpPeerMemberName(tcName) - - if pmm.certControl.CheckSecret(ns, svcName) { - return nil - } - - hostList := []string{ - svcName, - peerName, - fmt.Sprintf("%s.%s", svcName, ns), - fmt.Sprintf("%s.%s", peerName, ns), - fmt.Sprintf("*.%s.%s", peerName, ns), - } - - ipList := []string{ - "127.0.0.1", "::1", // able to access https endpoint via loopback network - } - - certOpts := &controller.TiDBClusterCertOptions{ - Namespace: ns, - Instance: tcName, - CommonName: svcName, - HostList: hostList, - IPList: ipList, - Component: "pump", - Suffix: "pump", - } - - return pmm.certControl.Create(controller.GetOwnerRef(tc), certOpts) -} - func (pmm *pumpMemberManager) pumpStatefulSetIsUpgrading(set *apps.StatefulSet, tc *v1alpha1.TidbCluster) (bool, error) { if statefulSetIsUpgrading(set) { return true, nil diff --git a/pkg/manager/member/template.go b/pkg/manager/member/template.go index cd0330dc78..48090d41ba 100644 --- a/pkg/manager/member/template.go +++ b/pkg/manager/member/template.go @@ -256,7 +256,7 @@ var pumpStartScriptTpl = template.Must(template.New("pump-start-script").Parse(` /pump \ -pd-urls={{ .Scheme }}://{{ .ClusterName }}-pd:2379 \ -L={{ .LogLevel }} \ --advertise-addr=` + "`" + `echo ${HOSTNAME}` + "`" + `.{{ .ClusterName }}-pump:8250 \ +-addr=` + "`" + `echo ${HOSTNAME}` + "`" + `.{{ .ClusterName }}-pump:8250 \ -config=/etc/pump/pump.toml \ -data-dir=/data \ -log-file= From 58cc2554424edb243c55e32ac5b57816683ca038 Mon Sep 17 00:00:00 2001 From: weekface Date: Wed, 18 Mar 2020 19:00:48 +0800 Subject: [PATCH 2/2] fix CI --- pkg/manager/member/template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/manager/member/template.go b/pkg/manager/member/template.go index 48090d41ba..cd0330dc78 100644 --- a/pkg/manager/member/template.go +++ b/pkg/manager/member/template.go @@ -256,7 +256,7 @@ var pumpStartScriptTpl = template.Must(template.New("pump-start-script").Parse(` /pump \ -pd-urls={{ .Scheme }}://{{ .ClusterName }}-pd:2379 \ -L={{ .LogLevel }} \ --addr=` + "`" + `echo ${HOSTNAME}` + "`" + `.{{ .ClusterName }}-pump:8250 \ +-advertise-addr=` + "`" + `echo ${HOSTNAME}` + "`" + `.{{ .ClusterName }}-pump:8250 \ -config=/etc/pump/pump.toml \ -data-dir=/data \ -log-file=