Flexible tidb initializer job with secret set outside of helm

charts/tidb-cluster/templates/NOTES.txt

@@ -3,17 +3,17 @@ Cluster Startup
      watch kubectl get pods --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -o wide
 2. List services in the tidb-cluster
      kubectl get services --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }}
-{{- if .Values.tidb.password }}
+{{- if .Values.tidb.passwordSecretName }}
 3. Wait until tidb-initializer pod becomes completed
      watch kubectl get po --namespace {{ .Release.Namespace }}  -l app.kubernetes.io/component=tidb-initializer
 4. Get the TiDB password
-     kubectl get secret -n {{ .Release.Namespace }} {{ .Values.clusterName }}-tidb -o jsonpath="{.data.password}" | base64 --decode | awk '{print $6}'
-{{- end -}}
+     kubectl get secret -n {{ .Release.Namespace }} {{ .Values.tidb.passwordSecretName }} -ojsonpath='{.data.root}' | base64 --decode
+{{- end }}
 
 Cluster access
 * Access tidb-cluster using the MySQL client
     kubectl port-forward -n {{ .Release.Namespace }} svc/{{ .Values.clusterName }}-tidb 4000:4000 &
-{{- if .Values.tidb.password }}
+{{- if .Values.tidb.passwordSecretName }}
     mysql -h 127.0.0.1 -P 4000 -u root -D test -p
 {{- else -}}
     mysql -h 127.0.0.1 -P 4000 -u root -D test

charts/tidb-cluster/templates/tidb-configmap.yaml

@@ -11,7 +11,10 @@ metadata:
 data:
   startup-script: |-
 {{ tuple "scripts/_start_tidb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-
+  {{- if .Values.tidb.initSql }}
+  init-sql: |-
+{{ .Values.tidb.initSql | indent 4 }}
+  {{- end }}
   config-file: |-
     {{- if .Values.tidb.config }}
 {{ .Values.tidb.config | indent 4 }}

charts/tidb-cluster/templates/tidb-initializer-job.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.tidb.password }}
+{{- if (.Values.tidb.passwordSecretName) or (.Values.tidb.initSql) }}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -10,6 +10,7 @@ metadata:
     app.kubernetes.io/component: tidb-initializer
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 spec:
+  backoffLimit: 1000
   template:
     metadata:
       labels:
@@ -23,22 +24,51 @@ spec:
         image: {{ .Values.mysqlClient.image }}
         imagePullPolicy: {{ .Values.mysqlClient.imagePullPolicy | default "IfNotPresent" }}
         command:
-        - /bin/sh
+        - python
         - -c
-        # Read sql from file to avoid special characters interpreted as builtin variable
-        # And also avoid plain text password show in Job and Pod spec
-        # Besides we can also add more SQL in the file for initialization, eg. create database, create user etc
         - |
-          until mysql -h {{ .Values.clusterName }}-tidb -P 4000 --connect-timeout=5 < /data/init-password.sql; do sleep 2; done
+          import os, MySQLdb
+          host = {{ printf "%s-tidb" .Values.clusterName | quote }}
+          port = 4000
+          password_dir = '/etc/tidb/password'
+          conn = MySQLdb.connect(host=host, port=port, user='root', connect_timeout=5)
+          for file in os.listdir(password_dir):
+              if file.startswith('.'):
+                  continue
+              user = file
+              with open(os.path.join(password_dir, file), 'r') as f:
+                  password = f.read()
+              if user == 'root':
+                  conn.cursor().execute("set password for 'root'@'%%' = %s;", (password,))
+              else:
+                  conn.cursor().execute("create user %s@'%%' identified by %s;", (user, password,))
+          conn.cursor().execute("flush privileges;")
+          conn.commit()
+          {{- if .Values.tidb.initSql }}
+          with open('/data/init.sql', 'r') as sql:
+              for line in sql.readlines():
+                  conn.cursor().execute(line)
+                  conn.commit()
+          {{- end }}
         volumeMounts:
         - name: password
+          mountPath: /etc/tidb/password
+          readOnly: true
+        {{- if .Values.tidb.initSql }}
+        - name: init-sql
           mountPath: /data
           readOnly: true
+        {{- end }}
       volumes:
       - name: password
         secret:
-          secretName: {{ .Values.clusterName }}-tidb
+          secretName: {{ .Values.tidb.passwordSecretName }}
+      {{- if .Values.tidb.initSql }}
+      - name: init-sql
+        configMap:
+          name: {{ .Values.clusterName }}-tidb
           items:
-          - key: password
-            path: init-password.sql
+          - key: init-sql
+            path: init.sql
+      {{- end }}
 {{- end }}

charts/tidb-cluster/values.yaml

@@ -145,10 +145,13 @@ tikvPromGateway:
 
 tidb:
   replicas: 2
-  # The password to access TiDB
-  # If set, the password will be stored both in helm and in a Secret
+  # The secret name of root password, you can create secret with following command:
+  # kubectl create secret generic tidb-secret --from-literal=root_password=<root-password>
   # If unset, the root password will be empty and you can set it after connecting
-  # password: "admin"
+  # passwordSecretName: tidb-secret
+  # initSql is the SQL statements executed after the TiDB cluster is bootstrapped.
+  # initSql: |-
+  #   create database app;
   image: pingcap/tidb:v2.1.0
   # Image pull policy.
   imagePullPolicy: IfNotPresent
@@ -187,8 +190,9 @@ tidb:
         memory: 5Mi
 
 # mysqlClient is used to set password for TiDB
+# it must has Python MySQL client installed
 mysqlClient:
-  image: pingcap/tidb-enterprise-tools:latest
+  image: tnir/mysqlclient
   imagePullPolicy: IfNotPresent
 
 monitor:

images/tidb-operator-e2e/tidb-cluster-values.yaml

@@ -116,8 +116,13 @@ tikvPromGateway:
 
 tidb:
   replicas: 2
-  # password is TiDB's password, if omit password, a random password is generated
-  password: "admin"
+  # The secret name of root password, you can create secret with following command:
+  # kubectl create secret generic tidb-secret --from-literal=root_password=<root-password>
+  # If unset, the root password will be empty and you can set it after connecting
+  # passwordSecretName: tidb-secret
+  # initSql is the SQL statements executed after the TiDB cluster is bootstrapped.
+  # initSql: |-
+  #   create database app;
   image: pingcap/tidb:v2.1.0
   imagePullPolicy: IfNotPresent
   logLevel: info
@@ -146,8 +151,9 @@ tidb:
   separateSlowLog: true
 
 # mysqlClient is used to set password for TiDB
+# it must has Python MySQL client installed
 mysqlClient:
-  image: pingcap/tidb-enterprise-tools:latest
+  image: tnir/mysqlclient
   imagePullPolicy: IfNotPresent
 
 monitor:

tests/e2e/create.go

@@ -30,6 +30,7 @@ import (
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/controller"
 	"github.com/pingcap/tidb-operator/pkg/label"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -58,12 +59,16 @@ type Response struct {
 func testCreate(ns, clusterName string) {
 	By(fmt.Sprintf("When create the TiDB cluster: %s/%s", ns, clusterName))
 	instanceName := getInstanceName(ns, clusterName)
+
 	cmdStr := fmt.Sprintf("helm install /charts/tidb-cluster -f /tidb-cluster-values.yaml"+
-		" -n %s --namespace=%s --set clusterName=%s",
-		instanceName, ns, clusterName)
+		" -n %s --namespace=%s --set clusterName=%s,tidb.passwordSecretName=%s",
+		instanceName, ns, clusterName, ns+"-"+clusterName)
 	_, err := execCmd(cmdStr)
 	Expect(err).NotTo(HaveOccurred())
 
+	err = createSecret(ns, clusterName)
+	Expect(err).NotTo(HaveOccurred())
+
 	By("Then all members should running")
 	err = wait.Poll(5*time.Second, 5*time.Minute, func() (bool, error) {
 		return allMembersRunning(ns, clusterName)
@@ -588,3 +593,19 @@ outerLoop:
 
 	return true, nil
 }
+
+func createSecret(ns, clusterName string) error {
+	secretName := ns + "-" + clusterName
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: ns,
+		},
+		Data: map[string][]byte{
+			"root": []byte(password),
+		},
+		Type: corev1.SecretTypeOpaque,
+	}
+	_, err := kubeCli.CoreV1().Secrets(ns).Create(&secret)
+	return err
+}

tests/e2e/test_helper.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/onsi/ginkgo" // revive:disable:dot-imports
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/label"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -99,6 +100,10 @@ func clearOperator() error {
 		if err != nil {
 			return err
 		}
+		err = kubeCli.CoreV1().Secrets(fixture.ns).Delete(fixture.ns+"-"+fixture.clusterName, nil)
+		if err != nil && !apierrs.IsNotFound(err) {
+			return err
+		}
 	}
 
 	_, err := execCmd(fmt.Sprintf("helm del --purge %s", operatorHelmName))