From 4b75cb6d4ae5277316054d30013f42e0445d83b9 Mon Sep 17 00:00:00 2001 From: tennix Date: Tue, 26 Feb 2019 15:49:04 +0800 Subject: [PATCH 1/5] Flexible tidb initializer job with secret set outside of helm --- .../templates/tidb-configmap.yaml | 5 +- .../templates/tidb-initializer-job.yaml | 51 ++++++++++++++----- charts/tidb-cluster/values.yaml | 11 ++-- 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/charts/tidb-cluster/templates/tidb-configmap.yaml b/charts/tidb-cluster/templates/tidb-configmap.yaml index 8a2100765f..5ee5a9f099 100644 --- a/charts/tidb-cluster/templates/tidb-configmap.yaml +++ b/charts/tidb-cluster/templates/tidb-configmap.yaml @@ -11,7 +11,10 @@ metadata: data: startup-script: |- {{ tuple "scripts/_start_tidb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - + {{- if .Values.tidb.initSql }} + init-sql: |- +{{ .Values.tidb.initSql | indent 4 }} + {{- end }} config-file: |- {{- if .Values.tidb.config }} {{ .Values.tidb.config | indent 4 }} diff --git a/charts/tidb-cluster/templates/tidb-initializer-job.yaml b/charts/tidb-cluster/templates/tidb-initializer-job.yaml index c4bdc76c5b..b642bb287f 100644 --- a/charts/tidb-cluster/templates/tidb-initializer-job.yaml +++ b/charts/tidb-cluster/templates/tidb-initializer-job.yaml @@ -1,4 +1,3 @@ -{{- if .Values.tidb.password }} apiVersion: batch/v1 kind: Job metadata: @@ -9,7 +8,10 @@ metadata: app.kubernetes.io/instance: {{ .Values.clusterName }} app.kubernetes.io/component: tidb-initializer helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + annotations: + helm.sh/hook: post-install spec: + backoffLimit: 1000 template: metadata: labels: @@ -23,22 +25,45 @@ spec: image: {{ .Values.mysqlClient.image }} imagePullPolicy: {{ .Values.mysqlClient.imagePullPolicy | default "IfNotPresent" }} command: - - /bin/sh + - python - -c - # Read sql from file to avoid special characters interpreted as builtin variable - # And also avoid plain text password show in Job and Pod spec - # Besides we can also add more SQL in the file for initialization, eg. create database, create user etc - | - until mysql -h {{ .Values.clusterName }}-tidb -P 4000 --connect-timeout=5 < /data/init-password.sql; do sleep 2; done + import os, MySQLdb + host = {{ .Values.clusterName | quote }} + port = 4000 + root_password = os.environ.get('ROOT_PASSWORD') + system_password = os.environ.get('SYSTEM_PASSWORD') + conn = MySQLdb.connect(host=host, port=port, user=user) + if root_password: + conn.cursor().execute("set password for 'root'@'%%' = '%s';", (root_password,)) + if system_password: + conn.cursor().execute("create user 'system'@'%%' identified by '%s';", (system_password,)) + conn.cursor().execute("flush privileges;") + conn.commit() + with open('/data/init.sql', 'r') as sql: + for line in sql.readlines(): + conn.cursor().execute(line) + conn.commit() + env: + - name: ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.tidb.passwordSecret }} + key: root + - name: SYSTEM_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.tidb.passwordSecret }} + key: system + optional: true volumeMounts: - - name: password + - name: init-sql mountPath: /data readOnly: true volumes: - - name: password - secret: - secretName: {{ .Values.clusterName }}-tidb + - name: init-sql + configMap: + name: {{ .Values.clusterName }}-tidb items: - - key: password - path: init-password.sql -{{- end }} + - key: init.sql + path: init.sql diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 5a6098862a..7eae12e5ec 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -145,10 +145,13 @@ tikvPromGateway: tidb: replicas: 2 - # The password to access TiDB - # If set, the password will be stored both in helm and in a Secret + # The secret name of root password, you can create secret with following command: + # kubectl create secret generic tidb-secret --from-literal=root= # If unset, the root password will be empty and you can set it after connecting - # password: "admin" + # passwordSecret: tidb-secret + # initSql is the SQL statements executed after the TiDB cluster is bootstrapped. + # initSql: |- + # create database app; image: pingcap/tidb:v2.1.0 # Image pull policy. imagePullPolicy: IfNotPresent @@ -178,7 +181,7 @@ tidb: # mysqlClient is used to set password for TiDB mysqlClient: - image: pingcap/tidb-enterprise-tools:latest + image: tnir/mysqlclient imagePullPolicy: IfNotPresent monitor: From 169215be986064b11326dec2899ab64dd2e06003 Mon Sep 17 00:00:00 2001 From: tennix Date: Tue, 26 Feb 2019 18:18:46 +0800 Subject: [PATCH 2/5] create initialize job only when secret or initSql is provided --- .../templates/tidb-initializer-job.yaml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/charts/tidb-cluster/templates/tidb-initializer-job.yaml b/charts/tidb-cluster/templates/tidb-initializer-job.yaml index b642bb287f..a2e6ba8c1a 100644 --- a/charts/tidb-cluster/templates/tidb-initializer-job.yaml +++ b/charts/tidb-cluster/templates/tidb-initializer-job.yaml @@ -1,3 +1,4 @@ +{{- if (.Values.tidb.passwordSecret) or (.Values.tidb.initSql) }} apiVersion: batch/v1 kind: Job metadata: @@ -8,8 +9,6 @@ metadata: app.kubernetes.io/instance: {{ .Values.clusterName }} app.kubernetes.io/component: tidb-initializer helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - annotations: - helm.sh/hook: post-install spec: backoffLimit: 1000 template: @@ -29,21 +28,23 @@ spec: - -c - | import os, MySQLdb - host = {{ .Values.clusterName | quote }} + host = {{ printf "%s-tidb" .Values.clusterName | quote }} port = 4000 root_password = os.environ.get('ROOT_PASSWORD') system_password = os.environ.get('SYSTEM_PASSWORD') - conn = MySQLdb.connect(host=host, port=port, user=user) + conn = MySQLdb.connect(host=host, port=port, user='root', connect_timeout=5) if root_password: - conn.cursor().execute("set password for 'root'@'%%' = '%s';", (root_password,)) + conn.cursor().execute("set password for 'root'@'%%' = %s;", (root_password,)) if system_password: - conn.cursor().execute("create user 'system'@'%%' identified by '%s';", (system_password,)) + conn.cursor().execute("create user 'system'@'%%' identified by %s;", (system_password,)) conn.cursor().execute("flush privileges;") conn.commit() + {{- if .Values.tidb.initSql }} with open('/data/init.sql', 'r') as sql: for line in sql.readlines(): conn.cursor().execute(line) conn.commit() + {{- end }} env: - name: ROOT_PASSWORD valueFrom: @@ -56,6 +57,7 @@ spec: name: {{ .Values.tidb.passwordSecret }} key: system optional: true + {{- if .Values.tidb.initSql }} volumeMounts: - name: init-sql mountPath: /data @@ -65,5 +67,7 @@ spec: configMap: name: {{ .Values.clusterName }}-tidb items: - - key: init.sql + - key: init-sql path: init.sql + {{- end }} +{{- end }} From bb63031f4fe7a037322f73544fc8b28065e9760e Mon Sep 17 00:00:00 2001 From: tennix Date: Tue, 26 Feb 2019 21:28:07 +0800 Subject: [PATCH 3/5] update document --- charts/tidb-cluster/templates/NOTES.txt | 8 ++++---- charts/tidb-cluster/templates/tidb-secret.yaml | 15 --------------- charts/tidb-cluster/values.yaml | 1 + 3 files changed, 5 insertions(+), 19 deletions(-) delete mode 100644 charts/tidb-cluster/templates/tidb-secret.yaml diff --git a/charts/tidb-cluster/templates/NOTES.txt b/charts/tidb-cluster/templates/NOTES.txt index 27576a2a49..e62dcf5032 100644 --- a/charts/tidb-cluster/templates/NOTES.txt +++ b/charts/tidb-cluster/templates/NOTES.txt @@ -3,17 +3,17 @@ Cluster Startup watch kubectl get pods --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -o wide 2. List services in the tidb-cluster kubectl get services --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -{{- if .Values.tidb.password }} +{{- if .Values.tidb.passwordSecret }} 3. Wait until tidb-initializer pod becomes completed watch kubectl get po --namespace {{ .Release.Namespace }} -l app.kubernetes.io/component=tidb-initializer 4. Get the TiDB password - kubectl get secret -n {{ .Release.Namespace }} {{ .Values.clusterName }}-tidb -o jsonpath="{.data.password}" | base64 --decode | awk '{print $6}' -{{- end -}} + kubectl get secret -n {{ .Release.Namespace }} {{ .Values.tidb.passwordSecret }} -ojsonpath='{.data.root}' | base64 --decode +{{- end }} Cluster access * Access tidb-cluster using the MySQL client kubectl port-forward -n {{ .Release.Namespace }} svc/{{ .Values.clusterName }}-tidb 4000:4000 & -{{- if .Values.tidb.password }} +{{- if .Values.tidb.passwordSecret }} mysql -h 127.0.0.1 -P 4000 -u root -D test -p {{- else -}} mysql -h 127.0.0.1 -P 4000 -u root -D test diff --git a/charts/tidb-cluster/templates/tidb-secret.yaml b/charts/tidb-cluster/templates/tidb-secret.yaml deleted file mode 100644 index e92697767f..0000000000 --- a/charts/tidb-cluster/templates/tidb-secret.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.tidb.password }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.clusterName }}-tidb - labels: - app.kubernetes.io/name: {{ template "chart.name" . }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Values.clusterName }} - app.kubernetes.io/component: tidb - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} -type: Opaque -data: - password: {{ printf "SET PASSWORD FOR 'root'@'%%' = '%s' ; FLUSH PRIVILEGES;" .Values.tidb.password | b64enc }} -{{- end }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 7eae12e5ec..860f060db7 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -180,6 +180,7 @@ tidb: # cloud.google.com/load-balancer-type: Internal # mysqlClient is used to set password for TiDB +# it must has Python MySQL client installed mysqlClient: image: tnir/mysqlclient imagePullPolicy: IfNotPresent From 94240397a9a8aa17e7d0a2e6f7c0a22e43c0bb0a Mon Sep 17 00:00:00 2001 From: tennix Date: Wed, 27 Feb 2019 19:23:33 +0800 Subject: [PATCH 4/5] allow setting arbitrary user's password and fix e2e --- charts/tidb-cluster/templates/NOTES.txt | 6 +-- .../templates/tidb-initializer-job.yaml | 41 ++++++++++--------- charts/tidb-cluster/values.yaml | 4 +- .../tidb-cluster-values.yaml | 12 ++++-- tests/e2e/create.go | 27 ++++++++++-- tests/e2e/test_helper.go | 5 +++ 6 files changed, 64 insertions(+), 31 deletions(-) diff --git a/charts/tidb-cluster/templates/NOTES.txt b/charts/tidb-cluster/templates/NOTES.txt index e62dcf5032..4ae6c548cd 100644 --- a/charts/tidb-cluster/templates/NOTES.txt +++ b/charts/tidb-cluster/templates/NOTES.txt @@ -3,17 +3,17 @@ Cluster Startup watch kubectl get pods --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -o wide 2. List services in the tidb-cluster kubectl get services --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} -{{- if .Values.tidb.passwordSecret }} +{{- if .Values.tidb.passwordSecretName }} 3. Wait until tidb-initializer pod becomes completed watch kubectl get po --namespace {{ .Release.Namespace }} -l app.kubernetes.io/component=tidb-initializer 4. Get the TiDB password - kubectl get secret -n {{ .Release.Namespace }} {{ .Values.tidb.passwordSecret }} -ojsonpath='{.data.root}' | base64 --decode + kubectl get secret -n {{ .Release.Namespace }} {{ .Values.tidb.passwordSecretName }} -ojsonpath='{.data.root}' | base64 --decode {{- end }} Cluster access * Access tidb-cluster using the MySQL client kubectl port-forward -n {{ .Release.Namespace }} svc/{{ .Values.clusterName }}-tidb 4000:4000 & -{{- if .Values.tidb.passwordSecret }} +{{- if .Values.tidb.passwordSecretName }} mysql -h 127.0.0.1 -P 4000 -u root -D test -p {{- else -}} mysql -h 127.0.0.1 -P 4000 -u root -D test diff --git a/charts/tidb-cluster/templates/tidb-initializer-job.yaml b/charts/tidb-cluster/templates/tidb-initializer-job.yaml index a2e6ba8c1a..647dcd68fc 100644 --- a/charts/tidb-cluster/templates/tidb-initializer-job.yaml +++ b/charts/tidb-cluster/templates/tidb-initializer-job.yaml @@ -1,4 +1,4 @@ -{{- if (.Values.tidb.passwordSecret) or (.Values.tidb.initSql) }} +{{- if (.Values.tidb.passwordSecretName) or (.Values.tidb.initSql) }} apiVersion: batch/v1 kind: Job metadata: @@ -30,13 +30,18 @@ spec: import os, MySQLdb host = {{ printf "%s-tidb" .Values.clusterName | quote }} port = 4000 - root_password = os.environ.get('ROOT_PASSWORD') - system_password = os.environ.get('SYSTEM_PASSWORD') + password_dir = '/etc/tidb/password' conn = MySQLdb.connect(host=host, port=port, user='root', connect_timeout=5) - if root_password: - conn.cursor().execute("set password for 'root'@'%%' = %s;", (root_password,)) - if system_password: - conn.cursor().execute("create user 'system'@'%%' identified by %s;", (system_password,)) + for file in os.listdir(password_dir): + if file.startswith('.'): + continue + user = file + with open(os.path.join(password_dir, file), 'r') as f: + password = f.read() + if user == 'root': + conn.cursor().execute("set password for 'root'@'%%' = %s;", (password,)) + else: + conn.cursor().execute("create user %s@'%%' identified by %s;", (user, password,)) conn.cursor().execute("flush privileges;") conn.commit() {{- if .Values.tidb.initSql }} @@ -45,24 +50,20 @@ spec: conn.cursor().execute(line) conn.commit() {{- end }} - env: - - name: ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.tidb.passwordSecret }} - key: root - - name: SYSTEM_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.tidb.passwordSecret }} - key: system - optional: true - {{- if .Values.tidb.initSql }} volumeMounts: + - name: password + mountPath: /etc/tidb/password + readOnly: true + {{- if .Values.tidb.initSql }} - name: init-sql mountPath: /data readOnly: true + {{- end }} volumes: + - name: password + secret: + secretName: {{ .Values.tidb.passwordSecretName }} + {{- if .Values.tidb.initSql }} - name: init-sql configMap: name: {{ .Values.clusterName }}-tidb diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 860f060db7..c3650e3ac3 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -146,9 +146,9 @@ tikvPromGateway: tidb: replicas: 2 # The secret name of root password, you can create secret with following command: - # kubectl create secret generic tidb-secret --from-literal=root= + # kubectl create secret generic tidb-secret --from-literal=root_password= # If unset, the root password will be empty and you can set it after connecting - # passwordSecret: tidb-secret + # passwordSecretName: tidb-secret # initSql is the SQL statements executed after the TiDB cluster is bootstrapped. # initSql: |- # create database app; diff --git a/images/tidb-operator-e2e/tidb-cluster-values.yaml b/images/tidb-operator-e2e/tidb-cluster-values.yaml index e0845060c0..45e4398f29 100644 --- a/images/tidb-operator-e2e/tidb-cluster-values.yaml +++ b/images/tidb-operator-e2e/tidb-cluster-values.yaml @@ -116,8 +116,13 @@ tikvPromGateway: tidb: replicas: 2 - # password is TiDB's password, if omit password, a random password is generated - password: "admin" + # The secret name of root password, you can create secret with following command: + # kubectl create secret generic tidb-secret --from-literal=root_password= + # If unset, the root password will be empty and you can set it after connecting + # passwordSecretName: tidb-secret + # initSql is the SQL statements executed after the TiDB cluster is bootstrapped. + # initSql: |- + # create database app; image: pingcap/tidb:v2.1.0 imagePullPolicy: IfNotPresent logLevel: info @@ -145,8 +150,9 @@ tidb: # cloud.google.com/load-balancer-type: Internal # mysqlClient is used to set password for TiDB +# it must has Python MySQL client installed mysqlClient: - image: pingcap/tidb-enterprise-tools:latest + image: tnir/mysqlclient imagePullPolicy: IfNotPresent monitor: diff --git a/tests/e2e/create.go b/tests/e2e/create.go index ed091e4315..fbad629ad8 100644 --- a/tests/e2e/create.go +++ b/tests/e2e/create.go @@ -15,6 +15,7 @@ package e2e import ( "database/sql" + "encoding/base64" "encoding/json" "fmt" "io/ioutil" @@ -30,6 +31,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" @@ -58,10 +60,13 @@ type Response struct { func testCreate(ns, clusterName string) { By(fmt.Sprintf("When create the TiDB cluster: %s/%s", ns, clusterName)) instanceName := getInstanceName(ns, clusterName) + err := createSecret(ns, clusterName) + Expect(err).NotTo(HaveOccurred()) + cmdStr := fmt.Sprintf("helm install /charts/tidb-cluster -f /tidb-cluster-values.yaml"+ - " -n %s --namespace=%s --set clusterName=%s", - instanceName, ns, clusterName) - _, err := execCmd(cmdStr) + " -n %s --namespace=%s --set clusterName=%s,tidb.passwordSecretName=%s", + instanceName, ns, clusterName, ns+"-"+clusterName) + _, err = execCmd(cmdStr) Expect(err).NotTo(HaveOccurred()) By("Then all members should running") @@ -588,3 +593,19 @@ outerLoop: return true, nil } + +func createSecret(ns, clusterName string) error { + secretName := ns + "-" + clusterName + secret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: ns, + }, + Data: map[string][]byte{ + "root": []byte(base64.StdEncoding.EncodeToString([]byte(password))), + }, + Type: corev1.SecretTypeOpaque, + } + _, err := kubeCli.CoreV1().Secrets(ns).Create(&secret) + return err +} diff --git a/tests/e2e/test_helper.go b/tests/e2e/test_helper.go index d62886fa6d..98ee8e1806 100644 --- a/tests/e2e/test_helper.go +++ b/tests/e2e/test_helper.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo" // revive:disable:dot-imports "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/label" + apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" @@ -99,6 +100,10 @@ func clearOperator() error { if err != nil { return err } + err = kubeCli.CoreV1().Secrets(fixture.ns).Delete(fixture.ns+"-"+fixture.clusterName, nil) + if err != nil && apierrs.IsNotFound(err) { + return err + } } _, err := execCmd(fmt.Sprintf("helm del --purge %s", operatorHelmName)) From b6000b54fad4b326d15c4794aab79431bcd8732d Mon Sep 17 00:00:00 2001 From: tennix Date: Wed, 27 Feb 2019 21:52:06 +0800 Subject: [PATCH 5/5] tiny fix --- tests/e2e/create.go | 10 +++++----- tests/e2e/test_helper.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/e2e/create.go b/tests/e2e/create.go index fbad629ad8..f9bb38c315 100644 --- a/tests/e2e/create.go +++ b/tests/e2e/create.go @@ -15,7 +15,6 @@ package e2e import ( "database/sql" - "encoding/base64" "encoding/json" "fmt" "io/ioutil" @@ -60,13 +59,14 @@ type Response struct { func testCreate(ns, clusterName string) { By(fmt.Sprintf("When create the TiDB cluster: %s/%s", ns, clusterName)) instanceName := getInstanceName(ns, clusterName) - err := createSecret(ns, clusterName) - Expect(err).NotTo(HaveOccurred()) cmdStr := fmt.Sprintf("helm install /charts/tidb-cluster -f /tidb-cluster-values.yaml"+ " -n %s --namespace=%s --set clusterName=%s,tidb.passwordSecretName=%s", instanceName, ns, clusterName, ns+"-"+clusterName) - _, err = execCmd(cmdStr) + _, err := execCmd(cmdStr) + Expect(err).NotTo(HaveOccurred()) + + err = createSecret(ns, clusterName) Expect(err).NotTo(HaveOccurred()) By("Then all members should running") @@ -602,7 +602,7 @@ func createSecret(ns, clusterName string) error { Namespace: ns, }, Data: map[string][]byte{ - "root": []byte(base64.StdEncoding.EncodeToString([]byte(password))), + "root": []byte(password), }, Type: corev1.SecretTypeOpaque, } diff --git a/tests/e2e/test_helper.go b/tests/e2e/test_helper.go index 98ee8e1806..110d628dbe 100644 --- a/tests/e2e/test_helper.go +++ b/tests/e2e/test_helper.go @@ -101,7 +101,7 @@ func clearOperator() error { return err } err = kubeCli.CoreV1().Secrets(fixture.ns).Delete(fixture.ns+"-"+fixture.clusterName, nil) - if err != nil && apierrs.IsNotFound(err) { + if err != nil && !apierrs.IsNotFound(err) { return err } }