Merge branch 'master' into fix759

pingcap · Aug 20, 2019 · 9703a3a · 9703a3a
2 parents fc1d770 + 89d41bd
commit 9703a3a
Show file tree

Hide file tree

Showing 113 changed files with 3,658 additions and 1,248 deletions.
diff --git a/Makefile b/Makefile
@@ -25,8 +25,9 @@ TEST_COVER_PACKAGES:=go list ./pkg/... | grep -vE "pkg/client" | grep -vE "pkg/t
 
 default: build
 
-docker-push: docker
+docker-push: docker backup-docker
 	docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest"
+	docker push "${DOCKER_REGISTRY}/pingcap/tidb-backup-manager:latest"
 
 docker: build
 	docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator

diff --git a/charts/tidb-backup/templates/backup-job.yaml b/charts/tidb-backup/templates/backup-job.yaml
@@ -26,6 +26,10 @@ spec:
       - name: backup
         image: {{ .Values.image.backup }}
         imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }}
+    {{- if .Values.resources }}
+        resources:
+{{ toYaml .Values.resources | indent 10 }}
+    {{- end }}
         command:
         - /bin/sh
         - -c

diff --git a/charts/tidb-backup/values.yaml b/charts/tidb-backup/values.yaml
@@ -23,6 +23,14 @@ extraLabels: {}
 # kubectl create secret generic backup-secret --namespace=<namespace> --from-literal=user=root --from-literal=password=<password>
 secretName: backup-secret
 
+resources:
+  limits:
+    cpu: 4000m
+    memory: 8Gi
+  requests:
+    cpu: 2000m
+    memory: 4Gi
+
 storage:
   className: local-storage
   size: 100Gi

diff --git a/charts/tidb-cluster/templates/_helpers.tpl b/charts/tidb-cluster/templates/_helpers.tpl
@@ -79,3 +79,34 @@ config-file: |-
 {{ include "tidb-configmap.data" . | sha256sum | trunc 8 }}
 {{- end -}}
 
+{{/*
+Encapsulate pump configmap data for consistent digest calculation
+*/}}
+{{- define "pump-configmap.data" -}}
+pump-config: |-
+    {{- if .Values.binlog.pump.config }}
+{{ .Values.binlog.pump.config | indent 2 }}
+    {{- else -}}
+{{ tuple "config/_pump-config.tpl" . | include "helm-toolkit.utils.template" | indent 2 }}
+    {{- end -}}
+{{- end -}}
+
+{{- define "pump-configmap.data-digest" -}}
+{{ include "pump-configmap.data" . | sha256sum | trunc 8 }}
+{{- end -}}
+
+{{/*
+Encapsulate drainer configmap data for consistent digest calculation
+*/}}
+{{- define "drainer-configmap.data" -}}
+drainer-config: |-
+    {{- if .Values.binlog.drainer.config }}
+{{ .Values.binlog.drainer.config | indent 2 }}
+    {{- else -}}
+{{ tuple "config/_drainer-config.tpl" . | include "helm-toolkit.utils.template" | indent 2 }}
+    {{- end -}}
+{{- end -}}
+
+{{- define "drainer-configmap.data-digest" -}}
+{{ include "drainer-configmap.data" . | sha256sum | trunc 8 }}
+{{- end -}}
diff --git a/charts/tidb-cluster/templates/drainer-configmap-rollout.yaml b/charts/tidb-cluster/templates/drainer-configmap-rollout.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.enableConfigMapRollout }}
+{{- if .Values.binlog.drainer.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "cluster.name" . }}-drainer-{{ template "drainer-configmap.data-digest" . }}
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: drainer
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+data:
+{{ include "drainer-configmap.data" . | indent 2 }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/tidb-cluster/templates/drainer-configmap.yaml b/charts/tidb-cluster/templates/drainer-configmap.yaml
@@ -10,6 +10,5 @@ metadata:
     app.kubernetes.io/component: drainer
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 data:
-  drainer-config: |-
-{{ tuple "config/_drainer-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ include "drainer-configmap.data" . | indent 2 }}
 {{- end -}}
diff --git a/charts/tidb-cluster/templates/drainer-statefulset.yaml b/charts/tidb-cluster/templates/drainer-statefulset.yaml
@@ -30,6 +30,14 @@ spec:
         app.kubernetes.io/managed-by: tidb-operator
         app.kubernetes.io/component: drainer
     spec:
+    {{- if .Values.binlog.drainer.affinity }}
+      affinity:
+{{ toYaml .Values.binlog.drainer.affinity | indent 8 }}
+    {{- end }}
+    {{- if .Values.binlog.drainer.tolerations }}
+      tolerations:
+{{ toYaml .Values.binlog.drainer.tolerations | indent 8 }}
+    {{- end }}
       containers:
       - name: drainer
         image: {{ .Values.binlog.drainer.image }}
@@ -50,7 +58,11 @@ spec:
       volumes:
       - name: config
         configMap:
+        {{- if .Values.enableConfigMapRollout }}
+          name: {{ template "cluster.name" . }}-drainer-{{ template "drainer-configmap.data-digest" . }}
+        {{- else }}
           name: {{ template "cluster.name" . }}-drainer
+        {{- end }}
           items:
           - key: drainer-config
             path: drainer.toml

diff --git a/charts/tidb-cluster/templates/pump-configmap-rollout.yaml b/charts/tidb-cluster/templates/pump-configmap-rollout.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.enableConfigMapRollout }}
+{{- if .Values.binlog.pump.create }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "cluster.name" . }}-pump-{{ template "pump-configmap.data-digest" . }}
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: pump
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+data:
+{{ include "pump-configmap.data" . | indent 2 }}
+{{- end }}
+{{- end }}
diff --git a/charts/tidb-cluster/templates/pump-configmap.yaml b/charts/tidb-cluster/templates/pump-configmap.yaml
@@ -10,6 +10,5 @@ metadata:
     app.kubernetes.io/component: pump
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 data:
-  pump-config: |-
-{{ tuple "config/_pump-config.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- end -}}
+{{ include "pump-configmap.data" . | indent 2 }}
+{{ end }}
diff --git a/charts/tidb-cluster/templates/pump-statefulset.yaml b/charts/tidb-cluster/templates/pump-statefulset.yaml
@@ -30,6 +30,14 @@ spec:
         app.kubernetes.io/managed-by: tidb-operator
         app.kubernetes.io/component: pump
     spec:
+    {{- if .Values.binlog.pump.affinity }}
+      affinity:
+{{ toYaml .Values.binlog.pump.affinity | indent 8 }}
+    {{- end }}
+    {{- if .Values.binlog.pump.tolerations }}
+      tolerations:
+{{ toYaml .Values.binlog.pump.tolerations | indent 8 }}
+    {{- end }}
       containers:
       - name: pump
         image: {{ .Values.binlog.pump.image }}
@@ -50,7 +58,11 @@ spec:
       volumes:
       - name: config
         configMap:
+        {{- if .Values.enableConfigMapRollout }}
+          name: {{ template "cluster.name" . }}-pump-{{ template "pump-configmap.data-digest" . }}
+        {{- else }}
           name: {{ template "cluster.name" . }}-pump
+        {{- end }}
           items:
           - key: pump-config
             path: pump.toml

diff --git a/charts/tidb-cluster/templates/scripts/_start_drainer.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_drainer.sh.tpl
@@ -26,6 +26,7 @@ done
 
 /drainer \
 -L={{ .Values.binlog.drainer.logLevel | default "info" }} \
+-pd-urls=http://{{ template "cluster.name" . }}-pd:2379 \
 -addr=`echo ${HOSTNAME}`.{{ template "cluster.name" . }}-drainer:8249 \
 -config=/etc/drainer/drainer.toml \
 -disable-detect={{ .Values.binlog.drainer.disableDetect | default false }} \

diff --git a/charts/tidb-cluster/templates/scripts/_start_pump.sh.tpl b/charts/tidb-cluster/templates/scripts/_start_pump.sh.tpl
@@ -1,6 +1,12 @@
 set -euo pipefail
 /pump \
+-pd-urls=http://{{ template "cluster.name" . }}-pd:2379 \
 -L={{ .Values.binlog.pump.logLevel | default "info" }} \
 -advertise-addr=`echo ${HOSTNAME}`.{{ template "cluster.name" . }}-pump:8250 \
 -config=/etc/pump/pump.toml \
 -log-file=
+
+if [ $? == 0 ]; then
+    echo $(date -u +"[%Y/%m/%d %H:%M:%S.%3N %:z]") "pump offline, please delete my pod"
+    tail -f /dev/null
+fi
diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml
@@ -423,13 +423,30 @@ binlog:
     # refer to https://kubernetes.io/docs/concepts/storage/storage-classes
     storageClassName: local-storage
     storage: 20Gi
+    # affinity for pump pod assignment, default: empty
+    # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+    affinity: {}
+    # tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints.
+    # refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration
+    tolerations: []
     syncLog: true
     # a integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored.
     # must bigger than 0
     gc: 7
     # number of seconds between heartbeat ticks (in 2 seconds)
     heartbeatInterval: 2
 
+    # Please refer to https://github.com/pingcap/tidb-binlog/blob/master/cmd/pump/pump.toml for the default
+    # pump configurations (change to the tags of your pump version),
+    # just follow the format in the file and configure in the 'config' section
+    # as below if you want to customize any configuration.
+    # config: |
+    #   gc = 7
+    #   heartbeat-interval = 2
+    #   [storage]
+    #   sync-log = true
+    #   stop-write-at-available-space = "10Gi"
+
   drainer:
     create: false
     image: pingcap/tidb-binlog:v3.0.1
@@ -441,6 +458,12 @@ binlog:
     # refer to https://kubernetes.io/docs/concepts/storage/storage-classes
     storageClassName: local-storage
     storage: 10Gi
+    # affinity for drainer pod assignment, default: empty
+    # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+    affinity: {}
+    # tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints.
+    # refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration
+    tolerations: []
     # the number of the concurrency of the downstream for synchronization. The bigger the value,
     # the better throughput performance of the concurrency (16 by default)
     workerCount: 16
@@ -475,6 +498,24 @@ binlog:
       # kafkaAddrs: "127.0.0.1:9092"
       # kafkaVersion: "0.8.2.0"
 
+    # Please refer to https://github.com/pingcap/tidb-binlog/blob/master/cmd/drainer/drainer.toml for the default
+    # drainer configurations (change to the tags of your drainer version),
+    # just follow the format in the file and configure in the 'config' section
+    # as below if you want to customize any configuration.
+    # config: |
+    #   worker-count = 16
+    #   detect-interval = 10
+    #   disable-dispatch = false
+    #   ignore-schemas = "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql"
+    #   safe-mode = false
+    #   txn-batch = 20
+    #   db-type = "mysql"
+    #   [syncer.to]
+    #   # host = "127.0.0.1"
+    #   # user = "root"
+    #   # password = ""
+    #   # port = 3306
+
 scheduledBackup:
   create: false
   # https://github.com/pingcap/tidb-cloud-backup

diff --git a/charts/tidb-drainer/.helmignore b/charts/tidb-drainer/.helmignore
@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
diff --git a/charts/tidb-drainer/Chart.yaml b/charts/tidb-drainer/Chart.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+description: A Helm chart for TiDB Binlog drainer.
+name: tidb-drainer
+version: dev
+home: https://github.com/pingcap/tidb-operator
+sources:
+  - https://github.com/pingcap/tidb-operator
+keywords:
+  - newsql
+  - htap
+  - database
+  - mysql
+  - cdc
diff --git a/charts/tidb-drainer/templates/NOTES.txt b/charts/tidb-drainer/templates/NOTES.txt
@@ -0,0 +1,6 @@
+StatefulSet {{ include "drainer.name" . }} installed.
+
+1. Watch if the drainer instance gets created:
+   watch kubectl get pod -n {{ .Release.Namespace }} {{ include "drainer.name" . }}-0
+2. Check if the drainer instance works properly:
+   kubectl logs -f -n {{ .Release.Namespace }} {{ include "drainer.name" . }}-0
diff --git a/charts/tidb-drainer/templates/_helpers.tpl b/charts/tidb-drainer/templates/_helpers.tpl
@@ -0,0 +1,25 @@
+{{- define "drainer.name" -}}
+{{ .Values.clusterName }}-{{ .Release.Name }}-drainer
+{{- end -}}
+
+{{/*
+Encapsulate config data for consistent digest calculation
+*/}}
+{{- define "drainer-configmap.data" -}}
+config-file: |-
+    {{- if .Values.config }}
+{{ .Values.config | indent 2 }}
+    {{- end -}}
+{{- end -}}
+
+{{- define "drainer-configmap.name" -}}
+{{ include "drainer.name" . }}-{{ include "drainer-configmap.data" . | sha256sum | trunc 8 }}
+{{- end -}}
+
+{{- define "helm-toolkit.utils.template" -}}
+{{- $name := index . 0 -}}
+{{- $context := index . 1 -}}
+{{- $last := base $context.Template.Name }}
+{{- $wtf := $context.Template.Name | replace $last $name -}}
+{{ include $wtf $context }}
+{{- end -}}
diff --git a/charts/tidb-drainer/templates/drainer-configmap.yaml b/charts/tidb-drainer/templates/drainer-configmap.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "drainer-configmap.name" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "drainer-configmap.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: drainer
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+data:
+{{ include "drainer-configmap.data" . | indent 2 }}
diff --git a/charts/tidb-drainer/templates/drainer-service.yaml b/charts/tidb-drainer/templates/drainer-service.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "drainer.name" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "drainer.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: drainer
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  clusterIP: None
+  ports:
+  - name: drainer
+    port: 8249
+  selector:
+    app.kubernetes.io/name: {{ include "drainer.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: drainer