From 59c9e4b4f672af03756c27d31007a6dd3120c027 Mon Sep 17 00:00:00 2001 From: Aylei Date: Fri, 24 May 2019 16:39:02 +0800 Subject: [PATCH 01/13] Refine backup and restore documentation Signed-off-by: Aylei --- charts/tidb-backup/templates/backup-job.yaml | 2 +- charts/tidb-backup/templates/backup-pvc.yaml | 3 +- charts/tidb-backup/values.yaml | 2 +- .../templates/scheduled-backup-pvc.yaml | 1 + docs/backup-restore.md | 109 ++++++++++++++++++ docs/operation-guide.md | 66 +---------- docs/references/tidb-backup-configuration.md | 76 ++++++++++++ 7 files changed, 193 insertions(+), 66 deletions(-) create mode 100644 docs/backup-restore.md create mode 100644 docs/references/tidb-backup-configuration.md diff --git a/charts/tidb-backup/templates/backup-job.yaml b/charts/tidb-backup/templates/backup-job.yaml index bc8f3fb0b0..0a9b580516 100644 --- a/charts/tidb-backup/templates/backup-job.yaml +++ b/charts/tidb-backup/templates/backup-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ .Values.clusterName }}-{{ .Values.name }} + name: {{ .Values.clusterName }}-{{ tpl .Values.name . }} labels: app.kubernetes.io/name: {{ template "chart.name" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} diff --git a/charts/tidb-backup/templates/backup-pvc.yaml b/charts/tidb-backup/templates/backup-pvc.yaml index 9368a55e04..154ff0b076 100644 --- a/charts/tidb-backup/templates/backup-pvc.yaml +++ b/charts/tidb-backup/templates/backup-pvc.yaml @@ -2,12 +2,13 @@ kind: PersistentVolumeClaim apiVersion: v1 metadata: - name: {{ .Values.name }} + name: {{ tpl .Values.name . }} labels: app.kubernetes.io/name: {{ template "chart.name" . }} app.kubernetes.io/managed-by: tidb-operator app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: backup + pingcap.com/backup-cluster-name: {{ .Values.clusterName }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} spec: accessModes: diff --git a/charts/tidb-backup/values.yaml b/charts/tidb-backup/values.yaml index 455bfb718b..fe4d0b6570 100644 --- a/charts/tidb-backup/values.yaml +++ b/charts/tidb-backup/values.yaml @@ -7,7 +7,7 @@ clusterName: demo mode: backup # backup | restore # name is the backup name -name: fullbackup-20190306 +name: fullbackup-{{ date "200601021504" .Release.Time }} image: pullPolicy: IfNotPresent binlog: pingcap/tidb-binlog:v2.1.8 diff --git a/charts/tidb-cluster/templates/scheduled-backup-pvc.yaml b/charts/tidb-cluster/templates/scheduled-backup-pvc.yaml index a7d9491fd2..f4838c8378 100644 --- a/charts/tidb-cluster/templates/scheduled-backup-pvc.yaml +++ b/charts/tidb-cluster/templates/scheduled-backup-pvc.yaml @@ -8,6 +8,7 @@ metadata: app.kubernetes.io/managed-by: tidb-operator app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: scheduled-backup + pingcap.com/backup-cluster-name: {{ template "cluster.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} spec: accessModes: diff --git a/docs/backup-restore.md b/docs/backup-restore.md new file mode 100644 index 0000000000..2a2f6c5eef --- /dev/null +++ b/docs/backup-restore.md @@ -0,0 +1,109 @@ +# Backup and Restore TiDB Cluster + +## Overview + +TiDB Operator supports two kinds of backup: + +* [Full backup](#full-backup)(scheduled or ad-hoc) via [`mydumper`](https://www.pingcap.com/docs/dev/reference/tools/mydumper/), which helps you logical backup the TiDB cluster. +* [Incremental backup](#incremental-backup) via [`TiDB-Binlog`](https://www.pingcap.com/docs/dev/reference/tools/tidb-binlog/overview/), which helps you synchronize the data in the TiDB cluster to other database or backup the data at real-time. + +Currently, tidb-operator only supports automatic [restore operation](#restore) for full backup taken by `mydumper`. Restore the backup data captured by `TiDB-Binlog` requires human intervention. + +## Full backup + +Full backup using `mydumper` to take the logical backup of TiDB cluster. The backup job will create a PVC to store backup data. + +By default, the backup uses PV to store the backup data. You can also store the backup data to [Google Cloud Storage](https://cloud.google.com/storage/) bucket or [Ceph Object Storage](https://ceph.com/ceph-storage/object-storage/) by changing the configuration. This way the PV temporarily stores backup data before it is placed in object storage. Refer to [TiDB cluster Backup configuration] for full configuration guide of backup and restore. + +You can either setup a scheduled full backup or take a full backup in ad-hoc manner. + +### Scheduled full backup + +Scheduled full backup is created along side the TiDB cluster, and it runs periodically like the crontab job. + +To configure a scheduled full backup, modify the `scheduledBackup` section in the `charts/tidb-cluster/values.yaml` of the tidb cluster: + +* Set `scheduledBackup.create` to `true` +* Set `scheduledBackup.storageClassName` to the PV storage class name used for backup data + +> **Note:** You must set the scheduled full backup PV's [reclaim policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy) to `Retain` to keep your backup data safe. + +* Configure `scheduledBackup.schedule` in the [Cron](https://en.wikipedia.org/wiki/Cron) format to define the scheduling +* `scheduledBakcup.user` and `scheduledBackup.password` must be set to the correct user which has the permission to read the database to be backuped. + +Then, create a new cluster with the scheduled full backup configured by `helm install`, or enabling scheduled full backup for existing cluster by `helm upgrade`: + +```shell +$ helm upgrade ${RELEASE_NAME} charts/tidb-cluster -f charts/tidb-cluster/values.yaml +``` + +### Ad-Hoc full backup + +Ad-hoc backup runs to complete for once. This functionality is encapsulated in another helm chart, `charts/tidb-backup`. According to the `mode` in `charts/tidb-backup/values.yaml`, this chart can perform either full backup or restore. We will cover restore operation in the [restore section](#restore) of this document. + +To create an ad-hoc full backup job, modify the `charts/tidb-backup/values.yaml`: + +* Set the `clusterName` to the target TiDB cluster name +* Set `mode` to `backup` +* Set `storage.className` to the PV storage class name used for backup data +* Adjust the `storage.size` according to your database size + +> **Note:** You must set the ad-hoc full backup PV's [reclaim policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy) to `Retain` to keep your backup data safe. + +Create a secret containing the user and password that has the permission to backup the database: + +```shell +$ kubectl create secret generic backup-secret -n ${namespace} --from-literal=user= --from-literal=password= +``` + +Then run the following command to create an ad-hoc backup job: + +```shell +$ helm install charts/tidb-backup --name= --namespace=${namespace} +``` + +### View backups + +For backups stored in PV, you can view the PVs by the following command: + +```shell +$ kubectl get pvc -n ${namespace} -l app.kubernetes.io/component=backup,pingcap.com/backup-cluster-name=${cluster_name} +``` + +If you store your backup data to [Google Cloud Storage] or [Ceph Object Storage], you may view the backups by the related GUI or CLI tool. + +## Restore + +The helm chart `charts/tidb-backup` helps restoring a TiDB cluster using backup data. To perform a restore operation, modify the `charts/tidb-backup/values.yaml`: + +* Set the `clusterName` to the target TiDB cluster name +* Set the `mode` to `restore` +* Set the `name` to the backup name you want to restore([view backups](#view-backups) helps you view all the backups available). If the backup is stored in `Google Cloud Storage` or `Ceph Object Storage`, you have to configure the corresponding section too(likely, you will continue to use the same configuration you set in the [adhoc full backup](#ad-hoc-full-backup)). + +Create a secret containing the user and password that has the permission to restore the database (skip this if you've already created one in the [adhoc full backup](#ad-hoc-full-backup) section): + +```shell +$ kubectl create secret generic backup-secret -n ${namespace} --from-literal=user= --from-literal=password= +``` + +Then, restore the backup: +```shell +$ helm install charts/tidb-backup --namespace=${namespace} +``` + +## Incremental backup + +Incremental backup leverage the [`TiDB-Binlog`](https://www.pingcap.com/docs/dev/reference/tools/tidb-binlog/overview/) tool to collect binlog data from TiDB and provide real-time backup and synchronization to downstream platforms. + +Incremental backup is disabled in the TiDB cluster by default. To create a TiDB cluster with incremental backup enabled or enable incremental backup in existing TiDB cluster, you have to modify the `charts/tidb-cluster/values.yaml`: + +* Set `binlog.pump.create` to `true` +* Set `binlog.drainer.create` to `true` +* Set `binlog.pump.storageClassName` and `binlog.drainer.storageClassName` to a proper `storageClass` available in your kubernetes cluster +* Set `binlog.drainer.destDBType` to your desired downstream, explained in detail below + +There's three types of downstream available for incremental backup: + +* PersistenceVolume: default downstream. You may consider configuring a large PersistenceVolume for `drainer` (the `binlog.drainer.storage` variable) in this case +* MySQL compatible database: enable by setting the `binlog.drainer.destDBType` to `mysql`. You have to configure the target address and credential in the `binlog.drainer.mysql` section too. +* Kafka: enable by setting the `binlog.drainer.destDBType` to `kafka`. You have to configure the zookeeper address and kafka address in the `binlog.drainer.kafka` section too. diff --git a/docs/operation-guide.md b/docs/operation-guide.md index 6a46c81211..d78e661a1d 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -222,68 +222,8 @@ To retrieve logs from multiple pods, [`stern`](https://github.com/wercker/stern) $ stern -n ${namespace} tidb -c slowlog ``` -## Backup +## Backup and Restore -Currently, TiDB Operator supports two kinds of backup: incremental backup via binlog and full backup(scheduled or ad-hoc) via [Mydumper](https://github.com/maxbube/mydumper). +TiDB Operator provides highly automated backup and recovery operations for TiDB cluster. You can easily take full backup or setup incremental backup of TiDB cluster, and restore the TiDB cluster when the cluster fails. -### Incremental backup - -To enable incremental backup, set `binlog.pump.create` and `binlog.drainer.create` to `true`. By default the incremental backup data is stored in protobuffer format in a PV. You can change `binlog.drainer.destDBType` from `pb` to `mysql` or `kafka` and configure the corresponding downstream. - -### Full backup - -Currently, full backup requires a PersistentVolume. The backup job will create a PVC to store backup data. - -By default, the backup uses PV to store the backup data. -> **Note:** You must set the ad-hoc full backup PV's [reclaim policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy) to `Retain` to keep your backup data safe. - -You can also store the backup data to [Google Cloud Storage](https://cloud.google.com/storage/) bucket or [Ceph object storage](https://ceph.com/ceph-storage/object-storage/) by configuring the corresponding section in `values.yaml`. This way the PV temporarily stores backup data before it is placed in object storage. - -The comments in `values.yaml` is self-explanatory for both GCP backup and Ceph backup. - -### Scheduled full backup - -Scheduled full backup can be ran periodically just like crontab job. - -To create a scheduled full backup job, modify `scheduledBackup` section in `values.yaml` file. - -* `create` must be set to `true` -* Set `storageClassName` to the PV storage class name used for backup data -* `schedule` takes the [Cron](https://en.wikipedia.org/wiki/Cron) format -* `user` and `password` must be set to the correct user which has the permission to read the database to be backuped. - -> **Note:** You must set the scheduled full backup PV's [reclaim policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy) to `Retain` to keep your backup data safe. - - -### Ad-Hoc full backup - -> **Note:** The rest of the document will use `values.yaml` to reference `charts/tidb-backup/values.yaml` - -Ad-Hoc full backup can be done once just like job. - -To create an ad-hoc full backup job, modify `backup` section in `values.yaml` file. - -* `mode` must be set to `backup` -* Set `storage.className` to the PV storage class name used for backup data - -Create a secret containing the user and password that has the permission to backup the database: - -```shell -$ kubectl create secret generic backup-secret -n ${namespace} --from-literal=user= --from-literal=password= -``` - -Then run the following command to create an ad-hoc backup job: - -```shell -$ helm install charts/tidb-backup --name= --namespace=${namespace} -``` - -## Restore - -Restore is similar to backup. See the `values.yaml` file for details. - -Modified the variables in `values.yaml` and then create restore job using the following command: - -```shell -$ helm install charts/tidb-backup --name= --namespace=${namespace} -``` +For detail operation guides of backup and restore, please refer to [Backup and Restore TiDB Cluster](./backup-restore.md). diff --git a/docs/references/tidb-backup-configuration.md b/docs/references/tidb-backup-configuration.md new file mode 100644 index 0000000000..672c5b238a --- /dev/null +++ b/docs/references/tidb-backup-configuration.md @@ -0,0 +1,76 @@ +# TiDB Backup Configuration Reference + +`TiDB-Backup` is a helm chart designed for TiDB cluster backup and restore via the [`mydumper`](https://www.pingcap.com/docs/dev/reference/tools/mydumper/) and [`loader`](https://www.pingcap.com/docs-cn/tools/loader/). This documentation will explain the configurations of `TiDB-Backup`, you may refer to [Restore and Backup TiDB cluster](#tidb-backup-configuration-reference) for user guide with example. + +## Configurations + +### `mode` + +- To choose the operation, backup or restore, required +- Default: "backup" + +### `clusterName` + +- The TiDB cluster name that should backup from or restore to, required +- Default: "demo" + +### `name` + +- The backup name +- Default: "fullbackup-${date}", date is the start time of backup, accurate to minute + +### `secretName` + +- The name of the secret which stores user and password used for backup/restore +- Default: "backup-secret" +- You can create the secret by `kubectl create secret generic backup-secret --from-literal=user=root --from-literal=password=` + +### `storage.className` + +- The storageClass used to store the backup data +- Default: "local-storage" + +### `storage.size` + +- The storage size of PersistenceVolume +- Default: "100Gi" + +### `backupOptions` + +- The options that passed to [`mydumper`](https://github.com/maxbube/mydumper/blob/master/docs/mydumper_usage.rst#options) +- Default: "--chunk-filesize=100" + +### `restoreOptions` + +- The options that passed to [`loader`](https://www.pingcap.com/docs-cn/tools/loader/) +- Default: "-t 16" + +### `gcp.bucket` + +- The GCP bucket name to store backup data + +> **Note**: Once you set any variables under `gcp` section, the backup data will be uploaded to Google Cloud Storage, namely, you have to keep the configuration intact. + +### `gcp.secretName` + +- The name of the secret which stores the gcp service account credentials json file +- You can create the secret by `kubectl create secret generic gcp-backup-secret --from-file=./credentials.json`. To download credentials json, refer to [Google Cloud Documentation](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually) + +### `ceph.endpoint` + +- The endpoint of ceph object storage + +> **Note**: Once you set any variables under `ceph` section, the backup data will be uploaded to ceph object storage, namely, you have to keep the configuration intact. + +### `ceph.bucket` + +- The bucket name of ceph object storage + +### `ceph.secretName` + +- The name of the secret which stores ceph object store access key and secret key +- You can create the secret by: + +```shell +$ kubectl create secret generic ceph-backup-secret --from-literal=access_key= --from-literal=secret_key= +``` From bb000678c871d50435279999667d0b27dc0c9110 Mon Sep 17 00:00:00 2001 From: Aylei Date: Fri, 24 May 2019 17:57:17 +0800 Subject: [PATCH 02/13] Fix dead link Signed-off-by: Aylei --- docs/backup-restore.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 2a2f6c5eef..d10a0ee2d0 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -13,7 +13,7 @@ Currently, tidb-operator only supports automatic [restore operation](#restore) f Full backup using `mydumper` to take the logical backup of TiDB cluster. The backup job will create a PVC to store backup data. -By default, the backup uses PV to store the backup data. You can also store the backup data to [Google Cloud Storage](https://cloud.google.com/storage/) bucket or [Ceph Object Storage](https://ceph.com/ceph-storage/object-storage/) by changing the configuration. This way the PV temporarily stores backup data before it is placed in object storage. Refer to [TiDB cluster Backup configuration] for full configuration guide of backup and restore. +By default, the backup uses PV to store the backup data. You can also store the backup data to [Google Cloud Storage](https://cloud.google.com/storage/) bucket or [Ceph Object Storage](https://ceph.com/ceph-storage/object-storage/) by changing the configuration. This way the PV temporarily stores backup data before it is placed in object storage. Refer to [TiDB cluster Backup configuration](./references/tidb-backup-configuration.md) for full configuration guide of backup and restore. You can either setup a scheduled full backup or take a full backup in ad-hoc manner. @@ -70,7 +70,7 @@ For backups stored in PV, you can view the PVs by the following command: $ kubectl get pvc -n ${namespace} -l app.kubernetes.io/component=backup,pingcap.com/backup-cluster-name=${cluster_name} ``` -If you store your backup data to [Google Cloud Storage] or [Ceph Object Storage], you may view the backups by the related GUI or CLI tool. +If you store your backup data to [Google Cloud Storage](https://cloud.google.com/storage/) or [Ceph Object Storage](https://ceph.com/ceph-storage/object-storage/), you may view the backups by the related GUI or CLI tool. ## Restore From 1fb922acf9b70d989488329591d8ddc962bc7c2c Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 09:40:19 +0800 Subject: [PATCH 03/13] Update docs/backup-restore.md Co-Authored-By: Lilian Lee --- docs/backup-restore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index d10a0ee2d0..509bed8015 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -1,4 +1,4 @@ -# Backup and Restore TiDB Cluster +# Backup and Restore a TiDB Cluster ## Overview From dd1cf84b056c4bee7e7b829719b4bcebfa04f1b8 Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 09:40:29 +0800 Subject: [PATCH 04/13] Update docs/backup-restore.md Co-Authored-By: Lilian Lee --- docs/backup-restore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 509bed8015..00c1d706e6 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -4,7 +4,7 @@ TiDB Operator supports two kinds of backup: -* [Full backup](#full-backup)(scheduled or ad-hoc) via [`mydumper`](https://www.pingcap.com/docs/dev/reference/tools/mydumper/), which helps you logical backup the TiDB cluster. +* [Full backup](#full-backup)(scheduled or ad-hoc) via [`mydumper`](https://www.pingcap.com/docs/dev/reference/tools/mydumper/), which helps you to logically back up a TiDB cluster. * [Incremental backup](#incremental-backup) via [`TiDB-Binlog`](https://www.pingcap.com/docs/dev/reference/tools/tidb-binlog/overview/), which helps you synchronize the data in the TiDB cluster to other database or backup the data at real-time. Currently, tidb-operator only supports automatic [restore operation](#restore) for full backup taken by `mydumper`. Restore the backup data captured by `TiDB-Binlog` requires human intervention. From 7748702d573aff1c19bb3eea59b56c33cc198028 Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 09:40:56 +0800 Subject: [PATCH 05/13] Update docs/backup-restore.md Co-Authored-By: Lilian Lee --- docs/backup-restore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 00c1d706e6..769833dbde 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -5,7 +5,7 @@ TiDB Operator supports two kinds of backup: * [Full backup](#full-backup)(scheduled or ad-hoc) via [`mydumper`](https://www.pingcap.com/docs/dev/reference/tools/mydumper/), which helps you to logically back up a TiDB cluster. -* [Incremental backup](#incremental-backup) via [`TiDB-Binlog`](https://www.pingcap.com/docs/dev/reference/tools/tidb-binlog/overview/), which helps you synchronize the data in the TiDB cluster to other database or backup the data at real-time. +* [Incremental backup](#incremental-backup) via [`TiDB-Binlog`](https://www.pingcap.com/docs/dev/reference/tools/tidb-binlog/overview/), which helps you replicate the data in a TiDB cluster to other databases or back up the data in real time. Currently, tidb-operator only supports automatic [restore operation](#restore) for full backup taken by `mydumper`. Restore the backup data captured by `TiDB-Binlog` requires human intervention. From 9039a0d14695b41571dcc4823d1bf2abf7309c13 Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 09:41:15 +0800 Subject: [PATCH 06/13] Update docs/backup-restore.md Co-Authored-By: Lilian Lee --- docs/backup-restore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 769833dbde..d3ea0bab15 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -7,7 +7,7 @@ TiDB Operator supports two kinds of backup: * [Full backup](#full-backup)(scheduled or ad-hoc) via [`mydumper`](https://www.pingcap.com/docs/dev/reference/tools/mydumper/), which helps you to logically back up a TiDB cluster. * [Incremental backup](#incremental-backup) via [`TiDB-Binlog`](https://www.pingcap.com/docs/dev/reference/tools/tidb-binlog/overview/), which helps you replicate the data in a TiDB cluster to other databases or back up the data in real time. -Currently, tidb-operator only supports automatic [restore operation](#restore) for full backup taken by `mydumper`. Restore the backup data captured by `TiDB-Binlog` requires human intervention. +Currently, TiDB Operator only supports automatic [restore operation](#restore) for full backup taken by `mydumper`. Restoring the backup data captured by TiDB Binlog requires manual intervention. ## Full backup From d0c7fba749f8ba137b63de38fbfd22a2c97912a1 Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 09:43:32 +0800 Subject: [PATCH 07/13] Update docs/backup-restore.md Co-Authored-By: Lilian Lee --- docs/backup-restore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index d3ea0bab15..968b166acb 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -15,7 +15,7 @@ Full backup using `mydumper` to take the logical backup of TiDB cluster. The bac By default, the backup uses PV to store the backup data. You can also store the backup data to [Google Cloud Storage](https://cloud.google.com/storage/) bucket or [Ceph Object Storage](https://ceph.com/ceph-storage/object-storage/) by changing the configuration. This way the PV temporarily stores backup data before it is placed in object storage. Refer to [TiDB cluster Backup configuration](./references/tidb-backup-configuration.md) for full configuration guide of backup and restore. -You can either setup a scheduled full backup or take a full backup in ad-hoc manner. +You can either set up a scheduled full backup or take a full backup in an ad-hoc manner. ### Scheduled full backup From 44711f21525b06d34fc8cfbddd778c95b9596f23 Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 09:44:10 +0800 Subject: [PATCH 08/13] Update docs/backup-restore.md Co-Authored-By: Lilian Lee --- docs/backup-restore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 968b166acb..e452b07232 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -21,7 +21,7 @@ You can either set up a scheduled full backup or take a full backup in an ad-hoc Scheduled full backup is created along side the TiDB cluster, and it runs periodically like the crontab job. -To configure a scheduled full backup, modify the `scheduledBackup` section in the `charts/tidb-cluster/values.yaml` of the tidb cluster: +To configure a scheduled full backup, modify the `scheduledBackup` section in the `charts/tidb-cluster/values.yaml` file of the TiDB cluster: * Set `scheduledBackup.create` to `true` * Set `scheduledBackup.storageClassName` to the PV storage class name used for backup data From 7a53ad3dbb76a05babf5d09b96212b2dd9253569 Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 09:45:16 +0800 Subject: [PATCH 09/13] Update docs/backup-restore.md Co-Authored-By: Lilian Lee --- docs/backup-restore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index e452b07232..dc4b987116 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -19,7 +19,7 @@ You can either set up a scheduled full backup or take a full backup in an ad-hoc ### Scheduled full backup -Scheduled full backup is created along side the TiDB cluster, and it runs periodically like the crontab job. +Scheduled full backup is created alongside the TiDB cluster, and it runs periodically like the crontab job. To configure a scheduled full backup, modify the `scheduledBackup` section in the `charts/tidb-cluster/values.yaml` file of the TiDB cluster: From 630237c10c4fdb366fe0b872dba92121c0338b5e Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 09:45:55 +0800 Subject: [PATCH 10/13] Update docs/backup-restore.md Co-Authored-By: Lilian Lee --- docs/backup-restore.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index dc4b987116..6f5eea4825 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -26,7 +26,9 @@ To configure a scheduled full backup, modify the `scheduledBackup` section in th * Set `scheduledBackup.create` to `true` * Set `scheduledBackup.storageClassName` to the PV storage class name used for backup data -> **Note:** You must set the scheduled full backup PV's [reclaim policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy) to `Retain` to keep your backup data safe. +> **Note:** +> +> You must set the scheduled full backup PV's [reclaim policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy) to `Retain` to keep your backup data safe. * Configure `scheduledBackup.schedule` in the [Cron](https://en.wikipedia.org/wiki/Cron) format to define the scheduling * `scheduledBakcup.user` and `scheduledBackup.password` must be set to the correct user which has the permission to read the database to be backuped. From 0176ac74751c73401ceda6a38849cb2c1db812ae Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 10:15:10 +0800 Subject: [PATCH 11/13] Refine backup documents Signed-off-by: Aylei --- docs/backup-restore.md | 51 +++++++++++--------- docs/operation-guide.md | 6 +-- docs/references/tidb-backup-configuration.md | 26 +++++----- 3 files changed, 46 insertions(+), 37 deletions(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index 6f5eea4825..b2d8aa3689 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -1,5 +1,4 @@ # Backup and Restore a TiDB Cluster - ## Overview TiDB Operator supports two kinds of backup: @@ -11,9 +10,9 @@ Currently, TiDB Operator only supports automatic [restore operation](#restore) f ## Full backup -Full backup using `mydumper` to take the logical backup of TiDB cluster. The backup job will create a PVC to store backup data. +Full backup uses `mydumper` to make a logical backup of TiDB cluster. The backup job will create a PVC([PersistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims), the same below) to store backup data. -By default, the backup uses PV to store the backup data. You can also store the backup data to [Google Cloud Storage](https://cloud.google.com/storage/) bucket or [Ceph Object Storage](https://ceph.com/ceph-storage/object-storage/) by changing the configuration. This way the PV temporarily stores backup data before it is placed in object storage. Refer to [TiDB cluster Backup configuration](./references/tidb-backup-configuration.md) for full configuration guide of backup and restore. +By default, the backup uses PV ([Persistent Volume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistent-volumes)) to store the backup data. You can also store the backup data to [Google Cloud Storage](https://cloud.google.com/storage/) bucket or [Ceph Object Storage](https://ceph.com/ceph-storage/object-storage/) by changing the configuration. This way the PV temporarily stores backup data before it is placed in object storage. Refer to [TiDB cluster Backup configuration](./references/tidb-backup-configuration.md) for full configuration guide of backup and restore. You can either set up a scheduled full backup or take a full backup in an ad-hoc manner. @@ -30,10 +29,14 @@ To configure a scheduled full backup, modify the `scheduledBackup` section in th > > You must set the scheduled full backup PV's [reclaim policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy) to `Retain` to keep your backup data safe. -* Configure `scheduledBackup.schedule` in the [Cron](https://en.wikipedia.org/wiki/Cron) format to define the scheduling -* `scheduledBakcup.user` and `scheduledBackup.password` must be set to the correct user which has the permission to read the database to be backuped. +* Configure `scheduledBackup.schedule` in the [Cron](https://en.wikipedia.org/wiki/Cron) format to define the scheduling. +* Create a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) containing the username and password that has the privilege to backup the database: + +```shell +$ kubectl create secret generic backup-secret -n ${namespace} --from-literal=user= --from-literal=password= +``` -Then, create a new cluster with the scheduled full backup configured by `helm install`, or enabling scheduled full backup for existing cluster by `helm upgrade`: +Then, create a new cluster with the scheduled full backup configured by `helm install`, or enabling scheduled full backup for the existing cluster by `helm upgrade`: ```shell $ helm upgrade ${RELEASE_NAME} charts/tidb-cluster -f charts/tidb-cluster/values.yaml @@ -43,16 +46,18 @@ $ helm upgrade ${RELEASE_NAME} charts/tidb-cluster -f charts/tidb-cluster/values Ad-hoc backup runs to complete for once. This functionality is encapsulated in another helm chart, `charts/tidb-backup`. According to the `mode` in `charts/tidb-backup/values.yaml`, this chart can perform either full backup or restore. We will cover restore operation in the [restore section](#restore) of this document. -To create an ad-hoc full backup job, modify the `charts/tidb-backup/values.yaml`: +To create an ad-hoc full backup job, modify the `charts/tidb-backup/values.yaml` file: -* Set the `clusterName` to the target TiDB cluster name +* Set `clusterName` to the target TiDB cluster name * Set `mode` to `backup` * Set `storage.className` to the PV storage class name used for backup data * Adjust the `storage.size` according to your database size -> **Note:** You must set the ad-hoc full backup PV's [reclaim policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy) to `Retain` to keep your backup data safe. +> **Note:** + +> You must set the ad-hoc full backup PV's [reclaim policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy) to `Retain` to keep your backup data safe. -Create a secret containing the user and password that has the permission to backup the database: +Create a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) containing the username and password that has the privilege to backup the database: ```shell $ kubectl create secret generic backup-secret -n ${namespace} --from-literal=user= --from-literal=password= @@ -66,23 +71,23 @@ $ helm install charts/tidb-backup --name= --namespace=${namespace} ### View backups -For backups stored in PV, you can view the PVs by the following command: +For backups stored in PV, you can view the PVs by using the following command: ```shell $ kubectl get pvc -n ${namespace} -l app.kubernetes.io/component=backup,pingcap.com/backup-cluster-name=${cluster_name} ``` -If you store your backup data to [Google Cloud Storage](https://cloud.google.com/storage/) or [Ceph Object Storage](https://ceph.com/ceph-storage/object-storage/), you may view the backups by the related GUI or CLI tool. +If you store your backup data to [Google Cloud Storage](https://cloud.google.com/storage/) or [Ceph Object Storage](https://ceph.com/ceph-storage/object-storage/), you can view the backups by using the related GUI or CLI tool. ## Restore -The helm chart `charts/tidb-backup` helps restoring a TiDB cluster using backup data. To perform a restore operation, modify the `charts/tidb-backup/values.yaml`: +The helm chart `charts/tidb-backup` helps restore a TiDB cluster using backup data. To perform a restore operation, modify the `charts/tidb-backup/values.yaml` file: -* Set the `clusterName` to the target TiDB cluster name -* Set the `mode` to `restore` -* Set the `name` to the backup name you want to restore([view backups](#view-backups) helps you view all the backups available). If the backup is stored in `Google Cloud Storage` or `Ceph Object Storage`, you have to configure the corresponding section too(likely, you will continue to use the same configuration you set in the [adhoc full backup](#ad-hoc-full-backup)). +* Set `clusterName` to the target TiDB cluster name +* Set `mode` to `restore` +* Set `name` to the backup name you want to restore([view backups](#view-backups) helps you view all the backups available). If the backup is stored in `Google Cloud Storage` or `Ceph Object Storage`, you must configure the corresponding section too(you might continue to use the same configuration you set in the [adhoc full backup](#ad-hoc-full-backup)). -Create a secret containing the user and password that has the permission to restore the database (skip this if you've already created one in the [adhoc full backup](#ad-hoc-full-backup) section): +Create a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) containing the user and password that has the privilege to restore the database (skip this if you have already created one in the [adhoc full backup](#ad-hoc-full-backup) section): ```shell $ kubectl create secret generic backup-secret -n ${namespace} --from-literal=user= --from-literal=password= @@ -95,17 +100,17 @@ $ helm install charts/tidb-backup --namespace=${namespace} ## Incremental backup -Incremental backup leverage the [`TiDB-Binlog`](https://www.pingcap.com/docs/dev/reference/tools/tidb-binlog/overview/) tool to collect binlog data from TiDB and provide real-time backup and synchronization to downstream platforms. +Incremental backup leverages the [TiDB Binlog](https://www.pingcap.com/docs/dev/reference/tools/tidb-binlog/overview/) tool to collect binlog data from TiDB and provide real-time backup and replication to downstream platforms. -Incremental backup is disabled in the TiDB cluster by default. To create a TiDB cluster with incremental backup enabled or enable incremental backup in existing TiDB cluster, you have to modify the `charts/tidb-cluster/values.yaml`: +Incremental backup is disabled in the TiDB cluster by default. To create a TiDB cluster with incremental backup enabled or enable incremental backup in existing TiDB cluster, modify the `charts/tidb-cluster/values.yaml` file: * Set `binlog.pump.create` to `true` * Set `binlog.drainer.create` to `true` * Set `binlog.pump.storageClassName` and `binlog.drainer.storageClassName` to a proper `storageClass` available in your kubernetes cluster * Set `binlog.drainer.destDBType` to your desired downstream, explained in detail below -There's three types of downstream available for incremental backup: +Three types of downstream platforms available for incremental backup: -* PersistenceVolume: default downstream. You may consider configuring a large PersistenceVolume for `drainer` (the `binlog.drainer.storage` variable) in this case -* MySQL compatible database: enable by setting the `binlog.drainer.destDBType` to `mysql`. You have to configure the target address and credential in the `binlog.drainer.mysql` section too. -* Kafka: enable by setting the `binlog.drainer.destDBType` to `kafka`. You have to configure the zookeeper address and kafka address in the `binlog.drainer.kafka` section too. +* PersistenceVolume: default downstream. You can consider configuring a large PV for `drainer` (the `binlog.drainer.storage` variable) in this case +* MySQL compatible database: enabled by setting the `binlog.drainer.destDBType` to `mysql`. You have to configure the target address and credential in the `binlog.drainer.mysql` section too. +* Kafka: enable by setting `binlog.drainer.destDBType` to `kafka`. You have to configure the zookeeper address and kafka address in the `binlog.drainer.kafka` section too. diff --git a/docs/operation-guide.md b/docs/operation-guide.md index d78e661a1d..1724091006 100644 --- a/docs/operation-guide.md +++ b/docs/operation-guide.md @@ -222,8 +222,8 @@ To retrieve logs from multiple pods, [`stern`](https://github.com/wercker/stern) $ stern -n ${namespace} tidb -c slowlog ``` -## Backup and Restore +## Backup and restore -TiDB Operator provides highly automated backup and recovery operations for TiDB cluster. You can easily take full backup or setup incremental backup of TiDB cluster, and restore the TiDB cluster when the cluster fails. +TiDB Operator provides highly automated backup and recovery operations for a TiDB cluster. You can easily take full backup or setup incremental backup of a TiDB cluster, and restore the TiDB cluster when the cluster fails. -For detail operation guides of backup and restore, please refer to [Backup and Restore TiDB Cluster](./backup-restore.md). +For detailed operation guides of backup and restore, refer to [Backup and Restore TiDB Cluster](./backup-restore.md). diff --git a/docs/references/tidb-backup-configuration.md b/docs/references/tidb-backup-configuration.md index 672c5b238a..a3c726b536 100644 --- a/docs/references/tidb-backup-configuration.md +++ b/docs/references/tidb-backup-configuration.md @@ -1,8 +1,8 @@ # TiDB Backup Configuration Reference -`TiDB-Backup` is a helm chart designed for TiDB cluster backup and restore via the [`mydumper`](https://www.pingcap.com/docs/dev/reference/tools/mydumper/) and [`loader`](https://www.pingcap.com/docs-cn/tools/loader/). This documentation will explain the configurations of `TiDB-Backup`, you may refer to [Restore and Backup TiDB cluster](#tidb-backup-configuration-reference) for user guide with example. +`TiDB-Backup` is a helm chart designed for TiDB cluster backup and restore via the [`mydumper`](https://www.pingcap.com/docs/dev/reference/tools/mydumper/) and [`loader`](https://www.pingcap.com/docs-cn/tools/loader/). This documentation explains `TiDB-Backup` configuration. Refer to [Restore and Backup TiDB cluster](#tidb-backup-configuration-reference) for user guide with example. -## Configurations +## Configuration ### `mode` @@ -11,7 +11,7 @@ ### `clusterName` -- The TiDB cluster name that should backup from or restore to, required +- The name of the TiDB cluster that data is backed up from or restore to, required - Default: "demo" ### `name` @@ -37,19 +37,21 @@ ### `backupOptions` -- The options that passed to [`mydumper`](https://github.com/maxbube/mydumper/blob/master/docs/mydumper_usage.rst#options) +- The options that are passed to [`mydumper`](https://github.com/maxbube/mydumper/blob/master/docs/mydumper_usage.rst#options) - Default: "--chunk-filesize=100" ### `restoreOptions` -- The options that passed to [`loader`](https://www.pingcap.com/docs-cn/tools/loader/) +- The options that are passed to [`loader`](https://www.pingcap.com/docs-cn/tools/loader/) - Default: "-t 16" ### `gcp.bucket` -- The GCP bucket name to store backup data +- The name of the GCP bucket used to store backup data -> **Note**: Once you set any variables under `gcp` section, the backup data will be uploaded to Google Cloud Storage, namely, you have to keep the configuration intact. +> **Note:** + +> Once you set any variables under `gcp` section, the backup data will be uploaded to Google Cloud Storage, namely, you have to keep the configuration intact. ### `gcp.secretName` @@ -60,7 +62,9 @@ - The endpoint of ceph object storage -> **Note**: Once you set any variables under `ceph` section, the backup data will be uploaded to ceph object storage, namely, you have to keep the configuration intact. +> **Note:** + +> Once you set any variables under `ceph` section, the backup data will be uploaded to ceph object storage, namely, you have to keep the configuration intact. ### `ceph.bucket` @@ -71,6 +75,6 @@ - The name of the secret which stores ceph object store access key and secret key - You can create the secret by: -```shell -$ kubectl create secret generic ceph-backup-secret --from-literal=access_key= --from-literal=secret_key= -``` + ```shell + $ kubectl create secret generic ceph-backup-secret --from-literal=access_key= --from-literal=secret_key= + ``` From 97562d1302fa00a1fc8ae229aab879e827939d81 Mon Sep 17 00:00:00 2001 From: Aylei Date: Mon, 27 May 2019 10:37:55 +0800 Subject: [PATCH 12/13] Fix code block indention Signed-off-by: Aylei --- docs/backup-restore.md | 12 ++++++------ docs/references/tidb-backup-configuration.md | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index b2d8aa3689..ff7ec3ca84 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -32,9 +32,9 @@ To configure a scheduled full backup, modify the `scheduledBackup` section in th * Configure `scheduledBackup.schedule` in the [Cron](https://en.wikipedia.org/wiki/Cron) format to define the scheduling. * Create a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) containing the username and password that has the privilege to backup the database: -```shell -$ kubectl create secret generic backup-secret -n ${namespace} --from-literal=user= --from-literal=password= -``` + ```shell + $ kubectl create secret generic backup-secret -n ${namespace} --from-literal=user= --from-literal=password= + ``` Then, create a new cluster with the scheduled full backup configured by `helm install`, or enabling scheduled full backup for the existing cluster by `helm upgrade`: @@ -85,7 +85,7 @@ The helm chart `charts/tidb-backup` helps restore a TiDB cluster using backup da * Set `clusterName` to the target TiDB cluster name * Set `mode` to `restore` -* Set `name` to the backup name you want to restore([view backups](#view-backups) helps you view all the backups available). If the backup is stored in `Google Cloud Storage` or `Ceph Object Storage`, you must configure the corresponding section too(you might continue to use the same configuration you set in the [adhoc full backup](#ad-hoc-full-backup)). +* Set `name` to the backup name you want to restore([view backups](#view-backups) helps you view all the backups available). If the backup is stored in `Google Cloud Storage` or `Ceph Object Storage`, you must configure the corresponding section too (you might continue to use the same configuration you set in the [adhoc full backup](#ad-hoc-full-backup)). Create a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/) containing the user and password that has the privilege to restore the database (skip this if you have already created one in the [adhoc full backup](#ad-hoc-full-backup) section): @@ -112,5 +112,5 @@ Incremental backup is disabled in the TiDB cluster by default. To create a TiDB Three types of downstream platforms available for incremental backup: * PersistenceVolume: default downstream. You can consider configuring a large PV for `drainer` (the `binlog.drainer.storage` variable) in this case -* MySQL compatible database: enabled by setting the `binlog.drainer.destDBType` to `mysql`. You have to configure the target address and credential in the `binlog.drainer.mysql` section too. -* Kafka: enable by setting `binlog.drainer.destDBType` to `kafka`. You have to configure the zookeeper address and kafka address in the `binlog.drainer.kafka` section too. +* MySQL compatible database: enabled by setting `binlog.drainer.destDBType` to `mysql`. You must configure the target address and credential in the `binlog.drainer.mysql` section too. +* Kafka: enable by setting `binlog.drainer.destDBType` to `kafka`. You must configure the zookeeper address and Kafka address in the `binlog.drainer.kafka` section too. diff --git a/docs/references/tidb-backup-configuration.md b/docs/references/tidb-backup-configuration.md index a3c726b536..a9ddf2daaa 100644 --- a/docs/references/tidb-backup-configuration.md +++ b/docs/references/tidb-backup-configuration.md @@ -76,5 +76,5 @@ - You can create the secret by: ```shell - $ kubectl create secret generic ceph-backup-secret --from-literal=access_key= --from-literal=secret_key= + $ kubectl create secret generic ceph-backup-secret -n ${namespace} --from-literal=access_key= --from-literal=secret_key= ``` From 61cd5dd4500935c681038a29828a7f9f39120752 Mon Sep 17 00:00:00 2001 From: Aylei Date: Thu, 30 May 2019 15:21:29 +0800 Subject: [PATCH 13/13] Add namespace parameter in tidb-backup configuration document --- docs/backup-restore.md | 2 +- docs/references/tidb-backup-configuration.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index ff7ec3ca84..1ba5f1b1f2 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -44,7 +44,7 @@ $ helm upgrade ${RELEASE_NAME} charts/tidb-cluster -f charts/tidb-cluster/values ### Ad-Hoc full backup -Ad-hoc backup runs to complete for once. This functionality is encapsulated in another helm chart, `charts/tidb-backup`. According to the `mode` in `charts/tidb-backup/values.yaml`, this chart can perform either full backup or restore. We will cover restore operation in the [restore section](#restore) of this document. +Ad-hoc backup is encapsulated in another helm chart, `charts/tidb-backup`. According to the `mode` in `charts/tidb-backup/values.yaml`, this chart can perform either full backup or restore. We will cover restore operation in the [restore section](#restore) of this document. To create an ad-hoc full backup job, modify the `charts/tidb-backup/values.yaml` file: diff --git a/docs/references/tidb-backup-configuration.md b/docs/references/tidb-backup-configuration.md index a9ddf2daaa..426427b0f8 100644 --- a/docs/references/tidb-backup-configuration.md +++ b/docs/references/tidb-backup-configuration.md @@ -23,7 +23,7 @@ - The name of the secret which stores user and password used for backup/restore - Default: "backup-secret" -- You can create the secret by `kubectl create secret generic backup-secret --from-literal=user=root --from-literal=password=` +- You can create the secret by `kubectl create secret generic backup-secret -n ${namespace} --from-literal=user=root --from-literal=password=` ### `storage.className` @@ -56,7 +56,7 @@ ### `gcp.secretName` - The name of the secret which stores the gcp service account credentials json file -- You can create the secret by `kubectl create secret generic gcp-backup-secret --from-file=./credentials.json`. To download credentials json, refer to [Google Cloud Documentation](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually) +- You can create the secret by `kubectl create secret generic gcp-backup-secret -n ${namespace} --from-file=./credentials.json`. To download credentials json, refer to [Google Cloud Documentation](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually) ### `ceph.endpoint`