From 3568f3693367b9544075d1ae3e7eff54ed1b475d Mon Sep 17 00:00:00 2001 From: pingcap-github-bot Date: Fri, 8 May 2020 17:09:35 +0800 Subject: [PATCH] backup: support gcp backup with br (#2267) (#2388) --- cmd/backup-manager/app/util/remote.go | 90 +++++++++++++++++++ docs/api-references/docs.md | 15 +++- images/tidb-backup-manager/entrypoint.sh | 7 +- manifests/crd.yaml | 6 ++ .../pingcap/v1alpha1/openapi_generated.go | 11 ++- pkg/apis/pingcap/v1alpha1/types.go | 6 +- 6 files changed, 126 insertions(+), 9 deletions(-) diff --git a/cmd/backup-manager/app/util/remote.go b/cmd/backup-manager/app/util/remote.go index fc9b91f3b6..9758cba109 100644 --- a/cmd/backup-manager/app/util/remote.go +++ b/cmd/backup-manager/app/util/remote.go @@ -21,7 +21,9 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "gocloud.dev/blob" + "gocloud.dev/blob/gcsblob" "gocloud.dev/blob/s3blob" + "gocloud.dev/gcp" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/backup/util" @@ -43,6 +45,18 @@ type s3Query struct { forcePathStyle bool } +type gcsQuery struct { + projectId string + location string + path string + bucket string + storageClass string + objectAcl string + bucketAcl string + secretName string + prefix string +} + // NewRemoteStorage creates new remote storage func NewRemoteStorage(backup *v1alpha1.Backup) (*blob.Bucket, error) { st := util.GetStorageType(backup.Spec.StorageProvider) @@ -54,6 +68,13 @@ func NewRemoteStorage(backup *v1alpha1.Backup) (*blob.Bucket, error) { return nil, err } return bucket, nil + case v1alpha1.BackupStorageTypeGcs: + qs := checkGcsConfig(backup.Spec.Gcs, true) + bucket, err := newGcsStorage(qs) + if err != nil { + return nil, err + } + return bucket, nil default: return nil, fmt.Errorf("storage %s not support yet", st) } @@ -67,6 +88,10 @@ func getRemoteStorage(provider v1alpha1.StorageProvider) ([]string, string, erro qs := checkS3Config(provider.S3, false) s, path := newS3StorageOption(qs) return s, path, nil + case v1alpha1.BackupStorageTypeGcs: + qs := checkGcsConfig(provider.Gcs, false) + s, path := newGcsStorageOption(qs) + return s, path, nil default: return nil, "", fmt.Errorf("storage %s not support yet", st) } @@ -131,6 +156,51 @@ func newS3Storage(qs *s3Query) (*blob.Bucket, error) { } +// newGcsStorage initialize a new gcs storage +func newGcsStorage(qs *gcsQuery) (*blob.Bucket, error) { + ctx := context.Background() + + // Your GCP credentials. + creds, err := gcp.DefaultCredentials(ctx) + if err != nil { + return nil, err + } + + // Create an HTTP client. + client, err := gcp.NewHTTPClient( + gcp.DefaultTransport(), + gcp.CredentialsTokenSource(creds)) + if err != nil { + return nil, err + } + + // Create a *blob.Bucket. + bucket, err := gcsblob.OpenBucket(ctx, client, qs.bucket, nil) + if err != nil { + return nil, err + } + return bucket, nil +} + +// newGcsStorageOption constructs the arg for --storage option and the remote path for br +func newGcsStorageOption(qs *gcsQuery) ([]string, string) { + var gcsoptions []string + var path string + if qs.prefix == "/" { + path = fmt.Sprintf("gcs://%s%s", qs.bucket, qs.prefix) + } else { + path = fmt.Sprintf("gcs://%s/%s", qs.bucket, qs.prefix) + } + gcsoptions = append(gcsoptions, fmt.Sprintf("--storage=%s", path)) + if qs.storageClass != "" { + gcsoptions = append(gcsoptions, fmt.Sprintf("--gcs.storage-class=%s", qs.storageClass)) + } + if qs.objectAcl != "" { + gcsoptions = append(gcsoptions, fmt.Sprintf("--gcs.predefined-acl=%s", qs.objectAcl)) + } + return gcsoptions, path +} + // checkS3Config constructs s3Query parameters func checkS3Config(s3 *v1alpha1.S3StorageProvider, fakeRegion bool) *s3Query { sqs := s3Query{} @@ -159,3 +229,23 @@ func checkS3Config(s3 *v1alpha1.S3StorageProvider, fakeRegion bool) *s3Query { return &sqs } + +// checkGcsConfig constructs gcsQuery parameters +func checkGcsConfig(gcs *v1alpha1.GcsStorageProvider, fakeRegion bool) *gcsQuery { + gqs := gcsQuery{} + + gqs.bucket = gcs.Bucket + gqs.location = gcs.Location + gqs.path = gcs.Path + gqs.projectId = gcs.ProjectId + gqs.storageClass = gcs.StorageClass + gqs.objectAcl = gcs.ObjectAcl + gqs.bucketAcl = gcs.BucketAcl + gqs.secretName = gcs.SecretName + gqs.prefix = gcs.Prefix + + gqs.prefix = strings.Trim(gqs.prefix, "/") + gqs.prefix += "/" + + return &gqs +} diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 3443f957f9..5117b603ec 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -3457,7 +3457,18 @@ string

SecretName is the name of secret which stores the -gcs service account credentials JSON .

+gcs service account credentials JSON.

+ + + + +prefix
+ +string + + + +

Prefix of the data path.

@@ -7346,7 +7357,7 @@ string -

Prefix for the keys.

+

Prefix of the data path.

diff --git a/images/tidb-backup-manager/entrypoint.sh b/images/tidb-backup-manager/entrypoint.sh index 53d474500e..6864ee80ea 100755 --- a/images/tidb-backup-manager/entrypoint.sh +++ b/images/tidb-backup-manager/entrypoint.sh @@ -15,6 +15,7 @@ set -e +export GOOGLE_APPLICATION_CREDENTIALS=/tmp/google-credentials.json echo "Create rclone.conf file." cat < /tmp/rclone.conf [s3] @@ -30,7 +31,7 @@ storage_class = ${AWS_STORAGE_CLASS} [gcs] type = google cloud storage project_number = ${GCS_PROJECT_ID} -service_account_file = /tmp/google-credentials.json +service_account_file = ${GOOGLE_APPLICATION_CREDENTIALS} object_acl = ${GCS_OBJECT_ACL} bucket_acl = ${GCS_BUCKET_ACL} location = ${GCS_LOCATION} @@ -43,11 +44,11 @@ EOF if [[ -n "${GCS_SERVICE_ACCOUNT_JSON_KEY:-}" ]]; then echo "Create google-credentials.json file." - cat < /tmp/google-credentials.json + cat < ${GOOGLE_APPLICATION_CREDENTIALS} ${GCS_SERVICE_ACCOUNT_JSON_KEY} EOF else - touch /tmp/google-credentials.json + touch ${GOOGLE_APPLICATION_CREDENTIALS} fi BACKUP_BIN=/tidb-backup-manager diff --git a/manifests/crd.yaml b/manifests/crd.yaml index e260934d10..65a7cb0a49 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -4523,6 +4523,8 @@ spec: type: string path: type: string + prefix: + type: string projectId: type: string secretName: @@ -4925,6 +4927,8 @@ spec: type: string path: type: string + prefix: + type: string projectId: type: string secretName: @@ -5371,6 +5375,8 @@ spec: type: string path: type: string + prefix: + type: string projectId: type: string secretName: diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 906ccbd2b9..aaab4d615f 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -1440,7 +1440,14 @@ func schema_pkg_apis_pingcap_v1alpha1_GcsStorageProvider(ref common.ReferenceCal }, "secretName": { SchemaProps: spec.SchemaProps{ - Description: "SecretName is the name of secret which stores the gcs service account credentials JSON .", + Description: "SecretName is the name of secret which stores the gcs service account credentials JSON.", + Type: []string{"string"}, + Format: "", + }, + }, + "prefix": { + SchemaProps: spec.SchemaProps{ + Description: "Prefix of the data path.", Type: []string{"string"}, Format: "", }, @@ -3584,7 +3591,7 @@ func schema_pkg_apis_pingcap_v1alpha1_S3StorageProvider(ref common.ReferenceCall }, "prefix": { SchemaProps: spec.SchemaProps{ - Description: "Prefix for the keys.", + Description: "Prefix of the data path.", Type: []string{"string"}, Format: "", }, diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 5f7ea3d7c1..34a8c37ccd 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -840,7 +840,7 @@ type S3StorageProvider struct { // SecretName is the name of secret which stores // S3 compliant storage access key and secret key. SecretName string `json:"secretName,omitempty"` - // Prefix for the keys. + // Prefix of the data path. Prefix string `json:"prefix,omitempty"` // SSE Sever-Side Encryption. SSE string `json:"sse,omitempty"` @@ -867,8 +867,10 @@ type GcsStorageProvider struct { // BucketAcl represents the access control list for new buckets BucketAcl string `json:"bucketAcl,omitempty"` // SecretName is the name of secret which stores the - // gcs service account credentials JSON . + // gcs service account credentials JSON. SecretName string `json:"secretName"` + // Prefix of the data path. + Prefix string `json:"prefix,omitempty"` } // +k8s:openapi-gen=true