backup: support gcp backup with br (pingcap#2267)

cmd/backup-manager/app/util/remote.go

@@ -21,7 +21,9 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"gocloud.dev/blob"
+	"gocloud.dev/blob/gcsblob"
 	"gocloud.dev/blob/s3blob"
+	"gocloud.dev/gcp"
 
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/backup/util"
@@ -43,6 +45,18 @@ type s3Query struct {
 	forcePathStyle bool
 }
 
+type gcsQuery struct {
+	projectId    string
+	location     string
+	path         string
+	bucket       string
+	storageClass string
+	objectAcl    string
+	bucketAcl    string
+	secretName   string
+	prefix       string
+}
+
 // NewRemoteStorage creates new remote storage
 func NewRemoteStorage(backup *v1alpha1.Backup) (*blob.Bucket, error) {
 	st := util.GetStorageType(backup.Spec.StorageProvider)
@@ -54,6 +68,13 @@ func NewRemoteStorage(backup *v1alpha1.Backup) (*blob.Bucket, error) {
 			return nil, err
 		}
 		return bucket, nil
+	case v1alpha1.BackupStorageTypeGcs:
+		qs := checkGcsConfig(backup.Spec.Gcs, true)
+		bucket, err := newGcsStorage(qs)
+		if err != nil {
+			return nil, err
+		}
+		return bucket, nil
 	default:
 		return nil, fmt.Errorf("storage %s not support yet", st)
 	}
@@ -67,6 +88,10 @@ func getRemoteStorage(provider v1alpha1.StorageProvider) ([]string, string, erro
 		qs := checkS3Config(provider.S3, false)
 		s, path := newS3StorageOption(qs)
 		return s, path, nil
+	case v1alpha1.BackupStorageTypeGcs:
+		qs := checkGcsConfig(provider.Gcs, false)
+		s, path := newGcsStorageOption(qs)
+		return s, path, nil
 	default:
 		return nil, "", fmt.Errorf("storage %s not support yet", st)
 	}
@@ -131,6 +156,51 @@ func newS3Storage(qs *s3Query) (*blob.Bucket, error) {
 
 }
 
+// newGcsStorage initialize a new gcs storage
+func newGcsStorage(qs *gcsQuery) (*blob.Bucket, error) {
+	ctx := context.Background()
+
+	// Your GCP credentials.
+	creds, err := gcp.DefaultCredentials(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create an HTTP client.
+	client, err := gcp.NewHTTPClient(
+		gcp.DefaultTransport(),
+		gcp.CredentialsTokenSource(creds))
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a *blob.Bucket.
+	bucket, err := gcsblob.OpenBucket(ctx, client, qs.bucket, nil)
+	if err != nil {
+		return nil, err
+	}
+	return bucket, nil
+}
+
+// newGcsStorageOption constructs the arg for --storage option and the remote path for br
+func newGcsStorageOption(qs *gcsQuery) ([]string, string) {
+	var gcsoptions []string
+	var path string
+	if qs.prefix == "/" {
+		path = fmt.Sprintf("gcs://%s%s", qs.bucket, qs.prefix)
+	} else {
+		path = fmt.Sprintf("gcs://%s/%s", qs.bucket, qs.prefix)
+	}
+	gcsoptions = append(gcsoptions, fmt.Sprintf("--storage=%s", path))
+	if qs.storageClass != "" {
+		gcsoptions = append(gcsoptions, fmt.Sprintf("--gcs.storage-class=%s", qs.storageClass))
+	}
+	if qs.objectAcl != "" {
+		gcsoptions = append(gcsoptions, fmt.Sprintf("--gcs.predefined-acl=%s", qs.objectAcl))
+	}
+	return gcsoptions, path
+}
+
 // checkS3Config constructs s3Query parameters
 func checkS3Config(s3 *v1alpha1.S3StorageProvider, fakeRegion bool) *s3Query {
 	sqs := s3Query{}
@@ -159,3 +229,23 @@ func checkS3Config(s3 *v1alpha1.S3StorageProvider, fakeRegion bool) *s3Query {
 
 	return &sqs
 }
+
+// checkGcsConfig constructs gcsQuery parameters
+func checkGcsConfig(gcs *v1alpha1.GcsStorageProvider, fakeRegion bool) *gcsQuery {
+	gqs := gcsQuery{}
+
+	gqs.bucket = gcs.Bucket
+	gqs.location = gcs.Location
+	gqs.path = gcs.Path
+	gqs.projectId = gcs.ProjectId
+	gqs.storageClass = gcs.StorageClass
+	gqs.objectAcl = gcs.ObjectAcl
+	gqs.bucketAcl = gcs.BucketAcl
+	gqs.secretName = gcs.SecretName
+	gqs.prefix = gcs.Prefix
+
+	gqs.prefix = strings.Trim(gqs.prefix, "/")
+	gqs.prefix += "/"
+
+	return &gqs
+}

docs/api-references/docs.md

@@ -3374,7 +3374,18 @@ string
 </td>
 <td>
 <p>SecretName is the name of secret which stores the
-gcs service account credentials JSON .</p>
+gcs service account credentials JSON.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>prefix</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>Prefix of the data path.</p>
 </td>
 </tr>
 </tbody>
@@ -7263,7 +7274,7 @@ string
 </em>
 </td>
 <td>
-<p>Prefix for the keys.</p>
+<p>Prefix of the data path.</p>
 </td>
 </tr>
 <tr>

images/tidb-backup-manager/entrypoint.sh

@@ -15,6 +15,7 @@
 
 set -e
 
+export GOOGLE_APPLICATION_CREDENTIALS=/tmp/google-credentials.json
 echo "Create rclone.conf file."
 cat <<EOF > /tmp/rclone.conf
 [s3]
@@ -30,7 +31,7 @@ storage_class = ${AWS_STORAGE_CLASS}
 [gcs]
 type = google cloud storage
 project_number = ${GCS_PROJECT_ID}
-service_account_file = /tmp/google-credentials.json
+service_account_file = ${GOOGLE_APPLICATION_CREDENTIALS}
 object_acl = ${GCS_OBJECT_ACL}
 bucket_acl = ${GCS_BUCKET_ACL}
 location =  ${GCS_LOCATION}
@@ -43,11 +44,11 @@ EOF
 
 if [[ -n "${GCS_SERVICE_ACCOUNT_JSON_KEY:-}" ]]; then
     echo "Create google-credentials.json file."
-    cat <<EOF > /tmp/google-credentials.json
+    cat <<EOF > ${GOOGLE_APPLICATION_CREDENTIALS}
     ${GCS_SERVICE_ACCOUNT_JSON_KEY}
 EOF
 else
-    touch /tmp/google-credentials.json
+    touch ${GOOGLE_APPLICATION_CREDENTIALS}
 fi
 
 BACKUP_BIN=/tidb-backup-manager

manifests/crd.yaml

@@ -4474,6 +4474,8 @@ spec:
                   type: string
                 path:
                   type: string
+                prefix:
+                  type: string
                 projectId:
                   type: string
                 secretName:
@@ -4876,6 +4878,8 @@ spec:
                   type: string
                 path:
                   type: string
+                prefix:
+                  type: string
                 projectId:
                   type: string
                 secretName:
@@ -5322,6 +5326,8 @@ spec:
                       type: string
                     path:
                       type: string
+                    prefix:
+                      type: string
                     projectId:
                       type: string
                     secretName:

pkg/apis/pingcap/v1alpha1/types.go

@@ -840,7 +840,7 @@ type S3StorageProvider struct {
 	// SecretName is the name of secret which stores
 	// S3 compliant storage access key and secret key.
 	SecretName string `json:"secretName,omitempty"`
-	// Prefix for the keys.
+	// Prefix of the data path.
 	Prefix string `json:"prefix,omitempty"`
 	// SSE Sever-Side Encryption.
 	SSE string `json:"sse,omitempty"`
@@ -867,8 +867,10 @@ type GcsStorageProvider struct {
 	// BucketAcl represents the access control list for new buckets
 	BucketAcl string `json:"bucketAcl,omitempty"`
 	// SecretName is the name of secret which stores the
-	// gcs service account credentials JSON .
+	// gcs service account credentials JSON.
 	SecretName string `json:"secretName"`
+	// Prefix of the data path.
+	Prefix string `json:"prefix,omitempty"`
 }
 
 // +k8s:openapi-gen=true