Merge pull request #331 from kargakis/gcs

Add GCS Fs implementation

README.md

@@ -259,6 +259,18 @@ system using InMemoryFile.
 Afero has experimental support for secure file transfer protocol (sftp). Which can
 be used to perform file operations over a encrypted channel.
 
+### GCSFs
+
+Afero has experimental support for Google Cloud Storage (GCS). You can either set the
+`GOOGLE_APPLICATION_CREDENTIALS_JSON` env variable to your JSON credentials or use `opts` in
+`NewGcsFS` to configure access to your GCS bucket.
+
+Some known limitations of the existing implementation:
+* No Chmod support - The GCS ACL could probably be mapped to *nix style permissions but that would add another level of complexity and is ignored in this version.
+* No Chtimes support - Could be simulated with attributes (gcs a/m-times are set implicitly) but that's is left for another version.
+* Not thread safe - Also assumes all file operations are done through the same instance of the GcsFs. File operations between different GcsFs instances are not guaranteed to be consistent.
+
+
 ## Filtering Backends
 
 ### BasePathFs

gcs-fake-service-account.json

@@ -0,0 +1,9 @@
+{
+  "type": "service_account",
+  "private_key_id": "abc",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDY3E8o1NEFcjMM\nHW/5ZfFJw29/8NEqpViNjQIx95Xx5KDtJ+nWn9+OW0uqsSqKlKGhAdAo+Q6bjx2c\nuXVsXTu7XrZUY5Kltvj94DvUa1wjNXs606r/RxWTJ58bfdC+gLLxBfGnB6CwK0YQ\nxnfpjNbkUfVVzO0MQD7UP0Hl5ZcY0Puvxd/yHuONQn/rIAieTHH1pqgW+zrH/y3c\n59IGThC9PPtugI9ea8RSnVj3PWz1bX2UkCDpy9IRh9LzJLaYYX9RUd7++dULUlat\nAaXBh1U6emUDzhrIsgApjDVtimOPbmQWmX1S60mqQikRpVYZ8u+NDD+LNw+/Eovn\nxCj2Y3z1AgMBAAECggEAWDBzoqO1IvVXjBA2lqId10T6hXmN3j1ifyH+aAqK+FVl\nGjyWjDj0xWQcJ9ync7bQ6fSeTeNGzP0M6kzDU1+w6FgyZqwdmXWI2VmEizRjwk+/\n/uLQUcL7I55Dxn7KUoZs/rZPmQDxmGLoue60Gg6z3yLzVcKiDc7cnhzhdBgDc8vd\nQorNAlqGPRnm3EqKQ6VQp6fyQmCAxrr45kspRXNLddat3AMsuqImDkqGKBmF3Q1y\nxWGe81LphUiRqvqbyUlh6cdSZ8pLBpc9m0c3qWPKs9paqBIvgUPlvOZMqec6x4S6\nChbdkkTRLnbsRr0Yg/nDeEPlkhRBhasXpxpMUBgPywKBgQDs2axNkFjbU94uXvd5\nznUhDVxPFBuxyUHtsJNqW4p/ujLNimGet5E/YthCnQeC2P3Ym7c3fiz68amM6hiA\nOnW7HYPZ+jKFnefpAtjyOOs46AkftEg07T9XjwWNPt8+8l0DYawPoJgbM5iE0L2O\nx8TU1Vs4mXc+ql9F90GzI0x3VwKBgQDqZOOqWw3hTnNT07Ixqnmd3dugV9S7eW6o\nU9OoUgJB4rYTpG+yFqNqbRT8bkx37iKBMEReppqonOqGm4wtuRR6LSLlgcIU9Iwx\nyfH12UWqVmFSHsgZFqM/cK3wGev38h1WBIOx3/djKn7BdlKVh8kWyx6uC8bmV+E6\nOoK0vJD6kwKBgHAySOnROBZlqzkiKW8c+uU2VATtzJSydrWm0J4wUPJifNBa/hVW\ndcqmAzXC9xznt5AVa3wxHBOfyKaE+ig8CSsjNyNZ3vbmr0X04FoV1m91k2TeXNod\njMTobkPThaNm4eLJMN2SQJuaHGTGERWC0l3T18t+/zrDMDCPiSLX1NAvAoGBAN1T\nVLJYdjvIMxf1bm59VYcepbK7HLHFkRq6xMJMZbtG0ryraZjUzYvB4q4VjHk2UDiC\nlhx13tXWDZH7MJtABzjyg+AI7XWSEQs2cBXACos0M4Myc6lU+eL+iA+OuoUOhmrh\nqmT8YYGu76/IBWUSqWuvcpHPpwl7871i4Ga/I3qnAoGBANNkKAcMoeAbJQK7a/Rn\nwPEJB+dPgNDIaboAsh1nZhVhN5cvdvCWuEYgOGCPQLYQF0zmTLcM+sVxOYgfy8mV\nfbNgPgsP5xmu6dw2COBKdtozw0HrWSRjACd1N4yGu75+wPCcX/gQarcjRcXXZeEa\nNtBLSfcqPULqD+h7br9lEJio\n-----END PRIVATE KEY-----\n",
+  "client_email": "123-abc@developer.gserviceaccount.com",
+  "client_id": "123-abc.apps.googleusercontent.com",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "http://localhost:8080/token"
+}

gcs.go

@@ -0,0 +1,115 @@
+// Copyright © 2021 Vasily Ovchinnikov <vasily@remerge.io>.
+//
+// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp
+// licensed under Apache License 2.0.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package afero
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/spf13/afero/gcsfs"
+
+	"cloud.google.com/go/storage"
+	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
+
+	"google.golang.org/api/option"
+)
+
+type GcsFs struct {
+	source *gcsfs.GcsFs
+}
+
+// NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client.
+// You can provide additional options to be passed to the client creation, as per
+// cloud.google.com/go/storage documentation
+func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
+	if json := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); json != "" {
+		opts = append(opts, option.WithCredentialsJSON([]byte(json)))
+	}
+	client, err := storage.NewClient(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewGcsFSFromClient(ctx, client)
+}
+
+// NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator.
+func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
+	client, err := storage.NewClient(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewGcsFSFromClientWithSeparator(ctx, client, folderSeparator)
+}
+
+// NewGcsFSFromClient creates a GCS file system from a given storage client
+func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) {
+	c := stiface.AdaptClient(client)
+
+	return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil
+}
+
+// NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator.
+func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) {
+	c := stiface.AdaptClient(client)
+
+	return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
+}
+
+// Wraps gcs.GcsFs and convert some return types to afero interfaces.
+
+func (fs *GcsFs) Name() string {
+	return fs.source.Name()
+}
+func (fs *GcsFs) Create(name string) (File, error) {
+	return fs.source.Create(name)
+}
+func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
+	return fs.source.Mkdir(name, perm)
+}
+func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
+	return fs.source.MkdirAll(path, perm)
+}
+func (fs *GcsFs) Open(name string) (File, error) {
+	return fs.source.Open(name)
+}
+func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
+	return fs.source.OpenFile(name, flag, perm)
+}
+func (fs *GcsFs) Remove(name string) error {
+	return fs.source.Remove(name)
+}
+func (fs *GcsFs) RemoveAll(path string) error {
+	return fs.source.RemoveAll(path)
+}
+func (fs *GcsFs) Rename(oldname, newname string) error {
+	return fs.source.Rename(oldname, newname)
+}
+func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
+	return fs.source.Stat(name)
+}
+func (fs *GcsFs) Chmod(name string, mode os.FileMode) error {
+	return fs.source.Chmod(name, mode)
+}
+func (fs *GcsFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return fs.source.Chtimes(name, atime, mtime)
+}
+func (fs *GcsFs) Chown(name string, uid, gid int) error {
+	return fs.source.Chown(name, uid, gid)
+}