From 9a79ce36cfd7cc6f58c314c8dfe49b493a659070 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 18 Oct 2017 23:34:24 -0700 Subject: [PATCH] Fix docs and add a validator tool for our api docs --- .gitignore | 3 +- .travis.yml | 9 +- Makefile | 17 ++ docs/API.md | 411 +++++++++++++++++++------------ docs/checker.go.template | 23 ++ docs/validator.go | 224 +++++++++++++++++ examples/s3/fputobject.go | 4 +- examples/s3/putobject-context.go | 4 +- examples/s3/removeobjects.go | 9 +- 9 files changed, 529 insertions(+), 175 deletions(-) create mode 100644 Makefile create mode 100644 docs/checker.go.template create mode 100644 docs/validator.go diff --git a/.gitignore b/.gitignore index acf19db3a7..fa967abd77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ -*.test \ No newline at end of file +*.test +validator diff --git a/.travis.yml b/.travis.yml index c6b9ec0649..561f55f03e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,4 @@ go: script: - diff -au <(gofmt -d .) <(printf "") -- go get -u github.com/cheggaaa/pb/... -- go get -u github.com/sirupsen/logrus/... -- go get -u github.com/dustin/go-humanize/... -- go vet ./... -- SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 go test -race -v ./... -- SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 go run functional_tests.go -- mkdir /tmp/examples \ - && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done +- make diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..bf7392d5a7 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +all: checks + +checks: + @go get -u github.com/go-ini/ini/... + @go get -u github.com/minio/go-homedir/... + @go get -u github.com/cheggaaa/pb/... + @go get -u github.com/sirupsen/logrus/... + @go get -u github.com/dustin/go-humanize/... + @go vet ./... + @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 go test -race -v ./... + @SERVER_ENDPOINT=play.minio.io:9000 ACCESS_KEY=Q3AM3UQ867SPQQA43P2F SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG ENABLE_HTTPS=1 go run functional_tests.go + @mkdir /tmp/examples && for i in $(echo examples/s3/*); do go build -o /tmp/examples/$(basename ${i:0:-3}) ${i}; done + @go get -u github.com/a8m/mark/... + @go get -u github.com/minio/cli/... + @go get -u golang.org/x/tools/cmd/goimports + @go get -u github.com/gernest/wow/... + @go build docs/validator.go && ./validator -m docs/API.md -t docs/checker.go.tpl diff --git a/docs/API.md b/docs/API.md index 67488b50e6..d1026ee228 100644 --- a/docs/API.md +++ b/docs/API.md @@ -123,7 +123,7 @@ __Example__ ```go -err := minioClient.MakeBucket("mybucket", "us-east-1") +err = minioClient.MakeBucket("mybucket", "us-east-1") if err != nil { fmt.Println(err) return @@ -133,7 +133,6 @@ fmt.Println("Successfully created mybucket.") ### ListBuckets() ([]BucketInfo, error) - Lists all buckets. | Param | Type | Description | @@ -154,7 +153,7 @@ __Example__ ```go buckets, err := minioClient.ListBuckets() - if err != nil { +if err != nil { fmt.Println(err) return } @@ -165,7 +164,6 @@ for _, bucket := range buckets { ### BucketExists(bucketName string) (found bool, err error) - Checks if a bucket exists. __Parameters__ @@ -200,8 +198,7 @@ if found { ### RemoveBucket(bucketName string) error - -Removes a bucket. +Removes a bucket, bucket should be empty to be successfully removed. __Parameters__ @@ -214,7 +211,7 @@ __Example__ ```go -err := minioClient.RemoveBucket("mybucket") +err = minioClient.RemoveBucket("mybucket") if err != nil { fmt.Println(err) return @@ -223,7 +220,6 @@ if err != nil { ### ListObjects(bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo - Lists objects in a bucket. __Parameters__ @@ -274,7 +270,6 @@ for object := range objectCh { ### ListObjectsV2(bucketName, prefix string, recursive bool, doneCh chan struct{}) <-chan ObjectInfo - Lists objects in a bucket using the recommended listing API v2 __Parameters__ @@ -315,7 +310,6 @@ for object := range objectCh { ### ListIncompleteUploads(bucketName, prefix string, recursive bool, doneCh chan struct{}) <- chan ObjectMultipartInfo - Lists partially uploaded objects in a bucket. @@ -369,7 +363,6 @@ for multiPartObject := range multiPartObjectCh { ### GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) - Returns a stream of the object data. Most of the common errors occur when reading the stream. @@ -401,7 +394,7 @@ __Example__ ```go -object, err := minioClient.GetObject("mybucket", "photo.jpg", minio.GetObjectOptions{}) +object, err := minioClient.GetObject("mybucket", "myobject", minio.GetObjectOptions{}) if err != nil { fmt.Println(err) return @@ -419,8 +412,7 @@ if _, err = io.Copy(localFile, object); err != nil { ### FGetObject(bucketName, objectName, filePath string, opts GetObjectOptions) error - Downloads and saves the object as a file in the local filesystem. - +Downloads and saves the object as a file in the local filesystem. __Parameters__ @@ -437,7 +429,7 @@ __Example__ ```go -err := minioClient.FGetObject("mybucket", "photo.jpg", "/tmp/photo.jpg", minio.GetObjectOptions{}) +err = minioClient.FGetObject("mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) if err != nil { fmt.Println(err) return @@ -445,7 +437,6 @@ if err != nil { ``` ### GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) - Identical to GetObject operation, but accepts a context for request cancellation. __Parameters__ @@ -471,18 +462,21 @@ __Example__ ```go -ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Seconds) +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) defer cancel() -object, err := minioClient.GetObjectWithContext(ctx, "mybucket", "photo.jpg", minio.GetObjectOptions{}) + +object, err := minioClient.GetObjectWithContext(ctx, "mybucket", "myobject", minio.GetObjectOptions{}) if err != nil { fmt.Println(err) return } + localFile, err := os.Create("/tmp/local-file.jpg") if err != nil { fmt.Println(err) return } + if _, err = io.Copy(localFile, object); err != nil { fmt.Println(err) return @@ -491,7 +485,7 @@ if _, err = io.Copy(localFile, object); err != nil { ### FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error - Identical to FGetObject operation, but allows request cancellation +Identical to FGetObject operation, but allows request cancellation. __Parameters__ @@ -509,9 +503,10 @@ __Example__ ```go -ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Seconds) +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) defer cancel() -err := minioClient.FGetObjectWithContext(ctx, "mybucket", "photo.jpg", "/tmp/photo.jpg", minio.GetObjectOptions{}) + +err = minioClient.FGetObjectWithContext(ctx, "mybucket", "myobject", "/tmp/myobject", minio.GetObjectOptions{}) if err != nil { fmt.Println(err) return @@ -520,7 +515,7 @@ if err != nil { ### FGetEncryptedObject(bucketName, objectName, filePath string, materials encrypt.Materials) error - Identical to FGetObject operation, but decrypts an encrypted request +Identical to FGetObject operation, but decrypts an encrypted request __Parameters__ @@ -538,23 +533,24 @@ __Example__ ```go // Generate a master symmetric key -key := encrypt.NewSymmetricKey("my-secret-key-00") +key := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) // Build the CBC encryption material cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) if err != nil { - log.Fatalln(err) + fmt.Println(err) + return } -err = minioClient.FGetEncryptedObject("mybucket", "photo.jpg", "/tmp/photo.jpg", cbcMaterials) +err = minioClient.FGetEncryptedObject("mybucket", "myobject", "/tmp/myobject", cbcMaterials) if err != nil { - log.Fatalln(err) + fmt.Println(err) + return } ``` ### PutObject(bucketName, objectName string, reader io.Reader, objectSize int64,opts PutObjectOptions) (n int, err error) - Uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than 64MiB in size, PutObject seamlessly uploads the object as parts of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. __Parameters__ @@ -590,7 +586,6 @@ if err != nil { fmt.Println(err) return } - defer file.Close() fileStat, err := file.Stat() @@ -604,12 +599,13 @@ if err != nil { fmt.Println(err) return } +fmt.Println("Successfully uploaded bytes: ", n) ``` -API methods PutObjectWithSize, PutObjectWithMetadata, PutObjectStreaming, and PutObjectWithProgress available in minio-go SDK release v3.0.2 are replaced by the new PutObject call variant that accepts a pointer to PutObjectOptions struct. + +API methods PutObjectWithSize, PutObjectWithMetadata, PutObjectStreaming, and PutObjectWithProgress available in minio-go SDK release v3.0.3 are replaced by the new PutObject call variant that accepts a pointer to PutObjectOptions struct. ### PutObjectWithContext(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64, opts PutObjectOptions) (n int, err error) - Identical to PutObject operation, but allows request cancellation. __Parameters__ @@ -625,12 +621,11 @@ __Parameters__ |`opts` | _minio.PutObjectOptions_ |Pointer to struct that allows user to set optional custom metadata, content-type, content-encoding,content-disposition and cache-control headers, pass encryption module for encrypting objects, and optionally configure number of threads for multipart put operation. | - __Example__ ```go -ctx, cancel := context.WithTimeout(context.Background(), 10 * time.Seconds) +ctx, cancel := context.WithTimeout(context.Background(), 10 * time.Second) defer cancel() file, err := os.Open("my-testfile") @@ -646,21 +641,22 @@ if err != nil { return } -n, err := minioClient.PutObjectWithContext(ctx, "mybucket", "myobject", file, fileStat.Size(), minio.PutObjectOptions{ContentType:"application/octet-stream"}) +n, err := minioClient.PutObjectWithContext(ctx, "my-bucketname", "my-objectname", file, fileStat.Size(), minio.PutObjectOptions{ + ContentType: "application/octet-stream", +}) if err != nil { fmt.Println(err) return } +fmt.Println("Successfully uploaded bytes: ", n) ``` ### CopyObject(dst DestinationInfo, src SourceInfo) error - Create or replace an object through server-side copying of an existing object. It supports conditional copying, copying a part of an object and server-side encryption of destination and decryption of source. See the `SourceInfo` and `DestinationInfo` types for further details. To copy multiple source objects into a single destination object see the `ComposeObject` API. - __Parameters__ @@ -674,7 +670,7 @@ __Example__ ```go -// Use-case 1: Simple copy object with no conditions, etc +// Use-case 1: Simple copy object with no conditions. // Source object src := minio.NewSourceInfo("my-sourcebucketname", "my-sourceobjectname", nil) @@ -686,13 +682,16 @@ if err != nil { } // Copy object call -err = s3Client.CopyObject(dst, src) +err = minioClient.CopyObject(dst, src) if err != nil { fmt.Println(err) return } +``` -// Use-case 2: Copy object with copy-conditions, and copying only part of the source object. +```go +// Use-case 2: +// Copy object with copy-conditions, and copying only part of the source object. // 1. that matches a given ETag // 2. and modified after 1st April 2014 // 3. but unmodified since 23rd April 2014 @@ -721,7 +720,7 @@ if err != nil { } // Copy object call -err = s3Client.CopyObject(dst, src) +err = minioClient.CopyObject(dst, src) if err != nil { fmt.Println(err) return @@ -730,9 +729,7 @@ if err != nil { ### ComposeObject(dst minio.DestinationInfo, srcs []minio.SourceInfo) error - -Create an object by concatenating a list of source objects using -server-side copying. +Create an object by concatenating a list of source objects using server-side copying. __Parameters__ @@ -753,14 +750,14 @@ decKey := minio.NewSSEInfo([]byte{1, 2, 3}, "") // Source objects to concatenate. We also specify decryption // key for each -src1 := minio.NewSourceInfo("bucket1", "object1", decKey) -src1.SetMatchETag("31624deb84149d2f8ef9c385918b653a") +src1 := minio.NewSourceInfo("bucket1", "object1", &decKey) +src1.SetMatchETagCond("31624deb84149d2f8ef9c385918b653a") -src2 := minio.NewSourceInfo("bucket2", "object2", decKey) -src2.SetMatchETag("f8ef9c385918b653a31624deb84149d2") +src2 := minio.NewSourceInfo("bucket2", "object2", &decKey) +src2.SetMatchETagCond("f8ef9c385918b653a31624deb84149d2") -src3 := minio.NewSourceInfo("bucket3", "object3", decKey) -src3.SetMatchETag("5918b653a31624deb84149d2f8ef9c38") +src3 := minio.NewSourceInfo("bucket3", "object3", &decKey) +src3.SetMatchETagCond("5918b653a31624deb84149d2f8ef9c38") // Create slice of sources. srcs := []minio.SourceInfo{src1, src2, src3} @@ -769,19 +766,24 @@ srcs := []minio.SourceInfo{src1, src2, src3} encKey := minio.NewSSEInfo([]byte{8, 9, 0}, "") // Create destination info -dst := minio.NewDestinationInfo("bucket", "object", encKey, nil) -err = s3Client.ComposeObject(dst, srcs) +dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) +if err != nil { + fmt.Println(err) + return +} + +// Compose object call by concatenating multiple source files. +err = minioClient.ComposeObject(dst, srcs) if err != nil { - log.Println(err) - return + fmt.Println(err) + return } -log.Println("Composed object successfully.") +fmt.Println("Composed object successfully.") ``` ### NewSourceInfo(bucket, object string, decryptSSEC *SSEInfo) SourceInfo - Construct a `SourceInfo` object that can be used as the source for server-side copying operations like `CopyObject` and `ComposeObject`. This object can be used to set copy-conditions on the source. __Parameters__ @@ -794,18 +796,47 @@ __Parameters__ __Example__ -``` go +```go // No decryption parameter. src := minio.NewSourceInfo("bucket", "object", nil) +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + +```go // With decryption parameter. -decKey := minio.NewSSEKey([]byte{1,2,3}, "") -src := minio.NewSourceInfo("bucket", "object", decKey) +decKey := minio.NewSSEInfo([]byte{1,2,3}, "") +src := minio.NewSourceInfo("bucket", "object", &decKey) + +// Destination object +dst, err := minio.NewDestinationInfo("my-bucketname", "my-objectname", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} ``` ### NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo, userMeta map[string]string) (DestinationInfo, error) - Construct a `DestinationInfo` object that can be used as the destination object for server-side copying operations like `CopyObject` and `ComposeObject`. __Parameters__ @@ -819,24 +850,48 @@ __Parameters__ __Example__ -``` go +```go // No encryption parameter. +src := minio.NewSourceInfo("bucket", "object", nil) dst, err := minio.NewDestinationInfo("bucket", "object", nil, nil) +if err != nil { + fmt.Println(err) + return +} + +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` + +```go +src := minio.NewSourceInfo("bucket", "object", nil) // With encryption parameter. encKey := minio.NewSSEInfo([]byte{1,2,3}, "") -dst, err := minio.NewDecryptionInfo("bucket", "object", encKey, nil) -``` +dst, err := minio.NewDestinationInfo("bucket", "object", &encKey, nil) +if err != nil { + fmt.Println(err) + return +} +// Copy object call +err = minioClient.CopyObject(dst, src) +if err != nil { + fmt.Println(err) + return +} +``` ### FPutObject(bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) - Uploads contents from a file to objectName. FPutObject uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than the 64MiB in size, FPutObject seamlessly uploads the object in chunks of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. - __Parameters__ @@ -852,15 +907,18 @@ __Example__ ```go -n, err := minioClient.FPutObject("mybucket", "myobject.csv", "/tmp/otherobject.csv", minio.PutObjectOptions{ContentType:"application/csv"}) +n, err := minioClient.FPutObject("my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ + ContentType: "application/csv", +}); if err != nil { fmt.Println(err) return } +fmt.Println("Successfully uploaded bytes: ", n) ``` + ### FPutObjectWithContext(ctx context.Context, bucketName, objectName, filePath, opts PutObjectOptions) (length int64, err error) - Identical to FPutObject operation, but allows request cancellation. __Parameters__ @@ -878,19 +936,20 @@ __Example__ ```go -ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Seconds) +ctx, cancel := context.WithTimeout(context.Background(), 100 * time.Second) defer cancel() + n, err := minioClient.FPutObjectWithContext(ctx, "mybucket", "myobject.csv", "/tmp/otherobject.csv", minio.PutObjectOptions{ContentType:"application/csv"}) if err != nil { fmt.Println(err) return } +fmt.Println("Successfully uploaded bytes: ", n) ``` + ### StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) - -Gets metadata of an object. - +Fetch metadata of an object. __Parameters__ @@ -923,7 +982,7 @@ __Example__ ```go -objInfo, err := minioClient.StatObject("mybucket", "photo.jpg", minio.GetObjectOptions{}) +objInfo, err := minioClient.StatObject("mybucket", "myobject", minio.StatObjectOptions{}) if err != nil { fmt.Println(err) return @@ -933,10 +992,8 @@ fmt.Println(objInfo) ### RemoveObject(bucketName, objectName string) error - Removes an object. - __Parameters__ @@ -947,17 +1004,16 @@ __Parameters__ ```go -err := minioClient.RemoveObject("mybucket", "photo.jpg") +err = minioClient.RemoveObject("mybucket", "myobject") if err != nil { fmt.Println(err) return } ``` + ### RemoveObjects(bucketName string, objectsCh chan string) (errorCh <-chan RemoveObjectError) - -Removes a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. -The errors observed are sent over the error channel. +Removes a list of objects obtained from an input channel. The call sends a delete request to the server up to 1000 objects at a time. The errors observed are sent over the error channel. __Parameters__ @@ -974,19 +1030,28 @@ __Return Values__ |`errorCh` | _<-chan minio.RemoveObjectError_ | Receive-only channel of errors observed during deletion. | - ```go -errorCh := minioClient.RemoveObjects("mybucket", objectsCh) -for e := range errorCh { - fmt.Println("Error detected during deletion: " + e.Err.Error()) +objectsCh := make(chan string) + +// Send object names that are needed to be removed to objectsCh +go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range minioClient.ListObjects("my-bucketname", "my-prefixname", true, nil) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key + } +}() + +for rErr := range minioClient.RemoveObjects("mybucket", objectsCh) { + fmt.Println("Error detected during deletion: ", rErr) } ``` - - ### RemoveIncompleteUpload(bucketName, objectName string) error - Removes a partially uploaded object. __Parameters__ @@ -1001,7 +1066,7 @@ __Example__ ```go -err := minioClient.RemoveIncompleteUpload("mybucket", "photo.jpg") +err = minioClient.RemoveIncompleteUpload("mybucket", "myobject") if err != nil { fmt.Println(err) return @@ -1028,8 +1093,22 @@ __Return Value__ ```go symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) -``` +// Build the CBC encryption material with symmetric key. +cbcMaterials, err := encrypt.NewCBCSecureMaterials(symKey) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully initialized Symmetric key CBC materials", cbcMaterials) + +object, err := minioClient.GetEncryptedObject("mybucket", "myobject", cbcMaterials) +if err != nil { + fmt.Println(err) + return +} +defer object.Close() +``` ### NewAsymmetricKey(privateKey []byte, publicKey[]byte) (*encrypt.AsymmetricKey, error) @@ -1047,25 +1126,43 @@ __Return Value__ |Param |Type |Description | |:---|:---| :---| |`asymmetricKey` | _*encrypt.AsymmetricKey_ | represents an asymmetric key structure which can be used to encrypt and decrypt data | -|`err` | _error_ | encountered errors | +|`err` | _error_ | Standard Error | ```go privateKey, err := ioutil.ReadFile("private.key") if err != nil { - log.Fatal(err) + fmt.Println(err) + return } publicKey, err := ioutil.ReadFile("public.key") if err != nil { - log.Fatal(err) + fmt.Println(err) + return } // Initialize the asymmetric key asymmetricKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) if err != nil { - log.Fatal(err) + fmt.Println(err) + return } + +// Build the CBC encryption material for asymmetric key. +cbcMaterials, err := encrypt.NewCBCSecureMaterials(asymmetricKey) +if err != nil { + fmt.Println(err) + return +} +fmt.Println("Successfully initialized Asymmetric key CBC materials", cbcMaterials) + +object, err := minioClient.GetEncryptedObject("mybucket", "myobject", cbcMaterials) +if err != nil { + fmt.Println(err) + return +} +defer object.Close() ``` @@ -1095,15 +1192,16 @@ __Example__ ```go // Generate a master symmetric key -key := encrypt.NewSymmetricKey("my-secret-key-00") +key := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) // Build the CBC encryption material cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) if err != nil { - t.Fatal(err) + fmt.Println(err) + return } -object, err := minioClient.GetEncryptedObject("mybucket", "photo.jpg", cbcMaterials) +object, err := minioClient.GetEncryptedObject("mybucket", "myobject", cbcMaterials) if err != nil { fmt.Println(err) return @@ -1115,6 +1213,7 @@ if err != nil { fmt.Println(err) return } +defer localFile.Close() if _, err = io.Copy(localFile, object); err != nil { fmt.Println(err) @@ -1142,25 +1241,29 @@ __Example__ // Load a private key privateKey, err := ioutil.ReadFile("private.key") if err != nil { - log.Fatal(err) + fmt.Println(err) + return } // Load a public key publicKey, err := ioutil.ReadFile("public.key") if err != nil { - log.Fatal(err) + fmt.Println(err) + return } // Build an asymmetric key -key, err := encrypt.NewAsymetricKey(privateKey, publicKey) +key, err := encrypt.NewAsymmetricKey(privateKey, publicKey) if err != nil { - log.Fatal(err) + fmt.Println(err) + return } // Build the CBC encryption module cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) if err != nil { - log.Fatal(err) + fmt.Println(err) + return } // Open a file to upload @@ -1177,10 +1280,11 @@ if err != nil { fmt.Println(err) return } +fmt.Println("Successfully uploaded encrypted bytes: ", n) ``` + ### FPutEncryptedObject(bucketName, objectName, filePath, encryptMaterials encrypt.Materials) (n int, err error) - Encrypt and upload an object from a file. __Parameters__ @@ -1200,36 +1304,42 @@ __Example__ // Load a private key privateKey, err := ioutil.ReadFile("private.key") if err != nil { - log.Fatal(err) + fmt.Println(err) + return } // Load a public key publicKey, err := ioutil.ReadFile("public.key") if err != nil { - log.Fatal(err) + fmt.Println(err) + return } // Build an asymmetric key -key, err := encrypt.NewAsymetricKey(privateKey, publicKey) +key, err := encrypt.NewAsymmetricKey(privateKey, publicKey) if err != nil { - log.Fatal(err) + fmt.Println(err) + return } // Build the CBC encryption module cbcMaterials, err := encrypt.NewCBCSecureMaterials(key) if err != nil { - log.Fatal(err) + fmt.Println(err) + return } + n, err := minioClient.FPutEncryptedObject("mybucket", "myobject.csv", "/tmp/otherobject.csv", cbcMaterials) if err != nil { fmt.Println(err) return } +fmt.Println("Successfully uploaded encrypted bytes: ", n) ``` + ### NewSSEInfo(key []byte, algo string) SSEInfo - Create a key object for use as encryption or decryption parameter in operations involving server-side-encryption with customer provided key (SSE-C). __Parameters__ @@ -1239,18 +1349,11 @@ __Parameters__ | `key` | _[]byte_ | Byte-slice of the raw, un-encoded binary key | | `algo` | _string_ | Algorithm to use in encryption or decryption with the given key. Can be empty (defaults to `AES256`) | -__Example__ - -``` go -// Key for use in encryption/decryption -keyInfo := minio.NewSSEInfo([]byte{1,2,3}, "") -``` ## 5. Presigned operations ### PresignedGetObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) - Generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. __Parameters__ @@ -1278,11 +1381,11 @@ if err != nil { fmt.Println(err) return } +fmt.Println("Successfully generated presigned URL", presignedURL) ``` ### PresignedPutObject(bucketName, objectName string, expiry time.Duration) (*url.URL, error) - Generates a presigned URL for HTTP PUT operations. Browsers/Mobile clients may point to this URL to upload objects directly to a bucket even if it is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. NOTE: you can upload to S3 only with specified object name. @@ -1308,12 +1411,11 @@ if err != nil { fmt.Println(err) return } -fmt.Println(presignedURL) +fmt.Println("Successfully generated presigned URL", presignedURL) ``` ### PresignedHeadObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) - Generates a presigned URL for HTTP HEAD operations. Browsers/Mobile clients may point to this URL to directly get metadata from objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which it is no longer operational. The default expiry is set to 7 days. __Parameters__ @@ -1340,23 +1442,18 @@ if err != nil { fmt.Println(err) return } +fmt.Println("Successfully generated presigned URL", presignedURL) ``` ### PresignedPostPolicy(PostPolicy) (*url.URL, map[string]string, error) - Allows setting policy conditions to a presigned URL for POST operations. Policies such as bucket name to receive object uploads, key name prefixes, expiry policy may be set. -Create policy : - ```go +// Initialize policy condition config. policy := minio.NewPostPolicy() -``` - -Apply upload policy restrictions: - -```go +// Apply upload policy restrictions: policy.SetBucket("mybucket") policy.SetKey("myobject") policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days @@ -1371,17 +1468,13 @@ policy.SetContentLengthRange(1024, 1024*1024) policy.SetUserMetadata("custom", "user") // Get the POST form key/value object: - url, formData, err := minioClient.PresignedPostPolicy(policy) if err != nil { fmt.Println(err) return } -``` -POST your content from the command line using `curl`: - -```go +// POST your content from the command line using `curl` fmt.Printf("curl ") for k, v := range formData { fmt.Printf("-F %s=%s ", k, v) @@ -1394,7 +1487,6 @@ fmt.Printf("%s\n", url) ### SetBucketPolicy(bucketname, objectPrefix string, policy policy.BucketPolicy) error - Set access permissions on bucket or an object prefix. Importing `github.com/minio/minio-go/pkg/policy` package is needed. @@ -1425,7 +1517,9 @@ __Example__ ```go -err := minioClient.SetBucketPolicy("mybucket", "myprefix", policy.BucketPolicyReadWrite) +// Sets 'mybucket' with a sub-directory 'myprefix' to be anonymously accessible for +// both read and write operations. +err = minioClient.SetBucketPolicy("mybucket", "myprefix", policy.BucketPolicyReadWrite) if err != nil { fmt.Println(err) return @@ -1434,7 +1528,6 @@ if err != nil { ### GetBucketPolicy(bucketName, objectPrefix string) (policy.BucketPolicy, error) - Get access permissions on a bucket or a prefix. Importing `github.com/minio/minio-go/pkg/policy` package is needed. @@ -1469,7 +1562,6 @@ fmt.Println("Access permissions for mybucket is", bucketPolicy) ### ListBucketPolicies(bucketName, objectPrefix string) (map[string]BucketPolicy, error) - Get access permissions rules associated to the specified bucket and prefix. __Parameters__ @@ -1504,8 +1596,7 @@ for resource, permission := range bucketPolicies { ### GetBucketNotification(bucketName string) (BucketNotification, error) - -Get all notification configurations related to the specified bucket. +Get notification configuration on a bucket. __Parameters__ @@ -1528,10 +1619,12 @@ __Example__ ```go bucketNotification, err := minioClient.GetBucketNotification("mybucket") if err != nil { - log.Fatalf("Failed to get bucket notification configurations for mybucket - %v", err) + fmt.Println("Failed to get bucket notification configurations for mybucket", err) + return } -for _, topicConfig := range bucketNotification.TopicConfigs { - for _, e := range topicConfig.Events { + +for _, queueConfig := range bucketNotification.QueueConfigs { + for _, e := range queueConfig.Events { fmt.Println(e + " event is enabled") } } @@ -1539,7 +1632,6 @@ for _, topicConfig := range bucketNotification.TopicConfigs { ### SetBucketNotification(bucketName string, bucketNotification BucketNotification) error - Set a new bucket notification on a bucket. __Parameters__ @@ -1561,24 +1653,25 @@ __Example__ ```go -topicArn := minio.NewArn("aws", "sns", "us-east-1", "804605494417", "PhotoUpdate") +queueArn := minio.NewArn("aws", "sqs", "us-east-1", "804605494417", "PhotoUpdate") -topicConfig := minio.NewNotificationConfig(topicArn) -topicConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) -topicConfig.AddFilterPrefix("photos/") -topicConfig.AddFilterSuffix(".jpg") +queueConfig := minio.NewNotificationConfig(queueArn) +queueConfig.AddEvents(minio.ObjectCreatedAll, minio.ObjectRemovedAll) +queueConfig.AddFilterPrefix("photos/") +queueConfig.AddFilterSuffix(".jpg") bucketNotification := minio.BucketNotification{} -bucketNotification.AddTopic(topicConfig) -err := c.SetBucketNotification(bucketName, bucketNotification) +bucketNotification.AddQueue(queueConfig) + +err = minioClient.SetBucketNotification("mybucket", bucketNotification) if err != nil { - fmt.Println("Unable to set the bucket notification: " + err) + fmt.Println("Unable to set the bucket notification: ", err) + return } ``` ### RemoveAllBucketNotification(bucketName string) error - Remove all configured bucket notifications on a bucket. __Parameters__ @@ -1599,18 +1692,16 @@ __Example__ ```go -err := c.RemoveAllBucketNotification(bucketName) +err = minioClient.RemoveAllBucketNotification("mybucket") if err != nil { fmt.Println("Unable to remove bucket notifications.", err) + return } ``` ### ListenBucketNotification(bucketName, prefix, suffix string, events []string, doneCh <-chan struct{}) <-chan NotificationInfo - -ListenBucketNotification API receives bucket notification events through the -notification channel. The returned notification channel has two fields -'Records' and 'Err'. +ListenBucketNotification API receives bucket notification events through the notification channel. The returned notification channel has two fields 'Records' and 'Err'. - 'Records' holds the notifications received from the server. - 'Err' indicates any error while processing the received notifications. @@ -1638,7 +1729,7 @@ __minio.NotificationInfo__ |Field |Type |Description | |`notificationInfo.Records` | _[]minio.NotificationEvent_ | Collection of notification events | -|`notificationInfo.Err` | _error_ | Carries any error occurred during the operation | +|`notificationInfo.Err` | _error_ | Carries any error occurred during the operation (Standard Error) | __Example__ @@ -1652,15 +1743,15 @@ doneCh := make(chan struct{}) defer close(doneCh) // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. -for notificationInfo := range minioClient.ListenBucketNotification("YOUR-BUCKET", "PREFIX", "SUFFIX", []string{ +for notificationInfo := range minioClient.ListenBucketNotification("mybucket", "myprefix/", ".mysuffix", []string{ "s3:ObjectCreated:*", "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", }, doneCh) { if notificationInfo.Err != nil { - log.Fatalln(notificationInfo.Err) + fmt.Println(notificationInfo.Err) } - log.Println(notificationInfo) + fmt.Println(notificationInfo) } ``` @@ -1668,7 +1759,7 @@ for notificationInfo := range minioClient.ListenBucketNotification("YOUR-BUCKET" ### SetAppInfo(appName, appVersion string) -Adds application details to User-Agent. +Add custom application details to User-Agent. __Parameters__ @@ -1688,8 +1779,7 @@ minioClient.SetAppInfo("myCloudApp", "1.0.0") ### SetCustomTransport(customHTTPTransport http.RoundTripper) -Overrides default HTTP transport. This is usually needed for debugging -or for adding custom TLS certificates. +Overrides default HTTP transport. This is usually needed for debugging or for adding custom TLS certificates. __Parameters__ @@ -1700,8 +1790,7 @@ __Parameters__ ### TraceOn(outputStream io.Writer) -Enables HTTP tracing. The trace is written to the io.Writer -provided. If outputStream is nil, trace is written to os.Stdout. +Enables HTTP tracing. The trace is written to the io.Writer provided. If outputStream is nil, trace is written to os.Stdout. __Parameters__ @@ -1717,7 +1806,7 @@ Disables HTTP tracing. ### SetS3TransferAccelerate(acceleratedEndpoint string) Set AWS S3 transfer acceleration endpoint for all API requests hereafter. -NOTE: This API applies only to AWS S3 and ignored with other S3 compatible object storage services. +NOTE: This API applies only to AWS S3 and is a no operation for S3 compatible object storage services. __Parameters__ diff --git a/docs/checker.go.template b/docs/checker.go.template new file mode 100644 index 0000000000..7fba8f6b5c --- /dev/null +++ b/docs/checker.go.template @@ -0,0 +1,23 @@ +// +build ignore + +package main + +import ( + "fmt" + + "github.com/minio/minio-go" +) + +func main() { + // Use a secure connection. + ssl := true + + // Initialize minio client object. + minioClient, err := minio.New("play.minio.io:9000", "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", ssl) + if err != nil { + fmt.Println(err) + return + } + + {{.Text}} +} diff --git a/docs/validator.go b/docs/validator.go new file mode 100644 index 0000000000..bc7eac21b9 --- /dev/null +++ b/docs/validator.go @@ -0,0 +1,224 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + + "github.com/a8m/mark" + "github.com/gernest/wow" + "github.com/gernest/wow/spin" + "github.com/minio/cli" +) + +func init() { + // Validate go binary. + if _, err := exec.LookPath("go"); err != nil { + panic(err) + } +} + +var globalFlags = []cli.Flag{ + cli.StringFlag{ + Name: "m", + Value: "API.md", + Usage: "Path to markdown api documentation.", + }, + cli.StringFlag{ + Name: "t", + Value: "checker.go.template", + Usage: "Template used for generating the programs.", + }, + cli.IntFlag{ + Name: "skip", + Value: 2, + Usage: "Skip entries before validating the code.", + }, +} + +func runGofmt(path string) (msg string, err error) { + cmdArgs := []string{"-s", "-w", "-l", path} + cmd := exec.Command("gofmt", cmdArgs...) + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + return string(stdoutStderr), nil +} + +func runGoImports(path string) (msg string, err error) { + cmdArgs := []string{"-w", path} + cmd := exec.Command("goimports", cmdArgs...) + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + return string(stdoutStderr), err + } + return string(stdoutStderr), nil +} + +func runGoBuild(path string) (msg string, err error) { + // Go build the path. + cmdArgs := []string{"build", "-o", "/dev/null", path} + cmd := exec.Command("go", cmdArgs...) + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + return string(stdoutStderr), err + } + return string(stdoutStderr), nil +} + +func validatorAction(ctx *cli.Context) error { + if !ctx.IsSet("m") || !ctx.IsSet("t") { + return nil + } + docPath := ctx.String("m") + var err error + docPath, err = filepath.Abs(docPath) + if err != nil { + return err + } + data, err := ioutil.ReadFile(docPath) + if err != nil { + return err + } + + templatePath := ctx.String("t") + templatePath, err = filepath.Abs(templatePath) + if err != nil { + return err + } + + skipEntries := ctx.Int("skip") + m := mark.New(string(data), &mark.Options{ + Gfm: true, // Github markdown support is enabled by default. + }) + + t, err := template.ParseFiles(templatePath) + if err != nil { + return err + } + + tmpDir, err := ioutil.TempDir("", "md-verifier") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + entryN := 1 + for i := mark.NodeText; i < mark.NodeCheckbox; i++ { + if mark.NodeCode != mark.NodeType(i) { + m.AddRenderFn(mark.NodeType(i), func(node mark.Node) (s string) { + return "" + }) + continue + } + m.AddRenderFn(mark.NodeCode, func(node mark.Node) (s string) { + p, ok := node.(*mark.CodeNode) + if !ok { + return + } + p.Text = strings.NewReplacer("<", "<", ">", ">", """, `"`, "&", "&").Replace(p.Text) + if skipEntries > 0 { + skipEntries-- + return + } + + testFilePath := filepath.Join(tmpDir, fmt.Sprintf("%d.go", entryN)) + w, werr := os.Create(testFilePath) + if werr != nil { + panic(werr) + } + t.Execute(w, p) + w.Sync() + w.Close() + entryN++ + + msg, err := runGofmt(testFilePath) + if err != nil { + fmt.Printf("Failed running gofmt on %s, with (%s):(%s)\n", testFilePath, msg, err) + os.Exit(-1) + } + + msg, err = runGoImports(testFilePath) + if err != nil { + fmt.Printf("Failed running gofmt on %s, with (%s):(%s)\n", testFilePath, msg, err) + os.Exit(-1) + } + + msg, err = runGoBuild(testFilePath) + if err != nil { + fmt.Printf("Failed running gobuild on %s, with (%s):(%s)\n", testFilePath, msg, err) + fmt.Printf("Code with possible issue in %s:\n%s", docPath, p.Text) + fmt.Printf("To test `go build %s`\n", testFilePath) + os.Exit(-1) + } + return + }) + } + + w := wow.New(os.Stdout, spin.Get(spin.Moon), fmt.Sprintf(" Running validation tests in %s", tmpDir)) + + w.Start() + // Render markdown executes our checker on each code blocks. + _ = m.Render() + w.PersistWith(spin.Get(spin.Runner), " Successfully finished tests") + w.Stop() + + return nil +} + +func main() { + app := cli.NewApp() + app.Action = validatorAction + app.HideVersion = true + app.HideHelpCommand = true + app.Usage = "Validates code block sections inside API.md" + app.Author = "Minio.io" + app.Flags = globalFlags + // Help template for validator + app.CustomAppHelpTemplate = `NAME: + {{.Name}} - {{.Usage}} + +USAGE: + {{.Name}} {{if .VisibleFlags}}[FLAGS] {{end}}COMMAND{{if .VisibleFlags}} [COMMAND FLAGS | -h]{{end}} [ARGUMENTS...] + +COMMANDS: + {{range .VisibleCommands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} + {{end}}{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +TEMPLATE: + Validator uses Go's 'text/template' formatting so you need to ensure + your template is formatted correctly, check 'docs/checker.go.template' + +USAGE: + go run docs/validator.go -m docs/API.md -t /tmp/mycode.go.template + +` + app.Run(os.Args) + +} diff --git a/examples/s3/fputobject.go b/examples/s3/fputobject.go index 5cd7264829..e06a2fc81c 100644 --- a/examples/s3/fputobject.go +++ b/examples/s3/fputobject.go @@ -38,7 +38,9 @@ func main() { log.Fatalln(err) } - if _, err := s3Client.FPutObject("my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ContentType: "application/csv"}); err != nil { + if _, err := s3Client.FPutObject("my-bucketname", "my-objectname", "my-filename.csv", minio.PutObjectOptions{ + ContentType: "application/csv", + }); err != nil { log.Fatalln(err) } log.Println("Successfully uploaded my-filename.csv") diff --git a/examples/s3/putobject-context.go b/examples/s3/putobject-context.go index 6bde996fc3..ffe57c355c 100644 --- a/examples/s3/putobject-context.go +++ b/examples/s3/putobject-context.go @@ -57,7 +57,9 @@ func main() { log.Fatalln(err) } - n, err := s3Client.PutObjectWithContext(ctx, "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + n, err := s3Client.PutObjectWithContext(ctx, "my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ + ContentType: "application/octet-stream", + }) if err != nil { log.Fatalln(err) } diff --git a/examples/s3/removeobjects.go b/examples/s3/removeobjects.go index 5946069293..be5fb3848a 100644 --- a/examples/s3/removeobjects.go +++ b/examples/s3/removeobjects.go @@ -20,7 +20,6 @@ package main import ( "log" - "strconv" "github.com/minio/minio-go" ) @@ -44,8 +43,12 @@ func main() { // Send object names that are needed to be removed to objectsCh go func() { defer close(objectsCh) - for i := 0; i < 10; i++ { - objectsCh <- "/path/to/my-objectname" + strconv.Itoa(i) + // List all objects from a bucket-name with a matching prefix. + for object := range s3Client.ListObjects("my-bucketname", "my-prefixname", true, doneCh) { + if object.Err != nil { + log.Fatalln(object.Err) + } + objectsCh <- object.Key } }()