From ea95513af0bb2ec413030e990cdd05fd4c425a37 Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Wed, 25 Mar 2020 12:24:40 -0700 Subject: [PATCH] chore: minor fix and refactory --- api/openapi-spec/swagger.json | 2 +- go.mod | 2 +- go.sum | 4 ++-- pkg/apiclient/cronworkflow/cron-workflow.swagger.json | 2 +- pkg/apiclient/workflow/workflow.swagger.json | 2 +- .../workflowarchive/workflow-archive.swagger.json | 2 +- .../workflowtemplate/workflow-template.swagger.json | 2 +- pkg/apis/workflow/v1alpha1/generated.proto | 2 +- pkg/apis/workflow/v1alpha1/workflow_types.go | 2 +- workflow/artifacts/artifacts.go | 2 +- workflow/artifacts/gcs/gcs.go | 11 ++++++----- 11 files changed, 17 insertions(+), 16 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index f7df4162e6cb..d76a8aed6f02 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -1809,7 +1809,7 @@ }, "io.argoproj.workflow.v1alpha1.GCSBucket": { "type": "object", - "title": "GCSBucket contains the access information for interfacring with a GCS bucket", + "title": "GCSBucket contains the access information for interfacring with a GCS bucket", "properties": { "bucket": { "type": "string", diff --git a/go.mod b/go.mod index ddde3037256e..a0bb95a0c7ba 100644 --- a/go.mod +++ b/go.mod @@ -76,7 +76,7 @@ require ( golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d golang.org/x/net v0.0.0-20200301022130-244492dfa37a golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224 // indirect + golang.org/x/tools v0.0.0-20200325194147-644a21fb1449 // indirect google.golang.org/api v0.20.0 google.golang.org/genproto v0.0.0-20200317114155-1f3552e48f24 google.golang.org/grpc v1.28.0 diff --git a/go.sum b/go.sum index 909f588f7646..ebbdbb55fa80 100644 --- a/go.sum +++ b/go.sum @@ -607,8 +607,8 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200317043434-63da46f3035e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224 h1:azwY/v0y0K4mFHVsg5+UrTgchqALYWpqVo6vL5OmkmI= -golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200325194147-644a21fb1449 h1:KOG4qpDXI4j4YMkZezw18+e2jkjrRw5f/n+3p9JiIjY= +golang.org/x/tools v0.0.0-20200325194147-644a21fb1449/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/apiclient/cronworkflow/cron-workflow.swagger.json b/pkg/apiclient/cronworkflow/cron-workflow.swagger.json index 649298fb90f6..d6a980844bcd 100644 --- a/pkg/apiclient/cronworkflow/cron-workflow.swagger.json +++ b/pkg/apiclient/cronworkflow/cron-workflow.swagger.json @@ -748,7 +748,7 @@ "title": "ServiceAccountKeySecret is the secret selector to the bucket's service account key" } }, - "title": "GCSBucket contains the access information for interfacring with a GCS bucket" + "title": "GCSBucket contains the access information for interfacring with a GCS bucket" }, "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.Gauge": { "type": "object", diff --git a/pkg/apiclient/workflow/workflow.swagger.json b/pkg/apiclient/workflow/workflow.swagger.json index 5e01e27e199e..84b1309e0e82 100644 --- a/pkg/apiclient/workflow/workflow.swagger.json +++ b/pkg/apiclient/workflow/workflow.swagger.json @@ -1024,7 +1024,7 @@ "title": "ServiceAccountKeySecret is the secret selector to the bucket's service account key" } }, - "title": "GCSBucket contains the access information for interfacring with a GCS bucket" + "title": "GCSBucket contains the access information for interfacring with a GCS bucket" }, "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.Gauge": { "type": "object", diff --git a/pkg/apiclient/workflowarchive/workflow-archive.swagger.json b/pkg/apiclient/workflowarchive/workflow-archive.swagger.json index 8024c9686704..30d9bc48abd7 100644 --- a/pkg/apiclient/workflowarchive/workflow-archive.swagger.json +++ b/pkg/apiclient/workflowarchive/workflow-archive.swagger.json @@ -455,7 +455,7 @@ "title": "ServiceAccountKeySecret is the secret selector to the bucket's service account key" } }, - "title": "GCSBucket contains the access information for interfacring with a GCS bucket" + "title": "GCSBucket contains the access information for interfacring with a GCS bucket" }, "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.Gauge": { "type": "object", diff --git a/pkg/apiclient/workflowtemplate/workflow-template.swagger.json b/pkg/apiclient/workflowtemplate/workflow-template.swagger.json index 186d0295b2b2..be7b258b7236 100644 --- a/pkg/apiclient/workflowtemplate/workflow-template.swagger.json +++ b/pkg/apiclient/workflowtemplate/workflow-template.swagger.json @@ -616,7 +616,7 @@ "title": "ServiceAccountKeySecret is the secret selector to the bucket's service account key" } }, - "title": "GCSBucket contains the access information for interfacring with a GCS bucket" + "title": "GCSBucket contains the access information for interfacring with a GCS bucket" }, "github.com.argoproj.argo.pkg.apis.workflow.v1alpha1.Gauge": { "type": "object", diff --git a/pkg/apis/workflow/v1alpha1/generated.proto b/pkg/apis/workflow/v1alpha1/generated.proto index b639a1ce013c..84587871a801 100644 --- a/pkg/apis/workflow/v1alpha1/generated.proto +++ b/pkg/apis/workflow/v1alpha1/generated.proto @@ -280,7 +280,7 @@ message GCSArtifact { optional string key = 2; } -// GCSBucket contains the access information for interfacring with a GCS bucket +// GCSBucket contains the access information for interfacring with a GCS bucket message GCSBucket { // Bucket is the name of the bucket optional string bucket = 1; diff --git a/pkg/apis/workflow/v1alpha1/workflow_types.go b/pkg/apis/workflow/v1alpha1/workflow_types.go index 0a40895e019b..bedefee42ced 100644 --- a/pkg/apis/workflow/v1alpha1/workflow_types.go +++ b/pkg/apis/workflow/v1alpha1/workflow_types.go @@ -1358,7 +1358,7 @@ func (h *HTTPArtifact) HasLocation() bool { return h != nil && h.URL != "" } -// GCSBucket contains the access information for interfacring with a GCS bucket +// GCSBucket contains the access information for interfacring with a GCS bucket type GCSBucket struct { // Bucket is the name of the bucket diff --git a/workflow/artifacts/artifacts.go b/workflow/artifacts/artifacts.go index ee3435441d07..d69ec1278217 100644 --- a/workflow/artifacts/artifacts.go +++ b/workflow/artifacts/artifacts.go @@ -137,7 +137,7 @@ func NewDriver(art *wfv1.Artifact, ri resource.Interface) (ArtifactDriver, error } if art.GCS != nil { - driver := gcs.GCSArtifactDriver{} + driver := gcs.ArtifactDriver{} if art.GCS.ServiceAccountKeySecret.Name != "" { serviceAccountKeyBytes, err := ri.GetSecret(art.GCS.ServiceAccountKeySecret.Name, art.GCS.ServiceAccountKeySecret.Key) if err != nil { diff --git a/workflow/artifacts/gcs/gcs.go b/workflow/artifacts/gcs/gcs.go index 5bfca1573efe..89702c513c5e 100644 --- a/workflow/artifacts/gcs/gcs.go +++ b/workflow/artifacts/gcs/gcs.go @@ -22,12 +22,12 @@ import ( wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" ) -// GCSArtifactDriver is a driver for GCS -type GCSArtifactDriver struct { +// ArtifactDriver is a driver for GCS +type ArtifactDriver struct { ServiceAccountKey string } -func (g *GCSArtifactDriver) newGCSClient() (*storage.Client, error) { +func (g *ArtifactDriver) newGCSClient() (*storage.Client, error) { if g.ServiceAccountKey != "" { return newGCSClientWithCredential(g.ServiceAccountKey) } @@ -58,7 +58,7 @@ func newGCSClientDefault() (*storage.Client, error) { } // Load function downloads objects from GCS -func (g *GCSArtifactDriver) Load(inputArtifact *wfv1.Artifact, path string) error { +func (g *ArtifactDriver) Load(inputArtifact *wfv1.Artifact, path string) error { err := wait.ExponentialBackoff(wait.Backoff{Duration: time.Second * 2, Factor: 2.0, Steps: 5, Jitter: 0.1}, func() (bool, error) { log.Infof("GCS Load path: %s, key: %s", path, inputArtifact.GCS.Key) @@ -146,7 +146,7 @@ func listByPrefix(client *storage.Client, bucket, prefix, delim string) ([]strin } // Save an artifact to GCS compliant storage, e.g., uploading a local file to GCS bucket -func (g *GCSArtifactDriver) Save(path string, outputArtifact *wfv1.Artifact) error { +func (g *ArtifactDriver) Save(path string, outputArtifact *wfv1.Artifact) error { err := wait.ExponentialBackoff(wait.Backoff{Duration: time.Second * 2, Factor: 2.0, Steps: 5, Jitter: 0.1}, func() (bool, error) { log.Infof("GCS Save path: %s, key: %s", path, outputArtifact.GCS.Key) @@ -155,6 +155,7 @@ func (g *GCSArtifactDriver) Save(path string, outputArtifact *wfv1.Artifact) err log.Warnf("Failed to create new GCS client: %v", err) return false, nil } + defer client.Close() err = uploadObjects(client, outputArtifact.GCS.Bucket, outputArtifact.GCS.Key, path) if err != nil { log.Warnf("Failed to upload objects: %v", err)