diff --git a/Gopkg.lock b/Gopkg.lock index d8bea715700a..c533e955a2bb 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -200,6 +200,7 @@ name = "github.com/golang/protobuf" packages = [ "proto", + "protoc-gen-go/descriptor", "ptypes", "ptypes/any", "ptypes/duration", @@ -217,6 +218,12 @@ pruneopts = "" revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" +[[projects]] + name = "github.com/googleapis/gax-go" + packages = ["."] + revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f" + version = "v2.0.0" + [[projects]] digest = "1:16b2837c8b3cf045fa2cdc82af0cf78b19582701394484ae76b2c3bc3c99ad73" name = "github.com/googleapis/gnostic" @@ -529,6 +536,26 @@ revision = "640f0ab560aeb89d523bb6ac322b1244d5c3796c" version = "v0.2.0" +[[projects]] + name = "go.opencensus.io" + packages = [ + ".", + "internal", + "internal/tagencoding", + "plugin/ochttp", + "plugin/ochttp/propagation/b3", + "stats", + "stats/internal", + "stats/view", + "tag", + "trace", + "trace/internal", + "trace/propagation", + "trace/tracestate" + ] + revision = "79993219becaa7e29e3b60cb67f5b8e82dee11d6" + version = "v0.17.0" + [[projects]] branch = "master" digest = "1:53c4b75f22ea7757dea07eae380ea42de547ae6865a5e3b41866754a8a8219c9" @@ -568,6 +595,9 @@ "http2", "http2/hpack", "idna", + "internal/timeseries", + "lex/httplex", + "trace" ] pruneopts = "" revision = "f9ce57c11b242f0f1599cf25c89d8cb02c45295a" @@ -642,6 +672,23 @@ pruneopts = "" revision = "ca6481ae56504398949d597084558e50ad07117a" +[[projects]] + branch = "master" + name = "google.golang.org/api" + packages = [ + "gensupport", + "googleapi", + "googleapi/internal/uritemplates", + "googleapi/transport", + "internal", + "iterator", + "option", + "storage/v1", + "transport/http", + "transport/http/internal/propagation" + ] + revision = "44c6748ece026e0fe668793d8f92e521356400a3" + [[projects]] digest = "1:c1771ca6060335f9768dff6558108bc5ef6c58506821ad43377ee23ff059e472" name = "google.golang.org/appengine" @@ -661,6 +708,49 @@ revision = "b1f26356af11148e710935ed1ac8a7f5702c7612" version = "v1.1.0" +[[projects]] + branch = "master" + name = "google.golang.org/genproto" + packages = [ + "googleapis/api/annotations", + "googleapis/iam/v1", + "googleapis/rpc/status" + ] + revision = "0e822944c569bf5c9afd034adaa56208bd2906ac" + +[[projects]] + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "codes", + "connectivity", + "credentials", + "encoding", + "encoding/proto", + "grpclog", + "internal", + "internal/backoff", + "internal/channelz", + "internal/envconfig", + "internal/grpcrand", + "internal/transport", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap" + ] + revision = "8dea3dc473e90c8179e519d91302d0597c0ca1d1" + version = "v1.15.0" + [[projects]] digest = "1:75fb3fcfc73a8c723efde7777b40e8e8ff9babf30d8c56160d01beffea8a95a6" name = "gopkg.in/inf.v0" diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index e56cbbfad65f..9c1ce74f8386 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -58,6 +58,10 @@ "description": "From allows an artifact to reference an artifact from a previous step", "type": "string" }, + "gcs": { + "description": "GCS contains GCS artifact location details", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GCSArtifact" + }, "git": { "description": "Git contains git artifact location details", "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GitArtifact" @@ -104,6 +108,10 @@ "description": "Artifactory contains artifactory artifact location details", "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.ArtifactoryArtifact" }, + "gcs": { + "description": "GCS contains GCS artifact location details", + "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GCSArtifact" + }, "git": { "description": "Git contains git artifact location details", "$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GitArtifact" @@ -221,6 +229,32 @@ } } }, + "io.argoproj.workflow.v1alpha1.GCSArtifact": { + "description": "GCSArtifact is the location of a GCS artifact", + "required": [ + "bucket", + "key" + ], + "properties": { + "bucket": { + "type": "string" + }, + "key": { + "type": "string" + } + } + }, + "io.argoproj.workflow.v1alpha1.GCSBucket": { + "description": "GCSBucket contains the access information required for acting with a GCS bucket", + "required": [ + "bucket" + ], + "properties": { + "bucket": { + "type": "string" + } + } + }, "io.argoproj.workflow.v1alpha1.GitArtifact": { "description": "GitArtifact is the location of an git artifact", "required": [ diff --git a/pkg/apis/workflow/v1alpha1/openapi_generated.go b/pkg/apis/workflow/v1alpha1/openapi_generated.go index 2ebb01d7e668..9287c54ec56d 100644 --- a/pkg/apis/workflow/v1alpha1/openapi_generated.go +++ b/pkg/apis/workflow/v1alpha1/openapi_generated.go @@ -21,6 +21,8 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.ArtifactoryAuth": schema_pkg_apis_workflow_v1alpha1_ArtifactoryAuth(ref), "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.DAGTask": schema_pkg_apis_workflow_v1alpha1_DAGTask(ref), "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.DAGTemplate": schema_pkg_apis_workflow_v1alpha1_DAGTemplate(ref), + "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GCSArtifact": schema_pkg_apis_workflow_v1alpha1_GCSArtifact(ref), + "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GCSBucket": schema_pkg_apis_workflow_v1alpha1_GCSBucket(ref), "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GitArtifact": schema_pkg_apis_workflow_v1alpha1_GitArtifact(ref), "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact": schema_pkg_apis_workflow_v1alpha1_HTTPArtifact(ref), "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.Inputs": schema_pkg_apis_workflow_v1alpha1_Inputs(ref), @@ -183,6 +185,12 @@ func schema_pkg_apis_workflow_v1alpha1_Artifact(ref common.ReferenceCallback) co Ref: ref("github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.RawArtifact"), }, }, + "gcs": { + SchemaProps: spec.SchemaProps{ + Description: "GCS contains GCS artifact location details", + Ref: ref("github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GCSArtifact"), + }, + }, "globalName": { SchemaProps: spec.SchemaProps{ Description: "GlobalName exports an output artifact to the global scope, making it available as '{{workflow.outputs.artifacts.XXXX}} and in workflow.status.outputs.artifacts", @@ -201,7 +209,7 @@ func schema_pkg_apis_workflow_v1alpha1_Artifact(ref common.ReferenceCallback) co }, }, Dependencies: []string{ - "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.ArchiveStrategy", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GitArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.RawArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.S3Artifact"}, + "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.ArchiveStrategy", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GCSArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GitArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.RawArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.S3Artifact"}, } } @@ -248,11 +256,17 @@ func schema_pkg_apis_workflow_v1alpha1_ArtifactLocation(ref common.ReferenceCall Ref: ref("github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.RawArtifact"), }, }, + "gcs": { + SchemaProps: spec.SchemaProps{ + Description: "GCS contains GCS artifact location details", + Ref: ref("github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GCSArtifact"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GitArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.RawArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.S3Artifact"}, + "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.ArtifactoryArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GCSArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.GitArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.HTTPArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.RawArtifact", "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1.S3Artifact"}, } } @@ -433,6 +447,52 @@ func schema_pkg_apis_workflow_v1alpha1_DAGTemplate(ref common.ReferenceCallback) } } +func schema_pkg_apis_workflow_v1alpha1_GCSArtifact(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GCSArtifact is the location of a GCS artifact", + Properties: map[string]spec.Schema{ + "bucket": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "key": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"bucket", "key"}, + }, + }, + Dependencies: []string{}, + } +} + +func schema_pkg_apis_workflow_v1alpha1_GCSBucket(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GCSBucket contains the access information required for acting with a GCS bucket", + Properties: map[string]spec.Schema{ + "bucket": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"bucket"}, + }, + }, + Dependencies: []string{}, + } +} + func schema_pkg_apis_workflow_v1alpha1_GitArtifact(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/pkg/apis/workflow/v1alpha1/types.go b/pkg/apis/workflow/v1alpha1/types.go index fbd07fe54174..d2815274acd8 100644 --- a/pkg/apis/workflow/v1alpha1/types.go +++ b/pkg/apis/workflow/v1alpha1/types.go @@ -318,6 +318,9 @@ type ArtifactLocation struct { // Raw contains raw artifact location details Raw *RawArtifact `json:"raw,omitempty"` + + // GCS contains GCS artifact location details + GCS *GCSArtifact `json:"gcs,omitempty"` } // Outputs hold parameters, artifacts, and results from a step @@ -594,6 +597,21 @@ func (s *S3Artifact) String() string { return fmt.Sprintf("%s://%s/%s/%s", protocol, s.Endpoint, s.Bucket, s.Key) } +// GCSBucket contains the access information required for acting with a GCS bucket +type GCSBucket struct { + Bucket string `json:"bucket"` +} + +// GCSArtifact is the location of a GCS artifact +type GCSArtifact struct { + GCSBucket `json:",inline"` + Key string `json:"key"` +} + +func (s *GCSArtifact) String() string { + return fmt.Sprintf("gs://%s/%s", s.Bucket, s.Key) +} + // GitArtifact is the location of an git artifact type GitArtifact struct { // Repo is the git repository @@ -818,7 +836,7 @@ func (args *Arguments) GetParameterByName(name string) *Parameter { // HasLocation whether or not an artifact has a location defined func (a *Artifact) HasLocation() bool { - return a.S3 != nil || a.Git != nil || a.HTTP != nil || a.Artifactory != nil || a.Raw != nil + return a.S3 != nil || a.Git != nil || a.HTTP != nil || a.Artifactory != nil || a.Raw != nil || a.GCS != nil } // GetTemplate retrieves a defined template by its name diff --git a/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go index ffa11d640f1b..6b6ccec7ba76 100644 --- a/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/workflow/v1alpha1/zz_generated.deepcopy.go @@ -125,6 +125,11 @@ func (in *ArtifactLocation) DeepCopyInto(out *ArtifactLocation) { *out = new(RawArtifact) **out = **in } + if in.GCS != nil { + in, out := &in.GCS, &out.GCS + *out = new(GCSArtifact) + **out = **in + } return } @@ -238,6 +243,39 @@ func (in *DAGTemplate) DeepCopy() *DAGTemplate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GCSArtifact) DeepCopyInto(out *GCSArtifact) { + *out = *in + out.GCSBucket = in.GCSBucket + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCSArtifact. +func (in *GCSArtifact) DeepCopy() *GCSArtifact { + if in == nil { + return nil + } + out := new(GCSArtifact) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GCSBucket) DeepCopyInto(out *GCSBucket) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCSBucket. +func (in *GCSBucket) DeepCopy() *GCSBucket { + if in == nil { + return nil + } + out := new(GCSBucket) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitArtifact) DeepCopyInto(out *GitArtifact) { *out = *in diff --git a/workflow/artifacts/gcs/gcs.go b/workflow/artifacts/gcs/gcs.go new file mode 100644 index 000000000000..5802076c3ae8 --- /dev/null +++ b/workflow/artifacts/gcs/gcs.go @@ -0,0 +1,126 @@ +package gcs + +import ( + "cloud.google.com/go/storage" + "context" + "errors" + argoErrors "github.com/argoproj/argo/errors" + wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" + log "github.com/sirupsen/logrus" + "io" + "os" +) + +type GCSArtifactDriver struct { + Context context.Context +} + +func (gcsDriver *GCSArtifactDriver) newGcsClient() (client *storage.Client, err error) { + gcsDriver.Context = context.Background() + client, err = storage.NewClient(gcsDriver.Context) + if err != nil { + return nil, argoErrors.InternalWrapError(err) + } + return + +} + +func (gcsDriver *GCSArtifactDriver) saveToFile(inputArtifact *wfv1.Artifact, filePath string) error { + + log.Infof("Loading from GCS (gs://%s/%s) to %s", + inputArtifact.GCS.Bucket, inputArtifact.GCS.Key, filePath) + + stat, err := os.Stat(filePath) + if err != nil && !os.IsNotExist(err) { + return err + } + + if stat.IsDir() { + return errors.New("output artifact path is a directory") + } + + outputFile, err := os.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return err + } + + gcsClient, err := gcsDriver.newGcsClient() + if err != nil { + return err + } + + bucket := gcsClient.Bucket(inputArtifact.GCS.Bucket) + object := bucket.Object(inputArtifact.GCS.Key) + + r, err := object.NewReader(gcsDriver.Context) + if err != nil { + return err + } + defer r.Close() + + _, err = io.Copy(outputFile, r) + if err != nil { + return err + } + + err = outputFile.Close() + if err != nil { + return err + } + return nil +} + +func (gcsDriver *GCSArtifactDriver) saveToGCS(outputArtifact *wfv1.Artifact, filePath string) error { + + log.Infof("Saving to GCS (gs://%s/%s)", + outputArtifact.GCS.Bucket, outputArtifact.GCS.Key) + + gcsClient, err := gcsDriver.newGcsClient() + if err != nil { + return err + } + + inputFile, err := os.Open(filePath) + if err != nil { + return err + } + + stat, err := os.Stat(filePath) + if err != nil { + return err + } + + if stat.IsDir() { + return errors.New("only single files can be saved to GCS, not entire directories") + } + + defer inputFile.Close() + + bucket := gcsClient.Bucket(outputArtifact.GCS.Bucket) + object := bucket.Object(outputArtifact.GCS.Key) + + w := object.NewWriter(gcsDriver.Context) + _, err = io.Copy(w, inputFile) + if err != nil { + return err + } + + err = w.Close() + if err != nil { + return err + } + return nil + +} + +func (gcsDriver *GCSArtifactDriver) Load(inputArtifact *wfv1.Artifact, path string) error { + + err := gcsDriver.saveToFile(inputArtifact, path) + return err +} + +func (gcsDriver *GCSArtifactDriver) Save(path string, outputArtifact *wfv1.Artifact) error { + + err := gcsDriver.saveToGCS(outputArtifact, path) + return err +} diff --git a/workflow/common/common.go b/workflow/common/common.go index 29ab0d0725ec..88e4be4722ad 100644 --- a/workflow/common/common.go +++ b/workflow/common/common.go @@ -1,6 +1,7 @@ package common import ( + "os" "time" "github.com/argoproj/argo/pkg/apis/workflow" @@ -36,6 +37,11 @@ const ( // DockerSockVolumeName is the volume name for the /var/run/docker.sock host path volume DockerSockVolumeName = "docker-sock" + // GoogleSecretVolumeName is the volume name for the /var/secrets/google volume + GoogleSecretVolumeName = "google-cloud-key" + // EvnVarGoogleSecret contains the name of the google credentials file used fro GCS access + EnvVarGoogleSecret = "GOOGLE_CREDENTIALS_SECRET" + // AnnotationKeyNodeName is the pod metadata annotation key containing the workflow node name AnnotationKeyNodeName = workflow.FullName + "/node-name" // AnnotationKeyNodeMessage is the pod metadata annotation key the executor will use to @@ -113,6 +119,10 @@ const ( LocalVarPodName = "pod.name" ) +var ( + GoogleSecretName = os.Getenv(EnvVarGoogleSecret) +) + // ExecutionControl contains execution control parameters for executor to decide how to execute the container type ExecutionControl struct { // Deadline is a max timestamp in which an executor can run the container before terminating it diff --git a/workflow/controller/config.go b/workflow/controller/config.go index 1559eb93b02d..0caa9141fb4e 100644 --- a/workflow/controller/config.go +++ b/workflow/controller/config.go @@ -67,6 +67,7 @@ type ArtifactRepository struct { S3 *S3ArtifactRepository `json:"s3,omitempty"` // Artifactory stores artifacts to JFrog Artifactory Artifactory *ArtifactoryArtifactRepository `json:"artifactory,omitempty"` + GCS *GCSArtifactRepository `json:"gcs,omitempty"` } // S3ArtifactRepository defines the controller configuration for an S3 artifact repository @@ -88,6 +89,11 @@ type ArtifactoryArtifactRepository struct { RepoURL string `json:"repoURL,omitempty"` } +// GCSArtifactRepository defines the controller configuration for a GCS artifact repository +type GCSArtifactRepository struct { + wfv1.GCSBucket `json:",inline"` +} + // ResyncConfig reloads the controller config from the configmap func (wfc *WorkflowController) ResyncConfig() error { cmClient := wfc.kubeclientset.CoreV1().ConfigMaps(wfc.namespace) diff --git a/workflow/controller/workflowpod.go b/workflow/controller/workflowpod.go index 3ca62c26a49a..e41610f60991 100644 --- a/workflow/controller/workflowpod.go +++ b/workflow/controller/workflowpod.go @@ -87,6 +87,25 @@ var ( execEnvVars = []apiv1.EnvVar{ envFromField(common.EnvVarPodName, "metadata.name"), } + + volumeMountGoogleSecret = apiv1.VolumeMount{ + Name: common.GoogleSecretVolumeName, + MountPath: "/var/secrets/google", + } + + googleCredentialSecretEnvVar = apiv1.EnvVar{ + Name: "GOOGLE_APPLICATION_CREDENTIALS", + Value: "/var/secrets/google/key.json", + } + + volumeGoogleSecret = apiv1.Volume{ + Name: common.GoogleSecretVolumeName, + VolumeSource: apiv1.VolumeSource{ + Secret: &apiv1.SecretVolumeSource{ + SecretName: common.GoogleSecretName, + }, + }, + } ) // envFromField is a helper to return a EnvVar with the name and field @@ -137,6 +156,10 @@ func (woc *wfOperationCtx) createWorkflowPod(nodeName string, mainCtr apiv1.Cont pod.ObjectMeta.Labels[common.LabelKeyControllerInstanceID] = woc.controller.Config.InstanceID } + if common.GoogleSecretName != "" { + pod.Spec.Volumes = append(pod.Spec.Volumes, volumeGoogleSecret) + } + if tmpl.GetType() != wfv1.TemplateTypeResource { // we do not need the wait container for resource templates because // argoexec runs as the main container and will perform the job of @@ -259,9 +282,8 @@ func (woc *wfOperationCtx) newInitContainer(tmpl *wfv1.Template) apiv1.Container ctr := woc.newExecContainer(common.InitContainerName, false) ctr.Command = []string{"argoexec"} ctr.Args = []string{"init"} - ctr.VolumeMounts = []apiv1.VolumeMount{ - volumeMountPodMetadata, - } + ctr.VolumeMounts = append(ctr.VolumeMounts, volumeMountPodMetadata) + return *ctr } @@ -269,7 +291,8 @@ func (woc *wfOperationCtx) newWaitContainer(tmpl *wfv1.Template) (*apiv1.Contain ctr := woc.newExecContainer(common.WaitContainerName, false) ctr.Command = []string{"argoexec"} ctr.Args = []string{"wait"} - ctr.VolumeMounts = woc.createVolumeMounts() + ctr.VolumeMounts = append(ctr.VolumeMounts, woc.createVolumeMounts()...) + return ctr, nil } @@ -332,6 +355,7 @@ func (woc *wfOperationCtx) newExecContainer(name string, privileged bool) *apiv1 Name: name, Image: woc.controller.executorImage(), ImagePullPolicy: woc.controller.executorImagePullPolicy(), + VolumeMounts: []apiv1.VolumeMount{}, Env: woc.createEnvVars(), SecurityContext: &apiv1.SecurityContext{ Privileged: &privileged, @@ -340,6 +364,11 @@ func (woc *wfOperationCtx) newExecContainer(name string, privileged bool) *apiv1 if woc.controller.Config.ExecutorResources != nil { exec.Resources = *woc.controller.Config.ExecutorResources } + + if common.GoogleSecretName != "" { + exec.VolumeMounts = append(exec.VolumeMounts, volumeMountGoogleSecret) + exec.Env = append(exec.Env, googleCredentialSecretEnvVar) + } return &exec } @@ -586,6 +615,13 @@ func (woc *wfOperationCtx) addArchiveLocation(pod *apiv1.Pod, tmpl *wfv1.Templat ArtifactoryAuth: woc.controller.Config.ArtifactRepository.Artifactory.ArtifactoryAuth, URL: artURL, } + } else if woc.controller.Config.ArtifactRepository.GCS != nil { + log.Debugf("Setting GCS artifact repository information") + artLocationKey := fmt.Sprintf("%s/%s", woc.wf.ObjectMeta.Name, pod.ObjectMeta.Name) + tmpl.ArchiveLocation.GCS = &wfv1.GCSArtifact{ + GCSBucket: woc.controller.Config.ArtifactRepository.GCS.GCSBucket, + Key: artLocationKey, + } } else { for _, art := range tmpl.Outputs.Artifacts { if !art.HasLocation() { diff --git a/workflow/executor/executor.go b/workflow/executor/executor.go index 017d89dfad09..a545e7ef58aa 100644 --- a/workflow/executor/executor.go +++ b/workflow/executor/executor.go @@ -24,6 +24,7 @@ import ( "github.com/argoproj/argo/util/retry" artifact "github.com/argoproj/argo/workflow/artifacts" "github.com/argoproj/argo/workflow/artifacts/artifactory" + "github.com/argoproj/argo/workflow/artifacts/gcs" "github.com/argoproj/argo/workflow/artifacts/git" "github.com/argoproj/argo/workflow/artifacts/http" "github.com/argoproj/argo/workflow/artifacts/raw" @@ -253,6 +254,10 @@ func (we *WorkflowExecutor) saveArtifact(tempOutArtDir string, mainCtrID string, } artifactoryURL.Path = path.Join(artifactoryURL.Path, fileName) art.Artifactory.URL = artifactoryURL.String() + } else if we.Template.ArchiveLocation.GCS != nil { + shallowCopy := *we.Template.ArchiveLocation.GCS + art.GCS = &shallowCopy + art.GCS.Key = path.Join(art.GCS.Key, fileName) } else { return errors.Errorf(errors.CodeBadRequest, "Unable to determine path to store %s. Archive location provided no information", art.Name) } @@ -435,6 +440,10 @@ func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriv } return &driver, nil } + if art.GCS != nil { + driver := gcs.GCSArtifactDriver{} + return &driver, nil + } if art.HTTP != nil { return &http.HTTPArtifactDriver{}, nil }