Added support for artifact path references (#1300)

* Added support for artifact path references Adds new `{{inputs.artifacts.<NAME>.path}}` and `{{outputs.artifacts.<NAME>.path}}` placeholders.
argoproj · Apr 5, 2019 · 4591e44 · 4591e44
1 parent 928e4df
commit 4591e44
Show file tree

Hide file tree

Showing 7 changed files with 197 additions and 1 deletion.
diff --git a/docs/variables.md b/docs/variables.md
@@ -28,6 +28,9 @@ The following variables are made available to reference various metadata of a wo
 | Variable | Description|
 |----------|------------|
 | `pod.name` | Pod name of the container/script |
+| `inputs.artifacts.<NAME>.path` | Local path of the input artifact |
+| `outputs.artifacts.<NAME>.path` | Local path of the output artifact |
+| `outputs.parameters.<NAME>.path` | Local path of the output parameter |
 
 ## Loops (withItems / withParam)
 | Variable | Description|

diff --git a/examples/artifact-path-placeholders.yaml b/examples/artifact-path-placeholders.yaml
@@ -0,0 +1,40 @@
+# This example demonstrates the how to refer to input and output artifact paths.
+# Referring to the path instead of copy/pasting it prevents errors when paths change.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: artifact-path-placeholders-
+spec:
+  entrypoint: head-lines
+  arguments:
+    parameters:
+    - name: lines-count
+      value: 3
+    artifacts:
+    - name: text
+      raw:
+        data: |
+          1
+          2
+          3
+          4
+          5
+  templates:
+  - name: head-lines
+    inputs:
+      parameters:
+      - name: lines-count
+      artifacts:
+      - name: text
+        path: /inputs/text/data
+    outputs:
+      parameters:
+      - name: actual-lines-count
+        valueFrom:
+          path: /outputs/actual-lines-count/data
+      artifacts:
+      - name: text
+        path: /outputs/text/data
+    container:
+      image: busybox
+      command: [sh, -c, 'head -n {{inputs.parameters.lines-count}} <"{{inputs.artifacts.text.path}}" | tee "{{outputs.artifacts.text.path}}" | wc -l > "{{outputs.parameters.actual-lines-count.path}}"']
diff --git a/workflow/common/common.go b/workflow/common/common.go
@@ -116,7 +116,7 @@ const (
 )
 
 // GlobalVarWorkflowRootTags is a list of root tags in workflow which could be used for variable reference
-var GlobalVarValidWorkflowVariablePrefix = []string{"item.", "steps.", "inputs.", "pod.", "workflow.", "tasks."}
+var GlobalVarValidWorkflowVariablePrefix = []string{"item.", "steps.", "inputs.", "outputs.", "pod.", "workflow.", "tasks."}
 
 // ExecutionControl contains execution control parameters for executor to decide how to execute the container
 type ExecutionControl struct {

diff --git a/workflow/common/util.go b/workflow/common/util.go
@@ -195,6 +195,22 @@ func substituteParams(tmpl *wfv1.Template, globalParams, localParams map[string]
 		}
 		replaceMap["inputs.parameters."+inParam.Name] = *inParam.Value
 	}
+	for _, inArt := range globalReplacedTmpl.Inputs.Artifacts {
+		if inArt.Path != "" {
+			replaceMap["inputs.artifacts."+inArt.Name+".path"] = inArt.Path
+		}
+	}
+	for _, outArt := range globalReplacedTmpl.Outputs.Artifacts {
+		if outArt.Path != "" {
+			replaceMap["outputs.artifacts."+outArt.Name+".path"] = outArt.Path
+		}
+	}
+	for _, param := range globalReplacedTmpl.Outputs.Parameters {
+		if param.ValueFrom != nil && param.ValueFrom.Path != "" {
+			replaceMap["outputs.parameters."+param.Name+".path"] = param.ValueFrom.Path
+		}
+	}
+
 	fstTmpl = fasttemplate.New(globalReplacedTmplStr, "{{", "}}")
 	s, err := Replace(fstTmpl, replaceMap, true)
 	if err != nil {

diff --git a/workflow/controller/operator_test.go b/workflow/controller/operator_test.go
@@ -947,3 +947,57 @@ func TestMetadataPassing(t *testing.T) {
 	assert.True(t, foundRev)
 	assert.Equal(t, "foo:bar", container.Image)
 }
+
+var ioPathPlaceholders = `
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: artifact-path-placeholders-
+spec:
+  entrypoint: head-lines
+  arguments:
+    parameters:
+    - name: lines-count
+      value: 3
+    artifacts:
+    - name: text
+      raw:
+        data: |
+          1
+          2
+          3
+          4
+          5
+  templates:
+  - name: head-lines
+    inputs:
+      parameters:
+      - name: lines-count
+      artifacts:
+      - name: text
+        path: /inputs/text/data
+    outputs:
+      parameters:
+      - name: actual-lines-count
+        valueFrom:
+          path: /outputs/actual-lines-count/data
+      artifacts:
+      - name: text
+        path: /outputs/text/data
+    container:
+      image: busybox
+      command: [sh, -c, 'head -n {{inputs.parameters.lines-count}} <"{{inputs.artifacts.text.path}}" | tee "{{outputs.artifacts.text.path}}" | wc -l > "{{outputs.parameters.actual-lines-count.path}}"']
+`
+
+func TestResolveIOPathPlaceholders(t *testing.T) {
+	wf := unmarshalWF(ioPathPlaceholders)
+	woc := newWoc(*wf)
+	woc.controller.Config.ArtifactRepository.S3 = new(S3ArtifactRepository)
+	woc.operate()
+	assert.Equal(t, wfv1.NodeRunning, woc.wf.Status.Phase)
+	pods, err := woc.controller.kubeclientset.CoreV1().Pods(wf.ObjectMeta.Namespace).List(metav1.ListOptions{})
+	assert.Nil(t, err)
+	assert.True(t, len(pods.Items) > 0, "pod was not created successfully")
+
+	assert.Equal(t, []string{"sh", "-c", "head -n 3 <\"/inputs/text/data\" | tee \"/outputs/text/data\" | wc -l > \"/outputs/actual-lines-count/data\""}, pods.Items[0].Spec.Containers[0].Command)
+}
diff --git a/workflow/validate/validate.go b/workflow/validate/validate.go
@@ -119,6 +119,18 @@ func (ctx *wfValidationCtx) validateTemplate(tmpl *wfv1.Template, args wfv1.Argu
 		localParams[common.LocalVarPodName] = placeholderValue
 		scope[common.LocalVarPodName] = placeholderValue
 	}
+	if tmpl.IsLeaf() {
+		for _, art := range tmpl.Outputs.Artifacts {
+			if art.Path != "" {
+				scope[fmt.Sprintf("outputs.artifacts.%s.path", art.Name)] = true
+			}
+		}
+		for _, param := range tmpl.Outputs.Parameters {
+			if param.ValueFrom != nil && param.ValueFrom.Path != "" {
+				scope[fmt.Sprintf("outputs.parameters.%s.path", param.Name)] = true
+			}
+		}
+	}
 
 	_, err = common.ProcessArgs(tmpl, args, ctx.globalParams, localParams, true)
 	if err != nil {
@@ -190,6 +202,7 @@ func validateInputs(tmpl *wfv1.Template) (map[string]interface{}, error) {
 			if art.Path == "" {
 				return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.path not specified", tmpl.Name, artRef)
 			}
+			scope[fmt.Sprintf("inputs.artifacts.%s.path", art.Name)] = true
 		} else {
 			if art.Path != "" {
 				return nil, errors.Errorf(errors.CodeBadRequest, "templates.%s.%s.path only valid in container/script templates", tmpl.Name, artRef)

diff --git a/workflow/validate/validate_test.go b/workflow/validate/validate_test.go
@@ -163,6 +163,76 @@ func TestUnresolved(t *testing.T) {
 	}
 }
 
+var ioArtifactPaths = `
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: artifact-path-placeholders-
+spec:
+  entrypoint: head-lines
+  arguments:
+    parameters:
+    - name: lines-count
+      value: 3
+    artifacts:
+    - name: text
+      raw:
+        data: |
+          1
+          2
+          3
+          4
+          5
+  templates:
+  - name: head-lines
+    inputs:
+      parameters:
+      - name: lines-count
+      artifacts:
+      - name: text
+        path: /inputs/text/data
+    outputs:
+      parameters:
+      - name: actual-lines-count
+        valueFrom:
+          path: /outputs/actual-lines-count/data
+      artifacts:
+      - name: text
+        path: /outputs/text/data
+    container:
+      image: busybox
+      command: [sh, -c, 'head -n {{inputs.parameters.lines-count}} <"{{inputs.artifacts.text.path}}" | tee "{{outputs.artifacts.text.path}}" | wc -l > "{{outputs.parameters.actual-lines-count.path}}"']
+`
+
+func TestResolveIOArtifactPathPlaceholders(t *testing.T) {
+	err := validate(ioArtifactPaths)
+	assert.Nil(t, err)
+}
+
+var outputParameterPath = `
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: get-current-date-
+spec:
+  entrypoint: get-current-date
+  templates:
+  - name: get-current-date
+    outputs:
+      parameters:
+      - name: current-date
+        valueFrom:
+          path: /tmp/current-date
+    container:
+      image: busybox
+      command: [sh, -c, 'date > {{outputs.parameters.current-date.path}}']
+`
+
+func TestResolveOutputParameterPathPlaceholder(t *testing.T) {
+	err := validate(outputParameterPath)
+	assert.Nil(t, err)
+}
+
 var stepOutputReferences = `
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow