Passing step results between steps

This PR enables passing step results between steps.
The replacements of stepresults needs to happen in the entrypointer.

docs/stepactions.md

@@ -181,7 +181,7 @@ spec:
     date | tee $(step.results.current-date-human-readable.path)
 ```
 
-`Results` from the above `StepAction` can be [fetched by the `Task`](#fetching-emitted-results-from-stepactions) in another `StepAction` via `$(steps.<stepName>.results.<resultName>)`.
+`Results` from the above `StepAction` can be [fetched by the `Task`](#fetching-emitted-results-from-stepactions) or in [another `Step/StepAction`](#passing-step-results-between-steps) via `$(steps.<stepName>.results.<resultName>)`.
 
 #### Fetching Emitted Results from StepActions
 
@@ -238,6 +238,64 @@ spec:
         name: kaniko-step-action
 ```
 
+#### Passing Results between Steps
+
+`StepResults` (i.e. results written to `$(step.results.<result-name>.path)`, NOT `$(results.<result-name>.path)`) can be shared with following steps via replacement variable `$(steps.<step-name>.results.<result-name>)`.
+
+Pipeline supports two new types of results and parameters: array `[]string` and object `map[string]string`.
+Array and Object result is a beta feature and can be enabled by setting `enable-api-fields` to `alpha` or `beta`.
+
+| Result Type | Parameter Type | Specification                                    | `enable-api-fields` |
+|-------------|----------------|--------------------------------------------------|---------------------|
+| string      | string         | `$(steps.<step-name>.results.<result-name>)`     | stable              |
+| array       | array          | `$(steps.<step-name>.results.<result-name>[*])`  | alpha or beta       |
+| array       | string         | `$(steps.<step-name>.results.<result-name>[i])`  | alpha or beta       |
+| object      | string         | `$(tasks.<task-name>.results.<result-name>.key)` | alpha or beta       |
+
+**Note:** Whole Array `Results` (using star notation) cannot be referred in `script` and `env`.
+
+The example below shows how you could pass `step results` from a `step` into following steps, in this case, into a `StepAction`.
+
+```yaml
+apiVersion: tekton.dev/v1
+kind: TaskRun
+metadata:
+  name: step-action-run
+spec:
+  TaskSpec:
+    steps:
+      - name: inline-step
+        results:
+          - name: result1
+            type: array
+          - name: result2
+            type: string
+          - name: result3
+            type: object
+            properties:
+              IMAGE_URL:
+                type: string
+              IMAGE_DIGEST:
+                type: string 
+        image: alpine
+        script: |
+          echo -n "[\"image1\", \"image2\", \"image3\"]" | tee $(step.results.result1.path)
+          echo -n "foo" | tee $(step.results.result2.path)
+          echo -n "{\"IMAGE_URL\":\"ar.com\", \"IMAGE_DIGEST\":\"sha234\"}" | tee $(step.results.result3.path) 
+      - name: action-runner
+        ref:
+          name: step-action
+        params:
+          - name: param1
+            value: $(steps.inline-step.results.result1[*])
+          - name: param2
+            value: $(steps.inline-step.results.result2)
+          - name: param3
+            value: $(steps.inline-step.results.result3[*])
+```
+
+**Note:** `Step Results` can only be referenced in a `Step's/StepAction's` `env`, `command` and `args`. Referencing in any other field will throw an error.
+
 ### Declaring WorkingDir
 
 You can declare `workingDir` in a `StepAction`:
@@ -463,9 +521,3 @@ spec:
 ```
 
 The default resolver type can be configured by the `default-resolver-type` field in the `config-defaults` ConfigMap (`alpha` feature). See [additional-configs.md](./additional-configs.md) for details.
-
-## Known Limitations
-
-### Cannot pass Step Results between Steps
-
-It's not currently possible to pass results produced by a `Step` into following `Steps`. We are working on this feature and it will be made available soon.

examples/v1/taskruns/alpha/stepaction-passing-results.yaml

@@ -0,0 +1,93 @@
+apiVersion: tekton.dev/v1alpha1
+kind: StepAction
+metadata:
+  name: step-action
+spec:
+  params:
+    - name: param1
+      type: array
+    - name: param2
+      type: string
+    - name: param3
+      type: object
+      properties:
+        IMAGE_URL:
+          type: string
+        IMAGE_DIGEST:
+          type: string
+  image: bash:3.2
+  env:
+    - name: STRINGPARAM
+      value: $(params.param2)
+  args: [
+    "$(params.param1[*])",
+    "$(params.param1[0])",
+    "$(params.param3.IMAGE_URL)",
+    "$(params.param3.IMAGE_DIGEST)",
+  ]
+  script: |
+    if [[ $1 != "image1" ]]; then
+        echo "Want: image1, Got: $1"
+      exit 1
+    fi
+    if [[ $2 != "image2" ]]; then
+        echo "Want: image2, Got: $2"
+      exit 1
+    fi
+    if [[ $3 != "image3" ]]; then
+        echo "Want: image3, Got: $3"
+      exit 1
+    fi
+    if [[ $4 != "image1" ]]; then
+        echo "Want: image1, Got: $4"
+      exit 1
+    fi
+    if [[ $5 != "ar.com" ]]; then
+        echo "Want: ar.com, Got: $5"
+      exit 1
+    fi
+    if [[ $6 != "sha234" ]]; then
+        echo "Want: sha234, Got: $6"
+      exit 1
+    fi
+    if [[ ${STRINGPARAM} != "foo" ]]; then
+        echo "Want: foo, Got: ${STRINGPARAM}"
+      exit 1
+    fi
+---
+apiVersion: tekton.dev/v1
+kind: TaskRun
+metadata:
+  name: step-action-run
+spec:
+  TaskSpec:
+    steps:
+      - name: inline-step
+        results:
+          - name: result1
+            type: array
+          - name: result2
+            type: string
+          - name: result3
+            type: object
+            properties:
+              IMAGE_URL:
+                type: string
+              IMAGE_DIGEST:
+                type: string
+        image: alpine
+        script: |
+          echo -n "[\"image1\", \"image2\", \"image3\"]" | tee $(step.results.result1.path)
+          echo -n "foo" | tee $(step.results.result2.path)
+          echo -n "{\"IMAGE_URL\":\"ar.com\", \"IMAGE_DIGEST\":\"sha234\"}" | tee $(step.results.result3.path)
+          cat /tekton/scripts/*
+      - name: action-runner
+        ref:
+          name: step-action
+        params:
+          - name: param1
+            value: $(steps.inline-step.results.result1[*])
+          - name: param2
+            value: $(steps.inline-step.results.result2)
+          - name: param3
+            value: $(steps.inline-step.results.result3[*])

pkg/apis/pipeline/v1/pipeline_validation.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/tektoncd/pipeline/pkg/apis/config"
 	"github.com/tektoncd/pipeline/pkg/apis/validate"
+	"github.com/tektoncd/pipeline/pkg/internal/resultref"
 	"github.com/tektoncd/pipeline/pkg/reconciler/pipeline/dag"
 	"github.com/tektoncd/pipeline/pkg/substitution"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
@@ -643,7 +644,7 @@ func validatePipelineResults(results []PipelineResult, tasks []PipelineTask, fin
 				"value").ViaFieldIndex("results", idx))
 		}
 
-		expressions = filter(expressions, looksLikeResultRef)
+		expressions = filter(expressions, resultref.LooksLikeResultRef)
 		resultRefs := NewResultRefs(expressions)
 		if len(expressions) != len(resultRefs) {
 			errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("expected all of the expressions %v to be result expressions but only %v were", expressions, resultRefs),
@@ -678,16 +679,16 @@ func taskContainsResult(resultExpression string, pipelineTaskNames sets.String,
 	for _, expression := range split {
 		if expression != "" {
 			value := stripVarSubExpression("$" + expression)
-			pipelineTaskName, _, _, _, err := parseExpression(value)
+			pr, err := resultref.ParseTaskExpression(value)
 
 			if err != nil {
 				return false
 			}
 
-			if strings.HasPrefix(value, "tasks") && !pipelineTaskNames.Has(pipelineTaskName) {
+			if strings.HasPrefix(value, "tasks") && !pipelineTaskNames.Has(pr.ResourceName) {
 				return false
 			}
-			if strings.HasPrefix(value, "finally") && !pipelineFinallyTaskNames.Has(pipelineTaskName) {
+			if strings.HasPrefix(value, "finally") && !pipelineFinallyTaskNames.Has(pr.ResourceName) {
 				return false
 			}
 		}

pkg/apis/pipeline/v1/pipelinerun_validation.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/tektoncd/pipeline/pkg/apis/config"
 	"github.com/tektoncd/pipeline/pkg/apis/validate"
+	"github.com/tektoncd/pipeline/pkg/internal/resultref"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"knative.dev/pkg/apis"
@@ -128,7 +129,7 @@ func (ps *PipelineRunSpec) validatePipelineRunParameters(ctx context.Context) (e
 		expressions, ok := param.GetVarSubstitutionExpressions()
 		if ok {
 			if LooksLikeContainsResultRefs(expressions) {
-				expressions = filter(expressions, looksLikeResultRef)
+				expressions = filter(expressions, resultref.LooksLikeResultRef)
 				resultRefs := NewResultRefs(expressions)
 				if len(resultRefs) > 0 {
 					errs = errs.Also(apis.ErrInvalidValue(fmt.Sprintf("cannot use result expressions in %v as PipelineRun parameter values", expressions),

pkg/apis/pipeline/v1/resultref.go

@@ -17,10 +17,10 @@ limitations under the License.
 package v1
 
 import (
-	"fmt"
 	"regexp"
-	"strconv"
 	"strings"
+
+	"github.com/tektoncd/pipeline/pkg/internal/resultref"
 )
 
 // ResultRef is a type that represents a reference to a task run result
@@ -32,25 +32,21 @@ type ResultRef struct {
 }
 
 const (
-	resultExpressionFormat = "tasks.<taskName>.results.<resultName>"
-	// Result expressions of the form <resultName>.<attribute> will be treated as object results.
-	// If a string result name contains a dot, brackets should be used to differentiate it from an object result.
-	// https://github.com/tektoncd/community/blob/main/teps/0075-object-param-and-result-types.md#collisions-with-builtin-variable-replacement
-	objectResultExpressionFormat = "tasks.<taskName>.results.<objectResultName>.<individualAttribute>"
 	// ResultTaskPart Constant used to define the "tasks" part of a pipeline result reference
-	ResultTaskPart = "tasks"
+	// retained because of backwards compatibility
+	ResultTaskPart = resultref.ResultTaskPart
 	// ResultFinallyPart Constant used to define the "finally" part of a pipeline result reference
-	ResultFinallyPart = "finally"
+	// retained because of backwards compatibility
+	ResultFinallyPart = resultref.ResultFinallyPart
 	// ResultResultPart Constant used to define the "results" part of a pipeline result reference
-	ResultResultPart = "results"
+	// retained because of backwards compatibility
+	ResultResultPart = resultref.ResultResultPart
 	// TODO(#2462) use one regex across all substitutions
 	// variableSubstitutionFormat matches format like $result.resultname, $result.resultname[int] and $result.resultname[*]
 	variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9]+|\*)\])?\)`
 	// exactVariableSubstitutionFormat matches strings that only contain a single reference to result or param variables, but nothing else
 	// i.e. `$(result.resultname)` is a match, but `foo $(result.resultname)` is not.
 	exactVariableSubstitutionFormat = `^\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9]+|\*)\])?\)$`
-	// arrayIndexing will match all `[int]` and `[*]` for parseExpression
-	arrayIndexing = `\[([0-9])*\*?\]`
 	// ResultNameFormat Constant used to define the regex Result.Name should follow
 	ResultNameFormat = `^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$`
 )
@@ -60,25 +56,22 @@ var VariableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
 var exactVariableSubstitutionRegex = regexp.MustCompile(exactVariableSubstitutionFormat)
 var resultNameFormatRegex = regexp.MustCompile(ResultNameFormat)
 
-// arrayIndexingRegex is used to match `[int]` and `[*]`
-var arrayIndexingRegex = regexp.MustCompile(arrayIndexing)
-
 // NewResultRefs extracts all ResultReferences from a param or a pipeline result.
 // If the ResultReference can be extracted, they are returned. Expressions which are not
 // results are ignored.
 func NewResultRefs(expressions []string) []*ResultRef {
 	var resultRefs []*ResultRef
 	for _, expression := range expressions {
-		pipelineTask, result, index, property, err := parseExpression(expression)
+		pr, err := resultref.ParseTaskExpression(expression)
 		// If the expression isn't a result but is some other expression,
-		// parseExpression will return an error, in which case we just skip that expression,
+		// parseTaskExpression will return an error, in which case we just skip that expression,
 		// since although it's not a result ref, it might be some other kind of reference
 		if err == nil {
 			resultRefs = append(resultRefs, &ResultRef{
-				PipelineTask: pipelineTask,
-				Result:       result,
-				ResultsIndex: index,
-				Property:     property,
+				PipelineTask: pr.ResourceName,
+				Result:       pr.ResultName,
+				ResultsIndex: pr.ArrayIdx,
+				Property:     pr.ObjectKey,
 			})
 		}
 	}
@@ -91,20 +84,13 @@ func NewResultRefs(expressions []string) []*ResultRef {
 // performing strict validation
 func LooksLikeContainsResultRefs(expressions []string) bool {
 	for _, expression := range expressions {
-		if looksLikeResultRef(expression) {
+		if resultref.LooksLikeResultRef(expression) {
 			return true
 		}
 	}
 	return false
 }
 
-// looksLikeResultRef attempts to check if the given string looks like it contains any
-// result references. Returns true if it does, false otherwise
-func looksLikeResultRef(expression string) bool {
-	subExpressions := strings.Split(expression, ".")
-	return len(subExpressions) >= 4 && (subExpressions[0] == ResultTaskPart || subExpressions[0] == ResultFinallyPart) && subExpressions[2] == ResultResultPart
-}
-
 func validateString(value string) []string {
 	expressions := VariableSubstitutionRegex.FindAllString(value, -1)
 	if expressions == nil {
@@ -121,54 +107,16 @@ func stripVarSubExpression(expression string) string {
 	return strings.TrimSuffix(strings.TrimPrefix(expression, "$("), ")")
 }
 
-// parseExpression parses "task name", "result name", "array index" (iff it's an array result) and "object key name" (iff it's an object result)
-// 1. Reference string result
-// - Input: tasks.myTask.results.aStringResult
-// - Output: "myTask", "aStringResult", nil, "", nil
-// 2. Reference Object value with key:
-// - Input: tasks.myTask.results.anObjectResult.key1
-// - Output: "myTask", "anObjectResult", nil, "key1", nil
-// 3. Reference array elements with array indexing :
-// - Input: tasks.myTask.results.anArrayResult[1]
-// - Output: "myTask", "anArrayResult", 1, "", nil
-// 4. Referencing whole array or object result:
-// - Input: tasks.myTask.results.Result[*]
-// - Output: "myTask", "Result", nil, "", nil
-// Invalid Case:
-// - Input: tasks.myTask.results.resultName.foo.bar
-// - Output: "", "", nil, "", error
-// TODO: may use regex for each type to handle possible reference formats
-func parseExpression(substitutionExpression string) (string, string, *int, string, error) {
-	if looksLikeResultRef(substitutionExpression) {
-		subExpressions := strings.Split(substitutionExpression, ".")
-		// For string result: tasks.<taskName>.results.<stringResultName>
-		// For array result: tasks.<taskName>.results.<arrayResultName>[index]
-		if len(subExpressions) == 4 {
-			resultName, stringIdx := ParseResultName(subExpressions[3])
-			if stringIdx != "" && stringIdx != "*" {
-				intIdx, _ := strconv.Atoi(stringIdx)
-				return subExpressions[1], resultName, &intIdx, "", nil
-			}
-			return subExpressions[1], resultName, nil, "", nil
-		} else if len(subExpressions) == 5 {
-			// For object type result: tasks.<taskName>.results.<objectResultName>.<individualAttribute>
-			return subExpressions[1], subExpressions[3], nil, subExpressions[4], nil
-		}
-	}
-	return "", "", nil, "", fmt.Errorf("must be one of the form 1). %q; 2). %q", resultExpressionFormat, objectResultExpressionFormat)
-}
-
 // ParseResultName parse the input string to extract resultName and result index.
 // Array indexing:
 // Input:  anArrayResult[1]
 // Output: anArrayResult, "1"
 // Array star reference:
 // Input:  anArrayResult[*]
 // Output: anArrayResult, "*"
+// retained for backwards compatibility
 func ParseResultName(resultName string) (string, string) {
-	stringIdx := strings.TrimSuffix(strings.TrimPrefix(arrayIndexingRegex.FindString(resultName), "["), "]")
-	resultName = arrayIndexingRegex.ReplaceAllString(resultName, "")
-	return resultName, stringIdx
+	return resultref.ParseResultName(resultName)
 }
 
 // PipelineTaskResultRefs walks all the places a result reference can be used