Link git outputs between tasks

- Implementation details
Pipelinerun creates pvc for the lifetime for object and uses that pvc as
scratch space to transfer git resources between them. This information
is passed to taskrun via resource paths. Paths are array of strings and
incase of inouts these paths will be considered as new source of
pipeline resource. In the case of outputs paths will be considered as
new destination directory.

Partially fixes tektoncd#148

docs/Concepts.md

@@ -37,6 +37,7 @@ A Task is a collection of sequential steps you would want to run as part of your
 A task will run inside a container on your cluster. A Task declares:
 
 1. Inputs the task needs.
+Task input resource can provide `targetPath` to initialize resource in specific directory. Resource will be placed under `/workspace/targetPath`. If `targetPath` is not specified then resource will be initialized under `/workspace`.
 1. Outputs the task will produce.
 1. Sequence of steps to execute. Each step is [a container image](./using.md#image-contract).
 
@@ -169,6 +170,12 @@ Creating a `TaskRun` will invoke a [Task](#task), running all of the steps until
 completion or failure. Creating a `TaskRun` will require satisfying all of the input
 requirements of the `Task`.
 
+`TaskRun` definition includes `inputs`, `outputs` for `Task` referred in spec. 
+
+Input resource holds reference to pipeline resource and `paths`. `paths` will be used by `TaskRun` as the resource's new source paths i.e., copy the resource from specified list of paths. `TaskRun` expects the folder and contents to be already present in specified paths.
+
+Output resource holds reference to pipeline resource and `paths` too. `paths` will be used by `TaskRun` as the resource's new destination paths i.e., copy the resource artificats to specified paths. `TaskRun` will be responsible for creating required directories and copying contents over.
+
 `TaskRuns` can be created directly by a user or by a [PipelineRun](#pipelinerun).
 
 #### PipelineRun

docs/using.md

@@ -106,6 +106,8 @@ steps:
     value: 'world'
 ```
 
+Pipeline Tasks are allowed to pass resources from previous tasks via `providedBy` field. This feature is implemented using Persistent Volume Claim under the hood but however has an implication that tasks cannot have any volume mounted under path `/pvc`.
+
 ### Conventions
 
 * `/workspace/<resource-name>`: [`PipelineResources` are made available in this mounted dir](#creating-resources)

examples/invocations/kritis-pvc-run.yaml

@@ -0,0 +1,25 @@
+apiVersion: pipeline.knative.dev/v1alpha1
+kind: PipelineRun
+metadata:
+  name: kritis-pipeline-pvc
+  namespace: default
+spec:
+  pipelineRef:
+    name: kritis-pipeline-pvc
+  triggerRef:
+    type: manual
+  resources:
+  - name: create-file-kritis
+    inputs:
+    - name: workspace
+      resourceRef:
+        name: kritis-resources-git
+    outputs:
+    - name: workspace
+      resourceRef:
+        name: kritis-resources-git
+  - name: check-kritis
+    inputs:
+    - name: workspace
+      resourceRef:
+        name: kritis-resources-git

examples/kritis-pvc-tasks.yaml

@@ -0,0 +1,42 @@
+apiVersion: pipeline.knative.dev/v1alpha1
+kind: Task
+metadata:
+  name: create-file
+  namespace: default
+spec:
+  inputs:
+    resources:
+    - name: workspace
+      type: git
+      targetPath: /damnworkspace
+  outputs:
+    resources:
+    - name: workspace
+      type: git
+  steps:
+  - name: read-docs-old
+    image: ubuntu
+    command: ["/bin/bash"]
+    args: ['-c', 'ls -la /workspace/damnworkspace/docs/install.md'] # tests that targetpath works
+  - name: write-new-stuff
+    image: ubuntu
+    command: ['bash']
+    args: ['-c', 'echo some stuff > /workspace/damnworkspace/stuff']
+
+---
+apiVersion: pipeline.knative.dev/v1alpha1
+kind: Task
+metadata:
+  name: check-stuff-file-exists
+  namespace: default
+spec:
+  inputs:
+    resources:
+    - name: workspace
+      type: git
+      targetPath: /newworkspace
+  steps:
+  - name: read
+    image: ubuntu
+    command: ["/bin/bash"]
+    args: ['-c', 'cat /workspace/newworkspace/stuff'] # tests that new targetpath and previous task output is dumped

examples/pipelines/kritis-pvc-pipeline.yaml

@@ -0,0 +1,16 @@
+apiVersion: pipeline.knative.dev/v1alpha1
+kind: Pipeline
+metadata:
+  name: kritis-pipeline-pvc
+  namespace: default
+spec:
+  tasks:
+  - name: create-file-kritis          # 1. create file
+    taskRef:
+      name: create-file
+  - name: check-kritis               # 2.  check file exists
+    taskRef:
+      name: check-stuff-file-exists
+    resources:
+      - name: workspace
+        providedBy: [create-file-kritis]

examples/pipelines/kritis-resources.yaml

@@ -37,4 +37,4 @@ spec:
   - name: token
     value: eyJhbGciOiJ.....
   - name: username
-    value: admin    
+    value: admin

pkg/apis/pipeline/v1alpha1/pipelinerun_types.go

@@ -17,10 +17,23 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
+
 	duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1"
 	"github.com/knative/pkg/webhook"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	pipelineRunControllerName = "PipelineRun"
+	groupVersionKind          = schema.GroupVersionKind{
+		Group:   SchemeGroupVersion.Group,
+		Version: SchemeGroupVersion.Version,
+		Kind:    pipelineRunControllerName,
+	}
 )
 
 // Assert that TaskRun implements the GenericCRD interface.
@@ -178,3 +191,38 @@ func (pr *PipelineRun) GetTaskRunRef() corev1.ObjectReference {
 
 // SetDefaults for pipelinerun
 func (pr *PipelineRun) SetDefaults() {}
+
+// GetPVC gets PVC for
+func (pr *PipelineRun) GetPVC() *corev1.PersistentVolumeClaim {
+	var pvcSizeBytes int64
+	// TODO(shashwathi): make this value configurable
+	pvcSizeBytes = 5 * 1024 * 1024 * 1024 // 5 GBs
+	return &corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:       pr.Namespace,
+			Name:            pr.GetPVCName(),
+			OwnerReferences: pr.GetOwnerReference(),
+		},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			// Multiple tasks should be allowed to read
+			AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
+			Resources: corev1.ResourceRequirements{
+				Requests: map[corev1.ResourceName]resource.Quantity{
+					corev1.ResourceStorage: *resource.NewQuantity(pvcSizeBytes, resource.BinarySI),
+				},
+			},
+		},
+	}
+}
+
+// GetOwnerReference gets the pipeline run as owner reference for any related objects
+func (pr *PipelineRun) GetOwnerReference() []metav1.OwnerReference {
+	return []metav1.OwnerReference{
+		*metav1.NewControllerRef(pr, groupVersionKind),
+	}
+}
+
+// GetPVCName provides name of PVC for corresponding PR
+func (pr *PipelineRun) GetPVCName() string {
+	return fmt.Sprintf("%s-pvc", pr.Name)
+}

pkg/apis/pipeline/v1alpha1/pipelinerun_types_test.go

@@ -83,3 +83,24 @@ func TestInitializeConditions(t *testing.T) {
 		t.Fatalf("PipelineRun status getting reset")
 	}
 }
+
+func Test_GetPVC(t *testing.T) {
+	p := &PipelineRun{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-name",
+			Namespace: "test-ns",
+		},
+	}
+	expectedPVC := corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "test-ns",
+			Name:      "test-name-pvc", // prname-pvc
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(p, groupVersionKind),
+			},
+		},
+	}
+	if d := cmp.Diff(p.GetPVC().ObjectMeta, expectedPVC.ObjectMeta); d != "" {
+		t.Fatalf("GetPVC mismatch; want %v got %v; diff %s", expectedPVC.ObjectMeta, p.GetPVC().ObjectMeta, d)
+	}
+}

pkg/apis/pipeline/v1alpha1/resource_types.go

@@ -93,6 +93,9 @@ var _ webhook.GenericCRD = (*PipelineResource)(nil)
 type TaskResource struct {
 	Name string               `json:"name"`
 	Type PipelineResourceType `json:"type"`
+	// +optional
+	// TargetPath is the path in workspace directory where the task resource will be copied.
+	TargetPath string `json:"targetPath"`
 }
 
 // +genclient
@@ -118,6 +121,8 @@ type PipelineResource struct {
 type TaskRunResource struct {
 	Name        string              `json:"name"`
 	ResourceRef PipelineResourceRef `json:"resourceRef"`
+	// +optional
+	Paths []string `json:"paths"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/pipeline/v1alpha1/taskrun_types.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
+
 	"github.com/knative/pkg/apis"
 	duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1"
 	"github.com/knative/pkg/webhook"
@@ -43,7 +45,7 @@ type TaskRunSpec struct {
 	// +optional
 	Generation int64 `json:"generation,omitempty"`
 	// +optional
-	ServiceAccount string `json:"serviceAccount"`
+	ServiceAccount string `json:"serviceAccount,omitempty"`
 }
 
 // TaskRunInputs holds the input values that this task was invoked with.
@@ -174,3 +176,13 @@ func (tr *TaskRun) GetBuildRef() corev1.ObjectReference {
 		Name:       tr.Name,
 	}
 }
+
+// GetPipelineRunPVCName for taskrun gets pipelinerun
+func (tr *TaskRun) GetPipelineRunPVCName() string {
+	for _, ref := range tr.GetOwnerReferences() {
+		if ref.Kind == pipelineRunControllerName {
+			return fmt.Sprintf("%s-pvc", ref.Name)
+		}
+	}
+	return ""
+}

pkg/apis/pipeline/v1alpha1/taskrun_types_test.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2018 The Knative Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestTaskRun_GetBuildRef(t *testing.T) {
+	tr := TaskRun{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "taskrunname",
+			Namespace: "testns",
+		},
+	}
+	expectedBuildRef := corev1.ObjectReference{
+		APIVersion: "build.knative.dev/v1alpha1",
+		Kind:       "Build",
+		Namespace:  "testns",
+		Name:       "taskrunname",
+	}
+	if d := cmp.Diff(tr.GetBuildRef(), expectedBuildRef); d != "" {
+		t.Fatalf("taskrun build ref mismatch: %s", d)
+	}
+}
+
+func TestTasRun_Valid_GetPipelineRunPVCName(t *testing.T) {
+	tr := TaskRun{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "taskrunname",
+			Namespace: "testns",
+			OwnerReferences: []metav1.OwnerReference{{
+				Kind: "PipelineRun",
+				Name: "testpr",
+			}},
+		},
+	}
+	expectedPVCName := "testpr-pvc"
+	if tr.GetPipelineRunPVCName() != expectedPVCName {
+		t.Fatalf("taskrun pipeline run mismatch: got %s ; expected %s", tr.GetPipelineRunPVCName(), expectedPVCName)
+	}
+}
+
+func TestTasRun_InvalidOwner_GetPipelineRunPVCName(t *testing.T) {
+	tr := TaskRun{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "taskrunname",
+			Namespace: "testns",
+			OwnerReferences: []metav1.OwnerReference{{
+				Kind: "SomeOtherOwner",
+				Name: "testpr",
+			}},
+		},
+	}
+	expectedPVCName := ""
+	if tr.GetPipelineRunPVCName() != expectedPVCName {
+		t.Fatalf("taskrun pipeline run pvc name mismatch: got %s ; expected %s", tr.GetPipelineRunPVCName(), expectedPVCName)
+	}
+}

pkg/apis/pipeline/v1alpha1/taskrun_validation_test.go

@@ -1,3 +1,18 @@
+/*
+Copyright 2018 The Knative Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
 package v1alpha1
 
 import (
@@ -120,8 +135,7 @@ func TestTaskRunSpec_Invalidate(t *testing.T) {
 				},
 			},
 			wantErr: apis.ErrMissingField("spec.trigger.triggerref.name"),
-		},
-	}
+		}}
 
 	for _, ts := range tests {
 		t.Run(ts.name, func(t *testing.T) {