From 7381daba06594864df6ea35f394d359ff78a3f12 Mon Sep 17 00:00:00 2001 From: Shash Reddy Date: Mon, 12 Nov 2018 18:16:21 -0500 Subject: [PATCH] Link git outputs between tasks - Implementation details Pipelinerun creates pvc for the lifetime for object and uses that pvc as scratch space to transfer git resources between them. This information is passed to taskrun via resource paths. Paths are array of strings and incase of inouts these paths will be considered as new source of pipeline resource. In the case of outputs paths will be considered as new destination directory. Partially fixes https://github.com/knative/build-pipeline/issues/148 --- docs/Concepts.md | 7 + docs/using.md | 2 + examples/invocations/kritis-pvc-run.yaml | 25 + examples/kritis-pvc-tasks.yaml | 42 ++ examples/pipelines/kritis-pvc-pipeline.yaml | 16 + examples/pipelines/kritis-resources.yaml | 2 +- .../pipeline/v1alpha1/pipelinerun_types.go | 48 ++ .../v1alpha1/pipelinerun_types_test.go | 21 + pkg/apis/pipeline/v1alpha1/resource_types.go | 5 + pkg/apis/pipeline/v1alpha1/taskrun_types.go | 14 +- .../pipeline/v1alpha1/taskrun_types_test.go | 77 ++++ .../v1alpha1/taskrun_validation_test.go | 18 +- .../v1alpha1/zz_generated.deepcopy.go | 13 +- .../v1alpha1/pipelinerun/pipelinerun.go | 63 ++- .../v1alpha1/pipelinerun/pipelinerun_test.go | 57 ++- .../resources/input_output_steps.go | 83 ++++ .../resources/input_output_steps_test.go | 195 ++++++++ .../pipelinerun/resources/pipelinestate.go | 3 +- .../resources/pipelinestate_test.go | 6 +- .../v1alpha1/pipelinerun/validate.go | 10 +- .../taskrun/resources/build_step_test.go | 219 +++++++++ .../taskrun/resources/input_resource_test.go | 63 ++- .../taskrun/resources/input_resources.go | 7 +- .../taskrun/resources/pre_post_build_step.go | 136 ++++++ pkg/reconciler/v1alpha1/taskrun/taskrun.go | 18 +- .../v1alpha1/taskrun/taskrun_test.go | 434 +++++++++++------- test/crd_checks.go | 4 +- test/helm_task_test.go | 6 +- test/pipelinerun_test.go | 275 ++++++++++- 29 files changed, 1616 insertions(+), 253 deletions(-) create mode 100644 examples/invocations/kritis-pvc-run.yaml create mode 100644 examples/kritis-pvc-tasks.yaml create mode 100644 examples/pipelines/kritis-pvc-pipeline.yaml create mode 100644 pkg/apis/pipeline/v1alpha1/taskrun_types_test.go create mode 100644 pkg/reconciler/v1alpha1/pipelinerun/resources/input_output_steps.go create mode 100644 pkg/reconciler/v1alpha1/pipelinerun/resources/input_output_steps_test.go create mode 100644 pkg/reconciler/v1alpha1/taskrun/resources/build_step_test.go create mode 100644 pkg/reconciler/v1alpha1/taskrun/resources/pre_post_build_step.go diff --git a/docs/Concepts.md b/docs/Concepts.md index 34a2c23d881..9a5cc68b33f 100644 --- a/docs/Concepts.md +++ b/docs/Concepts.md @@ -37,6 +37,7 @@ A Task is a collection of sequential steps you would want to run as part of your A task will run inside a container on your cluster. A Task declares: 1. Inputs the task needs. +Task input resource can provide `targetPath` to initialize resource in specific directory. Resource will be placed under `/workspace/targetPath`. If `targetPath` is not specified then resource will be initialized under `/workspace`. 1. Outputs the task will produce. 1. Sequence of steps to execute. Each step is [a container image](./using.md#image-contract). @@ -169,6 +170,12 @@ Creating a `TaskRun` will invoke a [Task](#task), running all of the steps until completion or failure. Creating a `TaskRun` will require satisfying all of the input requirements of the `Task`. +`TaskRun` definition includes `inputs`, `outputs` for `Task` referred in spec. + +Input resource holds reference to pipeline resource and `paths`. `paths` will be used by `TaskRun` as the resource's new source paths i.e., copy the resource from specified list of paths. `TaskRun` expects the folder and contents to be already present in specified paths. + +Output resource holds reference to pipeline resource and `paths` too. `paths` will be used by `TaskRun` as the resource's new destination paths i.e., copy the resource artificats to specified paths. `TaskRun` will be responsible for creating required directories and copying contents over. + `TaskRuns` can be created directly by a user or by a [PipelineRun](#pipelinerun). #### PipelineRun diff --git a/docs/using.md b/docs/using.md index 4cd7ec62dc5..f03cc685197 100644 --- a/docs/using.md +++ b/docs/using.md @@ -106,6 +106,8 @@ steps: value: 'world' ``` +Pipeline Tasks are allowed to pass resources from previous tasks via `providedBy` field. This feature is implemented using Persistent Volume Claim under the hood but however has an implication that tasks cannot have any volume mounted under path `/pvc`. + ### Conventions * `/workspace/`: [`PipelineResources` are made available in this mounted dir](#creating-resources) diff --git a/examples/invocations/kritis-pvc-run.yaml b/examples/invocations/kritis-pvc-run.yaml new file mode 100644 index 00000000000..15338377f9f --- /dev/null +++ b/examples/invocations/kritis-pvc-run.yaml @@ -0,0 +1,25 @@ +apiVersion: pipeline.knative.dev/v1alpha1 +kind: PipelineRun +metadata: + name: kritis-pipeline-pvc + namespace: default +spec: + pipelineRef: + name: kritis-pipeline-pvc + triggerRef: + type: manual + resources: + - name: create-file-kritis + inputs: + - name: workspace + resourceRef: + name: kritis-resources-git + outputs: + - name: workspace + resourceRef: + name: kritis-resources-git + - name: check-kritis + inputs: + - name: workspace + resourceRef: + name: kritis-resources-git \ No newline at end of file diff --git a/examples/kritis-pvc-tasks.yaml b/examples/kritis-pvc-tasks.yaml new file mode 100644 index 00000000000..60b43ec9286 --- /dev/null +++ b/examples/kritis-pvc-tasks.yaml @@ -0,0 +1,42 @@ +apiVersion: pipeline.knative.dev/v1alpha1 +kind: Task +metadata: + name: create-file + namespace: default +spec: + inputs: + resources: + - name: workspace + type: git + targetPath: /damnworkspace + outputs: + resources: + - name: workspace + type: git + steps: + - name: read-docs-old + image: ubuntu + command: ["/bin/bash"] + args: ['-c', 'ls -la /workspace/damnworkspace/docs/install.md'] # tests that targetpath works + - name: write-new-stuff + image: ubuntu + command: ['bash'] + args: ['-c', 'echo some stuff > /workspace/damnworkspace/stuff'] + +--- +apiVersion: pipeline.knative.dev/v1alpha1 +kind: Task +metadata: + name: check-stuff-file-exists + namespace: default +spec: + inputs: + resources: + - name: workspace + type: git + targetPath: /newworkspace + steps: + - name: read + image: ubuntu + command: ["/bin/bash"] + args: ['-c', 'cat /workspace/newworkspace/stuff'] # tests that new targetpath and previous task output is dumped diff --git a/examples/pipelines/kritis-pvc-pipeline.yaml b/examples/pipelines/kritis-pvc-pipeline.yaml new file mode 100644 index 00000000000..de0e8d36995 --- /dev/null +++ b/examples/pipelines/kritis-pvc-pipeline.yaml @@ -0,0 +1,16 @@ +apiVersion: pipeline.knative.dev/v1alpha1 +kind: Pipeline +metadata: + name: kritis-pipeline-pvc + namespace: default +spec: + tasks: + - name: create-file-kritis # 1. create file + taskRef: + name: create-file + - name: check-kritis # 2. check file exists + taskRef: + name: check-stuff-file-exists + resources: + - name: workspace + providedBy: [create-file-kritis] \ No newline at end of file diff --git a/examples/pipelines/kritis-resources.yaml b/examples/pipelines/kritis-resources.yaml index 3e47609217a..e272d628712 100644 --- a/examples/pipelines/kritis-resources.yaml +++ b/examples/pipelines/kritis-resources.yaml @@ -37,4 +37,4 @@ spec: - name: token value: eyJhbGciOiJ..... - name: username - value: admin \ No newline at end of file + value: admin \ No newline at end of file diff --git a/pkg/apis/pipeline/v1alpha1/pipelinerun_types.go b/pkg/apis/pipeline/v1alpha1/pipelinerun_types.go index 9573c7ecef4..099b4222012 100644 --- a/pkg/apis/pipeline/v1alpha1/pipelinerun_types.go +++ b/pkg/apis/pipeline/v1alpha1/pipelinerun_types.go @@ -17,10 +17,23 @@ limitations under the License. package v1alpha1 import ( + "fmt" + duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" "github.com/knative/pkg/webhook" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + pipelineRunControllerName = "PipelineRun" + groupVersionKind = schema.GroupVersionKind{ + Group: SchemeGroupVersion.Group, + Version: SchemeGroupVersion.Version, + Kind: pipelineRunControllerName, + } ) // Assert that TaskRun implements the GenericCRD interface. @@ -178,3 +191,38 @@ func (pr *PipelineRun) GetTaskRunRef() corev1.ObjectReference { // SetDefaults for pipelinerun func (pr *PipelineRun) SetDefaults() {} + +// GetPVC gets PVC for +func (pr *PipelineRun) GetPVC() *corev1.PersistentVolumeClaim { + var pvcSizeBytes int64 + // TODO(shashwathi): make this value configurable + pvcSizeBytes = 5 * 1024 * 1024 * 1024 // 5 GBs + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: pr.Namespace, + Name: pr.GetPVCName(), + OwnerReferences: pr.GetOwnerReference(), + }, + Spec: corev1.PersistentVolumeClaimSpec{ + // Multiple tasks should be allowed to read + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: *resource.NewQuantity(pvcSizeBytes, resource.BinarySI), + }, + }, + }, + } +} + +// GetOwnerReference gets the pipeline run as owner reference for any related objects +func (pr *PipelineRun) GetOwnerReference() []metav1.OwnerReference { + return []metav1.OwnerReference{ + *metav1.NewControllerRef(pr, groupVersionKind), + } +} + +// GetPVCName provides name of PVC for corresponding PR +func (pr *PipelineRun) GetPVCName() string { + return fmt.Sprintf("%s-pvc", pr.Name) +} diff --git a/pkg/apis/pipeline/v1alpha1/pipelinerun_types_test.go b/pkg/apis/pipeline/v1alpha1/pipelinerun_types_test.go index d61db17cc80..fa8192d2d21 100644 --- a/pkg/apis/pipeline/v1alpha1/pipelinerun_types_test.go +++ b/pkg/apis/pipeline/v1alpha1/pipelinerun_types_test.go @@ -83,3 +83,24 @@ func TestInitializeConditions(t *testing.T) { t.Fatalf("PipelineRun status getting reset") } } + +func Test_GetPVC(t *testing.T) { + p := &PipelineRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-name", + Namespace: "test-ns", + }, + } + expectedPVC := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-name-pvc", // prname-pvc + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(p, groupVersionKind), + }, + }, + } + if d := cmp.Diff(p.GetPVC().ObjectMeta, expectedPVC.ObjectMeta); d != "" { + t.Fatalf("GetPVC mismatch; want %v got %v; diff %s", expectedPVC.ObjectMeta, p.GetPVC().ObjectMeta, d) + } +} diff --git a/pkg/apis/pipeline/v1alpha1/resource_types.go b/pkg/apis/pipeline/v1alpha1/resource_types.go index 277bab12e1a..0264f758a0d 100644 --- a/pkg/apis/pipeline/v1alpha1/resource_types.go +++ b/pkg/apis/pipeline/v1alpha1/resource_types.go @@ -93,6 +93,9 @@ var _ webhook.GenericCRD = (*PipelineResource)(nil) type TaskResource struct { Name string `json:"name"` Type PipelineResourceType `json:"type"` + // +optional + // TargetPath is the path in workspace directory where the task resource will be copied. + TargetPath string `json:"targetPath"` } // +genclient @@ -118,6 +121,8 @@ type PipelineResource struct { type TaskRunResource struct { Name string `json:"name"` ResourceRef PipelineResourceRef `json:"resourceRef"` + // +optional + Paths []string `json:"paths"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/pipeline/v1alpha1/taskrun_types.go b/pkg/apis/pipeline/v1alpha1/taskrun_types.go index b03362e642a..95ef449d7e0 100644 --- a/pkg/apis/pipeline/v1alpha1/taskrun_types.go +++ b/pkg/apis/pipeline/v1alpha1/taskrun_types.go @@ -17,6 +17,8 @@ limitations under the License. package v1alpha1 import ( + "fmt" + "github.com/knative/pkg/apis" duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" "github.com/knative/pkg/webhook" @@ -43,7 +45,7 @@ type TaskRunSpec struct { // +optional Generation int64 `json:"generation,omitempty"` // +optional - ServiceAccount string `json:"serviceAccount"` + ServiceAccount string `json:"serviceAccount,omitempty"` } // TaskRunInputs holds the input values that this task was invoked with. @@ -174,3 +176,13 @@ func (tr *TaskRun) GetBuildRef() corev1.ObjectReference { Name: tr.Name, } } + +// GetPipelineRunPVCName for taskrun gets pipelinerun +func (tr *TaskRun) GetPipelineRunPVCName() string { + for _, ref := range tr.GetOwnerReferences() { + if ref.Kind == pipelineRunControllerName { + return fmt.Sprintf("%s-pvc", ref.Name) + } + } + return "" +} diff --git a/pkg/apis/pipeline/v1alpha1/taskrun_types_test.go b/pkg/apis/pipeline/v1alpha1/taskrun_types_test.go new file mode 100644 index 00000000000..7230d6a30e9 --- /dev/null +++ b/pkg/apis/pipeline/v1alpha1/taskrun_types_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2018 The Knative Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestTaskRun_GetBuildRef(t *testing.T) { + tr := TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "taskrunname", + Namespace: "testns", + }, + } + expectedBuildRef := corev1.ObjectReference{ + APIVersion: "build.knative.dev/v1alpha1", + Kind: "Build", + Namespace: "testns", + Name: "taskrunname", + } + if d := cmp.Diff(tr.GetBuildRef(), expectedBuildRef); d != "" { + t.Fatalf("taskrun build ref mismatch: %s", d) + } +} + +func TestTasRun_Valid_GetPipelineRunPVCName(t *testing.T) { + tr := TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "taskrunname", + Namespace: "testns", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "PipelineRun", + Name: "testpr", + }}, + }, + } + expectedPVCName := "testpr-pvc" + if tr.GetPipelineRunPVCName() != expectedPVCName { + t.Fatalf("taskrun pipeline run mismatch: got %s ; expected %s", tr.GetPipelineRunPVCName(), expectedPVCName) + } +} + +func TestTasRun_InvalidOwner_GetPipelineRunPVCName(t *testing.T) { + tr := TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "taskrunname", + Namespace: "testns", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "SomeOtherOwner", + Name: "testpr", + }}, + }, + } + expectedPVCName := "" + if tr.GetPipelineRunPVCName() != expectedPVCName { + t.Fatalf("taskrun pipeline run pvc name mismatch: got %s ; expected %s", tr.GetPipelineRunPVCName(), expectedPVCName) + } +} diff --git a/pkg/apis/pipeline/v1alpha1/taskrun_validation_test.go b/pkg/apis/pipeline/v1alpha1/taskrun_validation_test.go index 511ef9418ba..b6a21e0f025 100644 --- a/pkg/apis/pipeline/v1alpha1/taskrun_validation_test.go +++ b/pkg/apis/pipeline/v1alpha1/taskrun_validation_test.go @@ -1,3 +1,18 @@ +/* +Copyright 2018 The Knative Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package v1alpha1 import ( @@ -120,8 +135,7 @@ func TestTaskRunSpec_Invalidate(t *testing.T) { }, }, wantErr: apis.ErrMissingField("spec.trigger.triggerref.name"), - }, - } + }} for _, ts := range tests { t.Run(ts.name, func(t *testing.T) { diff --git a/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go index 66db01149c4..05b8a5e594c 100644 --- a/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go @@ -978,7 +978,9 @@ func (in *TaskRunInputs) DeepCopyInto(out *TaskRunInputs) { if in.Resources != nil { in, out := &in.Resources, &out.Resources *out = make([]TaskRunResource, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Params != nil { in, out := &in.Params, &out.Params @@ -1037,7 +1039,9 @@ func (in *TaskRunOutputs) DeepCopyInto(out *TaskRunOutputs) { if in.Resources != nil { in, out := &in.Resources, &out.Resources *out = make([]TaskRunResource, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Params != nil { in, out := &in.Params, &out.Params @@ -1061,6 +1065,11 @@ func (in *TaskRunOutputs) DeepCopy() *TaskRunOutputs { func (in *TaskRunResource) DeepCopyInto(out *TaskRunResource) { *out = *in out.ResourceRef = in.ResourceRef + if in.Paths != nil { + in, out := &in.Paths, &out.Paths + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/reconciler/v1alpha1/pipelinerun/pipelinerun.go b/pkg/reconciler/v1alpha1/pipelinerun/pipelinerun.go index bfb9b2f9b0a..03b51c719f9 100644 --- a/pkg/reconciler/v1alpha1/pipelinerun/pipelinerun.go +++ b/pkg/reconciler/v1alpha1/pipelinerun/pipelinerun.go @@ -22,6 +22,8 @@ import ( "reflect" "time" + "k8s.io/client-go/kubernetes" + "github.com/knative/build-pipeline/pkg/apis/pipeline" "github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1" "github.com/knative/build-pipeline/pkg/reconciler" @@ -35,7 +37,6 @@ import ( "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/cache" informers "github.com/knative/build-pipeline/pkg/client/informers/externalversions/pipeline/v1alpha1" @@ -55,14 +56,6 @@ const ( pipelineRunControllerName = "PipelineRun" ) -var ( - groupVersionKind = schema.GroupVersionKind{ - Group: v1alpha1.SchemeGroupVersion.Group, - Version: v1alpha1.SchemeGroupVersion.Version, - Kind: pipelineRunControllerName, - } -) - // Reconciler implements controller.Reconciler for Configuration resources. type Reconciler struct { *reconciler.Base @@ -199,9 +192,15 @@ func (c *Reconciler) reconcile(ctx context.Context, pr *v1alpha1.PipelineRun) er return fmt.Errorf("error getting Tasks and/or TaskRuns for Pipeline %s: %s", p.Name, err) } prtr := resources.GetNextTask(pr.Name, pipelineState, c.Logger) + + if err := getOrCreatePVC(pr, c.KubeClientSet); err != nil { + c.Logger.Infof("PipelineRun failed to create/get volume %s", pr.Name) + return fmt.Errorf("Failed to create/get persistent volume claim %s for task %q: %v", pr.Name, err, pr.Name) + } + if prtr != nil { c.Logger.Infof("Creating a new TaskRun object %s", prtr.TaskRunName) - prtr.TaskRun, err = c.createTaskRun(prtr.Task, prtr.TaskRunName, pr, prtr.PipelineTask, serviceAccount) + prtr.TaskRun, err = c.createTaskRun(c.Logger, prtr.Task, prtr.TaskRunName, pr, prtr.PipelineTask, serviceAccount) if err != nil { c.Recorder.Eventf(pr, corev1.EventTypeWarning, "TaskRunCreationFailed", "Failed to create TaskRun %q: %v", prtr.TaskRunName, err) return fmt.Errorf("error creating TaskRun called %s for PipelineTask %s from PipelineRun %s: %s", prtr.TaskRunName, prtr.PipelineTask.Name, pr.Name, err) @@ -228,14 +227,12 @@ func UpdateTaskRunsStatus(pr *v1alpha1.PipelineRun, pipelineState []*resources.P } } -func (c *Reconciler) createTaskRun(t *v1alpha1.Task, trName string, pr *v1alpha1.PipelineRun, pt *v1alpha1.PipelineTask, sa string) (*v1alpha1.TaskRun, error) { +func (c *Reconciler) createTaskRun(logger *zap.SugaredLogger, t *v1alpha1.Task, trName string, pr *v1alpha1.PipelineRun, pt *v1alpha1.PipelineTask, sa string) (*v1alpha1.TaskRun, error) { tr := &v1alpha1.TaskRun{ ObjectMeta: metav1.ObjectMeta{ - Name: trName, - Namespace: t.Namespace, - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(pr, groupVersionKind), - }, + Name: trName, + Namespace: t.Namespace, + OwnerReferences: pr.GetOwnerReference(), Labels: map[string]string{ pipeline.GroupName + pipeline.PipelineLabelKey: pr.Spec.PipelineRef.Name, pipeline.GroupName + pipeline.PipelineRunLabelKey: pr.Name, @@ -251,24 +248,8 @@ func (c *Reconciler) createTaskRun(t *v1alpha1.Task, trName string, pr *v1alpha1 ServiceAccount: sa, }, } - var resources v1alpha1.PipelineTaskResource - for _, ptr := range pr.Spec.PipelineTaskResources { - if ptr.Name == pt.Name { - resources = ptr - } - } - for _, isb := range resources.Inputs { - tr.Spec.Inputs.Resources = append(tr.Spec.Inputs.Resources, v1alpha1.TaskRunResource{ - ResourceRef: isb.ResourceRef, - Name: isb.Name, - }) - } - for _, osb := range resources.Outputs { - tr.Spec.Outputs.Resources = append(tr.Spec.Outputs.Resources, v1alpha1.TaskRunResource{ - ResourceRef: osb.ResourceRef, - Name: osb.Name, - }) - } + resources.WrapSteps(&tr.Spec, pr.Spec.PipelineTaskResources, pt) + return c.PipelineClientSet.PipelineV1alpha1().TaskRuns(t.Namespace).Create(tr) } @@ -283,3 +264,17 @@ func (c *Reconciler) updateStatus(pr *v1alpha1.PipelineRun) (*v1alpha1.PipelineR } return newPr, nil } + +func getOrCreatePVC(pr *v1alpha1.PipelineRun, c kubernetes.Interface) error { + if _, err := c.CoreV1().PersistentVolumeClaims(pr.Namespace).Get(pr.GetPVCName(), metav1.GetOptions{}); err != nil { + if errors.IsNotFound(err) { + pvc := pr.GetPVC() + if _, err := c.CoreV1().PersistentVolumeClaims(pr.Namespace).Create(pvc); err != nil { + return fmt.Errorf("failed to claim Persistent Volume %q due to error: %s", pr.Name, err) + } + return nil + } + return fmt.Errorf("failed to get claim Persistent Volume %q due to error: %s", pr.Name, err) + } + return nil +} diff --git a/pkg/reconciler/v1alpha1/pipelinerun/pipelinerun_test.go b/pkg/reconciler/v1alpha1/pipelinerun/pipelinerun_test.go index aea39b76a27..b524518d74c 100644 --- a/pkg/reconciler/v1alpha1/pipelinerun/pipelinerun_test.go +++ b/pkg/reconciler/v1alpha1/pipelinerun/pipelinerun_test.go @@ -59,6 +59,19 @@ func TestReconcile(t *testing.T) { ResourceRef: v1alpha1.PipelineResourceRef{ Name: "some-image", }, + }, { + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "some-repo", + }, + }}, + }, { + Name: "unit-test-2", + Inputs: []v1alpha1.TaskResourceBinding{{ + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "some-repo", + }, }}, }}, }, @@ -82,9 +95,16 @@ func TestReconcile(t *testing.T) { Name: "templatedparam", Value: "${inputs.workspace.revision}", }}, + }, { + Name: "unit-test-2", + TaskRef: v1alpha1.TaskRef{Name: "unit-test-followup-task"}, + ResourceDependencies: []v1alpha1.ResourceDependency{{ + Name: "workspace", + ProvidedBy: []string{"unit-test-1"}, + }}, }}, - }}, - } + }, + }} ts := []*v1alpha1.Task{{ ObjectMeta: metav1.ObjectMeta{ Name: "unit-test-task", @@ -108,6 +128,22 @@ func TestReconcile(t *testing.T) { Resources: []v1alpha1.TaskResource{{ Name: "image-to-use", Type: "image", + }, { + Name: "workspace", + Type: "git", + }}, + }, + }, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "unit-test-followup-task", + Namespace: "foo", + }, + Spec: v1alpha1.TaskSpec{ + Inputs: &v1alpha1.Inputs{ + Resources: []v1alpha1.TaskResource{{ + Name: "workspace", + Type: "git", }}, }, }, @@ -217,17 +253,26 @@ func TestReconcile(t *testing.T) { }, Outputs: v1alpha1.TaskRunOutputs{ Resources: []v1alpha1.TaskRunResource{{ - ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "some-image", - }, - Name: "image-to-use", + Name: "image-to-use", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "some-image"}, + Paths: []string{"/pvc/unit-test-1/image-to-use"}, + }, { + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "some-repo"}, + Paths: []string{"/pvc/unit-test-1/workspace"}, }}, }, }, } + + // ignore IgnoreUnexported ignore both after and before steps fields if d := cmp.Diff(actual, expectedTaskRun); d != "" { t.Errorf("expected to see TaskRun %v created. Diff %s", expectedTaskRun, d) } + // test taskrun is able to recreate correct pipeline-pvc-name + if expectedTaskRun.GetPipelineRunPVCName() != "test-pipeline-run-success-pvc" { + t.Errorf("expected to see TaskRun PVC name set to %q created but got %s", "test-pipeline-run-success-pvc", expectedTaskRun.GetPipelineRunPVCName()) + } // This PipelineRun is in progress now and the status should reflect that condition := reconciledRun.Status.GetCondition(duckv1alpha1.ConditionSucceeded) diff --git a/pkg/reconciler/v1alpha1/pipelinerun/resources/input_output_steps.go b/pkg/reconciler/v1alpha1/pipelinerun/resources/input_output_steps.go new file mode 100644 index 00000000000..e48b098db66 --- /dev/null +++ b/pkg/reconciler/v1alpha1/pipelinerun/resources/input_output_steps.go @@ -0,0 +1,83 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package resources + +import ( + "path/filepath" + + "github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1" +) + +var ( + pvcDir = "/pvc" +) + +// GetOutputSteps function reads output task resources and constructs post task step +// postSteps contains array of resource names and named path(pvcdir + name of task + name of resource) under which the output resource will be dumped in PVC +func GetOutputSteps(taskResources []v1alpha1.TaskResourceBinding, taskName string) []v1alpha1.TaskRunResource { + var taskOutputResources []v1alpha1.TaskRunResource + + for _, outputRes := range taskResources { + taskOutputResources = append(taskOutputResources, v1alpha1.TaskRunResource{ + ResourceRef: outputRes.ResourceRef, + Name: outputRes.Name, + Paths: []string{filepath.Join(pvcDir, taskName, outputRes.Name)}, + }) + } + return taskOutputResources +} + +// GetInputSteps function reads input bindings and constructs pre build step +// with information to create build step to setup altered inputs. +func GetInputSteps(taskResources []v1alpha1.TaskResourceBinding, pt *v1alpha1.PipelineTask) []v1alpha1.TaskRunResource { + var taskInputResources []v1alpha1.TaskRunResource + + for _, inputResource := range taskResources { + taskInputResource := v1alpha1.TaskRunResource{ + ResourceRef: inputResource.ResourceRef, + Name: inputResource.Name, + } + + var stepSourceNames []string + for _, resourceDep := range pt.ResourceDependencies { + if resourceDep.Name == inputResource.Name { + for _, constr := range resourceDep.ProvidedBy { + stepSourceNames = append(stepSourceNames, filepath.Join(pvcDir, constr, inputResource.Name)) + } + } + } + if len(stepSourceNames) > 0 { + taskInputResource.Paths = append(taskInputResource.Paths, stepSourceNames...) + } + taskInputResources = append(taskInputResources, taskInputResource) + } + return taskInputResources +} + +// WrapSteps input and resources for taskrun along with presteps , poststeps +func WrapSteps(tr *v1alpha1.TaskRunSpec, pipelineResources []v1alpha1.PipelineTaskResource, pt *v1alpha1.PipelineTask) { + if pt == nil { + return + } + for _, prTask := range pipelineResources { + if prTask.Name == pt.Name { + // Add presteps to setup updated input + tr.Inputs.Resources = append(tr.Inputs.Resources, GetInputSteps(prTask.Inputs, pt)...) + // Add poststeps to setup outputs + tr.Outputs.Resources = append(tr.Outputs.Resources, GetOutputSteps(prTask.Outputs, prTask.Name)...) + } + } +} diff --git a/pkg/reconciler/v1alpha1/pipelinerun/resources/input_output_steps_test.go b/pkg/reconciler/v1alpha1/pipelinerun/resources/input_output_steps_test.go new file mode 100644 index 00000000000..bdb01f9be50 --- /dev/null +++ b/pkg/reconciler/v1alpha1/pipelinerun/resources/input_output_steps_test.go @@ -0,0 +1,195 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package resources_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1" + "github.com/knative/build-pipeline/pkg/reconciler/v1alpha1/pipelinerun/resources" +) + +func Test_GetOutputSteps(t *testing.T) { + tcs := []struct { + name string + taskResourceBinding []v1alpha1.TaskResourceBinding + expectedtaskOuputResources []v1alpha1.TaskRunResource + pipelineTaskName string + }{{ + name: "output", + taskResourceBinding: []v1alpha1.TaskResourceBinding{{ + Name: "test-output", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "resource1", + }, + }}, + expectedtaskOuputResources: []v1alpha1.TaskRunResource{{ + Name: "test-output", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + Paths: []string{"/pvc/test-taskname/test-output"}, + }}, + pipelineTaskName: "test-taskname", + }, { + name: "multiple-outputs", + taskResourceBinding: []v1alpha1.TaskResourceBinding{{ + Name: "test-output", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + }, { + Name: "test-output-2", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource2"}, + }}, + expectedtaskOuputResources: []v1alpha1.TaskRunResource{{ + Name: "test-output", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + Paths: []string{"/pvc/test-multiple-outputs/test-output"}, + }, { + Name: "test-output-2", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource2"}, + Paths: []string{"/pvc/test-multiple-outputs/test-output-2"}, + }}, + pipelineTaskName: "test-multiple-outputs", + }} + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + postTasks := resources.GetOutputSteps(tc.taskResourceBinding, tc.pipelineTaskName) + if d := cmp.Diff(postTasks, tc.expectedtaskOuputResources); d != "" { + t.Errorf("error comparing post steps: %s", d) + } + }) + } +} + +func Test_GetInputSteps(t *testing.T) { + tcs := []struct { + name string + taskResourceBinding []v1alpha1.TaskResourceBinding + pipelineTask *v1alpha1.PipelineTask + expectedtaskInputResources []v1alpha1.TaskRunResource + }{ + { + name: "task-with-a-constraint", + taskResourceBinding: []v1alpha1.TaskResourceBinding{{ + Name: "test-input", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + }}, + pipelineTask: &v1alpha1.PipelineTask{ + ResourceDependencies: []v1alpha1.ResourceDependency{{ + Name: "test-input", + ProvidedBy: []string{"prev-task-1"}, + }}, + }, + expectedtaskInputResources: []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + Name: "test-input", + Paths: []string{"/pvc/prev-task-1/test-input"}, + }}, + }, { + name: "task-with-no-input-constraint", + taskResourceBinding: []v1alpha1.TaskResourceBinding{{ + Name: "test-input", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "resource1", + }, + }}, + expectedtaskInputResources: []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + Name: "test-input", + }}, + pipelineTask: &v1alpha1.PipelineTask{ + Name: "sample-test-task", + ResourceDependencies: []v1alpha1.ResourceDependency{{ + Name: "test-input", + }}, + }, + }, { + name: "task-with-multiple-constraints", + taskResourceBinding: []v1alpha1.TaskResourceBinding{{ + Name: "test-input", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + }}, + pipelineTask: &v1alpha1.PipelineTask{ + ResourceDependencies: []v1alpha1.ResourceDependency{{ + Name: "test-input", + ProvidedBy: []string{"prev-task-1", "prev-task-2"}, + }}, + }, + expectedtaskInputResources: []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + Name: "test-input", + Paths: []string{"/pvc/prev-task-1/test-input", "/pvc/prev-task-2/test-input"}, + }}, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + taskInputResources := resources.GetInputSteps(tc.taskResourceBinding, tc.pipelineTask) + if d := cmp.Diff(tc.expectedtaskInputResources, taskInputResources); d != "" { + t.Errorf("error comparing task resource inputs: %s", d) + } + + }) + } +} + +func Test_WrapSteps(t *testing.T) { + taskRunSpec := &v1alpha1.TaskRunSpec{} + pipelineResources := []v1alpha1.PipelineTaskResource{{ + Name: "test-task", + Inputs: []v1alpha1.TaskResourceBinding{{ + Name: "test-input", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + }, { + Name: "test-input-2", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + }}, + Outputs: []v1alpha1.TaskResourceBinding{{ + Name: "test-output", + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + }}, + }} + + pt := &v1alpha1.PipelineTask{ + Name: "test-task", + ResourceDependencies: []v1alpha1.ResourceDependency{{ + Name: "test-input", + ProvidedBy: []string{"prev-task"}, + }}, + } + + resources.WrapSteps(taskRunSpec, pipelineResources, pt) + + expectedtaskInputResources := []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + Name: "test-input", + Paths: []string{"/pvc/prev-task/test-input"}, + }, { + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + Name: "test-input-2", + }} + expectedtaskOuputResources := []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{Name: "resource1"}, + Name: "test-output", + Paths: []string{"/pvc/test-task/test-output"}, + }} + + if d := cmp.Diff(taskRunSpec.Inputs.Resources, expectedtaskInputResources); d != "" { + t.Errorf("error comparing input resources: %s", d) + } + if d := cmp.Diff(taskRunSpec.Outputs.Resources, expectedtaskOuputResources); d != "" { + t.Errorf("error comparing output resources: %s", d) + } +} diff --git a/pkg/reconciler/v1alpha1/pipelinerun/resources/pipelinestate.go b/pkg/reconciler/v1alpha1/pipelinerun/resources/pipelinestate.go index 59223397447..d5f98f6c6bd 100644 --- a/pkg/reconciler/v1alpha1/pipelinerun/resources/pipelinestate.go +++ b/pkg/reconciler/v1alpha1/pipelinerun/resources/pipelinestate.go @@ -19,12 +19,11 @@ package resources import ( "fmt" + "github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1" duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - - "github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1" ) const ( diff --git a/pkg/reconciler/v1alpha1/pipelinerun/resources/pipelinestate_test.go b/pkg/reconciler/v1alpha1/pipelinerun/resources/pipelinestate_test.go index 3694b7e3279..880f944e8e6 100644 --- a/pkg/reconciler/v1alpha1/pipelinerun/resources/pipelinestate_test.go +++ b/pkg/reconciler/v1alpha1/pipelinerun/resources/pipelinestate_test.go @@ -19,10 +19,10 @@ package resources import ( "testing" - "go.uber.org/zap" - "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -247,7 +247,7 @@ func TestGetPipelineState(t *testing.T) { TaskRunName: "pipelinerun-mytask2", TaskRun: nil, }} - if d := cmp.Diff(pipelineState, expectedState); d != "" { + if d := cmp.Diff(pipelineState, expectedState, cmpopts.IgnoreUnexported(v1alpha1.TaskRunSpec{})); d != "" { t.Fatalf("Expected to get current pipeline state %v, but actual differed: %s", expectedState, d) } } diff --git a/pkg/reconciler/v1alpha1/pipelinerun/validate.go b/pkg/reconciler/v1alpha1/pipelinerun/validate.go index 808d5bd93b0..caee21a4f9a 100644 --- a/pkg/reconciler/v1alpha1/pipelinerun/validate.go +++ b/pkg/reconciler/v1alpha1/pipelinerun/validate.go @@ -67,6 +67,7 @@ func validatePipelineTaskAndTask(c *Reconciler, ptask v1alpha1.PipelineTask, tas inputMapping[r.Name] = rr.Spec.Type } } + for _, r := range outputs { outputMapping[r.Name] = "" // TODO(#213): if this is the empty string should it be an error? or maybe let the lookup fail? @@ -78,16 +79,15 @@ func validatePipelineTaskAndTask(c *Reconciler, ptask v1alpha1.PipelineTask, tas outputMapping[r.Name] = rr.Spec.Type } } - if task.Spec.Inputs != nil { for _, inputResource := range task.Spec.Inputs.Resources { inputResourceType, ok := inputMapping[inputResource.Name] if !ok { - return fmt.Errorf("resource %q not provided for pipeline task %q (task %q)", inputResource.Name, ptask.Name, task.Name) + return fmt.Errorf("input resource %q not provided for pipeline task %q (task %q)", inputResource.Name, ptask.Name, task.Name) } // Validate the type of resource match if inputResource.Type != inputResourceType { - return fmt.Errorf("resource %q for pipeline task %q (task %q) should be type %q but was %q", inputResource.Name, ptask.Name, task.Name, inputResourceType, inputResource.Type) + return fmt.Errorf("input resource %q for pipeline task %q (task %q) should be type %q but was %q", inputResource.Name, ptask.Name, task.Name, inputResourceType, inputResource.Type) } } for _, inputResourceParam := range task.Spec.Inputs.Params { @@ -102,10 +102,10 @@ func validatePipelineTaskAndTask(c *Reconciler, ptask v1alpha1.PipelineTask, tas for _, outputResource := range task.Spec.Outputs.Resources { outputResourceType, ok := outputMapping[outputResource.Name] if !ok { - return fmt.Errorf("resource %q not provided for pipeline task %q (task %q)", outputResource.Name, ptask.Name, task.Name) + return fmt.Errorf("output resource %q not provided for pipeline task %q (task %q)", outputResource.Name, ptask.Name, task.Name) } if outputResource.Type != outputResourceType { - return fmt.Errorf("resource %q for pipeline task %q (task %q) should be type %q but was %q", outputResource.Name, ptask.Name, task.Name, outputResourceType, outputResource.Type) + return fmt.Errorf("output resource %q for pipeline task %q (task %q) should be type %q but was %q", outputResource.Name, ptask.Name, task.Name, outputResourceType, outputResource.Type) } } } diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/build_step_test.go b/pkg/reconciler/v1alpha1/taskrun/resources/build_step_test.go new file mode 100644 index 00000000000..7904e79da2f --- /dev/null +++ b/pkg/reconciler/v1alpha1/taskrun/resources/build_step_test.go @@ -0,0 +1,219 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1" + "github.com/knative/build-pipeline/pkg/reconciler/v1alpha1/taskrun/resources" + buildv1alpha1 "github.com/knative/build/pkg/apis/build/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestPostBuildSteps(t *testing.T) { + taskrun := &v1alpha1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-taskrun-run-output-steps", + Namespace: "foo", + }, + Spec: v1alpha1.TaskRunSpec{ + TaskRef: v1alpha1.TaskRef{ + Name: "", + APIVersion: "a1", + }, + Outputs: v1alpha1.TaskRunOutputs{ + Resources: []v1alpha1.TaskRunResource{{ + Name: "source-workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "source-git", + }, + Paths: []string{"test-path"}, + }, { + Name: "source-git-workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "source-git-2", + }, + Paths: []string{"test-path-2"}, + }, { + Name: "non-existent-source-no-workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "source-git-non-existent", + }, + Paths: []string{"non-existent-path"}, + }}, + }, + }, + } + + b := &buildv1alpha1.Build{ + Spec: buildv1alpha1.BuildSpec{ + Sources: []buildv1alpha1.SourceSpec{{ + Name: "source-git", + }, { + Name: "source-git-2", + TargetPath: "prev-task", + }, { + Name: "source-git-no-step", + }}, + }, + } + + needPVC := resources.AddAfterSteps(taskrun.Spec, b, "pipelinerun-pvc") + + expectedSteps := []corev1.Container{{ + Name: "source-mkdir-source-git", + Image: "busybox", + Command: []string{"mkdir"}, + Args: []string{"-p", "test-path"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "pipelinerun-pvc", + MountPath: "/pvc", + }}, + }, { + Name: "source-copy-source-git", + Image: "busybox", + Command: []string{"cp"}, + Args: []string{"-r", "/workspace/.", "test-path"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "pipelinerun-pvc", + MountPath: "/pvc", + }}, + }, { + Name: "source-mkdir-source-git-2", + Image: "busybox", + Command: []string{"mkdir"}, + Args: []string{"-p", "test-path-2"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "pipelinerun-pvc", + MountPath: "/pvc", + }}, + }, { + Name: "source-copy-source-git-2", + Image: "busybox", + Command: []string{"cp"}, + Args: []string{"-r", "/workspace/prev-task/.", "test-path-2"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "pipelinerun-pvc", + MountPath: "/pvc", + }}, + }} + if !needPVC { + t.Errorf("post build steps need pvc should be true but got %t", needPVC) + } + if d := cmp.Diff(b.Spec.Steps, expectedSteps); d != "" { + t.Fatalf("post build steps mismatch: %s", d) + } +} + +func TestPreBuildSteps(t *testing.T) { + taskrun := &v1alpha1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-taskrun-run-input-steps", + Namespace: "foo", + }, + Spec: v1alpha1.TaskRunSpec{ + TaskRef: v1alpha1.TaskRef{ + Name: "", + APIVersion: "a1", + }, + Inputs: v1alpha1.TaskRunInputs{ + Resources: []v1alpha1.TaskRunResource{{ + Name: "source-workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "source-git", + }, + Paths: []string{"test-path"}, + }, { + Name: "source-git-workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "source-another-git", + }, + Paths: []string{"prev-task-1", "prev-task-2"}, + }, { + Name: "non-existent-source-no-workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "source-git-non-existent", + }, + Paths: []string{"non-existent-path"}, + }}, + }, + }, + } + b := &buildv1alpha1.Build{ + Spec: buildv1alpha1.BuildSpec{ + Sources: []buildv1alpha1.SourceSpec{{ + Name: "source-git", + }, { + Name: "source-another-git", + TargetPath: "new-workspace", + }, { + Name: "source-git-no-step", + }}, + }, + } + + needPVC := resources.AddBeforeSteps(taskrun.Spec, b, "pipelinerun-pvc") + expectedSteps := []corev1.Container{{ + Name: "source-copy-source-another-git-0", + Image: "busybox", + Command: []string{"cp"}, + Args: []string{"-r", "prev-task-1/.", "/workspace/new-workspace"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "pipelinerun-pvc", + MountPath: "/pvc", + }}, + }, { + Name: "source-copy-source-another-git-1", + Image: "busybox", + Command: []string{"cp"}, + Args: []string{"-r", "prev-task-2/.", "/workspace/new-workspace"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "pipelinerun-pvc", + MountPath: "/pvc", + }}, + }, { + Name: "source-copy-source-git-0", + Image: "busybox", + Command: []string{"cp"}, + Args: []string{"-r", "test-path/.", "/workspace"}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "pipelinerun-pvc", + MountPath: "/pvc", + }}, + }} + if !needPVC { + t.Errorf("pre build steps need pvc should be true but got %t", needPVC) + } + if d := cmp.Diff(b.Spec.Steps, expectedSteps); d != "" { + t.Fatalf("pre build steps mismatch: %s", d) + } +} + +func Test_PVC_Volume(t *testing.T) { + expectedVolume := corev1.Volume{ + Name: "test-pvc", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "test-pvc"}, + }, + } + if d := cmp.Diff(expectedVolume, resources.GetPVCVolume("test-pvc")); d != "" { + t.Fatalf("PVC volume mismatch: %s", d) + } +} diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go b/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go index 00a185395c5..e9aaee85753 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/input_resource_test.go @@ -191,12 +191,13 @@ func TestAddResourceToBuild(t *testing.T) { }}, }, Spec: buildv1alpha1.BuildSpec{ - Source: &buildv1alpha1.SourceSpec{ + Sources: []buildv1alpha1.SourceSpec{{ Git: &buildv1alpha1.GitSourceSpec{ Url: "https://github.com/grafeas/kritis", Revision: "master", }, - }, + Name: "the-git", + }}, }, }, }, { @@ -214,7 +215,7 @@ func TestAddResourceToBuild(t *testing.T) { Inputs: v1alpha1.TaskRunInputs{ Resources: []v1alpha1.TaskRunResource{{ ResourceRef: v1alpha1.PipelineResourceRef{ - Name: "the-git-with-branch", + Name: "the-git", }, Name: "workspace", }}, @@ -239,13 +240,65 @@ func TestAddResourceToBuild(t *testing.T) { }}, }, Spec: buildv1alpha1.BuildSpec{ - Source: &buildv1alpha1.SourceSpec{ + Sources: []buildv1alpha1.SourceSpec{{ Git: &buildv1alpha1.GitSourceSpec{ Url: "https://github.com/grafeas/kritis", - Revision: "branch", + Revision: "master", + }, + Name: "the-git", + }}, + }, + }, + }, { + desc: "set revision to default value", + task: task, + taskRun: &v1alpha1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "build-from-repo-run", + Namespace: "marshmallow", + }, + Spec: v1alpha1.TaskRunSpec{ + TaskRef: v1alpha1.TaskRef{ + Name: "simpleTask", + }, + Inputs: v1alpha1.TaskRunInputs{ + Resources: []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "the-git-with-branch", + }, + Name: "workspace", + }}, + }, + }, + }, + build: build(), + wantErr: false, + want: &buildv1alpha1.Build{ + TypeMeta: metav1.TypeMeta{ + Kind: "Build", + APIVersion: "build.knative.dev/v1alpha1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "build-from-repo", + Namespace: "marshmallow", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "pipeline.knative.dev/v1alpha1", + Kind: "TaskRun", + Name: "build-from-repo-run", + Controller: &boolTrue, + BlockOwnerDeletion: &boolTrue, }, }, }, + Spec: buildv1alpha1.BuildSpec{ + Sources: []buildv1alpha1.SourceSpec{{ + Git: &buildv1alpha1.GitSourceSpec{ + Url: "https://github.com/grafeas/kritis", + Revision: "branch", + }, + Name: "the-git-with-branch", + }}, + }, }, }, { desc: "invalid resource name", diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/input_resources.go b/pkg/reconciler/v1alpha1/taskrun/resources/input_resources.go index 0432597e09a..2b055ff20b5 100644 --- a/pkg/reconciler/v1alpha1/taskrun/resources/input_resources.go +++ b/pkg/reconciler/v1alpha1/taskrun/resources/input_resources.go @@ -75,7 +75,11 @@ func AddInputResource( Url: gitResource.URL, Revision: gitResource.Revision, } - build.Spec.Source = &buildv1alpha1.SourceSpec{Git: gitSourceSpec} + build.Spec.Sources = append(build.Spec.Sources, buildv1alpha1.SourceSpec{ + Git: gitSourceSpec, + TargetPath: input.TargetPath, + Name: gitResource.Name, + }) } case v1alpha1.PipelineResourceTypeCluster: clusterResource, err := v1alpha1.NewClusterResource(resource) @@ -85,7 +89,6 @@ func AddInputResource( addClusterBuildStep(build, clusterResource) } } - return build, nil } diff --git a/pkg/reconciler/v1alpha1/taskrun/resources/pre_post_build_step.go b/pkg/reconciler/v1alpha1/taskrun/resources/pre_post_build_step.go new file mode 100644 index 00000000000..ccc78f8798c --- /dev/null +++ b/pkg/reconciler/v1alpha1/taskrun/resources/pre_post_build_step.go @@ -0,0 +1,136 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "path/filepath" + + "fmt" + + "github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1" + buildv1alpha1 "github.com/knative/build/pkg/apis/build/v1alpha1" + corev1 "k8s.io/api/core/v1" +) + +var ( + pvcDir = "/pvc" + workspaceDir = "/workspace" +) + +// AddAfterSteps adds steps to build that can run after all build steps have completed +// if source has targetPath specified then step will use that path as source dir for copy command else +// it uses /workspace as default directory +func AddAfterSteps( + taskRunSpec v1alpha1.TaskRunSpec, + b *buildv1alpha1.Build, + pvcName string, +) bool { + var postBuildSteps = make(map[string][]string) + var needPVC bool + for _, postStepResource := range taskRunSpec.Outputs.Resources { + postBuildSteps[postStepResource.ResourceRef.Name] = postStepResource.Paths + } + for _, source := range b.Spec.Sources { + if paths, ok := postBuildSteps[source.Name]; ok { + var newSteps []corev1.Container + for _, path := range paths { + var sPath string + if source.TargetPath == "" { + sPath = workspaceDir + } else { + sPath = filepath.Join(workspaceDir, source.TargetPath) + } + newSteps = append(newSteps, []corev1.Container{{ + Name: fmt.Sprintf("source-mkdir-%s", source.Name), + Image: "busybox", + Command: []string{"mkdir"}, + Args: []string{"-p", path}, + VolumeMounts: []corev1.VolumeMount{getPvcMount(pvcName)}, + }, { + Name: fmt.Sprintf("source-copy-%s", source.Name), + Image: "busybox", + Command: []string{"cp"}, + Args: []string{"-r", fmt.Sprintf("%s/.", sPath), path}, + VolumeMounts: []corev1.VolumeMount{getPvcMount(pvcName)}, + }, + }...) + } + if len(newSteps) > 0 { + needPVC = true + b.Spec.Steps = append(b.Spec.Steps, newSteps...) + } + } + } + return needPVC +} + +// AddBeforeSteps adds steps to run after sources are mounted but before actual build steps to override +// the source folder with previous task's source +func AddBeforeSteps( + taskRunSpec v1alpha1.TaskRunSpec, + b *buildv1alpha1.Build, + pvcName string, +) bool { + var needPVC bool + var preSteps = make(map[string][]string) + for _, preStepResource := range taskRunSpec.Inputs.Resources { + preSteps[preStepResource.ResourceRef.Name] = preStepResource.Paths + } + + for _, source := range b.Spec.Sources { + if paths, ok := preSteps[source.Name]; ok { + var newSteps []corev1.Container + for i, path := range paths { + var dPath string + if source.TargetPath == "" { + dPath = workspaceDir + } else { + dPath = filepath.Join(workspaceDir, source.TargetPath) + } + newSteps = append(newSteps, corev1.Container{ + Name: fmt.Sprintf("source-copy-%s-%d", source.Name, i), + Image: "busybox", + Command: []string{"cp"}, + Args: []string{"-r", fmt.Sprintf("%s/.", path), dPath}, + VolumeMounts: []corev1.VolumeMount{getPvcMount(pvcName)}, + }) + } + if len(newSteps) > 0 { + needPVC = true + b.Spec.Steps = append(newSteps, b.Spec.Steps...) + } + } + } + return needPVC +} + +func getPvcMount(name string) corev1.VolumeMount { + return corev1.VolumeMount{ + Name: name, // taskrun pvc name + MountPath: pvcDir, // nothing should be mounted here + } +} + +// GetPVCVolume gets pipelinerun pvc +func GetPVCVolume(name string) corev1.Volume { + return corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: name}, + }, + } +} diff --git a/pkg/reconciler/v1alpha1/taskrun/taskrun.go b/pkg/reconciler/v1alpha1/taskrun/taskrun.go index 1deb9a7db3f..92e70ef1ef3 100644 --- a/pkg/reconciler/v1alpha1/taskrun/taskrun.go +++ b/pkg/reconciler/v1alpha1/taskrun/taskrun.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "github.com/knative/build-pipeline/pkg/apis/pipeline" @@ -209,7 +210,7 @@ func (c *Reconciler) reconcile(ctx context.Context, tr *v1alpha1.TaskRun) error pvc, err := c.KubeClientSet.CoreV1().PersistentVolumeClaims(tr.Namespace).Get(tr.Name, metav1.GetOptions{}) if errors.IsNotFound(err) { // Create a persistent volume claim to hold Build logs - pvc, err = c.createPVC(tr) + pvc, err = createPVC(c.KubeClientSet, tr) if err != nil { return fmt.Errorf("Failed to create persistent volume claim %s for task %q: %v", tr.Name, err, tr.Name) } @@ -301,10 +302,10 @@ func (c *Reconciler) updateStatus(taskrun *v1alpha1.TaskRun) (*v1alpha1.TaskRun, return newtaskrun, nil } -// createVolume will create a persistent volume mount for tr which +// createPVC will create a persistent volume mount for tr which // will be used to gather logs using the entrypoint wrapper -func (c *Reconciler) createPVC(tr *v1alpha1.TaskRun) (*corev1.PersistentVolumeClaim, error) { - v, err := c.KubeClientSet.CoreV1().PersistentVolumeClaims(tr.Namespace).Create( +func createPVC(kc kubernetes.Interface, tr *v1alpha1.TaskRun) (*corev1.PersistentVolumeClaim, error) { + v, err := kc.CoreV1().PersistentVolumeClaims(tr.Namespace).Create( &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: tr.Namespace, @@ -373,6 +374,13 @@ func (c *Reconciler) createBuild(ctx context.Context, tr *v1alpha1.TaskRun, pvcN return nil, err } + pipelineRunpvcName := tr.GetPipelineRunPVCName() + beforeStepsNeedPVC := resources.AddAfterSteps(tr.Spec, build, pipelineRunpvcName) + afterStepsNeedPVC := resources.AddBeforeSteps(tr.Spec, build, pipelineRunpvcName) + // attach PVC volume to build if output or inputs resource steps requir + if beforeStepsNeedPVC || afterStepsNeedPVC { + build.Spec.Volumes = append(build.Spec.Volumes, resources.GetPVCVolume(pipelineRunpvcName)) + } var defaults []v1alpha1.TaskParam if t.Spec.Inputs != nil { defaults = append(defaults, t.Spec.Inputs.Params...) @@ -421,6 +429,7 @@ func CreateRedirectedBuild(ctx context.Context, bs *buildv1alpha1.BuildSpec, pvc }, Spec: *bs, } + // Add the volume used for storing the binary and logs b.Spec.Volumes = append(b.Spec.Volumes, corev1.Volume{ Name: entrypoint.MountName, @@ -430,6 +439,7 @@ func CreateRedirectedBuild(ctx context.Context, bs *buildv1alpha1.BuildSpec, pvc }, }, }) + // Pass service account name from taskrun to build // if task specifies service account name override with taskrun SA b.Spec.ServiceAccountName = tr.Spec.ServiceAccount diff --git a/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go b/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go index 80720015cec..9d5b143a8d6 100644 --- a/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go +++ b/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go @@ -35,6 +35,7 @@ import ( "github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1" "github.com/knative/build-pipeline/pkg/reconciler/v1alpha1/taskrun" "github.com/knative/build-pipeline/pkg/reconciler/v1alpha1/taskrun/config" + "github.com/knative/build-pipeline/pkg/reconciler/v1alpha1/taskrun/resources" "github.com/knative/build-pipeline/test" ) @@ -104,6 +105,35 @@ var simpleTask = &v1alpha1.Task{ }, } +var outputTask = &v1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-output-task", + Namespace: "foo", + }, + Spec: v1alpha1.TaskSpec{ + Steps: []corev1.Container{{ + Name: "simple-step", + Image: "foo", + Command: []string{"/mycmd"}, + }}, + Inputs: &v1alpha1.Inputs{ + Resources: []v1alpha1.TaskResource{{ + Name: gitResource.Name, + Type: v1alpha1.PipelineResourceTypeGit, + }, { + Name: anotherGitResource.Name, + Type: v1alpha1.PipelineResourceTypeGit, + }}, + }, + Outputs: &v1alpha1.Outputs{ + Resources: []v1alpha1.TaskResource{{ + Name: gitResource.Name, + Type: v1alpha1.PipelineResourceTypeGit, + }}, + }, + }, +} + var saTask = &v1alpha1.Task{ ObjectMeta: metav1.ObjectMeta{ Name: "test-with-sa", @@ -193,6 +223,19 @@ var gitResource = &v1alpha1.PipelineResource{ }, } +var anotherGitResource = &v1alpha1.PipelineResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "another-git-resource", + Namespace: "foo", + }, + Spec: v1alpha1.PipelineResourceSpec{ + Type: "git", + Params: []v1alpha1.Param{{ + Name: "URL", + Value: "https://foobar.git", + }}, + }, +} var imageResource = &v1alpha1.PipelineResource{ ObjectMeta: metav1.ObjectMeta{ Name: "image-resource", @@ -300,15 +343,13 @@ func TestReconcile(t *testing.T) { Value: "foo", }, }, - Resources: []v1alpha1.TaskRunResource{ - { - ResourceRef: v1alpha1.PipelineResourceRef{ - Name: gitResource.Name, - APIVersion: "a1", - }, - Name: gitResource.Name, + Resources: []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: gitResource.Name, + APIVersion: "a1", }, - }, + Name: gitResource.Name, + }}, }, }, }, { @@ -322,201 +363,278 @@ func TestReconcile(t *testing.T) { APIVersion: "a1", }, Inputs: v1alpha1.TaskRunInputs{ - Resources: []v1alpha1.TaskRunResource{ - { - ResourceRef: v1alpha1.PipelineResourceRef{ - Name: gitResource.Name, - APIVersion: "a1", - }, + Resources: []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: gitResource.Name, + APIVersion: "a1", + }, + Name: gitResource.Name, + }}, + }, + }, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-taskrun-input-output", + Namespace: "foo", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "PipelineRun", + Name: "test", + }}, + }, + Spec: v1alpha1.TaskRunSpec{ + TaskRef: v1alpha1.TaskRef{ + Name: outputTask.Name, + }, + Inputs: v1alpha1.TaskRunInputs{ + Resources: []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{ Name: gitResource.Name, }, - }, + Name: gitResource.Name, + Paths: []string{"source-folder"}, + }, { + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: anotherGitResource.Name, + }, + Name: anotherGitResource.Name, + Paths: []string{"source-folder"}, + }}, + }, + Outputs: v1alpha1.TaskRunOutputs{ + Resources: []v1alpha1.TaskRunResource{{ + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: gitResource.Name, + }, + Name: gitResource.Name, + Paths: []string{"output-folder"}, + }}, }, }, }} d := test.Data{ TaskRuns: taskruns, - Tasks: []*v1alpha1.Task{simpleTask, saTask, templatedTask, defaultTemplatedTask}, - PipelineResources: []*v1alpha1.PipelineResource{gitResource, imageResource}, + Tasks: []*v1alpha1.Task{simpleTask, saTask, templatedTask, defaultTemplatedTask, outputTask}, + PipelineResources: []*v1alpha1.PipelineResource{gitResource, anotherGitResource, imageResource}, } testcases := []struct { name string taskRun *v1alpha1.TaskRun wantedBuildSpec buildv1alpha1.BuildSpec - }{ - { - name: "success", - taskRun: taskruns[0], - wantedBuildSpec: buildv1alpha1.BuildSpec{ - Steps: []corev1.Container{ - entrypointCopyStep, - { - Name: "simple-step", - Image: "foo", - Command: []string{entrypointLocation}, - Args: []string{}, - Env: []corev1.EnvVar{ - { - Name: "ENTRYPOINT_OPTIONS", - Value: `{"args":["/mycmd"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, - }, + }{{ + name: "success", + taskRun: taskruns[0], + wantedBuildSpec: buildv1alpha1.BuildSpec{ + Steps: []corev1.Container{ + entrypointCopyStep, + { + Name: "simple-step", + Image: "foo", + Command: []string{entrypointLocation}, + Args: []string{}, + Env: []corev1.EnvVar{ + { + Name: "ENTRYPOINT_OPTIONS", + Value: `{"args":["/mycmd"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, }, - VolumeMounts: []corev1.VolumeMount{toolsMount}, }, + VolumeMounts: []corev1.VolumeMount{toolsMount}, }, - Volumes: []corev1.Volume{ - getToolsVolume(taskruns[0].Name), - }, + }, + Volumes: []corev1.Volume{ + getToolsVolume(taskruns[0].Name), }, }, - { - name: "serviceaccount", - taskRun: taskruns[1], - wantedBuildSpec: buildv1alpha1.BuildSpec{ - ServiceAccountName: "test-sa", - Steps: []corev1.Container{ - entrypointCopyStep, - { - Name: "sa-step", - Image: "foo", - Command: []string{entrypointLocation}, - Args: []string{}, - Env: []corev1.EnvVar{ - { - Name: "ENTRYPOINT_OPTIONS", - Value: `{"args":["/mycmd"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, - }, + }, { + name: "serviceaccount", + taskRun: taskruns[1], + wantedBuildSpec: buildv1alpha1.BuildSpec{ + ServiceAccountName: "test-sa", + Steps: []corev1.Container{ + entrypointCopyStep, + { + Name: "sa-step", + Image: "foo", + Command: []string{entrypointLocation}, + Args: []string{}, + Env: []corev1.EnvVar{ + { + Name: "ENTRYPOINT_OPTIONS", + Value: `{"args":["/mycmd"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, }, - VolumeMounts: []corev1.VolumeMount{toolsMount}, }, + VolumeMounts: []corev1.VolumeMount{toolsMount}, }, - Volumes: []corev1.Volume{ - getToolsVolume(taskruns[1].Name), - }, + }, + Volumes: []corev1.Volume{ + getToolsVolume(taskruns[1].Name), }, }, - { - name: "params", - taskRun: taskruns[2], - wantedBuildSpec: buildv1alpha1.BuildSpec{ - Source: &buildv1alpha1.SourceSpec{ - Git: &buildv1alpha1.GitSourceSpec{ - Url: "https://foo.git", - Revision: "master", - }, + }, { + name: "params", + taskRun: taskruns[2], + wantedBuildSpec: buildv1alpha1.BuildSpec{ + Sources: []buildv1alpha1.SourceSpec{{ + Git: &buildv1alpha1.GitSourceSpec{ + Url: "https://foo.git", + Revision: "master", }, - Steps: []corev1.Container{ - entrypointCopyStep, - { - Name: "mycontainer", - Image: "myimage", - Command: []string{entrypointLocation}, - Args: []string{}, - Env: []corev1.EnvVar{ - { - Name: "ENTRYPOINT_OPTIONS", - Value: `{"args":["/mycmd","--my-arg=foo","--my-additional-arg=gcr.io/kristoff/sven"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, - }, + Name: "git-resource", + }}, + Steps: []corev1.Container{ + entrypointCopyStep, + { + Name: "mycontainer", + Image: "myimage", + Command: []string{entrypointLocation}, + Args: []string{}, + Env: []corev1.EnvVar{ + { + Name: "ENTRYPOINT_OPTIONS", + Value: `{"args":["/mycmd","--my-arg=foo","--my-additional-arg=gcr.io/kristoff/sven"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, }, - VolumeMounts: []corev1.VolumeMount{toolsMount}, }, - { - Name: "myothercontainer", - Image: "myotherimage", - Command: []string{entrypointLocation}, - Args: []string{}, - Env: []corev1.EnvVar{ - { - Name: "ENTRYPOINT_OPTIONS", - Value: `{"args":["/mycmd","--my-other-arg=https://foo.git"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, - }, + VolumeMounts: []corev1.VolumeMount{toolsMount}, + }, { + Name: "myothercontainer", + Image: "myotherimage", + Command: []string{entrypointLocation}, + Args: []string{}, + Env: []corev1.EnvVar{ + { + Name: "ENTRYPOINT_OPTIONS", + Value: `{"args":["/mycmd","--my-other-arg=https://foo.git"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, }, - VolumeMounts: []corev1.VolumeMount{toolsMount}, }, - }, - Volumes: []corev1.Volume{ - getToolsVolume(taskruns[2].Name), - }, + VolumeMounts: []corev1.VolumeMount{toolsMount}, + }}, + Volumes: []corev1.Volume{ + getToolsVolume(taskruns[2].Name), }, }, - { - name: "input-overrides-default-params", - taskRun: taskruns[3], - wantedBuildSpec: buildv1alpha1.BuildSpec{ - Steps: []corev1.Container{ - entrypointCopyStep, - { - Name: "mycontainer", - Image: "myimage", - Command: []string{entrypointLocation}, - Args: []string{}, - Env: []corev1.EnvVar{ - { - Name: "ENTRYPOINT_OPTIONS", - Value: `{"args":["/mycmd","--my-arg=foo"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, - }, + }, { + name: "input-overrides-default-params", + taskRun: taskruns[3], + wantedBuildSpec: buildv1alpha1.BuildSpec{ + Steps: []corev1.Container{ + entrypointCopyStep, + { + Name: "mycontainer", + Image: "myimage", + Command: []string{entrypointLocation}, + Args: []string{}, + Env: []corev1.EnvVar{ + { + Name: "ENTRYPOINT_OPTIONS", + Value: `{"args":["/mycmd","--my-arg=foo"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, }, - VolumeMounts: []corev1.VolumeMount{toolsMount}, }, - { - Name: "myothercontainer", - Image: "myotherimage", - Command: []string{entrypointLocation}, - Args: []string{}, - Env: []corev1.EnvVar{ - { - Name: "ENTRYPOINT_OPTIONS", - Value: `{"args":["/mycmd","--my-other-arg=https://foo.git"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, - }, + VolumeMounts: []corev1.VolumeMount{toolsMount}, + }, { + Name: "myothercontainer", + Image: "myotherimage", + Command: []string{entrypointLocation}, + Args: []string{}, + Env: []corev1.EnvVar{ + { + Name: "ENTRYPOINT_OPTIONS", + Value: `{"args":["/mycmd","--my-other-arg=https://foo.git"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, }, - VolumeMounts: []corev1.VolumeMount{toolsMount}, }, - }, - Volumes: []corev1.Volume{ - getToolsVolume(taskruns[3].Name), + VolumeMounts: []corev1.VolumeMount{toolsMount}, }, }, + Volumes: []corev1.Volume{getToolsVolume(taskruns[3].Name)}, }, - { - name: "default-params", - taskRun: taskruns[4], - wantedBuildSpec: buildv1alpha1.BuildSpec{ - Steps: []corev1.Container{ - entrypointCopyStep, - { - Name: "mycontainer", - Image: "myimage", - Command: []string{entrypointLocation}, - Args: []string{}, - Env: []corev1.EnvVar{ - { - Name: "ENTRYPOINT_OPTIONS", - Value: `{"args":["/mycmd","--my-arg=mydefault"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, - }, + }, { + name: "default-params", + taskRun: taskruns[4], + wantedBuildSpec: buildv1alpha1.BuildSpec{ + Steps: []corev1.Container{ + entrypointCopyStep, + { + Name: "mycontainer", + Image: "myimage", + Command: []string{entrypointLocation}, + Args: []string{}, + Env: []corev1.EnvVar{ + { + Name: "ENTRYPOINT_OPTIONS", + Value: `{"args":["/mycmd","--my-arg=mydefault"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, }, - VolumeMounts: []corev1.VolumeMount{toolsMount}, }, - { - Name: "myothercontainer", - Image: "myotherimage", - Command: []string{entrypointLocation}, - Args: []string{}, - Env: []corev1.EnvVar{ - { - Name: "ENTRYPOINT_OPTIONS", - Value: `{"args":["/mycmd","--my-other-arg=https://foo.git"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, - }, + VolumeMounts: []corev1.VolumeMount{toolsMount}, + }, { + Name: "myothercontainer", + Image: "myotherimage", + Command: []string{entrypointLocation}, + Args: []string{}, + Env: []corev1.EnvVar{ + { + Name: "ENTRYPOINT_OPTIONS", + Value: `{"args":["/mycmd","--my-other-arg=https://foo.git"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, }, - VolumeMounts: []corev1.VolumeMount{toolsMount}, }, + VolumeMounts: []corev1.VolumeMount{toolsMount}, + }, + }, + Volumes: []corev1.Volume{getToolsVolume(taskruns[4].Name)}, + }, + }, { + name: "wrap-steps", + taskRun: taskruns[5], + wantedBuildSpec: buildv1alpha1.BuildSpec{ + Sources: []buildv1alpha1.SourceSpec{{ + Git: &buildv1alpha1.GitSourceSpec{Url: "https://foo.git", Revision: "master"}, + Name: "git-resource", + }, { + Git: &buildv1alpha1.GitSourceSpec{Url: "https://foobar.git", Revision: "master"}, + Name: "another-git-resource", + }}, + Steps: []corev1.Container{ + { + Name: "source-copy-another-git-resource-0", + Image: "busybox", + Command: []string{"cp"}, Args: []string{"-r", "source-folder/.", "/workspace"}, + VolumeMounts: []corev1.VolumeMount{{Name: "test-pvc", MountPath: "/pvc"}}, }, - Volumes: []corev1.Volume{ - getToolsVolume(taskruns[4].Name), + { + Name: "source-copy-git-resource-0", + Image: "busybox", + Command: []string{"cp"}, Args: []string{"-r", "source-folder/.", "/workspace"}, + VolumeMounts: []corev1.VolumeMount{{Name: "test-pvc", MountPath: "/pvc"}}, }, + entrypointCopyStep, { + Name: "simple-step", + Image: "foo", + Command: []string{entrypointLocation}, + Args: []string{}, + Env: []corev1.EnvVar{ + { + Name: "ENTRYPOINT_OPTIONS", + Value: `{"args":["/mycmd"],"process_log":"/tools/process-log.txt","marker_file":"/tools/marker-file.txt"}`, + }, + }, + VolumeMounts: []corev1.VolumeMount{toolsMount}, + }, { + Name: "source-mkdir-git-resource", + Image: "busybox", + Command: []string{"mkdir"}, + Args: []string{"-p", "output-folder"}, + VolumeMounts: []corev1.VolumeMount{{Name: "test-pvc", MountPath: "/pvc"}}, + }, { + Name: "source-copy-git-resource", + Image: "busybox", + Command: []string{"cp"}, + Args: []string{"-r", "/workspace/.", "output-folder"}, + VolumeMounts: []corev1.VolumeMount{{Name: "test-pvc", MountPath: "/pvc"}}, + }}, + Volumes: []corev1.Volume{ + getToolsVolume(taskruns[5].Name), + resources.GetPVCVolume(taskruns[5].GetPipelineRunPVCName()), }, }, - } + }} for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { c, _, clients := test.GetTaskRunController(d) diff --git a/test/crd_checks.go b/test/crd_checks.go index 56bea537c2d..d3953aba850 100644 --- a/test/crd_checks.go +++ b/test/crd_checks.go @@ -73,12 +73,12 @@ func WaitForPodState(c *clients, name string, namespace string, inState func(r * // interval until inState returns `true` indicating it is done, returns an // error or timeout. desc will be used to name the metric that is emitted to // track how long it took for name to get into the state checked by inState. -func WaitForPipelineRunState(c *clients, name string, inState func(r *v1alpha1.PipelineRun) (bool, error), desc string) error { +func WaitForPipelineRunState(c *clients, name string, polltimeout time.Duration, inState func(r *v1alpha1.PipelineRun) (bool, error), desc string) error { metricName := fmt.Sprintf("WaitForPipelineRunState/%s/%s", name, desc) _, span := trace.StartSpan(context.Background(), metricName) defer span.End() - return wait.PollImmediate(interval, timeout, func() (bool, error) { + return wait.PollImmediate(interval, polltimeout, func() (bool, error) { r, err := c.PipelineRunClient.Get(name, metav1.GetOptions{}) if err != nil { return true, err diff --git a/test/helm_task_test.go b/test/helm_task_test.go index 4f0c7c4427f..97ca1bc3994 100644 --- a/test/helm_task_test.go +++ b/test/helm_task_test.go @@ -85,7 +85,7 @@ func TestHelmDeployPipelineRun(t *testing.T) { } // Verify status of PipelineRun (wait for it) - if err := WaitForPipelineRunState(c, helmDeployPipelineRunName, func(pr *v1alpha1.PipelineRun) (bool, error) { + if err := WaitForPipelineRunState(c, helmDeployPipelineRunName, timeout, func(pr *v1alpha1.PipelineRun) (bool, error) { c := pr.Status.GetCondition(duckv1alpha1.ConditionSucceeded) if c != nil { if c.Status == corev1.ConditionTrue { @@ -255,8 +255,8 @@ func getHelmDeployPipeline(namespace string) *v1alpha1.Pipeline { Name: helmDeployTaskName, }, ResourceDependencies: []v1alpha1.ResourceDependency{{ - Name: "workspace", - ProvidedBy: []string{createImageTaskName}, + Name: "workspace", + // ProvidedBy: []string{createImageTaskName}, //TODO: https://github.com/knative/build-pipeline/issues/148 }}, Params: []v1alpha1.Param{{ Name: "pathToHelmCharts", diff --git a/test/pipelinerun_test.go b/test/pipelinerun_test.go index f442d1ef99a..bd9002684c2 100644 --- a/test/pipelinerun_test.go +++ b/test/pipelinerun_test.go @@ -34,33 +34,42 @@ import ( "github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1" ) +var ( + pipelineRunTimeout = 10 * time.Minute +) + func TestPipelineRun(t *testing.T) { type tests struct { name string testSetup func(c *clients, namespace string, index int) expectedTaskRuns []string expectedNumberOfEvents int + pipelineRunFunc func(int, string) *v1alpha1.PipelineRun } tds := []tests{{ - name: "multiple tasks", + name: "fan-in and fan-out", testSetup: func(c *clients, namespace string, index int) { t.Helper() - task := getHelloWorldTask(namespace, []string{"echo", taskOutput}) - task.Name = getName(hwTaskName, index) - if _, err := c.TaskClient.Create(task); err != nil { - t.Fatalf("Failed to create Task `%s`: %s", task.Name, err) + for _, task := range getFanInFanOutTasks(namespace) { + if _, err := c.TaskClient.Create(task); err != nil { + t.Fatalf("Failed to create Task `%s`: %s", task.Name, err) + } } - if _, err := c.PipelineClient.Create(getHelloWorldPipeline(index, namespace)); err != nil { - t.Fatalf("Failed to create Pipeline `%s`: %s", getName(hwPipelineName, index), err) + if _, err := c.PipelineResourceClient.Create(getFanInFanOutGitResource(namespace)); err != nil { + t.Fatalf("Failed to create Pipeline Resource `%s`: %s", kanikoResourceName, err) } if _, err := c.PipelineParamsClient.Create(getHelloWorldPipelineParams(index, namespace)); err != nil { t.Fatalf("Failed to create PipelineParams `%s`: %s", getName(hwPipelineParamsName, index), err) } + if _, err := c.PipelineClient.Create(getFanInFanOutPipeline(index, namespace)); err != nil { + t.Fatalf("Failed to create Pipeline `%s`: %s", getName(hwPipelineName, index), err) + } }, - expectedTaskRuns: []string{hwPipelineTaskName1, hwPipelineTaskName2}, - // 1 from PipelineRun and 2 from Tasks defined in pipelinerun - expectedNumberOfEvents: 3, + pipelineRunFunc: getFanInFanOutPipelineRun, + expectedTaskRuns: []string{"create-file-kritis", "create-fan-out-1", "create-fan-out-2", "check-fan-in"}, + // 1 from PipelineRun and 4 from Tasks defined in pipelinerun + expectedNumberOfEvents: 5, }, { name: "service account propagation", testSetup: func(c *clients, namespace string, index int) { @@ -119,6 +128,7 @@ func TestPipelineRun(t *testing.T) { expectedTaskRuns: []string{hwPipelineTaskName1}, // 1 from PipelineRun and 1 from Tasks defined in pipelinerun expectedNumberOfEvents: 2, + pipelineRunFunc: getHelloWorldPipelineRun, }} for i, td := range tds { @@ -138,12 +148,12 @@ func TestPipelineRun(t *testing.T) { td.testSetup(c, namespace, i) prName := fmt.Sprintf("%s%d", hwPipelineRunName, i) - if _, err := c.PipelineRunClient.Create(getHelloWorldPipelineRun(i, namespace)); err != nil { + if _, err := c.PipelineRunClient.Create(td.pipelineRunFunc(i, namespace)); err != nil { t.Fatalf("Failed to create PipelineRun `%s`: %s", prName, err) } logger.Infof("Waiting for PipelineRun %s in namespace %s to complete", prName, namespace) - if err := WaitForPipelineRunState(c, prName, func(tr *v1alpha1.PipelineRun) (bool, error) { + if err := WaitForPipelineRunState(c, prName, pipelineRunTimeout, func(tr *v1alpha1.PipelineRun) (bool, error) { c := tr.Status.GetCondition(duckv1alpha1.ConditionSucceeded) if c != nil { if c.IsTrue() { @@ -186,11 +196,13 @@ func TestPipelineRun(t *testing.T) { logger.Infof("Making sure %d events were created from taskrun and pipelinerun with kinds %v", td.expectedNumberOfEvents, matchKinds) - events := collectMatchingEvents(t, c.KubeClient, namespace, matchKinds, "Succeeded") + events, err := collectMatchingEvents(c.KubeClient, namespace, matchKinds, "Succeeded") + if err != nil { + t.Fatalf("Failed to collect matching events: %q", err) + } if len(events) != td.expectedNumberOfEvents { - t.Fatalf("Expected %d number of successful events from pipelinerun and taskrun but got %d", td.expectedNumberOfEvents, len(events)) + t.Fatalf("Expected %d number of successful events from pipelinerun and taskrun but got %d; list of receieved events : %#v", td.expectedNumberOfEvents, len(events), events) } - logger.Infof("Successfully finished test %q", td.name) }) } @@ -213,7 +225,115 @@ func getHelloWorldPipelineWithSingularTask(suffix int, namespace string) *v1alph } } -func getHelloWorldPipeline(suffix int, namespace string) *v1alpha1.Pipeline { +func getFanInFanOutTasks(namespace string) []*v1alpha1.Task { + return []*v1alpha1.Task{ + &v1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "create-file", + }, + Spec: v1alpha1.TaskSpec{ + Inputs: &v1alpha1.Inputs{ + Resources: []v1alpha1.TaskResource{{ + Name: "workspace", + Type: v1alpha1.PipelineResourceTypeGit, + TargetPath: "brandnewspace", + }}, + }, + Steps: []corev1.Container{{ + Name: "write-data-task-0-step-0", + Image: "ubuntu", + Command: []string{"/bin/bash"}, + Args: []string{"-c", "echo stuff > /workspace/brandnewspace/stuff"}, + }, { + Name: "write-data-task-0-step-1", + Image: "ubuntu", + Command: []string{"/bin/bash"}, + Args: []string{"-c", "echo other > /workspace/brandnewspace/other"}, + }}, + }, + }, &v1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "check-create-files-exists", + }, + Spec: v1alpha1.TaskSpec{ + Inputs: &v1alpha1.Inputs{ + Resources: []v1alpha1.TaskResource{{ + Name: "workspace", + Type: v1alpha1.PipelineResourceTypeGit, + }}, + }, + Steps: []corev1.Container{{ + Name: "read-from-task-0", + Image: "ubuntu", + Command: []string{"bash"}, + Args: []string{"-c", "[[ stuff == $(cat /workspace/stuff) ]]"}, + }, { + Name: "write-data-task-1", + Image: "ubuntu", + Command: []string{"/bin/bash"}, + Args: []string{"-c", "echo something > /workspace/something"}, + }}, + }, + }, &v1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "check-create-files-exists-2", + }, + Spec: v1alpha1.TaskSpec{ + Inputs: &v1alpha1.Inputs{ + Resources: []v1alpha1.TaskResource{{ + Name: "workspace", + Type: v1alpha1.PipelineResourceTypeGit, + }}, + }, + Steps: []corev1.Container{{ + Name: "read-from-task-0", + Image: "ubuntu", + Command: []string{"bash"}, + Args: []string{"-c", "[[ other == $(cat /workspace/other) ]]"}, + }, { + Name: "write-data-task-1", + Image: "ubuntu", + Command: []string{"/bin/bash"}, + Args: []string{"-c", "echo else > /workspace/else"}, + }}, + }, + }, &v1alpha1.Task{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "read-files", + }, + Spec: v1alpha1.TaskSpec{ + Inputs: &v1alpha1.Inputs{ + Resources: []v1alpha1.TaskResource{{ + Name: "workspace", + Type: v1alpha1.PipelineResourceTypeGit, + TargetPath: "readingspace", + }}, + }, + Steps: []corev1.Container{{ + Name: "read-from-task-0", + Image: "ubuntu", + Command: []string{"bash"}, + Args: []string{"-c", "[[ stuff == $(cat /workspace/readingspace/stuff) ]]"}, + }, { + Name: "read-from-task-1", + Image: "ubuntu", + Command: []string{"bash"}, + Args: []string{"-c", "[[ something == $(cat /workspace/readingspace/something) ]]"}, + }, { + Name: "read-from-task-2", + Image: "ubuntu", + Command: []string{"bash"}, + Args: []string{"-c", "[[ else == $(cat /workspace/readingspace/else) ]]"}, + }}, + }, + }} +} + +func getFanInFanOutPipeline(suffix int, namespace string) *v1alpha1.Pipeline { return &v1alpha1.Pipeline{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, @@ -221,15 +341,56 @@ func getHelloWorldPipeline(suffix int, namespace string) *v1alpha1.Pipeline { }, Spec: v1alpha1.PipelineSpec{ Tasks: []v1alpha1.PipelineTask{{ - Name: hwPipelineTaskName1, + Name: "create-file-kritis", TaskRef: v1alpha1.TaskRef{ - Name: getName(hwTaskName, suffix), + Name: "create-file", }, }, { - Name: hwPipelineTaskName2, + Name: "create-fan-out-1", TaskRef: v1alpha1.TaskRef{ - Name: getName(hwTaskName, suffix), + Name: "check-create-files-exists", }, + ResourceDependencies: []v1alpha1.ResourceDependency{{ + Name: "workspace", + ProvidedBy: []string{"create-file-kritis"}, + }}, + }, { + Name: "create-fan-out-2", + TaskRef: v1alpha1.TaskRef{ + Name: "check-create-files-exists-2", + }, + ResourceDependencies: []v1alpha1.ResourceDependency{{ + Name: "workspace", + ProvidedBy: []string{"create-file-kritis"}, + }}, + }, { + Name: "check-fan-in", + TaskRef: v1alpha1.TaskRef{ + Name: "read-files", + }, + ResourceDependencies: []v1alpha1.ResourceDependency{{ + Name: "workspace", + ProvidedBy: []string{"create-fan-out-2", "create-fan-out-1"}, + }}, + }}, + }, + } +} + +func getFanInFanOutGitResource(namespace string) *v1alpha1.PipelineResource { + return &v1alpha1.PipelineResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kritis-resource-git", + Namespace: namespace, + }, + Spec: v1alpha1.PipelineResourceSpec{ + Type: v1alpha1.PipelineResourceTypeGit, + Params: []v1alpha1.Param{{ + Name: "Url", + Value: "https://github.com/grafeas/kritis", + }, { + Name: "Revision", + Value: "master", }}, }, } @@ -256,6 +417,74 @@ func getPipelineRunServiceAccount(suffix int, namespace string) *corev1.ServiceA }}, } } +func getFanInFanOutPipelineRun(suffix int, namespace string) *v1alpha1.PipelineRun { + return &v1alpha1.PipelineRun{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: getName(hwPipelineRunName, suffix), + }, + Spec: v1alpha1.PipelineRunSpec{ + PipelineRef: v1alpha1.PipelineRef{ + Name: getName(hwPipelineName, suffix), + }, + PipelineTriggerRef: v1alpha1.PipelineTriggerRef{ + Type: v1alpha1.PipelineTriggerTypeManual, + }, + PipelineTaskResources: []v1alpha1.PipelineTaskResource{ + { + Name: "create-file-kritis", + Inputs: []v1alpha1.TaskResourceBinding{{ + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "kritis-resource-git", + }, + }}, + Outputs: []v1alpha1.TaskResourceBinding{{ + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "kritis-resource-git", + }, + }}, + }, { + Name: "create-fan-out-1", + Inputs: []v1alpha1.TaskResourceBinding{{ + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "kritis-resource-git", + }, + }}, + Outputs: []v1alpha1.TaskResourceBinding{{ + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "kritis-resource-git", + }, + }}, + }, { + Name: "create-fan-out-2", + Inputs: []v1alpha1.TaskResourceBinding{{ + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "kritis-resource-git", + }, + }}, + Outputs: []v1alpha1.TaskResourceBinding{{ + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "kritis-resource-git", + }, + }}, + }, { + Name: "check-fan-in", + Inputs: []v1alpha1.TaskResourceBinding{{ + Name: "workspace", + ResourceRef: v1alpha1.PipelineResourceRef{ + Name: "kritis-resource-git", + }, + }}, + }}, + }, + } +} func getPipelineRunSecret(suffix int, namespace string) *corev1.Secret { // Generated by: @@ -319,14 +548,14 @@ func newHostPathType(pathType string) *corev1.HostPathType { // collectMatchingEvents collects list of events under 5 seconds that match // 1. matchKinds which is a map of Kind of Object with name of objects // 2. reason which is the expected reason of event -func collectMatchingEvents(t *testing.T, kubeClient *knativetest.KubeClient, namespace string, kinds map[string][]string, reason string) []*corev1.Event { +func collectMatchingEvents(kubeClient *knativetest.KubeClient, namespace string, kinds map[string][]string, reason string) ([]*corev1.Event, error) { var events []*corev1.Event watchEvents, err := kubeClient.Kube.CoreV1().Events(namespace).Watch(metav1.ListOptions{}) // close watchEvents channel defer watchEvents.Stop() if err != nil { - t.Errorf("failed to create watch on events: %v", err) + return events, err } // create timer to not wait for events longer than 5 seconds @@ -344,7 +573,7 @@ func collectMatchingEvents(t *testing.T, kubeClient *knativetest.KubeClient, nam } } case <-timer.C: - return events + return events, nil } } }