feat(pipelineloop) : Support latest tektoncd pipeline version. (#602)

* Support latest tektoncd pipeline version. * update readme for support latest tektoncd pipeline version.
kubeflow · Jun 2, 2021 · 052726a · 052726a
1 parent 7c42346
commit 052726a
Show file tree

Hide file tree

Showing 7 changed files with 255 additions and 32 deletions.
diff --git a/tekton-catalog/pipeline-loops/README.md b/tekton-catalog/pipeline-loops/README.md
@@ -52,4 +52,90 @@
 - Edit feature-flags configmap, ensure "data.enable-custom-tasks" is "true":
 `kubectl edit cm feature-flags -n tekton-pipelines`
 
-- Run the E2E example: `kubectl apply -f examples/loop-example-basic.yaml`
+- Run the E2E example: 
+
+  `kubectl apply -f examples/loop-example-basic.yaml`
+
+
+- Tekton now supports custom task as embedded spec, it requires tekton version >= v0.25
+
+  - Install Tekton version >= v0.25
+  Or directly from source as,
+  ```
+    git clone https://github.com/tektoncd/pipeline.git
+    cd pipeline
+    make apply
+  ```
+
+  - To use the `taskSpec` example as below
+
+    e.g.
+
+    ```yaml
+    apiVersion: tekton.dev/v1beta1
+    kind: PipelineRun
+    metadata:
+      name: pr-loop-example
+    spec:
+      pipelineSpec:
+        tasks:
+          - name: first-task
+            taskSpec:
+              steps:
+                - name: echo
+                  image: ubuntu
+                  imagePullPolicy: IfNotPresent
+                  script: |
+                    #!/usr/bin/env bash
+                    echo "I am the first task before the loop task"
+          - name: loop-task
+            runAfter:
+              - first-task
+            params:
+              - name: message
+                value:
+                  - I am the first one
+                  - I am the second one
+                  - I am the third one
+            taskSpec:
+              apiVersion: custom.tekton.dev/v1alpha1
+              kind: PipelineLoop
+              spec: # Following is the embedded spec.
+                iterateParam: message
+                pipelineSpec:
+                  params:
+                    - name: message
+                      type: string
+                  tasks:
+                    - name: echo-loop-task
+                      params:
+                        - name: message
+                          value: $(params.message)
+                      taskSpec:
+                        params:
+                          - name: message
+                            type: string
+                        steps:
+                          - name: echo
+                            image: ubuntu
+                            imagePullPolicy: IfNotPresent
+                            script: |
+                              #!/usr/bin/env bash
+                              echo "$(params.message)"
+          - name: last-task
+            runAfter:
+              - loop-task
+            taskSpec:
+              steps:
+                - name: echo
+                  image: ubuntu
+                  imagePullPolicy: IfNotPresent
+                  script: |
+                    #!/usr/bin/env bash
+                    echo "I am the last task after the loop task"
+    ```
+
+  - To run the above example:
+
+    `kubectl apply -f examples/loop-example-basic_taskspec.yaml`
+
diff --git a/tekton-catalog/pipeline-loops/cmd/webhook/main.go b/tekton-catalog/pipeline-loops/cmd/webhook/main.go
@@ -23,14 +23,14 @@ import (
 	pipelineloopv1alpha1 "github.com/kubeflow/kfp-tekton/tekton-catalog/pipeline-loops/pkg/apis/pipelineloop/v1alpha1"
 	defaultconfig "github.com/tektoncd/pipeline/pkg/apis/config"
 	"github.com/tektoncd/pipeline/pkg/contexts"
-	"github.com/tektoncd/pipeline/pkg/system"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"knative.dev/pkg/configmap"
 	"knative.dev/pkg/controller"
 	"knative.dev/pkg/injection"
 	"knative.dev/pkg/injection/sharedmain"
 	"knative.dev/pkg/logging"
 	"knative.dev/pkg/signals"
+	"knative.dev/pkg/system"
 	"knative.dev/pkg/webhook"
 	"knative.dev/pkg/webhook/certificates"
 	"knative.dev/pkg/webhook/resourcesemantics"
@@ -111,7 +111,7 @@ func main() {
 	}
 
 	// Scope informers to the webhook's namespace instead of cluster-wide
-	ctx := injection.WithNamespaceScope(signals.NewContext(), system.GetNamespace())
+	ctx := injection.WithNamespaceScope(signals.NewContext(), system.Namespace())
 
 	// Set up a signal context with our webhook options
 	ctx = webhook.WithOptions(ctx, webhook.Options{

diff --git a/tekton-catalog/pipeline-loops/examples/loop-example-basic_taskspec.yaml b/tekton-catalog/pipeline-loops/examples/loop-example-basic_taskspec.yaml
@@ -0,0 +1,61 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: pr-loop-example
+spec:
+  pipelineSpec:
+    tasks:
+      - name: first-task
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              imagePullPolicy: IfNotPresent
+              script: |
+                #!/usr/bin/env bash
+                echo "I am the first task before the loop task"
+      - name: loop-task
+        runAfter:
+          - first-task
+        params:
+          - name: message
+            value:
+              - I am the first one
+              - I am the second one
+              - I am the third one
+        taskSpec:
+          apiVersion: custom.tekton.dev/v1alpha1
+          kind: PipelineLoop
+          spec:
+            iterateParam: message
+            pipelineSpec:
+              params:
+                - name: message
+                  type: string
+              tasks:
+                - name: echo-loop-task
+                  params:
+                    - name: message
+                      value: $(params.message)
+                  taskSpec:
+                    params:
+                      - name: message
+                        type: string
+                    steps:
+                      - name: echo
+                        image: ubuntu
+                        imagePullPolicy: IfNotPresent
+                        script: |
+                          #!/usr/bin/env bash
+                          echo "$(params.message)"
+      - name: last-task
+        runAfter:
+          - loop-task
+        taskSpec:
+          steps:
+            - name: echo
+              image: ubuntu
+              imagePullPolicy: IfNotPresent
+              script: |
+                #!/usr/bin/env bash
+                echo "I am the last task after the loop task"
diff --git a/tekton-catalog/pipeline-loops/examples/simplepipelineloop_taskspec.yaml b/tekton-catalog/pipeline-loops/examples/simplepipelineloop_taskspec.yaml
@@ -0,0 +1,58 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: simpletask
+spec:
+  params:
+    - name: word
+      type: string
+    - name: suffix
+      type: string
+  steps:
+    - name: echo
+      image: ubuntu
+      script: |
+        #!/usr/bin/env bash
+        echo "$(params.word)$(params.suffix)"
+---
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: demo-pipeline
+spec:
+  params:
+  - name: word
+    default: "wordi"
+  - name: suffix
+    default: "suffixi"
+  tasks:
+  - name: simplepipelinetask
+    taskRef:
+      name: simpletask
+    params:
+    - name: word
+      value: $(params.word)
+    - name: suffix
+      value: $(params.suffix)
+---
+apiVersion: tekton.dev/v1alpha1
+kind: Run
+metadata:
+  name: simpletasklooprun02
+spec:
+  params:
+    - name: word
+      value:
+        - jump
+        - land
+        - roll
+    - name: suffix
+      value: ing
+  spec:
+    apiVersion: custom.tekton.dev/v1alpha1
+    kind: PipelineLoop
+    spec:
+      pipelineRef:
+        name: demo-pipeline
+      iterateParam: word
+      timeout: 60s
diff --git a/tekton-catalog/pipeline-loops/go.mod b/tekton-catalog/pipeline-loops/go.mod
@@ -3,31 +3,19 @@ module github.com/kubeflow/kfp-tekton/tekton-catalog/pipeline-loops
 go 1.13
 
 require (
-	github.com/google/go-cmp v0.5.4
+	github.com/google/go-cmp v0.5.5
 	github.com/hashicorp/go-multierror v1.1.0
-	github.com/tektoncd/pipeline v0.20.0
+	github.com/tektoncd/pipeline v0.24.2-0.20210526163752-2d05c9b8a427
 	go.uber.org/zap v1.16.0
 	gomodules.xyz/jsonpatch/v2 v2.1.0
-	k8s.io/api v0.18.12
-	k8s.io/apimachinery v0.19.0
-	k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible
-	knative.dev/pkg v0.0.0-20210107022335-51c72e24c179
+	k8s.io/api v0.19.7
+	k8s.io/apimachinery v0.19.7
+	k8s.io/client-go v0.19.7
+	knative.dev/pkg v0.0.0-20210331065221-952fdd90dbb0
 )
 
-// Knative deps
+// Knative deps (release-0.20)
 replace (
-	contrib.go.opencensus.io/exporter/stackdriver => contrib.go.opencensus.io/exporter/stackdriver v0.12.9-0.20191108183826-59d068f8d8ff
+	contrib.go.opencensus.io/exporter/stackdriver => contrib.go.opencensus.io/exporter/stackdriver v0.13.4
 	github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v38.2.0+incompatible
-	github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.4.0+incompatible
-)
-
-// Pin k8s deps to v0.18.8
-replace (
-	k8s.io/api => k8s.io/api v0.18.8
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.8
-	k8s.io/apimachinery => k8s.io/apimachinery v0.18.8
-	k8s.io/apiserver => k8s.io/apiserver v0.18.8
-	k8s.io/client-go => k8s.io/client-go v0.18.8
-	k8s.io/code-generator => k8s.io/code-generator v0.18.8
-	k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29
 )
diff --git a/tekton-catalog/pipeline-loops/pkg/reconciler/pipelinelooprun/pipelinelooprun.go b/tekton-catalog/pipeline-loops/pkg/reconciler/pipelinelooprun/pipelinelooprun.go
@@ -85,16 +85,29 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, run *v1alpha1.Run) pkgre
 	var merr error
 	logger := logging.FromContext(ctx)
 	logger.Infof("Reconciling Run %s/%s at %v", run.Namespace, run.Name, time.Now())
-
+	if run.Spec.Ref != nil && run.Spec.Spec != nil {
+		logger.Errorf("Run %s/%s can provide one of Run.Spec.Ref/Run.Spec.Spec", run.Namespace, run.Name)
+	}
+	if run.Spec.Spec == nil && run.Spec.Ref == nil {
+		logger.Errorf("Run %s/%s does not provide a spec or ref.", run.Namespace, run.Name)
+		return nil
+	}
 	// Check that the Run references a PipelineLoop CRD.  The logic is controller.go should ensure that only this type of Run
 	// is reconciled this controller but it never hurts to do some bullet-proofing.
-	if run.Spec.Ref == nil ||
-		run.Spec.Ref.APIVersion != pipelineloopv1alpha1.SchemeGroupVersion.String() ||
-		run.Spec.Ref.Kind != pipelineloop.PipelineLoopControllerName {
-		logger.Errorf("Received control for a Run %s/%s that does not reference a PipelineLoop custom CRD", run.Namespace, run.Name)
+	if run.Spec.Ref != nil &&
+		(run.Spec.Ref.APIVersion != pipelineloopv1alpha1.SchemeGroupVersion.String() ||
+			run.Spec.Ref.Kind != pipelineloop.PipelineLoopControllerName) {
+		logger.Errorf("Received control for a Run %s/%s/%v that does not reference a PipelineLoop custom CRD ref", run.Namespace, run.Name, run.Spec.Ref)
 		return nil
 	}
 
+	if run.Spec.Spec != nil &&
+		(run.Spec.Spec.APIVersion != pipelineloopv1alpha1.SchemeGroupVersion.String() ||
+			run.Spec.Spec.Kind != pipelineloop.PipelineLoopControllerName) {
+		logger.Errorf("Received control for a Run %s/%s that does not reference a PipelineLoop custom CRD spec", run.Namespace, run.Name)
+		return nil
+	}
+	logger.Infof("Received control for a Run %s/%s %-v", run.Namespace, run.Name, run.Spec.Spec)
 	// If the Run has not started, initialize the Condition and set the start time.
 	if !run.HasStarted() {
 		logger.Infof("Starting new Run %s/%s", run.Namespace, run.Name)
@@ -303,6 +316,23 @@ func (c *Reconciler) getPipelineLoop(ctx context.Context, run *v1alpha1.Run) (*m
 		}
 		pipelineLoopMeta = tl.ObjectMeta
 		pipelineLoopSpec = tl.Spec
+	} else if run.Spec.Spec != nil {
+		err := json.Unmarshal(run.Spec.Spec.Spec.Raw, &pipelineLoopSpec)
+		if err != nil {
+			run.Status.MarkRunFailed(pipelineloopv1alpha1.PipelineLoopRunReasonCouldntGetPipelineLoop.String(),
+				"Error unmarshal PipelineLoop spec for Run %s/%s: %s",
+				run.Namespace, run.Name, err)
+			return nil, nil, fmt.Errorf("Error unmarshal PipelineLoop spec for Run %s: %w", fmt.Sprintf("%s/%s", run.Namespace, run.Name), err)
+		}
+		pipelineloopObject := &pipelineloopv1alpha1.PipelineLoop{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       run.Spec.Spec.Kind,
+				APIVersion: run.Spec.Spec.APIVersion,
+			},
+			Spec: pipelineLoopSpec,
+		}
+		pipelineloopObject.Name = run.Name
+		pipelineLoopMeta = metav1.ObjectMeta{Name: run.Name}
 	} else {
 		// Run does not require name but for PipelineLoop it does.
 		run.Status.MarkRunFailed(pipelineloopv1alpha1.PipelineLoopRunReasonCouldntGetPipelineLoop.String(),
@@ -623,9 +653,9 @@ func Find(slice []string, val string) (int, bool) {
 func getPipelineRunLabels(run *v1alpha1.Run, iterationStr string) map[string]string {
 	// Propagate labels from Run to PipelineRun.
 	labels := make(map[string]string, len(run.ObjectMeta.Labels)+1)
-	ignnoreLabelsKey := []string{"tekton.dev/pipelineRun", "tekton.dev/pipelineTask", "tekton.dev/pipeline", "custom.tekton.dev/pipelineLoopIteration"}
+	ignoreLabelsKey := []string{"tekton.dev/pipelineRun", "tekton.dev/pipelineTask", "tekton.dev/pipeline", "custom.tekton.dev/pipelineLoopIteration"}
 	for key, val := range run.ObjectMeta.Labels {
-		if _, found := Find(ignnoreLabelsKey, key); !found {
+		if _, found := Find(ignoreLabelsKey, key); !found {
 			labels[key] = val
 		}
 	}

diff --git a/tekton-catalog/pipeline-loops/pkg/reconciler/pipelinelooprun/pipelinelooprun_test.go b/tekton-catalog/pipeline-loops/pkg/reconciler/pipelinelooprun/pipelinelooprun_test.go
@@ -33,7 +33,6 @@ import (
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
 	ttesting "github.com/tektoncd/pipeline/pkg/reconciler/testing"
-	"github.com/tektoncd/pipeline/pkg/system"
 	"github.com/tektoncd/pipeline/test/diff"
 	"github.com/tektoncd/pipeline/test/names"
 	corev1 "k8s.io/api/core/v1"
@@ -46,6 +45,7 @@ import (
 	"knative.dev/pkg/controller"
 	"knative.dev/pkg/logging"
 	"knative.dev/pkg/reconciler"
+	"knative.dev/pkg/system"
 )
 
 var (
@@ -132,7 +132,7 @@ func getPipelineLoopController(t *testing.T, d test.Data, pipelineloops []*pipel
 		}
 	}
 
-	configMapWatcher := informer.NewInformedWatcher(c.Kube, system.GetNamespace())
+	configMapWatcher := informer.NewInformedWatcher(c.Kube, system.Namespace())
 	ctl := NewController(namespace)(ctx, configMapWatcher)
 
 	if la, ok := ctl.Reconciler.(reconciler.LeaderAware); ok {