Notebook tests should build a docker image to run the notebook in.

* kubeflow#613 currently the way we run notebook tests is
  by firing off a K8s job on the KF cluster which runs the notebook.

  * The K8s job uses init containers to pull in source code and install
    dependencies like papermill.

  * This is a bit brittle.

* To fix this we will instead use Tekton to build a docker image that
  takes the notebook image and then adds the notebook code to it.

* Dockerfile.notebook_runner dockerfile to build the test image.

* Add tekton tasks to build the image.

images/Dockerfile.py3

@@ -108,3 +108,4 @@ RUN go get -u github.com/jstemmer/go-junit-report
 # need to clone the repo just to get access to them
 RUN mkdir -p /srcCache/kubeflow/testing
 COPY py /srcCache/kubeflow/testing/py
+COPY notebook_testing /srcCache/kubeflow/testing/notebook_testing

notebook_testing/Dockerfile.notebook_runner

@@ -0,0 +1,9 @@
+# Dockerfile for building docker images to run notebooks
+# in.
+ARG BASE_IMAGE=gcr.io/kubeflow-images-public/tensorflow-1.15.2-notebook-cpu:1.0.0
+
+FROM $BASE_IMAGE
+
+# Copy any source code into /src
+COPY . /src
+

notebook_testing/README.md

@@ -0,0 +1,3 @@
+# Notebook testing
+
+This directory contains some resources used for testing notebooks.

tekton/runs/nb-test-run.yaml

@@ -34,15 +34,22 @@ spec:
       - name: url
         value: https://github.com/kubeflow/examples.git
       - name: revision
-        value: refs/pull/803/head
+        value: master
   - name: testing-repo
     resourceSpec:
       type: git
       params:
       - name: url
-        value: https://github.com/kubeflow/testing.git
+        value: https://github.com/jlewi/testing.git
       - name: revision
-        #value: master
-        value: refs/pull/676/head
+        value: gcp_blueprints
+      #  value: refs/pull/676/head
+  # The image we want to build
+  - name:  image
+    resourceSpec:
+      type: image
+      params:
+      - name: url  
+        value: gcr.io/kubeflow-ci-deployment/mnist-test
   pipelineRef:
     name: notebook-test

tekton/templates/pipelines/notebook-test-pipeline.yaml

@@ -28,32 +28,44 @@ spec:
     type: string
     description: Directory to write outputs to in local FS.
   resources:
-  - name: examples-repo
+  - name: notebook-repo
     type: git
   - name: testing-repo
     type: git
+  - name: image
+    type: image
   tasks:
-  - name: mnist-gcp
+  # Build the docker image containing the source
+  - name: build-image
     resources:
       inputs:
-      - name: examples-repo
-        resource: examples-repo
-      - name: kf-testing-repo
+      - name: notebook-repo
+        resource: notebook-repo
+      - name: testing-repo
         resource: testing-repo
     taskRef:
-      name: nb-tests
-    params:
-    - name: notebook-path
-      value: 'kubeflow/examples/mnist/mnist_gcp.ipynb'
-    - name: junit-path
-      value: $(params.output-workspace)/$(params.junit-path)/junit_mnist-gcp.xml
-    - name: test-target-name
-      value: $(params.test-target-name)
-    - name: output-workspace
-      value: $(params.output-workspace)
-    - name: artifacts-gcs
-      value: $(params.artifacts-gcs)
-    - name: testing-cluster-pattern
-      value: $(params.testing-cluster-pattern)
-    - name: testing-cluster-location
-      value: $(params.testing-cluster-location)
+      name: notebook-test-builder
+  # - name: run-notebook
+  #   resources:
+  #     inputs:
+  #     - name: examples-repo
+  #       resource: examples-repo
+  #     - name: kf-testing-repo
+  #       resource: testing-repo
+  #   taskRef:
+  #     name: nb-tests
+  #   params:
+  #   - name: notebook-path
+  #     value: 'kubeflow/examples/mnist/mnist_gcp.ipynb'
+  #   - name: junit-path
+  #     value: $(params.output-workspace)/$(params.junit-path)/junit_mnist-gcp.xml
+  #   - name: test-target-name
+  #     value: $(params.test-target-name)
+  #   - name: output-workspace
+  #     value: $(params.output-workspace)
+  #   - name: artifacts-gcs
+  #     value: $(params.artifacts-gcs)
+  #   - name: testing-cluster-pattern
+  #     value: $(params.testing-cluster-pattern)
+  #   - name: testing-cluster-location
+  #     value: $(params.testing-cluster-location)

tekton/templates/tasks/notebook-test-task.yaml

@@ -122,3 +122,76 @@ spec:
   - name: gcp-credentials
     secret:
       secretName: gcp-credentials
+
+---
+
+# This task builds a docker image to run notebooks in.
+# It takes as a base image a notebook image and adds
+# the source code for the notebook to it.
+apiVersion: tekton.dev/v1alpha1
+kind: Task
+metadata:
+  name: notebook-test-builder
+  annotations:
+    # This gets passed down to the individual pods
+    sidecar.istio.io/inject: "false"
+spec:
+  inputs:
+    params:
+    #- description: docker target arg
+    #  name: docker_target
+    #  type: string
+    #- description: The build context used by Kaniko
+    #  name: path_to_context
+    #  type: string
+    #- description: The path to the dockerfile to build
+    #  name: path_to_docker_file
+    #  type: string
+    - name: test-image
+      type: string
+      default: gcr.io/kubeflow-ci/test-worker-py3:6f0d932-dirty@sha256:06ebe5412d638e3e51bdd792aecbafdc4ee1e7146ff367a7be346cd726738cbb # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"test-image","value":"gcr.io/kubeflow-ci/test-worker-py3:6f0d932-dirty@sha256:06ebe5412d638e3e51bdd792aecbafdc4ee1e7146ff367a7be346cd726738cbb"}]}}
+      description: The docker image to run the tests in
+    resources:
+    # Repo containing the notebook code 
+    - name: notebook-repo
+      type: git
+    - name: image
+      type: image  
+  steps:
+  # We need to setup the directory where we will build the docker image
+  - name: setup
+    image: $(inputs.params.test-image)
+    # Need to use script as workaround not to error out in tests.
+    # If any of the steps returns non-zero codes, subsequent steps will not be run.
+    script: |
+      #!/usr/bin/env bash
+      set -x
+      mkdir -p /workspace/build
+      cd /workspace/build
+      # Copy the source code
+      cp -r /workspace/$(inputs.resources.notebook-repo.name) .
+      cp -r /srcCache/kubeflow/testing/notebook_testing/Dockerfile.notebook_runner ./Dockerfile.notebook_runner
+  - name: build-push
+    image: gcr.io/kaniko-project/executor:v0.11.0
+    command:
+    - /kaniko/executor
+    - --dockerfile=/workspace/build/Dockerfile.notebook_runner
+    #- --target=$(inputs.params.docker_target)
+    - --destination=$(inputs.resources.image.url)
+    - --context=/workspace/build
+    - --digest-file=/workspace/image-digest    
+    # TODO(jlewi): Does kaniko work with workload identity
+    #env:
+    #- name: GOOGLE_APPLICATION_CREDENTIALS
+    #  value: /secret/user-gcp-sa.json
+    resources:
+      requests:
+        cpu: 7
+        memory: 16Gi
+    #volumeMounts:
+    #- mountPath: /secret
+    #  name: gcp-credentials  
+  #volumes:
+  #- name: gcp-credentials
+  #  secret:
+  #    secretName: user-gcp-sa