Fix the E2E tests

test/README.md

@@ -183,8 +183,7 @@ go test -v -tags=e2e -count=1 ./test -run ^TestTaskRun$
 
 ### Running YAML tests
 
-To run the YAML e2e tests, from the examples folder, you need to have a valid
-cluster and run the following command:
+To run the YAML e2e tests, run the following command:
 
 ```bash
 ./test/e2e-yaml-tests.sh

test/e2e-common.sh

@@ -14,11 +14,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This script calls out to scripts in knative/test-infra to setup a cluster
-# and deploy the Pipeline CRD to it for running integration tests.
+# Helper functions for E2E tests.
 
 source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/e2e-tests.sh
 
+function teardown() {
+    subheader "Tearing down Pipeline CRD"
+    ko delete --ignore-not-found=true -f config/
+    # teardown will be called when run against an existing cluster to cleanup before
+    # continuing, so we must wait for the cleanup to complete or the subsequent attempt
+    # to deploy to the same namespace will fail
+    wait_until_object_does_not_exist namespace knative-build-pipeline
+}
+
+function output_yaml_test_results() {
+  # If formatting fails for any reason, use yaml as a fall back.
+  kubectl get $1.pipeline.knative.dev -o=custom-columns-file=${REPO_ROOT_DIR}/test/columns.txt || \
+    kubectl get $1.pipeline.knative.dev -oyaml
+}
+
+# Called by `fail_test` (provided by `e2e-tests.sh`) to dump info on test failure
+function dump_extra_cluster_state() {
+  echo ">>> Pipeline controller log:"
+  kubectl -n knative-build-pipeline logs $(get_app_pod build-pipeline-controller knative-build-pipeline)
+  echo ">>> Pipeline webhook log:"
+  kubectl -n knative-build-pipeline logs $(get_app_pod build-pipeline-webhook knative-build-pipeline)
+}
+
 function validate_run() {
   # Wait for tests to finish.
   echo ">> Waiting for tests to finish"
@@ -61,18 +83,21 @@ function validate_run() {
       fi
     done
   done
+  return ${failed}
 }
 
 function run_yaml_tests() {
   echo ">> Starting tests"
 
-  sed -i.bak 's/christiewilson-catfactory/${KO_DOCKER_REPO}/' examples/resources.yaml
-
-  ko apply -R -f examples/ || return 1
+  find ${REPO_ROOT_DIR}/examples/ -name *.yaml -exec cat {} \; \
+      | sed 's/christiewilson-catfactory/${KO_DOCKER_REPO}/' \
+      | ko apply -f - \
+      || return 1
 
-  failed=$(validate_run)
+  if validate_run $1; then
+    echo ">> All YAML tests passed"
+    return 0
+  fi
 
-  (( failed )) && return 1
-  echo ">> All YAML tests passed"
-  return 0
-}
+  return 1
+}

test/e2e-tests-yaml.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script calls out to scripts in knative/test-infra to setup a cluster
+# and deploy the Pipeline CRD to it for running integration tests.
+
+source $(dirname $0)/e2e-common.sh
+
+# Script entry point.
+
+initialize $@
+
+header "Setting up environment"
+
+# Handle failures ourselves, so we can dump useful info.
+set +o errexit
+set +o pipefail
+
+echo ">> Deploying Pipeline CRD"
+ko apply -f config/ || fail_test "Build pipeline installation failed"
+
+for res in pipelineresources tasks pipelines taskruns pipelineruns; do
+  kubectl delete --ignore-not-found=true ${res}.pipeline.knative.dev --all
+done
+
+# Run the tests
+failed=0
+for test in taskrun pipelinerun; do
+  header "Running YAML e2e tests for ${test}s"
+  if ! run_yaml_tests ${test}; then
+    echo "ERROR: one or more YAML tests failed"
+    failed=1
+    output_yaml_test_results ${test}
+  fi
+done
+
+(( failed )) && fail_test
+
+success

test/e2e-tests.sh

@@ -17,28 +17,8 @@
 # This script calls out to scripts in knative/test-infra to setup a cluster
 # and deploy the Pipeline CRD to it for running integration tests.
 
-source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/e2e-tests.sh
 source $(dirname $0)/e2e-common.sh
 
-# Helper functions.
-
-function teardown() {
-    header "Tearing down Pipeline CRD"
-    ko delete --ignore-not-found=true -f config/
-    # teardown will be called when run against an existing cluster to cleanup before
-    # continuing, so we must wait for the cleanup to complete or the subsequent attempt
-    # to deploy to the same namespace will fail
-    wait_until_object_does_not_exist namespace knative-build-pipeline
-}
-
-# Called by `fail_test` (provided by `e2e-tests.sh`) to dump info on test failure
-function dump_extra_cluster_state() {
-  echo ">>> Pipeline controller log:"
-  kubectl -n knative-build-pipeline logs $(get_app_pod build-pipeline-controller knative-build-pipeline)
-  echo ">>> Pipeline webhook log:"
-  kubectl -n knative-build-pipeline logs $(get_app_pod build-pipeline-webhook knative-build-pipeline)
-}
-
 # Script entry point.
 
 initialize $@
@@ -61,11 +41,10 @@ failed=0
 header "Running Go e2e tests"
 go_test_e2e -timeout=20m ./test || failed=1
 
-# Run the smoke tests for the examples dir to make sure they are valid
 # Run these _after_ the integration tests b/c they don't quite work all the way
 # and they cause a lot of noise in the logs, making it harder to debug integration
 # test failures.
-$(dirname $0)/e2e-yaml-tests.sh
+${REPO_ROOT_DIR}/test/e2e-tests-yaml.sh --run-tests || failed=1
 
 (( failed )) && fail_test
 success

test/presubmit-tests.sh

@@ -44,11 +44,6 @@ function unit_tests() {
   report_go_test ./...
 }
 
-function integration_tests() {
-  local options=""
-  ./test/e2e-tests.sh ${options}
-}
-
 # We use the default integration test runner.
 
 main $@