diff --git a/test/README.md b/test/README.md index e05d737d4a8..e604c4ea168 100644 --- a/test/README.md +++ b/test/README.md @@ -183,8 +183,7 @@ go test -v -tags=e2e -count=1 ./test -run ^TestTaskRun$ ### Running YAML tests -To run the YAML e2e tests, from the examples folder, you need to have a valid -cluster and run the following command: +To run the YAML e2e tests, run the following command: ```bash ./test/e2e-yaml-tests.sh diff --git a/test/e2e-common.sh b/test/e2e-common.sh index d6ddbea9df4..621f1f995c2 100755 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -14,11 +14,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script calls out to scripts in knative/test-infra to setup a cluster -# and deploy the Pipeline CRD to it for running integration tests. +# Helper functions for E2E tests. source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/e2e-tests.sh +function teardown() { + subheader "Tearing down Pipeline CRD" + ko delete --ignore-not-found=true -f config/ + # teardown will be called when run against an existing cluster to cleanup before + # continuing, so we must wait for the cleanup to complete or the subsequent attempt + # to deploy to the same namespace will fail + wait_until_object_does_not_exist namespace knative-build-pipeline +} + +function output_yaml_test_results() { + # If formatting fails for any reason, use yaml as a fall back. + kubectl get $1.pipeline.knative.dev -o=custom-columns-file=${REPO_ROOT_DIR}/test/columns.txt || \ + kubectl get $1.pipeline.knative.dev -oyaml +} + +# Called by `fail_test` (provided by `e2e-tests.sh`) to dump info on test failure +function dump_extra_cluster_state() { + echo ">>> Pipeline controller log:" + kubectl -n knative-build-pipeline logs $(get_app_pod build-pipeline-controller knative-build-pipeline) + echo ">>> Pipeline webhook log:" + kubectl -n knative-build-pipeline logs $(get_app_pod build-pipeline-webhook knative-build-pipeline) +} + function validate_run() { # Wait for tests to finish. echo ">> Waiting for tests to finish" @@ -61,18 +83,21 @@ function validate_run() { fi done done + return ${failed} } function run_yaml_tests() { echo ">> Starting tests" - sed -i.bak 's/christiewilson-catfactory/${KO_DOCKER_REPO}/' examples/resources.yaml - - ko apply -R -f examples/ || return 1 + find ${REPO_ROOT_DIR}/examples/ -name *.yaml -exec cat {} \; \ + | sed 's/christiewilson-catfactory/${KO_DOCKER_REPO}/' \ + | ko apply -f - \ + || return 1 - failed=$(validate_run) + if validate_run $1; then + echo ">> All YAML tests passed" + return 0 + fi - (( failed )) && return 1 - echo ">> All YAML tests passed" - return 0 -} \ No newline at end of file + return 1 +} diff --git a/test/e2e-tests-yaml.sh b/test/e2e-tests-yaml.sh new file mode 100755 index 00000000000..969d7c35392 --- /dev/null +++ b/test/e2e-tests-yaml.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script calls out to scripts in knative/test-infra to setup a cluster +# and deploy the Pipeline CRD to it for running integration tests. + +source $(dirname $0)/e2e-common.sh + +# Script entry point. + +initialize $@ + +header "Setting up environment" + +# Handle failures ourselves, so we can dump useful info. +set +o errexit +set +o pipefail + +echo ">> Deploying Pipeline CRD" +ko apply -f config/ || fail_test "Build pipeline installation failed" + +for res in pipelineresources tasks pipelines taskruns pipelineruns; do + kubectl delete --ignore-not-found=true ${res}.pipeline.knative.dev --all +done + +# Run the tests +failed=0 +for test in taskrun pipelinerun; do + header "Running YAML e2e tests for ${test}s" + if ! run_yaml_tests ${test}; then + echo "ERROR: one or more YAML tests failed" + failed=1 + output_yaml_test_results ${test} + fi +done + +(( failed )) && fail_test + +success diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 8c945339b0a..26e979f72f2 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -17,28 +17,8 @@ # This script calls out to scripts in knative/test-infra to setup a cluster # and deploy the Pipeline CRD to it for running integration tests. -source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/e2e-tests.sh source $(dirname $0)/e2e-common.sh -# Helper functions. - -function teardown() { - header "Tearing down Pipeline CRD" - ko delete --ignore-not-found=true -f config/ - # teardown will be called when run against an existing cluster to cleanup before - # continuing, so we must wait for the cleanup to complete or the subsequent attempt - # to deploy to the same namespace will fail - wait_until_object_does_not_exist namespace knative-build-pipeline -} - -# Called by `fail_test` (provided by `e2e-tests.sh`) to dump info on test failure -function dump_extra_cluster_state() { - echo ">>> Pipeline controller log:" - kubectl -n knative-build-pipeline logs $(get_app_pod build-pipeline-controller knative-build-pipeline) - echo ">>> Pipeline webhook log:" - kubectl -n knative-build-pipeline logs $(get_app_pod build-pipeline-webhook knative-build-pipeline) -} - # Script entry point. initialize $@ @@ -61,11 +41,10 @@ failed=0 header "Running Go e2e tests" go_test_e2e -timeout=20m ./test || failed=1 -# Run the smoke tests for the examples dir to make sure they are valid # Run these _after_ the integration tests b/c they don't quite work all the way # and they cause a lot of noise in the logs, making it harder to debug integration # test failures. -$(dirname $0)/e2e-yaml-tests.sh +${REPO_ROOT_DIR}/test/e2e-tests-yaml.sh --run-tests || failed=1 (( failed )) && fail_test success diff --git a/test/e2e-yaml-tests.sh b/test/e2e-yaml-tests.sh deleted file mode 100755 index d10128b7b2e..00000000000 --- a/test/e2e-yaml-tests.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 The Knative Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script calls out to scripts in knative/test-infra to setup a cluster -# and deploy the Pipeline CRD to it for running integration tests. - -source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/e2e-tests.sh -source $(dirname $0)/e2e-common.sh - -# Handle failures ourselves, so we can dump useful info. -set +o errexit -set +o pipefail - -echo ">> Deploying Pipeline CRD" -ko apply -f config/ || fail_test "Build pipeline installation failed" - -kubectl delete --ignore-not-found=true pipelineresources.pipeline.knative.dev --all -kubectl delete --ignore-not-found=true tasks.pipeline.knative.dev --all -kubectl delete --ignore-not-found=true pipelines.pipeline.knative.dev --all -kubectl delete --ignore-not-found=true taskruns.pipeline.knative.dev --all -kubectl delete --ignore-not-found=true pipelineruns.pipeline.knative.dev --all - -# Run the tests -failed=0 - -# Run the smoke tests for the examples dir to make sure they are valid -# Run these _after_ the integration tests b/c they don't quite work all the way -# and they cause a lot of noise in the logs, making it harder to debug integration -# test failures. -header "Running YAML e2e tests for taskruns" -if ! run_yaml_tests "taskrun"; then - failed=1 - echo "ERROR: one or more YAML tests failed" - # If formatting fails for any reason, use yaml as a fall back. - kubectl get taskrun.pipeline.knative.dev -o=custom-columns-file=./test/columns.txt || \ - kubectl get taskrun.pipeline.knative.dev -oyaml -fi -header "Running YAML e2e tests for pipelineruns" -if ! run_yaml_tests "pipelinerun"; then - failed=1 - echo "ERROR: one or more YAML tests failed" - # If formatting fails for any reason, use yaml as a fall back. - kubectl get pipelinerun.pipeline.knative.dev -o=custom-columns-file=./test/columns.txt || \ - kubectl get pipelinerun.pipeline.knative.dev -oyaml -fi diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh index 3190487179f..55d1ad20ef9 100755 --- a/test/presubmit-tests.sh +++ b/test/presubmit-tests.sh @@ -44,11 +44,6 @@ function unit_tests() { report_go_test ./... } -function integration_tests() { - local options="" - ./test/e2e-tests.sh ${options} -} - # We use the default integration test runner. main $@