forked from tektoncd/pipeline
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
In tektoncd#2540 we are seeing that some yaml tests are timing out, but it's hard to see what yaml tests are failing. This commit moves the logic out of bash and into individual go tests - now we will run an individual go test for each yaml example, completing all v1alpha1 before all v1beta1 and cleaning up in between. The output will still be challenging to read since it will be interleaved, however the failures should at least be associated with a specific yaml file. This also makes it easier to run all tests locally, though if you interrupt the tests you end up with your cluster in a bad state and it might be good to update these to execute each example in a separate namespace (in which case we could run all of v1alpha1 and v1beta1 at the same time as well!)
- Loading branch information
1 parent
e83aa5f
commit 2fb6e49
Showing
3 changed files
with
243 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,241 @@ | ||
| /* | ||
| Copyright 2020 The Tekton Authors | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package test | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "path" | ||
| "path/filepath" | ||
| "runtime" | ||
| "strings" | ||
| "sync" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "os" | ||
| "os/exec" | ||
| ) | ||
|
|
||
| const ( | ||
| timeoutSeconds = 600 | ||
| sleepBetween = 10 | ||
| ) | ||
|
|
||
| // cmd will run the command c with args and if input is provided, that will be piped | ||
| // into the process as input | ||
| func cmd(t *testing.T, c string, args []string, input string) (string, error) { | ||
| t.Helper() | ||
| binary, err := exec.LookPath(c) | ||
| if err != nil { | ||
| t.Fatalf("couldn't find %s, plz install", c) | ||
| } | ||
|
|
||
| t.Logf("Executing %s %v", binary, args) | ||
|
|
||
| cmd := exec.Command(binary, args...) | ||
| cmd.Env = os.Environ() | ||
|
|
||
| if input != "" { | ||
| cmd.Stdin = strings.NewReader(input) | ||
| } | ||
|
|
||
| var stderr, stdout bytes.Buffer | ||
| cmd.Stderr = &stderr | ||
| cmd.Stdout = &stdout | ||
| if err := cmd.Run(); err != nil { | ||
| t.Fatalf("couldn't run command %s %v: %v, %s", c, args, err, stderr.String()) | ||
| return "", err | ||
| } | ||
|
|
||
| return stdout.String(), nil | ||
| } | ||
|
|
||
| // resetState will erase all types that the yaml examples could have concievably created. | ||
| func resetState(t *testing.T) { | ||
| t.Helper() | ||
|
|
||
| crdTypes := []string{ | ||
| "conditions.tekton.dev", | ||
| "pipelineresources.tekton.dev", | ||
| "tasks.tekton.dev", | ||
| "clustertasks.tekton.dev", | ||
| "pipelines.tekton.dev", | ||
| "taskruns.tekton.dev", | ||
| "pipelineruns.tekton.dev", | ||
| "services", | ||
| "pods", | ||
| "configmaps", | ||
| "secrets", | ||
| "serviceaccounts", | ||
| "persistentvolumeclaims", | ||
| } | ||
| for _, c := range crdTypes { | ||
| cmd(t, "kubectl", []string{"delete", "--ignore-not-found=true", c, "--all"}, "") | ||
| } | ||
| } | ||
|
|
||
| // getYamls will look in the directory in examples indicated by version and run for yaml files | ||
| func getYamls(t *testing.T, version, run string) []string { | ||
| t.Helper() | ||
| _, filename, _, _ := runtime.Caller(0) | ||
|
|
||
| // Don't read recursively; the only dir within these dirs is no-ci which doesn't | ||
| // want any tests run against it | ||
| files, err := ioutil.ReadDir(path.Join(path.Dir(filename), version, run)) | ||
| if err != nil { | ||
| t.Fatalf("Couldn't read yaml files from %s/%s/%s: %v", path.Dir(filename), version, run, err) | ||
| } | ||
| yamls := []string{} | ||
| for _, f := range files { | ||
| if matches, _ := filepath.Match("*.yaml", f.Name()); matches { | ||
| yamls = append(yamls, f.Name()) | ||
| } | ||
| } | ||
| return yamls | ||
| } | ||
|
|
||
| // replaceDockerRepo will look in the content f and replace the hard coded docker | ||
| // repo with the on provided via the KO_DOCKER_REPO environment variable | ||
| func replaceDockerRepo(t *testing.T, f string) string { | ||
| t.Helper() | ||
| r := os.Getenv("KO_DOCKER_REPO") | ||
| if r == "" { | ||
| t.Fatalf("KO_DOCKER_REPO must be set") | ||
| } | ||
| read, err := ioutil.ReadFile(f) | ||
| if err != nil { | ||
| t.Fatalf("couldnt read contents of %s: %v", f, err) | ||
| } | ||
| return strings.Replace(string(read), "gcr.io/christiewilson-catfactory", r, -1) | ||
| } | ||
|
|
||
| // logRun will retrieve the entire yaml of run and log it | ||
| func logRun(t *testing.T, run string) { | ||
| t.Helper() | ||
| yaml, err := cmd(t, "kubectl", []string{"get", run, "-o", "yaml"}, "") | ||
| if err == nil { | ||
| t.Logf(yaml) | ||
| } | ||
| } | ||
|
|
||
| // pollRun will use kubectl to query the specified run to see if it | ||
| // has completed. It will timeout after timeoutSeconds. | ||
| func pollRun(t *testing.T, run string, wg *sync.WaitGroup) { | ||
| t.Helper() | ||
| for i := 0; i < (timeoutSeconds / sleepBetween); i++ { | ||
| status, err := cmd(t, "kubectl", []string{"get", run, "--output=jsonpath={.status.conditions[*].status}"}, "") | ||
| if err != nil { | ||
| t.Fatalf("couldnt get status of %s: %v", run, err) | ||
| wg.Done() | ||
| return | ||
| } | ||
|
|
||
| switch status { | ||
| case "", "Unknown": | ||
| // Not finished running yet | ||
| time.Sleep(sleepBetween * time.Second) | ||
| case "True": | ||
| t.Logf("%s completed successfully", run) | ||
| wg.Done() | ||
| return | ||
| default: | ||
| t.Errorf("%s has failed with status %s", run, status) | ||
| logRun(t, run) | ||
|
|
||
| wg.Done() | ||
| return | ||
| } | ||
| } | ||
| t.Errorf("%s did not complete within %d seconds", run, timeoutSeconds) | ||
| logRun(t, run) | ||
| wg.Done() | ||
| } | ||
|
|
||
| // waitForAllRuns will use kubectl to poll all runs in runs until completed, | ||
| // failed, or timed out | ||
| func waitForAllRuns(t *testing.T, runs []string) { | ||
| t.Helper() | ||
|
|
||
| var wg sync.WaitGroup | ||
| for _, run := range runs { | ||
| wg.Add(1) | ||
| go pollRun(t, run, &wg) | ||
| } | ||
| wg.Wait() | ||
| } | ||
|
|
||
| // getRuns will look for "run" in the provided ko output to determine the names | ||
| // of any runs created | ||
| func getRuns(k string) []string { | ||
| runs := []string{} | ||
| for _, s := range strings.Split(k, "\n") { | ||
| name := strings.TrimSuffix(s, " created") | ||
| if strings.Contains(name, "run") { | ||
| runs = append(runs, name) | ||
| } | ||
| } | ||
| return runs | ||
| } | ||
|
|
||
| // runTests will use ko to create all yamls in the directory indicated by version | ||
| // and run, and wait for all runs (PipelineRuns and TaskRuns) created | ||
| func runTests(t *testing.T, version, run string) { | ||
| yamls := getYamls(t, version, run) | ||
| for _, yaml := range yamls { | ||
| y := yaml | ||
|
|
||
| t.Run(fmt.Sprintf("%s/%s", run, y), func(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| t.Logf("Applying %s", y) | ||
| content := replaceDockerRepo(t, fmt.Sprintf("%s/%s/%s", version, run, y)) | ||
| output, err := cmd(t, "ko", []string{"create", "-f", "-"}, content) | ||
| if err == nil { | ||
| runs := getRuns(output) | ||
|
|
||
| if len(runs) == 0 { | ||
| t.Fatalf("no runs were created for %s, output %s", y, output) | ||
| } | ||
|
|
||
| t.Logf("Waiting for created runs %v", runs) | ||
| waitForAllRuns(t, runs) | ||
| } | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestYaml(t *testing.T) { | ||
| versions := []string{"v1alpha1", "v1beta1"} | ||
| runs := []string{"taskruns", "pipelineruns"} | ||
|
|
||
| t.Cleanup(func() { resetState(t) }) | ||
| for i, version := range versions { | ||
| t.Run(version, func(t *testing.T) { | ||
| for _, run := range runs { | ||
| runTests(t, version, run) | ||
| } | ||
| }) | ||
| // Since the v1alpha1 and v1beta1 tests are mostly copies of each other, | ||
| // we need to wipe them out between invocations | ||
| if i == 0 { | ||
| t.Logf("deleting all %s types between tests", version) | ||
| resetState(t) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters