From 6eeb99c53c9decc40d8c623085602a15b59203aa Mon Sep 17 00:00:00 2001 From: Alex Collins Date: Tue, 7 Apr 2020 14:13:19 -0700 Subject: [PATCH] test: Fix teething problems --- go.mod | 2 +- go.sum | 4 ++-- test/e2e/cli_test.go | 14 +++++------ test/e2e/fixtures/e2e_suite.go | 43 ++++++++++++++++------------------ 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 07081b02a23c..f651e74254db 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ require ( golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 golang.org/x/net v0.0.0-20200301022130-244492dfa37a golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - golang.org/x/tools v0.0.0-20200407143752-a3568bac92ae // indirect + golang.org/x/tools v0.0.0-20200407191807-cd5a53e07f8a // indirect google.golang.org/api v0.20.0 google.golang.org/genproto v0.0.0-20200317114155-1f3552e48f24 google.golang.org/grpc v1.28.0 diff --git a/go.sum b/go.sum index 51d9daeaf756..efafb7d5e3f3 100644 --- a/go.sum +++ b/go.sum @@ -618,8 +618,8 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200317043434-63da46f3035e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200407143752-a3568bac92ae h1:nMRC2i7oRGagyN5g3chxBF7gb2do/tIJ3ihoEnvYvsU= -golang.org/x/tools v0.0.0-20200407143752-a3568bac92ae/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200407191807-cd5a53e07f8a h1:3CGHZdxu1QXyxpZvKA3QHjIyj6ia66IIHQ3O0lKHgyQ= +golang.org/x/tools v0.0.0-20200407191807-cd5a53e07f8a/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/test/e2e/cli_test.go b/test/e2e/cli_test.go index 5fc9520a95f3..ad8301fe0c47 100644 --- a/test/e2e/cli_test.go +++ b/test/e2e/cli_test.go @@ -228,7 +228,7 @@ func (s *CLISuite) TestRoot() { s.Given().CronWorkflow("@testdata/basic.yaml"). When(). CreateCronWorkflow(). - RunCli([]string{"submit", "--from", "cronwf/test-cron-wf-basic"}, func(t *testing.T, output string, err error) { + RunCli([]string{"submit", "--from", "cronwf/test-cron-wf-basic", "-l", "argo-e2e=true"}, func(t *testing.T, output string, err error) { assert.NoError(t, err) assert.Contains(t, output, "Name: test-cron-wf-basic-") r := regexp.MustCompile(`Name:\s+?(test-cron-wf-basic-[a-z0-9]+)`) @@ -238,7 +238,7 @@ func (s *CLISuite) TestRoot() { } createdWorkflowName = res[1] }). - WaitForWorkflowName(createdWorkflowName, 15*time.Second). + WaitForWorkflowName(createdWorkflowName, 20*time.Second). Then(). ExpectWorkflowName(createdWorkflowName, func(t *testing.T, metadata *corev1.ObjectMeta, status *wfv1.WorkflowStatus) { assert.Equal(t, wfv1.NodeSucceeded, status.Phase) @@ -320,7 +320,7 @@ func (s *CLISuite) TestWorkflowDelete() { Workflow("@smoke/basic.yaml"). When(). SubmitWorkflow(). - WaitForWorkflow(15*time.Second). + WaitForWorkflow(20*time.Second). Given(). RunCli([]string{"delete", "basic"}, func(t *testing.T, output string, err error) { if assert.NoError(t, err) { @@ -333,7 +333,7 @@ func (s *CLISuite) TestWorkflowDelete() { Workflow("@smoke/basic.yaml"). When(). SubmitWorkflow(). - WaitForWorkflow(15*time.Second). + WaitForWorkflow(20*time.Second). Given(). RunCli([]string{"delete", "--all", "-l", "argo-e2e"}, func(t *testing.T, output string, err error) { if assert.NoError(t, err) { @@ -354,7 +354,7 @@ func (s *CLISuite) TestWorkflowDelete() { } }). When(). - WaitForWorkflow(15*time.Second). + WaitForWorkflow(20*time.Second). Given(). RunCli([]string{"delete", "--completed", "-l", "argo-e2e"}, func(t *testing.T, output string, err error) { if assert.NoError(t, err) { @@ -367,7 +367,7 @@ func (s *CLISuite) TestWorkflowDelete() { Workflow("@smoke/basic.yaml"). When(). SubmitWorkflow(). - WaitForWorkflow(15*time.Second). + WaitForWorkflow(20*time.Second). Given(). RunCli([]string{"delete", "--older", "1d", "-l", "argo-e2e"}, func(t *testing.T, output string, err error) { if assert.NoError(t, err) { @@ -559,7 +559,7 @@ func (s *CLISuite) TestWorkflowResubmit() { Workflow("@testdata/exit-1.yaml"). When(). SubmitWorkflow(). - WaitForWorkflow(15*time.Second). + WaitForWorkflow(20*time.Second). Given(). RunCli([]string{"resubmit", "--memoized", "exit-1"}, func(t *testing.T, output string, err error) { if assert.NoError(t, err) { diff --git a/test/e2e/fixtures/e2e_suite.go b/test/e2e/fixtures/e2e_suite.go index b540188409bb..e31a8fd4e4c9 100644 --- a/test/e2e/fixtures/e2e_suite.go +++ b/test/e2e/fixtures/e2e_suite.go @@ -29,7 +29,7 @@ const Label = "argo-e2e" // Cron tests run in parallel, so use a different label so they are not deleted when a new test runs const LabelCron = Label + "-cron" -var gitBranch string +var imageTag string var k3d bool func init() { @@ -37,13 +37,16 @@ func init() { if err != nil { panic(err) } - gitBranch = strings.TrimSpace(output) + imageTag = strings.TrimSpace(output) + if imageTag == "master" { + imageTag = "latest" + } context, err := runCli("kubectl", "config", "current-context") if err != nil { panic(err) } k3d = strings.TrimSpace(context) == "k3s-default" - log.WithFields(log.Fields{"gitBranch": gitBranch, "k3d": k3d}).Info() + log.WithFields(log.Fields{"imageTag": imageTag, "k3d": k3d}).Info() } type E2ESuite struct { @@ -62,7 +65,7 @@ type E2ESuite struct { images map[string]bool // Guard-rail. // The number of archived workflows. If is changes between two tests, we have a problem. - numArchivedWorkflows int + numWorkflows int } func (s *E2ESuite) SetupSuite() { @@ -113,28 +116,24 @@ func (s *E2ESuite) BeforeTest(suiteName, testName string) { s.DeleteResources(Label) s.images = s.listImages() s.importImages() - numArchivedWorkflows := s.countArchivedWorkflows() - if s.numArchivedWorkflows > 0 && s.numArchivedWorkflows != numArchivedWorkflows { - s.T().Fatal("there should almost never be a change to the number of archived workflows between tests, this means the last test (no the current test) is bad and needs fixing - note this guard-rail does not work across test suites") + numWorkflows := s.countWorkflows() + if s.numWorkflows > 0 && s.numWorkflows != numWorkflows { + s.T().Fatal("there should almost never be a change to the number of workflows between tests, this means the last test (not the current test) is bad and needs fixing - note this guard-rail does not work across test suites") } - s.numArchivedWorkflows = numArchivedWorkflows + s.numWorkflows = numWorkflows } -func (s *E2ESuite) countArchivedWorkflows() int { - if s.Persistence.IsEnabled() { - workflows, err := s.Persistence.workflowArchive.ListWorkflows(Namespace, time.Time{}, time.Time{}, nil, 0, 0) - s.CheckError(err) - return len(workflows) - } - return 0 +func (s *E2ESuite) countWorkflows() int { + workflows, err := s.wfClient.List(metav1.ListOptions{}) + s.CheckError(err) + return len(workflows.Items) } func (s *E2ESuite) importImages() { // If we are running K3D we should re-import these prior to running tests, as they may have been evicted. if k3d { - images := s.listImages() - for _, n := range []string{"docker.io/argoproj/argoexec:" + gitBranch, "docker.io/library/cowsay:v1"} { - if !images[n] { + for _, n := range []string{"docker.io/argoproj/argoexec:" + imageTag, "docker.io/library/cowsay:v1"} { + if !s.images[n] { _, err := runCli("k3d", "import-images", n) s.CheckError(err) } @@ -326,11 +325,9 @@ func (s *E2ESuite) AfterTest(_, _ string) { // Using an arbitrary image will result in slow and flakey tests as we can't really predict when they'll be // downloaded or evicted. To keep tests fast and reliable you must use whitelisted images. imageWhitelist := map[string]bool{ - "docker.io/argoproj/argoexec:" + gitBranch: true, - "docker.io/library/cowsay:v1": true, - "docker.io/library/python:alpine3.6": true, - // why this different name? - "python:alpine3.6": true, + "docker.io/argoproj/argoexec:" + imageTag: true, + "docker.io/library/cowsay:v1": true, + "docker.io/library/python:alpine3.6": true, } for n := range s.listImages() { if !s.images[n] && !imageWhitelist[n] {