test: Fix teething problems (#2630)

go.mod

@@ -80,7 +80,7 @@ require (
 	golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4
 	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/tools v0.0.0-20200407143752-a3568bac92ae // indirect
+	golang.org/x/tools v0.0.0-20200407191807-cd5a53e07f8a // indirect
 	google.golang.org/api v0.20.0
 	google.golang.org/genproto v0.0.0-20200317114155-1f3552e48f24
 	google.golang.org/grpc v1.28.0

go.sum

@@ -618,8 +618,8 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200317043434-63da46f3035e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200407143752-a3568bac92ae h1:nMRC2i7oRGagyN5g3chxBF7gb2do/tIJ3ihoEnvYvsU=
-golang.org/x/tools v0.0.0-20200407143752-a3568bac92ae/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200407191807-cd5a53e07f8a h1:3CGHZdxu1QXyxpZvKA3QHjIyj6ia66IIHQ3O0lKHgyQ=
+golang.org/x/tools v0.0.0-20200407191807-cd5a53e07f8a/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

test/e2e/cli_test.go

@@ -228,7 +228,7 @@ func (s *CLISuite) TestRoot() {
 		s.Given().CronWorkflow("@testdata/basic.yaml").
 			When().
 			CreateCronWorkflow().
-			RunCli([]string{"submit", "--from", "cronwf/test-cron-wf-basic"}, func(t *testing.T, output string, err error) {
+			RunCli([]string{"submit", "--from", "cronwf/test-cron-wf-basic", "-l", "argo-e2e=true"}, func(t *testing.T, output string, err error) {
 				assert.NoError(t, err)
 				assert.Contains(t, output, "Name:                test-cron-wf-basic-")
 				r := regexp.MustCompile(`Name:\s+?(test-cron-wf-basic-[a-z0-9]+)`)
@@ -238,7 +238,7 @@ func (s *CLISuite) TestRoot() {
 				}
 				createdWorkflowName = res[1]
 			}).
-			WaitForWorkflowName(createdWorkflowName, 15*time.Second).
+			WaitForWorkflowName(createdWorkflowName, 20*time.Second).
 			Then().
 			ExpectWorkflowName(createdWorkflowName, func(t *testing.T, metadata *corev1.ObjectMeta, status *wfv1.WorkflowStatus) {
 				assert.Equal(t, wfv1.NodeSucceeded, status.Phase)
@@ -320,7 +320,7 @@ func (s *CLISuite) TestWorkflowDelete() {
 			Workflow("@smoke/basic.yaml").
 			When().
 			SubmitWorkflow().
-			WaitForWorkflow(15*time.Second).
+			WaitForWorkflow(20*time.Second).
 			Given().
 			RunCli([]string{"delete", "basic"}, func(t *testing.T, output string, err error) {
 				if assert.NoError(t, err) {
@@ -333,7 +333,7 @@ func (s *CLISuite) TestWorkflowDelete() {
 			Workflow("@smoke/basic.yaml").
 			When().
 			SubmitWorkflow().
-			WaitForWorkflow(15*time.Second).
+			WaitForWorkflow(20*time.Second).
 			Given().
 			RunCli([]string{"delete", "--all", "-l", "argo-e2e"}, func(t *testing.T, output string, err error) {
 				if assert.NoError(t, err) {
@@ -354,7 +354,7 @@ func (s *CLISuite) TestWorkflowDelete() {
 				}
 			}).
 			When().
-			WaitForWorkflow(15*time.Second).
+			WaitForWorkflow(20*time.Second).
 			Given().
 			RunCli([]string{"delete", "--completed", "-l", "argo-e2e"}, func(t *testing.T, output string, err error) {
 				if assert.NoError(t, err) {
@@ -367,7 +367,7 @@ func (s *CLISuite) TestWorkflowDelete() {
 			Workflow("@smoke/basic.yaml").
 			When().
 			SubmitWorkflow().
-			WaitForWorkflow(15*time.Second).
+			WaitForWorkflow(20*time.Second).
 			Given().
 			RunCli([]string{"delete", "--older", "1d", "-l", "argo-e2e"}, func(t *testing.T, output string, err error) {
 				if assert.NoError(t, err) {
@@ -559,7 +559,7 @@ func (s *CLISuite) TestWorkflowResubmit() {
 		Workflow("@testdata/exit-1.yaml").
 		When().
 		SubmitWorkflow().
-		WaitForWorkflow(15*time.Second).
+		WaitForWorkflow(20*time.Second).
 		Given().
 		RunCli([]string{"resubmit", "--memoized", "exit-1"}, func(t *testing.T, output string, err error) {
 			if assert.NoError(t, err) {

test/e2e/fixtures/e2e_suite.go

@@ -29,21 +29,24 @@ const Label = "argo-e2e"
 // Cron tests run in parallel, so use a different label so they are not deleted when a new test runs
 const LabelCron = Label + "-cron"
 
-var gitBranch string
+var imageTag string
 var k3d bool
 
 func init() {
 	output, err := runCli("git", "rev-parse", "--abbrev-ref=loose", "HEAD")
 	if err != nil {
 		panic(err)
 	}
-	gitBranch = strings.TrimSpace(output)
+	imageTag = strings.TrimSpace(output)
+	if imageTag == "master" {
+		imageTag = "latest"
+	}
 	context, err := runCli("kubectl", "config", "current-context")
 	if err != nil {
 		panic(err)
 	}
 	k3d = strings.TrimSpace(context) == "k3s-default"
-	log.WithFields(log.Fields{"gitBranch": gitBranch, "k3d": k3d}).Info()
+	log.WithFields(log.Fields{"imageTag": imageTag, "k3d": k3d}).Info()
 }
 
 type E2ESuite struct {
@@ -62,7 +65,7 @@ type E2ESuite struct {
 	images map[string]bool
 	// Guard-rail.
 	// The number of archived workflows. If is changes between two tests, we have a problem.
-	numArchivedWorkflows int
+	numWorkflows int
 }
 
 func (s *E2ESuite) SetupSuite() {
@@ -113,28 +116,24 @@ func (s *E2ESuite) BeforeTest(suiteName, testName string) {
 	s.DeleteResources(Label)
 	s.images = s.listImages()
 	s.importImages()
-	numArchivedWorkflows := s.countArchivedWorkflows()
-	if s.numArchivedWorkflows > 0 && s.numArchivedWorkflows != numArchivedWorkflows {
-		s.T().Fatal("there should almost never be a change to the number of archived workflows between tests, this means the last test (no the current test) is bad and needs fixing - note this guard-rail does not work across test suites")
+	numWorkflows := s.countWorkflows()
+	if s.numWorkflows > 0 && s.numWorkflows != numWorkflows {
+		s.T().Fatal("there should almost never be a change to the number of workflows between tests, this means the last test (not the current test) is bad and needs fixing - note this guard-rail does not work across test suites")
 	}
-	s.numArchivedWorkflows = numArchivedWorkflows
+	s.numWorkflows = numWorkflows
 }
 
-func (s *E2ESuite) countArchivedWorkflows() int {
-	if s.Persistence.IsEnabled() {
-		workflows, err := s.Persistence.workflowArchive.ListWorkflows(Namespace, time.Time{}, time.Time{}, nil, 0, 0)
-		s.CheckError(err)
-		return len(workflows)
-	}
-	return 0
+func (s *E2ESuite) countWorkflows() int {
+	workflows, err := s.wfClient.List(metav1.ListOptions{})
+	s.CheckError(err)
+	return len(workflows.Items)
 }
 
 func (s *E2ESuite) importImages() {
 	// If we are running K3D we should re-import these prior to running tests, as they may have been evicted.
 	if k3d {
-		images := s.listImages()
-		for _, n := range []string{"docker.io/argoproj/argoexec:" + gitBranch, "docker.io/library/cowsay:v1"} {
-			if !images[n] {
+		for _, n := range []string{"docker.io/argoproj/argoexec:" + imageTag, "docker.io/library/cowsay:v1"} {
+			if !s.images[n] {
 				_, err := runCli("k3d", "import-images", n)
 				s.CheckError(err)
 			}
@@ -326,11 +325,9 @@ func (s *E2ESuite) AfterTest(_, _ string) {
 	// Using an arbitrary image will result in slow and flakey tests as we can't really predict when they'll be
 	// downloaded or evicted. To keep tests fast and reliable you must use whitelisted images.
 	imageWhitelist := map[string]bool{
-		"docker.io/argoproj/argoexec:" + gitBranch: true,
-		"docker.io/library/cowsay:v1":              true,
-		"docker.io/library/python:alpine3.6":       true,
-		// why this different name?
-		"python:alpine3.6": true,
+		"docker.io/argoproj/argoexec:" + imageTag: true,
+		"docker.io/library/cowsay:v1":             true,
+		"docker.io/library/python:alpine3.6":      true,
 	}
 	for n := range s.listImages() {
 		if !s.images[n] && !imageWhitelist[n] {