Merge pull request #4886 from aledbf/flake

Fix flaking e2e tests
kubernetes · Jan 7, 2020 · e093130 · e093130
2 parents 5f6c4cf + 5ce93d9
commit e093130
Show file tree

Hide file tree

Showing 10 changed files with 22 additions and 14 deletions.
diff --git a/rootfs/etc/nginx/lua/test/monitor_test.lua b/rootfs/etc/nginx/lua/test/monitor_test.lua
@@ -96,7 +96,7 @@ describe("Monitor", function()
 
       monitor.flush()
 
-      local expected_payload = '[{"host":"example.com","method":"GET","requestLength":256,"status":"200","upstreamResponseLength":456,"upstreamLatency":0.01,"upstreamResponseTime":0.02,"path":"\\/","requestTime":0.04,"ingress":"example","namespace":"default","service":"http-svc","responseLength":512},{"host":"example.com","method":"POST","requestLength":256,"status":"201","upstreamResponseLength":456,"upstreamLatency":0.01,"upstreamResponseTime":0.02,"path":"\\/","requestTime":0.04,"ingress":"example","namespace":"default","service":"http-svc","responseLength":512}]'
+      local expected_payload = '[{"requestLength":256,"ingress":"example","status":"200","service":"http-svc","requestTime":0.04,"namespace":"default","host":"example.com","method":"GET","upstreamResponseTime":0.02,"upstreamResponseLength":456,"upstreamLatency":0.01,"path":"\\/","responseLength":512},{"requestLength":256,"ingress":"example","status":"201","service":"http-svc","requestTime":0.04,"namespace":"default","host":"example.com","method":"POST","upstreamResponseTime":0.02,"upstreamResponseLength":456,"upstreamLatency":0.01,"path":"\\/","responseLength":512}]'
 
       assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/prometheus-nginx.socket")
       assert.stub(tcp_mock.send).was_called_with(tcp_mock, expected_payload)

diff --git a/test/e2e/defaultbackend/custom_default_backend.go b/test/e2e/defaultbackend/custom_default_backend.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -37,19 +38,21 @@ var _ = framework.IngressNginxDescribe("Custom Default Backend", func() {
 	BeforeEach(func() {
 		f.NewEchoDeploymentWithReplicas(1)
 
-		framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
+		err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
 			func(deployment *appsv1.Deployment) error {
 				args := deployment.Spec.Template.Spec.Containers[0].Args
-				args = append(args, fmt.Sprintf("--default-backend-service=$(POD_NAMESPACE)/%v", framework.EchoService))
+				args = append(args, fmt.Sprintf("--default-backend-service=%v/%v", f.Namespace, framework.EchoService))
 				deployment.Spec.Template.Spec.Containers[0].Args = args
 				_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(deployment)
 
 				return err
 			})
+		Expect(err).NotTo(HaveOccurred())
+		time.Sleep(1 * time.Second)
 
 		f.WaitForNginxServer("_",
 			func(server string) bool {
-				return strings.Contains(server, "set $proxy_upstream_name \"upstream-default-backend\"")
+				return strings.Contains(server, `set $proxy_upstream_name "upstream-default-backend"`)
 			})
 	})
 

diff --git a/test/e2e/framework/deployment.go b/test/e2e/framework/deployment.go
@@ -423,7 +423,8 @@ func (f *Framework) ScaleDeploymentToZero(name string) {
 	Expect(d).NotTo(BeNil(), "expected a deployment but none returned")
 
 	d.Spec.Replicas = NewInt32(0)
+	f.EnsureDeployment(d)
 
-	d = f.EnsureDeployment(d)
-	Expect(d).NotTo(BeNil(), "expected a deployment but none returned")
+	err = WaitForEndpoints(f.KubeClientSet, DefaultTimeout, name, f.Namespace, 0)
+	Expect(err).NotTo(HaveOccurred(), "failed to wait for no endpoints")
 }
diff --git a/test/e2e/gracefulshutdown/shutdown.go b/test/e2e/gracefulshutdown/shutdown.go
@@ -69,13 +69,14 @@ var _ = framework.IngressNginxDescribe("Shutdown ingress controller", func() {
 	}
 
 	It("should shutdown after waiting 60 seconds for pending connections to be closed", func() {
-		framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
+		err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
 			func(deployment *appsv1.Deployment) error {
 				grace := int64(3600)
 				deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = &grace
 				_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(deployment)
 				return err
 			})
+		Expect(err).NotTo(HaveOccurred())
 
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-send-timeout": "600",
@@ -127,13 +128,14 @@ var _ = framework.IngressNginxDescribe("Shutdown ingress controller", func() {
 	})
 
 	It("should shutdown after waiting 150 seconds for pending connections to be closed", func() {
-		framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
+		err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
 			func(deployment *appsv1.Deployment) error {
 				grace := int64(3600)
 				deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = &grace
 				_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(deployment)
 				return err
 			})
+		Expect(err).NotTo(HaveOccurred())
 
 		annotations := map[string]string{
 			"nginx.ingress.kubernetes.io/proxy-send-timeout": "600",

diff --git a/test/e2e/settings/default_ssl_certificate.go b/test/e2e/settings/default_ssl_certificate.go
@@ -46,7 +46,7 @@ var _ = framework.IngressNginxDescribe("default-ssl-certificate", func() {
 			f.Namespace)
 		Expect(err).NotTo(HaveOccurred())
 
-		framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
+		err = framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
 			func(deployment *appsv1.Deployment) error {
 				args := deployment.Spec.Template.Spec.Containers[0].Args
 				args = append(args, "--default-ssl-certificate=$(POD_NAMESPACE)/"+secretName)
@@ -55,6 +55,7 @@ var _ = framework.IngressNginxDescribe("default-ssl-certificate", func() {
 
 				return err
 			})
+		Expect(err).NotTo(HaveOccurred(), "unexpected error updating ingress controller deployment flags")
 
 		// this asserts that it configures default custom ssl certificate without an ingress at all
 		framework.WaitForTLS(f.GetURL(framework.HTTPS), tlsConfig)

diff --git a/test/e2e/settings/disable_catch_all.go b/test/e2e/settings/disable_catch_all.go
@@ -37,7 +37,7 @@ var _ = framework.IngressNginxDescribe("Disabled catch-all", func() {
 	BeforeEach(func() {
 		f.NewEchoDeploymentWithReplicas(1)
 
-		framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
+		err := framework.UpdateDeployment(f.KubeClientSet, f.Namespace, "nginx-ingress-controller", 1,
 			func(deployment *appsv1.Deployment) error {
 				args := deployment.Spec.Template.Spec.Containers[0].Args
 				args = append(args, "--disable-catch-all=true")
@@ -46,6 +46,7 @@ var _ = framework.IngressNginxDescribe("Disabled catch-all", func() {
 
 				return err
 			})
+		Expect(err).NotTo(HaveOccurred(), "unexpected error updating ingress controller deployment flags")
 	})
 
 	AfterEach(func() {

diff --git a/test/e2e/settings/ingress_class.go b/test/e2e/settings/ingress_class.go
@@ -86,7 +86,7 @@ var _ = framework.IngressNginxDescribe("Ingress class", func() {
 
 					return err
 				})
-			Expect(err).To(BeNil())
+			Expect(err).NotTo(HaveOccurred(), "unexpected error updating ingress controller deployment flags")
 		})
 
 		It("should ignore Ingress with no class", func() {

diff --git a/test/e2e/settings/pod_security_policy.go b/test/e2e/settings/pod_security_policy.go
@@ -73,7 +73,7 @@ var _ = framework.IngressNginxDescribe("Pod Security Policies", func() {
 
 				return err
 			})
-		Expect(err).NotTo(HaveOccurred())
+		Expect(err).NotTo(HaveOccurred(), "unexpected error updating ingress controller deployment flags")
 
 		f.NewEchoDeployment()
 	})

diff --git a/test/e2e/settings/pod_security_policy_volumes.go b/test/e2e/settings/pod_security_policy_volumes.go
@@ -96,7 +96,7 @@ var _ = framework.IngressNginxDescribe("Pod Security Policies with volumes", fun
 
 				return err
 			})
-		Expect(err).NotTo(HaveOccurred())
+		Expect(err).NotTo(HaveOccurred(), "unexpected error updating ingress controller deployment")
 
 		f.NewEchoDeployment()
 

diff --git a/test/e2e/status/update.go b/test/e2e/status/update.go
@@ -109,7 +109,7 @@ var _ = framework.IngressNginxDescribe("Status Update [Status]", func() {
 			}
 		}()
 
-		err = wait.Poll(10*time.Second, framework.DefaultTimeout, func() (done bool, err error) {
+		err = wait.Poll(10*time.Second, 4*time.Minute, func() (done bool, err error) {
 			ing, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(host, metav1.GetOptions{})
 			if err != nil {
 				return false, nil