Wait for load balancer http resources

cmd/e2e-test/basic_test.go

@@ -294,7 +294,7 @@ func TestFrontendResourceDeletion(t *testing.T) {
 			}
 			// Wait for unused frontend resources to be deleted.
 			if err := e2e.WaitForFrontendResourceDeletion(ctx, Framework.Cloud, gclb, deleteOptions); err != nil {
-				t.Errorf("e2e.WaitForIngressDeletion(..., %q, _) = %v, want nil", ingKey, err)
+				t.Errorf("e2e.WaitForFrontendResourceDeletion(..., %q, _) = %v, want nil", ingKey, err)
 			}
 			if gclb, err = e2e.WhiteboxTest(ing, s, Framework.Cloud, ""); err != nil {
 				t.Fatalf("e2e.WhiteboxTest(%s, ...) = %v, want nil", ingKey, err)
@@ -325,6 +325,10 @@ func TestFrontendResourceDeletion(t *testing.T) {
 			if ing, err = e2e.WaitForIngress(s, ing, &e2e.WaitForIngressOptions{ExpectUnreachable: true}); err != nil {
 				t.Fatalf("error waiting for Ingress %s to stabilize: %v", ingKey, err)
 			}
+			if ing, err = e2e.WaitForHTTPResourceAnnotations(s, ing); err != nil {
+				t.Fatalf("error waiting for http annotations on Ingress %s: %v", ingKey, err)
+			}
+
 			gclb, err = e2e.WhiteboxTest(ing, s, Framework.Cloud, "")
 			if err != nil {
 				t.Fatalf("e2e.WhiteboxTest(%s, ...)", ingKey)

pkg/e2e/helpers.go

@@ -60,6 +60,9 @@ const (
 	k8sApiPoolInterval = 10 * time.Second
 	k8sApiPollTimeout  = 30 * time.Minute
 
+	updateIngressPollInterval = 30 * time.Second
+	updateIngressPollTimeout  = 15 * time.Minute
+
 	healthyState = "HEALTHY"
 )
 
@@ -141,6 +144,33 @@ func WaitForIngress(s *Sandbox, ing *v1beta1.Ingress, options *WaitForIngressOpt
 	return ing, err
 }
 
+// WaitForHTTPResourceAnnotations waits for http forwarding rule annotation to
+// be added on ingress.
+// This is to handle a special case where ingress updates from https only to http
+// or both http and https enabled. Turns out port 80 on ingress VIP is accessible
+// even when http forwarding rule and target proxy do not exist. So, ingress
+// validator thinks that http load balancer is configured when https only
+// configuration exists.
+// TODO(smatti): Remove this when the above issue is fixed.
+func WaitForHTTPResourceAnnotations(s *Sandbox, ing *v1beta1.Ingress) (*v1beta1.Ingress, error) {
+	ingKey := fmt.Sprintf("%s/%s", s.Namespace, ing.Name)
+	klog.V(3).Infof("Waiting for HTTP annotations to be added on Ingress %s", ingKey)
+	err := wait.Poll(updateIngressPollInterval, updateIngressPollTimeout, func() (bool, error) {
+		var err error
+		crud := IngressCRUD{s.f.Clientset}
+		if ing, err = crud.Get(s.Namespace, ing.Name); err != nil {
+			return true, err
+		}
+		if _, ok := ing.Annotations[annotations.HttpForwardingRuleKey]; !ok {
+			klog.V(3).Infof("HTTP forwarding rule annotation not found on ingress %s, retrying", ingKey)
+			return false, nil
+		}
+		klog.V(3).Infof("HTTP forwarding rule annotation found on ingress %s", ingKey)
+		return true, nil
+	})
+	return ing, err
+}
+
 // WaitForFinalizer waits for Finalizer to be added.
 // Note that this is used only for upgrade tests.
 func WaitForFinalizer(s *Sandbox, ing *v1beta1.Ingress) (*v1beta1.Ingress, error) {