Support eventual consistency in conformance tests

conformance/tests/httproute-cross-namespace.go

@@ -17,15 +17,12 @@ limitations under the License.
 package tests
 
 import (
-	"net/url"
 	"testing"
 
-	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/types"
 
 	"sigs.k8s.io/gateway-api/conformance/utils/http"
 	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
-	"sigs.k8s.io/gateway-api/conformance/utils/roundtripper"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
 )
 
@@ -43,19 +40,8 @@ var HTTPRouteCrossNamespace = suite.ConformanceTest{
 		gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeReady(t, suite.Client, suite.ControllerName, gwNN, routeNN)
 
 		t.Run("Simple HTTP request should reach web-backend", func(t *testing.T) {
-			t.Logf("Making request to http://%s", gwAddr)
-			cReq, cRes, err := suite.RoundTripper.CaptureRoundTrip(roundtripper.Request{
-				URL:      url.URL{Scheme: "http", Host: gwAddr},
-				Protocol: "HTTP",
-			})
-
-			require.NoErrorf(t, err, "error making request")
-
-			http.ExpectResponse(t, cReq, cRes, http.ExpectedResponse{
-				Request: http.ExpectedRequest{
-					Method: "GET",
-					Path:   "/",
-				},
+			http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, gwAddr, http.ExpectedResponse{
+				Request:    http.ExpectedRequest{Path: "/"},
 				StatusCode: 200,
 				Backend:    "web-backend",
 				Namespace:  "gateway-conformance-web-backend",

conformance/tests/httproute-matching-across-routes.go

@@ -109,7 +109,7 @@ var HTTPRouteMatchingAcrossRoutes = suite.ConformanceTest{
 			tc := testCases[i]
 			t.Run(testName(tc, i), func(t *testing.T) {
 				t.Parallel()
-				http.MakeRequestAndExpectResponse(t, suite.RoundTripper, gwAddr, tc)
+				http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, gwAddr, tc)
 			})
 		}
 	},

conformance/tests/httproute-matching.go

@@ -72,7 +72,7 @@ var HTTPRouteMatching = suite.ConformanceTest{
 			tc := testCases[i]
 			t.Run(testName(tc, i), func(t *testing.T) {
 				t.Parallel()
-				http.MakeRequestAndExpectResponse(t, suite.RoundTripper, gwAddr, tc)
+				http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, gwAddr, tc)
 			})
 		}
 	},

conformance/tests/httproute-simple-same-namespace.go

@@ -17,16 +17,13 @@ limitations under the License.
 package tests
 
 import (
-	"net/url"
 	"testing"
 
-	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/types"
 
 	"sigs.k8s.io/gateway-api/apis/v1alpha2"
 	"sigs.k8s.io/gateway-api/conformance/utils/http"
 	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
-	"sigs.k8s.io/gateway-api/conformance/utils/roundtripper"
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
 )
 
@@ -45,19 +42,8 @@ var HTTPRouteSimpleSameNamespace = suite.ConformanceTest{
 		gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeReady(t, suite.Client, suite.ControllerName, gwNN, routeNN)
 
 		t.Run("Simple HTTP request should reach infra-backend", func(t *testing.T) {
-			t.Logf("Making request to http://%s", gwAddr)
-			cReq, cRes, err := suite.RoundTripper.CaptureRoundTrip(roundtripper.Request{
-				URL:      url.URL{Scheme: "http", Host: gwAddr},
-				Protocol: "HTTP",
-			})
-
-			require.NoErrorf(t, err, "error making request")
-
-			http.ExpectResponse(t, cReq, cRes, http.ExpectedResponse{
-				Request: http.ExpectedRequest{
-					Method: "GET",
-					Path:   "/",
-				},
+			http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, gwAddr, http.ExpectedResponse{
+				Request:    http.ExpectedRequest{Path: "/"},
 				StatusCode: 200,
 				Backend:    "infra-backend-v1",
 				Namespace:  "gateway-conformance-infra",

conformance/utils/http/http.go

@@ -20,6 +20,7 @@ import (
 	"net/url"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -44,9 +45,15 @@ type ExpectedRequest struct {
 	Headers map[string]string
 }
 
-// MakeRequestAndExpectResponse makes a request with the given parameters and
-// verifies the response matches the provided ExpectedResponse.
-func MakeRequestAndExpectResponse(t *testing.T, r roundtripper.RoundTripper, gwAddr string, expected ExpectedResponse) {
+const maxConsistencyPeriodPerRequest = 60 * time.Second
+const numConsistencyChecksPerRequest = 3
+
+// MakeRequestAndExpectEventuallyConsistentResponse makes a request with the given parameters,
+// understanding that the request may fail for some amount of time.
+//
+// Once the request succeeds consistently with the response having the expected status code, make
+// additional assertions on the response body using the provided ExpectedResponse.
+func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripper.RoundTripper, gwAddr string, expected ExpectedResponse) {
 	t.Helper()
 
 	if expected.Request.Method == "" {
@@ -72,12 +79,49 @@ func MakeRequestAndExpectResponse(t *testing.T, r roundtripper.RoundTripper, gwA
 			req.Headers[name] = []string{value}
 		}
 	}
-	cReq, cRes, err := r.CaptureRoundTrip(req)
-	require.NoErrorf(t, err, "error making request")
 
+	cReq, cRes := WaitForConsistency(t, r, req, expected, numConsistencyChecksPerRequest)
 	ExpectResponse(t, cReq, cRes, expected)
 }
 
+// WaitForConsistency repeats the provided request until it completes with a response having
+// the expected status code consistently. The provided threshold determines how many times in
+// a row this must occur to be considered "consistent".
+func WaitForConsistency(t *testing.T, r roundtripper.RoundTripper, req roundtripper.Request, expected ExpectedResponse, threshold int) (*roundtripper.CapturedRequest, *roundtripper.CapturedResponse) {
+	var (
+		cReq         *roundtripper.CapturedRequest
+		cRes         *roundtripper.CapturedResponse
+		err          error
+		numSuccesses int
+	)
+
+	require.Eventually(t, func() bool {
+		cReq, cRes, err = r.CaptureRoundTrip(req)
+		if err != nil {
+			numSuccesses = 0
+			t.Logf("Request failed, not ready yet: %v", err.Error())
+			return false
+		}
+
+		if cRes.StatusCode != expected.StatusCode {
+			numSuccesses = 0
+			t.Logf("Expected response to have status %d but got %d, not ready yet", expected.StatusCode, cRes.StatusCode)
+			return false
+		}
+
+		numSuccesses++
+		if numSuccesses < threshold {
+			t.Logf("Request has passed %d times in a row of the desired %d, not ready yet", numSuccesses, threshold)
+			return false
+		}
+
+		t.Logf("Request has passed %d times in a row of the desired %d, ready!", numSuccesses, threshold)
+		return true
+	}, maxConsistencyPeriodPerRequest, 1*time.Second, "error making request, never got expected status")
+
+	return cReq, cRes
+}
+
 // ExpectResponse verifies that a captured request and response match the
 // provided ExpectedResponse.
 func ExpectResponse(t *testing.T, cReq *roundtripper.CapturedRequest, cRes *roundtripper.CapturedResponse, expected ExpectedResponse) {