e2e: eg controlplane metrics (envoyproxy#2106)

* e2e: eg controlplane metrics Signed-off-by: zirain <zirain2009@gmail.com> * update Signed-off-by: zirain <zirain2009@gmail.com> * update Signed-off-by: zirain <zirain2009@gmail.com> * update name Signed-off-by: zirain <zirain2009@gmail.com> * rename Signed-off-by: zirain <zirain2009@gmail.com> * wait more Signed-off-by: zirain <zirain2009@gmail.com> * add comment Signed-off-by: zirain <zirain2009@gmail.com> * update Signed-off-by: zirain <zirain2009@gmail.com> --------- Signed-off-by: zirain <zirain2009@gmail.com> (cherry picked from commit 046a593) Signed-off-by: Arko Dasgupta <arko@tetrate.io>
arkodg · Nov 1, 2023 · 42c1ac1 · 42c1ac1
1 parent eb07c53
commit 42c1ac1
Show file tree

Hide file tree

Showing 3 changed files with 107 additions and 8 deletions.
diff --git a/test/e2e/testdata/prometheus.yaml b/test/e2e/testdata/prometheus.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: envoy-gateway-metrics-lb
+  namespace: envoy-gateway-system
+  labels:
+    control-plane: envoy-gateway
+spec:
+  selector:
+    control-plane: envoy-gateway
+    app.kubernetes.io/instance: eg
+  ports:
+    - name: http-metrics
+      port: 19001
+      protocol: TCP
+      targetPort: 19001
+  type: LoadBalancer
diff --git a/test/e2e/tests/controlplane.go b/test/e2e/tests/controlplane.go
@@ -0,0 +1,76 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+//go:build e2e
+// +build e2e
+
+package tests
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/gateway-api/conformance/utils/suite"
+)
+
+func init() {
+	ConformanceTests = append(ConformanceTests, ControlPlaneMetricTest)
+}
+
+var ControlPlaneMetricTest = suite.ConformanceTest{
+	ShortName:   "ControlPlane",
+	Description: "Make sure control plane prometheus endpoint is working",
+	Manifests:   []string{"testdata/prometheus.yaml"},
+	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
+		t.Run("Prometheus", func(t *testing.T) {
+			nn := types.NamespacedName{Name: "envoy-gateway-metrics-lb", Namespace: "envoy-gateway-system"}
+			if err := wait.PollUntilContextTimeout(context.TODO(), time.Second, time.Minute, true,
+				func(_ context.Context) (done bool, err error) {
+					svc := corev1.Service{}
+					if err := suite.Client.Get(context.Background(), nn, &svc); err != nil {
+						return false, nil
+					}
+
+					host := ""
+					switch svc.Spec.Type {
+					case corev1.ServiceTypeLoadBalancer:
+						for _, ing := range svc.Status.LoadBalancer.Ingress {
+							if ing.IP != "" {
+								host = ing.IP
+								break
+							}
+						}
+					default:
+						// do nothing
+					}
+
+					if host == "" {
+						return false, nil
+					}
+
+					return true, nil
+				}); err != nil {
+				t.Errorf("failed to get service %s : %v", nn.String(), err)
+			}
+
+			// too much flakes in the test if timeout is 1 minute
+			// this should not take so long, but we give it a long timeout to be safe, and poll every second
+			if err := wait.PollUntilContextTimeout(context.TODO(), time.Second, 2*time.Minute, true,
+				func(_ context.Context) (done bool, err error) {
+					if err := ScrapeMetrics(t, suite.Client, nn, 19001, "/metrics"); err != nil {
+						t.Logf("failed to get metric: %v", err)
+						return false, nil
+					}
+					return true, nil
+				}); err != nil {
+				t.Errorf("failed to scrape metrics: %v", err)
+			}
+		})
+	},
+}
diff --git a/test/e2e/tests/metric.go b/test/e2e/tests/metric.go
@@ -59,7 +59,7 @@ var MetricTest = suite.ConformanceTest{
 					if err := ScrapeMetrics(t, suite.Client, types.NamespacedName{
 						Namespace: "envoy-gateway-system",
 						Name:      "same-namespace-gw-metrics",
-					}, "/stats/prometheus"); err != nil {
+					}, 19001, "/stats/prometheus"); err != nil {
 						t.Logf("failed to get metric: %v", err)
 						return false, nil
 					}
@@ -93,7 +93,7 @@ var MetricTest = suite.ConformanceTest{
 					if err := ScrapeMetrics(t, suite.Client, types.NamespacedName{
 						Namespace: "monitoring",
 						Name:      "otel-collecot-prometheus",
-					}, "/metrics"); err != nil {
+					}, 19001, "/metrics"); err != nil {
 						t.Logf("failed to get metric: %v", err)
 						return false, nil
 					}
@@ -105,19 +105,25 @@ var MetricTest = suite.ConformanceTest{
 	},
 }
 
-func ScrapeMetrics(t *testing.T, c client.Client, nn types.NamespacedName, path string) error {
+func ScrapeMetrics(t *testing.T, c client.Client, nn types.NamespacedName, port int32, path string) error {
 	svc := corev1.Service{}
 	if err := c.Get(context.Background(), nn, &svc); err != nil {
 		return err
 	}
 	host := ""
-	for _, ing := range svc.Status.LoadBalancer.Ingress {
-		if ing.IP != "" {
-			host = ing.IP
-			break
+	switch svc.Spec.Type {
+	case corev1.ServiceTypeLoadBalancer:
+		for _, ing := range svc.Status.LoadBalancer.Ingress {
+			if ing.IP != "" {
+				host = ing.IP
+				break
+			}
 		}
+	default:
+		host = fmt.Sprintf("%s.%s.svc", nn.Name, nn.Namespace)
 	}
-	url := fmt.Sprintf("http://%s:19001%s", host, path)
+
+	url := fmt.Sprintf("http://%s:%d%s", host, port, path)
 	t.Logf("try to request: %s", url)
 
 	httpClient := http.Client{