From 42c1ac14951feb3f25f1efc21362b5cccad3237a Mon Sep 17 00:00:00 2001 From: zirain Date: Mon, 30 Oct 2023 21:21:49 -0500 Subject: [PATCH] e2e: eg controlplane metrics (#2106) * e2e: eg controlplane metrics Signed-off-by: zirain * update Signed-off-by: zirain * update Signed-off-by: zirain * update name Signed-off-by: zirain * rename Signed-off-by: zirain * wait more Signed-off-by: zirain * add comment Signed-off-by: zirain * update Signed-off-by: zirain --------- Signed-off-by: zirain (cherry picked from commit 046a593fe66b4a30c9964edb29e3b2c1d7375078) Signed-off-by: Arko Dasgupta --- test/e2e/testdata/prometheus.yaml | 17 +++++++ test/e2e/tests/controlplane.go | 76 +++++++++++++++++++++++++++++++ test/e2e/tests/metric.go | 22 +++++---- 3 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 test/e2e/testdata/prometheus.yaml create mode 100644 test/e2e/tests/controlplane.go diff --git a/test/e2e/testdata/prometheus.yaml b/test/e2e/testdata/prometheus.yaml new file mode 100644 index 00000000000..d33db1d2dea --- /dev/null +++ b/test/e2e/testdata/prometheus.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: envoy-gateway-metrics-lb + namespace: envoy-gateway-system + labels: + control-plane: envoy-gateway +spec: + selector: + control-plane: envoy-gateway + app.kubernetes.io/instance: eg + ports: + - name: http-metrics + port: 19001 + protocol: TCP + targetPort: 19001 + type: LoadBalancer diff --git a/test/e2e/tests/controlplane.go b/test/e2e/tests/controlplane.go new file mode 100644 index 00000000000..7149c2546b6 --- /dev/null +++ b/test/e2e/tests/controlplane.go @@ -0,0 +1,76 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build e2e +// +build e2e + +package tests + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +func init() { + ConformanceTests = append(ConformanceTests, ControlPlaneMetricTest) +} + +var ControlPlaneMetricTest = suite.ConformanceTest{ + ShortName: "ControlPlane", + Description: "Make sure control plane prometheus endpoint is working", + Manifests: []string{"testdata/prometheus.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + t.Run("Prometheus", func(t *testing.T) { + nn := types.NamespacedName{Name: "envoy-gateway-metrics-lb", Namespace: "envoy-gateway-system"} + if err := wait.PollUntilContextTimeout(context.TODO(), time.Second, time.Minute, true, + func(_ context.Context) (done bool, err error) { + svc := corev1.Service{} + if err := suite.Client.Get(context.Background(), nn, &svc); err != nil { + return false, nil + } + + host := "" + switch svc.Spec.Type { + case corev1.ServiceTypeLoadBalancer: + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.IP != "" { + host = ing.IP + break + } + } + default: + // do nothing + } + + if host == "" { + return false, nil + } + + return true, nil + }); err != nil { + t.Errorf("failed to get service %s : %v", nn.String(), err) + } + + // too much flakes in the test if timeout is 1 minute + // this should not take so long, but we give it a long timeout to be safe, and poll every second + if err := wait.PollUntilContextTimeout(context.TODO(), time.Second, 2*time.Minute, true, + func(_ context.Context) (done bool, err error) { + if err := ScrapeMetrics(t, suite.Client, nn, 19001, "/metrics"); err != nil { + t.Logf("failed to get metric: %v", err) + return false, nil + } + return true, nil + }); err != nil { + t.Errorf("failed to scrape metrics: %v", err) + } + }) + }, +} diff --git a/test/e2e/tests/metric.go b/test/e2e/tests/metric.go index 0f312dc07b5..2c9a8c296c3 100644 --- a/test/e2e/tests/metric.go +++ b/test/e2e/tests/metric.go @@ -59,7 +59,7 @@ var MetricTest = suite.ConformanceTest{ if err := ScrapeMetrics(t, suite.Client, types.NamespacedName{ Namespace: "envoy-gateway-system", Name: "same-namespace-gw-metrics", - }, "/stats/prometheus"); err != nil { + }, 19001, "/stats/prometheus"); err != nil { t.Logf("failed to get metric: %v", err) return false, nil } @@ -93,7 +93,7 @@ var MetricTest = suite.ConformanceTest{ if err := ScrapeMetrics(t, suite.Client, types.NamespacedName{ Namespace: "monitoring", Name: "otel-collecot-prometheus", - }, "/metrics"); err != nil { + }, 19001, "/metrics"); err != nil { t.Logf("failed to get metric: %v", err) return false, nil } @@ -105,19 +105,25 @@ var MetricTest = suite.ConformanceTest{ }, } -func ScrapeMetrics(t *testing.T, c client.Client, nn types.NamespacedName, path string) error { +func ScrapeMetrics(t *testing.T, c client.Client, nn types.NamespacedName, port int32, path string) error { svc := corev1.Service{} if err := c.Get(context.Background(), nn, &svc); err != nil { return err } host := "" - for _, ing := range svc.Status.LoadBalancer.Ingress { - if ing.IP != "" { - host = ing.IP - break + switch svc.Spec.Type { + case corev1.ServiceTypeLoadBalancer: + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.IP != "" { + host = ing.IP + break + } } + default: + host = fmt.Sprintf("%s.%s.svc", nn.Name, nn.Namespace) } - url := fmt.Sprintf("http://%s:19001%s", host, path) + + url := fmt.Sprintf("http://%s:%d%s", host, port, path) t.Logf("try to request: %s", url) httpClient := http.Client{