OSSM-8225 Add test for mesh member metrics (#750)

* [OSSM-8225] Add test for mesh member metrics data

* Unify ClusterWide SMCP

pkg/tests/ossm/discovery_selectors_test.go

@@ -24,7 +24,6 @@ import (
 	"github.com/maistra/maistra-test-tool/pkg/util/ns"
 	"github.com/maistra/maistra-test-tool/pkg/util/oc"
 	"github.com/maistra/maistra-test-tool/pkg/util/pod"
-	"github.com/maistra/maistra-test-tool/pkg/util/template"
 	. "github.com/maistra/maistra-test-tool/pkg/util/test"
 	"github.com/maistra/maistra-test-tool/pkg/util/version"
 )
@@ -41,14 +40,15 @@ func TestDiscoverySelectors(t *testing.T) {
 		t.LogStep("Apply cluster-wide SMCP and standard SMMR")
 		oc.RecreateNamespace(t, meshNamespace)
 		oc.ApplyString(t, meshNamespace, smmr)
-		oc.ApplyString(t, meshNamespace, template.Run(t, clusterWideSMCP, DefaultSMCP()))
-		oc.WaitSMCPReady(t, meshNamespace, DefaultSMCP().Name)
+		DeployClusterWideControlPlane(t)
 		oc.WaitSMMRReady(t, meshNamespace)
 
 		t.LogStep("Install httpbin and sleep pod")
 		app.InstallAndWaitReady(t, app.Sleep(ns.Foo), app.Httpbin(ns.MeshExternal))
 		t.Cleanup(func() {
 			app.Uninstall(t, app.Sleep(ns.Foo), app.Httpbin(ns.MeshExternal))
+			oc.RecreateNamespace(t, ns.Foo)
+			oc.RecreateNamespace(t, ns.MeshExternal)
 			// since the smcp clusterwide mode was used, clean the namespace
 			t.LogStepf("Delete namespace %s", meshNamespace)
 			oc.RecreateNamespace(t, meshNamespace)
@@ -93,28 +93,3 @@ spec:
 				"Expected Httpbin to not be discovered, but it was."))
 	})
 }
-
-const (
-	clusterWideSMCP = `
-apiVersion: maistra.io/v2
-kind: ServiceMeshControlPlane
-metadata:
-  name: {{ .Name }}
-spec:
-  version: {{ .Version }}
-  mode: ClusterWide
-  tracing:
-    type: None
-  addons:
-    grafana:
-      enabled: false
-    kiali:
-      enabled: false
-    prometheus:
-      enabled: false
-  {{ if .Rosa }} 
-  security:
-    identity:
-      type: ThirdParty
-  {{ end }}`
-)

pkg/tests/ossm/operator/clusterwide_mode_test.go

@@ -69,7 +69,14 @@ func TestClusterWideMode(t *testing.T) {
 		oc.RecreateNamespace(t, meshNamespace)
 
 		t.LogStep("Install cluster-wide SMCP")
-		oc.ApplyTemplate(t, meshNamespace, clusterWideSMCP, ossm.DefaultSMCP())
+		ossm.DeployClusterWideControlPlane(t)
+		oc.Patch(t, meshNamespace, "smcp", smcpName, "merge", `
+spec:
+  general:
+    logging:
+      componentLevels:
+        default: info
+`)
 		oc.WaitSMCPReady(t, meshNamespace, smcpName)
 
 		t.NewSubTest("check Gateway API settings").Run(func(t TestHelper) {
@@ -517,8 +524,7 @@ spec:
 			t.Log("Check whether the cluster wide feature works with profiles")
 
 			t.LogStep("Delete SMCP and SMMR")
-			oc.DeleteFromTemplate(t, meshNamespace, clusterWideSMCP, ossm.DefaultSMCP())
-			oc.DeleteFromString(t, meshNamespace, defaultSMMR)
+			oc.RecreateNamespace(t, meshNamespace)
 
 			t.LogStep("Deploy SMCP with the profile")
 			oc.ApplyTemplate(t,
@@ -667,37 +673,6 @@ func deleteUserAndAdminRole(t TestHelper, namespaces ...string) {
 }
 
 const (
-	clusterWideSMCP = `
-apiVersion: maistra.io/v2
-kind: ServiceMeshControlPlane
-metadata:
-  name: {{ .Name }}
-spec:
-  version: {{ .Version }}
-  mode: ClusterWide
-  general:
-    logging:
-      componentLevels:
-        default: info
-  tracing:
-    sampling: 10000
-  policy:
-    type: Istiod
-  addons:
-    grafana:
-      enabled: true
-    kiali:
-      enabled: true
-    prometheus:
-      enabled: true
-  telemetry:
-    type: Istiod
-  {{ if .Rosa }} 
-  security:
-    identity:
-      type: ThirdParty
-  {{ end }}`
-
 	clusterWideSMCPWithProfile = `
 apiVersion: maistra.io/v2
 kind: ServiceMeshControlPlane

pkg/tests/ossm/prometheus_scoped_scraping_test.go

@@ -46,7 +46,7 @@ func TestOperatorCanUpdatePrometheusConfigMap(t *testing.T) {
 		}
 
 		t.Cleanup(func() {
-			oc.ApplyString(t, meshNamespace, smmr)
+			oc.RecreateNamespace(t, meshNamespace)
 		})
 
 		DeployControlPlane(t)
@@ -142,34 +142,15 @@ func TestOperatorCanUpdatePrometheusConfigMap(t *testing.T) {
 		})
 
 		t.NewSubTest("query istio_request_total").Run(func(t test.TestHelper) {
-			ns := "bookinfo"
+			t.Log("This test checks if Prometheus is scraping data from mesh member application when SMCP is MultiTenant mode")
 			t.Cleanup(func() {
-				oc.RecreateNamespace(t, ns)
+				oc.RecreateNamespace(t, ns.Bookinfo)
 			})
 
 			t.LogStep("Install bookinfo")
-			app.InstallAndWaitReady(t, app.Bookinfo(ns))
-
-			count := 10
-			t.LogStepf("Generate %d requests to product page", count)
-			productPageURL := app.BookinfoProductPageURL(t, meshNamespace)
-			for i := 0; i < count; i++ {
-				curl.Request(t, productPageURL, nil)
-			}
-
-			t.LogStep(`Check if the "istio_request_total metric is in Prometheus"`)
-			retry.UntilSuccess(t, func(t test.TestHelper) {
-				oc.Exec(t,
-					prometheusPodSelector,
-					"prometheus-proxy",
-					"curl localhost:9090/api/v1/query --data-urlencode 'query=istio_requests_total'",
-					assert.OutputContains(
-						`"__name__":"istio_requests_total"`,
-						`Found the "istio_request_total" metric`,
-						`Expected to find the "istio_request_total" metric, but found none`,
-					),
-				)
-			})
+			app.InstallAndWaitReady(t, app.Bookinfo(ns.Bookinfo))
+			waitUntilPrometheusTargetReady(t, "productpage")
+			testBookinfoAppReportIstioRequestsTotal(t)
 		})
 
 		t.NewSubTest("when removing SMMR").Run(func(t test.TestHelper) {
@@ -197,6 +178,40 @@ func TestOperatorCanUpdatePrometheusConfigMap(t *testing.T) {
 	})
 }
 
+func TestPrometheusScrapingAppDataWithClusterWideSmcp(t *testing.T) {
+	test.NewTest(t).Groups(test.Full, test.ARM).Run(func(t test.TestHelper) {
+		t.Log("This test checks if Prometheus is scraping data from mesh member application when SMCP is ClusterWide mode")
+		t.Log("Related issue: https://issues.redhat.com/browse/OSSM-8205")
+
+		if env.GetSMCPVersion().LessThan(version.SMCP_2_4) {
+			t.Skip("Test only valid in SMCP versions v2.4+")
+		}
+
+		t.Cleanup(func() {
+			oc.RecreateNamespace(t, meshNamespace)
+			oc.RecreateNamespace(t, ns.Bookinfo)
+		})
+
+		t.LogStep("Install SMCP in ClusterWide mode")
+		DeployClusterWideControlPlane(t)
+		oc.Label(t, "", "Namespace", ns.Bookinfo, "istio-injection=enabled")
+		oc.WaitSMMRReady(t, meshNamespace)
+		t.LogStep("Install bookinfo")
+		app.InstallAndWaitReady(t, app.Bookinfo(ns.Bookinfo))
+		waitUntilPrometheusTargetReady(t, "productpage")
+		testBookinfoAppReportIstioRequestsTotal(t)
+	})
+}
+
+func testBookinfoAppReportIstioRequestsTotal(t test.TestHelper) {
+	t.LogStep("Test that the `istio_requests_total` metric exist for bookinfo")
+	generateBookinfoTraffic(t)
+	checkIstioRequestsTotalInPrometheus(t, "productpage")
+	checkIstioRequestsTotalInPrometheus(t, "details")
+	checkIstioRequestsTotalInPrometheus(t, "reviews")
+	checkIstioRequestsTotalInPrometheus(t, "ratings")
+}
+
 func checkPermissionError(t test.TestHelper) {
 	t.LogStep("Check the Prometheus log to see if there is any permission error")
 	oc.Logs(t,
@@ -281,3 +296,44 @@ func assertConfigMapDoesNotContainNamespace(ns string) common.CheckFunc {
 func generateNamespace() string {
 	return fmt.Sprintf("namespace-%d", rand.Int())
 }
+
+func generateBookinfoTraffic(t test.TestHelper) {
+	count := 10
+	t.LogStepf("Generate %d requests to product page", count)
+	productPageURL := app.BookinfoProductPageURL(t, meshNamespace)
+	for i := 0; i < count; i++ {
+		curl.Request(t, productPageURL, nil)
+	}
+}
+
+func checkIstioRequestsTotalInPrometheus(t test.TestHelper, app string) {
+	query := "istio_requests_total"
+	expectedOutput := fmt.Sprintf(`"app":"%s"`, app)
+	t.LogStep(`Check if the "istio_request_total metric is in Prometheus"`)
+	retry.UntilSuccess(t, func(t test.TestHelper) {
+		oc.Exec(t,
+			prometheusPodSelector,
+			"prometheus-proxy",
+			fmt.Sprintf("curl localhost:9090/api/v1/query --data-urlencode 'query=%s'", query),
+			assert.OutputContains(
+				expectedOutput,
+				fmt.Sprintf("Successfully fetched %s query with expected output", query),
+				fmt.Sprintf("Expected to find %s results after query %s, but found none", expectedOutput, query)),
+		)
+	})
+}
+
+func waitUntilPrometheusTargetReady(t test.TestHelper, app string) {
+	t.LogStep(`Wait till targets are available in Prometheus"`)
+	retry.UntilSuccess(t, func(t test.TestHelper) {
+		oc.Exec(t,
+			prometheusPodSelector,
+			"prometheus-proxy",
+			"curl localhost:9090/api/v1/targets?state=active",
+			assert.OutputContains(
+				fmt.Sprintf(`"app":"%s"`, app),
+				fmt.Sprintf("The %s prometheus target is ready", app),
+				fmt.Sprintf("The %s prometheus target is not ready", app)),
+		)
+	})
+}

pkg/tests/ossm/setup.go

@@ -28,10 +28,11 @@ import (
 )
 
 type SMCP struct {
-	Name      string
-	Namespace string
-	Version   version.Version
-	Rosa      bool
+	Name          string
+	Namespace     string
+	Version       version.Version
+	Rosa          bool
+	ClusterWideCp bool
 
 	ClusterWideProxy bool
 	HttpProxy        string
@@ -74,10 +75,21 @@ var (
 
 func DefaultSMCP() SMCP {
 	return SMCP{
-		Name:      env.GetDefaultSMCPName(),
-		Namespace: env.GetDefaultMeshNamespace(),
-		Version:   env.GetSMCPVersion(),
-		Rosa:      env.IsRosa(),
+		Name:          env.GetDefaultSMCPName(),
+		Namespace:     env.GetDefaultMeshNamespace(),
+		Version:       env.GetSMCPVersion(),
+		Rosa:          env.IsRosa(),
+		ClusterWideCp: false,
+	}
+}
+
+func DefaultClusterWideSMCP() SMCP {
+	return SMCP{
+		Name:          env.GetDefaultSMCPName(),
+		Namespace:     env.GetDefaultMeshNamespace(),
+		Version:       env.GetSMCPVersion(),
+		Rosa:          env.IsRosa(),
+		ClusterWideCp: true,
 	}
 }
 
@@ -116,6 +128,13 @@ func DeployControlPlane(t test.TestHelper) {
 	oc.WaitSMMRReady(t, meshNamespace)
 }
 
+func DeployClusterWideControlPlane(t test.TestHelper) {
+	t.T().Helper()
+	t.LogStep("Apply ClusterWide SMCP")
+	InstallSMCPCustom(t, meshNamespace, DefaultClusterWideSMCP())
+	oc.WaitSMCPReady(t, meshNamespace, DefaultClusterWideSMCP().Name)
+}
+
 func InstallSMCP(t test.TestHelper, ns string) {
 	InstallSMCPVersion(t, ns, env.GetSMCPVersion())
 }

pkg/tests/ossm/yaml/smcp.yaml

@@ -50,3 +50,6 @@ spec:
     identity:
       type: ThirdParty
   {{ end }}
+  {{ if .ClusterWideCp }} 
+  mode: ClusterWide
+  {{ end }}