Skip to content

Commit

Permalink
[OSSM-8136] Refactor TestOpenShiftMonitoring test to not use standalo…
Browse files Browse the repository at this point in the history 
…ne Kiali CR
  • Loading branch information
@mkralik3
mkralik3 committed Sep 16, 2024
1 parent 869a55c commit 287968e
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 48 deletions.
3 changes: 0 additions & 3 deletions pkg/tests/tasks/observability/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,6 @@ var (
//go:embed yaml/federated-monitoring-mesh.tmpl.yaml
federatedMonitoringMeshTmpl string

//go:embed yaml/kiali-user-workload-monitoring.tmpl.yaml
kialiUserWorkloadMonitoringTmpl string

//go:embed yaml/kiali-cluster-monitoring-view.tmpl.yaml
kialiClusterMonitoringView string

Expand Down
91 changes: 75 additions & 16 deletions pkg/tests/tasks/observability/openshift_monitoring_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,39 +64,39 @@ func TestOpenShiftMonitoring(t *testing.T) {

t.Cleanup(func() {
oc.DeleteFromTemplate(t, monitoringNs, clusterMonitoringConfigTmpl, map[string]bool{"Enabled": false})
oc.DeleteFromTemplate(t, meshNamespace, kialiUserWorkloadMonitoringTmpl, kialiValues)
})

t.LogStep("Waiting until user workload monitoring stack is up and running")
oc.ApplyTemplate(t, monitoringNs, clusterMonitoringConfigTmpl, map[string]bool{"Enabled": true})
oc.WaitPodsExist(t, userWorkloadMonitoringNs)
oc.WaitAllPodsReady(t, userWorkloadMonitoringNs)

t.LogStep("Deploying Kiali")
oc.ApplyTemplate(t, meshNamespace, kialiUserWorkloadMonitoringTmpl, kialiValues)
oc.WaitPodRunning(t, pod.MatchingSelector("app=kiali", meshNamespace))
kialiPodName := shell.Executef(t, "oc get pod -l app=kiali -n %s -o jsonpath='{.items[0].metadata.name}'", meshNamespace)

t.LogStep("Grant cluster-monitoring-view to Kiali")
oc.ApplyTemplate(t, meshNamespace, kialiClusterMonitoringView, kialiValues)

t.NewSubTest("SMCP manageNetworkPolicy false").Run(func(t test.TestHelper) {
t.Cleanup(func() {
oc.DeleteFromString(t, meshNamespace, istiodMonitor)
oc.DeleteFromString(t, ns.Foo, istioProxyMonitor)
oc.DeleteFromString(t, meshNamespace, enableTrafficMetrics)
app.Uninstall(t, app.Httpbin(ns.Foo))
oc.DeleteFromTemplate(t, meshNamespace, meshTmpl, meshValues)
oc.RecreateNamespace(t, ns.Foo)
oc.RecreateNamespace(t, meshNamespace)
})

t.LogStep("Deploying SMCP")
oc.ApplyTemplate(t, meshNamespace, meshTmpl, meshValues)
oc.WaitSMCPReady(t, meshNamespace, smcpName)

t.LogStep("Wait kiali is ready")
waitKialiAndVerifyIsReconciled(t)
kialiPodName := shell.Executef(t, "oc get pod -l app=kiali -n %s -o jsonpath='{.items[0].metadata.name}'", meshNamespace)

t.LogStep("Patch kiali to use user workload monitoring")
patchKiali(t)

t.LogStep("Grant cluster-monitoring-view to Kiali")
oc.ApplyTemplate(t, meshNamespace, kialiClusterMonitoringView, kialiValues)

t.LogStep("Wait until the old Kiali pod has been deleted")
oc.WaitUntilResourceExist(t, meshNamespace, "pod", kialiPodName)
kialiPodName = shell.Executef(t, "oc get pod -l app=kiali -n %s -o jsonpath='{.items[0].metadata.name}'", meshNamespace)

t.LogStep("Fetch Kiali token")
kialiToken := fetchKialiToken(t)
Expand All @@ -113,6 +113,7 @@ func TestOpenShiftMonitoring(t *testing.T) {
oc.ApplyString(t, meshNamespace, istiodMonitor)
oc.ApplyString(t, ns.Foo, istioProxyMonitor)

waitUntilAllPrometheusTargetReady(t, kialiToken)
generateTrafficAndcheckMetrics(t, kialiToken)
})

Expand All @@ -123,18 +124,27 @@ func TestOpenShiftMonitoring(t *testing.T) {
oc.DeleteFromString(t, meshNamespace, enableTrafficMetrics)
app.Uninstall(t, app.Httpbin(ns.Foo))
oc.DeleteFromTemplate(t, meshNamespace, networkPolicy, map[string]string{"namespace": meshNamespace})
oc.DeleteFromTemplate(t, meshNamespace, meshTmpl, meshValues)
oc.RecreateNamespace(t, ns.Foo)
oc.RecreateNamespace(t, meshNamespace)
})

t.LogStep("Deploying SMCP")
meshValues["manageNetworkPolicy"] = "true"
oc.ApplyTemplate(t, meshNamespace, meshTmpl, meshValues)
oc.WaitSMCPReady(t, meshNamespace, smcpName)

t.LogStep("Wait kiali is ready")
waitKialiAndVerifyIsReconciled(t)
kialiPodName := shell.Executef(t, "oc get pod -l app=kiali -n %s -o jsonpath='{.items[0].metadata.name}'", meshNamespace)

t.LogStep("Patch kiali to use user workload monitoring")
patchKiali(t)

t.LogStep("Grant cluster-monitoring-view to Kiali")
oc.ApplyTemplate(t, meshNamespace, kialiClusterMonitoringView, kialiValues)

t.LogStep("Wait until the old Kiali pod has been deleted")
oc.WaitUntilResourceExist(t, meshNamespace, "pod", kialiPodName)
kialiPodName = shell.Executef(t, "oc get pod -l app=kiali -n %s -o jsonpath='{.items[0].metadata.name}'", meshNamespace)

t.LogStep("Fetch Kiali token")
kialiToken := fetchKialiToken(t)
Expand All @@ -155,6 +165,7 @@ func TestOpenShiftMonitoring(t *testing.T) {
oc.ApplyString(t, meshNamespace, istiodMonitor)
oc.ApplyString(t, ns.Foo, istioProxyMonitor)

waitUntilAllPrometheusTargetReady(t, kialiToken)
generateTrafficAndcheckMetrics(t, kialiToken)
})
})
Expand Down Expand Up @@ -193,9 +204,12 @@ func generateTrafficAndcheckMetrics(t test.TestHelper, thanosToken string) {
t.LogStep("Generate some ingress traffic")
oc.ApplyFile(t, ns.Foo, "https://raw.githubusercontent.com/maistra/istio/maistra-2.6/samples/httpbin/httpbin-gateway.yaml")
httpbinURL := fmt.Sprintf("http://%s/headers", istio.GetIngressGatewayHost(t, meshNamespace))
retry.UntilSuccess(t, func(t test.TestHelper) {
curl.Request(t, httpbinURL, nil, assert.ResponseStatus(http.StatusOK))
})

for i := 0; i < 5; i++ {
retry.UntilSuccess(t, func(t test.TestHelper) {
curl.Request(t, httpbinURL, nil, assert.ResponseStatus(http.StatusOK))
})
}

t.LogStep("Check istiod metrics")
checkMetricExists(t, meshNamespace, "pilot_info", thanosToken)
Expand All @@ -218,12 +232,36 @@ func checkMetricExists(t test.TestHelper, ns, metricName, token string) {
})
}

func waitUntilAllPrometheusTargetReady(t test.TestHelper, token string) {
waitUntilPrometheusTargetReady(t, "serviceMonitor", meshNamespace, "istiod-monitor", token)
waitUntilPrometheusTargetReady(t, "podMonitor", ns.Foo, "istio-proxies-monitor", token)
}

func waitUntilPrometheusTargetReady(t test.TestHelper, monitorType string, ns string, targetName string, token string) {
retry.UntilSuccess(t, func(t test.TestHelper) {
oc.Exec(t,
pod.MatchingSelectorFirst("app.kubernetes.io/instance=thanos-querier", monitoringNs),
"thanos-query",
prometheusActiveTargetQuery(token),
assert.OutputContains(
fmt.Sprintf(`"scrapePool":"%s/%s/%s`, monitorType, ns, targetName),
fmt.Sprintf("The %s %s prometheus target is ready in namespace %s", monitorType, targetName, ns),
fmt.Sprintf("The %s %s prometheus target is not ready yet in namespace %s", monitorType, targetName, ns)),
)
})
}

func prometheusQuery(ns, metricName, token string) string {
return fmt.Sprintf(
`curl -X GET -kG "https://localhost:9091/api/v1/query?namespace=%s&query=%s" --data-urlencode "query=up" -H "Authorization: Bearer %s"`,
ns, metricName, token)
}

func prometheusActiveTargetQuery(token string) string {
return fmt.Sprintf(
`curl -X GET -kG "https://localhost:9091/api/v1/targets?state=active" --data-urlencode "query=up" -H "Authorization: Bearer %s"`, token)
}

func fetchKialiToken(t test.TestHelper) string {
var kialiToken string
retry.UntilSuccess(t, func(t test.TestHelper) {
Expand All @@ -235,3 +273,24 @@ func fetchKialiToken(t test.TestHelper) string {
})
return kialiToken
}

func patchKiali(t test.TestHelper) {
t.LogStep("Wait until Kiali is ready")
shell.Execute(t,
fmt.Sprintf("oc patch kiali/kiali-user-workload-monitoring -n %s --type merge --patch '%s' || true", meshNamespace, kialiPrometheusPatch))

}

const kialiPrometheusPatch = `
spec:
external_services:
prometheus:
auth:
type: bearer
use_kiali_token: true
query_scope:
mesh_id: "unique-mesh-id"
thanos_proxy:
enabled: true
url: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
`
29 changes: 0 additions & 29 deletions pkg/tests/tasks/observability/yaml/kiali-user-workload-monitoring.tmpl.yaml
View file

This file was deleted.

0 comments on commit 287968e

Please sign in to comment.