Included few more mgmt cluster metrics

workloads/kube-burner-ocp-wrapper/metrics-profiles/hosted-cluster-metrics.yml

@@ -31,6 +31,22 @@
 - query: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")
   metricName: nodeMemoryUtilization-Infra
 
+- query: node_memory_MemAvailable_bytes{} and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")
+  metricName: nodeMemoryAvailable-Workers
+
+- query: node_memory_MemTotal_bytes{} and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")
+  metricName: nodeMemoryTotal-Workers
+
+- query: (max(sum(irate(node_cpu_seconds_total{}[2m])) by (mode,instance) and on (instance) label_replace(bottomk(1, min_over_time(sum(irate(node_cpu_seconds_total{mode=~"idle"}[2m])) by (mode,instance)[{{ .elapsed }}:])), "instance", "$1", "instance", "(.+)")) by (mode)) > 0
+  metricName: nodeCPU-MostUtilizedWorkers
+
+- query: bottomk(1,min_over_time(node_memory_MemAvailable_bytes{}[{{ .elapsed }}:]))
+  metricName: nodeMemoryAvailable-MostUtilizedWorker
+
+- query: avg(node_memory_MemTotal_bytes{} and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)"))
+  metricName: nodeMemoryTotal-AggregatedWorkers
+  instant: true  
+
 # Cluster metrics
 
 - query: sum(kube_namespace_status_phase) by (phase) > 0

workloads/kube-burner-ocp-wrapper/metrics-profiles/mc-metrics.yml

@@ -2,17 +2,80 @@
 # They should be preffixed with mgmt- to distinguish from the hosted cluster ones
 # Only collecting container and worker nodes CPU/Memory metrics
 
-# ControlPlane Containers & pod metrics
-
-- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~".+{{.HCP_NAMESPACE}}"}[2m]) * 100) by (container, pod, namespace, node)) > 0
-  metricName: mgmt-containerCPU
-
-- query: sum(container_memory_rss{name!="",container!="POD",namespace=~".+{{.HCP_NAMESPACE}}"}) by (container, pod, namespace, node)
-  metricName: mgmt-containerMemory
-
 # Management Node metrics: CPU & Memory
 - query: (sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) > 0
   metricName: mgmt-nodeCPU-Workers
 
 - query: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")
   metricName: mgmt-nodeMemoryUtilization-Workers
+
+- query: (max((sum(irate(node_cpu_seconds_total{}[2m])) by (mode,instance) and on (instance) label_replace(bottomk(1, min_over_time(sum(irate(node_cpu_seconds_total{mode=~"idle",instance=~"{{.MGMT_WORKER_NODES}}"}[2m])) by (mode,instance)[{{ .elapsed }}:])), "instance", "$1", "instance", "(.+)"))) by (mode, instance)) > 0
+  metricName: mgmtNodeCPU-MostUtilizedWorker
+
+- query: bottomk(1,min_over_time(node_memory_MemAvailable_bytes{instance=~"{{.MGMT_WORKER_NODES}}"}[{{ .elapsed }}:]))
+  metricName: mgmtNodeMemoryAvailable-MostUtilizedWorker
+
+- query: (avg(node_memory_MemTotal_bytes{instance=~"{{.MGMT_WORKER_NODES}}"}) by (instance))
+  metricName: mgmtNodeMemoryTotal
+
+- query: (avg((sum(irate(node_cpu_seconds_total{}[2m])) by (mode,instance) and on (instance) label_replace(cluster:nodes_roles{label_node_role_kubernetes_io_master!=""}, "instance", "$1", "node", "(.+)"))) by (mode, instance)) > 0
+  metricName: mgmtMasterCPU-Aggregated
+
+- query: (avg(node_memory_MemAvailable_bytes{} and on (instance) label_replace(cluster:nodes_roles{label_node_role_kubernetes_io_master!=""}, "instance", "$1", "node", "(.+)")) by (instance))
+  metricName: mgmtMasterMemoryAvailable-Aggregated
+
+- query: (avg(node_memory_MemTotal_bytes{} and on (instance) label_replace(cluster:nodes_roles{label_node_role_kubernetes_io_master!=""}, "instance", "$1", "node", "(.+)")) by (instance))
+  metricName: mgmtMasterMemoryTotal
+
+- query: kube_node_role{}
+  metricName: mgmtNodeRoles
+
+- query: irate(node_disk_reads_completed_total{instance=~"{{.MGMT_WORKER_NODES}}"}[2m])
+  metricName: mgmtNodeDiskReads
+
+- query: irate(node_disk_writes_completed_total{instance=~"{{.MGMT_WORKER_NODES}}"}[2m])
+  metricName: mgmtNodeDiskWrites
+# ControlPlane Containers metrics
+- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|.*apiserver|ovn-kubernetes|sdn|ingress|.*controller-manager|.*scheduler|image-registry|monitoring)"}[2m]) * 100)  by (container, pod, namespace, node)) > 0
+  metricName: mgmt-containerCPU
+
+- query: sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|.*apiserver|ovn-kubernetes|sdn|ingress|.*controller-manager|.*scheduler|image-registry|monitoring)"}) by (container, pod, namespace, node)
+  metricName: mgmt-containerMemory
+
+# MC Etcd metrics
+- query: sum(rate(etcd_server_leader_changes_seen_total{namespace=~"openshift-etcd"}[2m]))
+  metricName: mgmt-etcdLeaderChangesRate
+
+- query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{namespace=~"openshift-etcd"}[2m]))
+  metricName: mgmt-99thEtcdDiskBackendCommitDurationSeconds
+
+- query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{namespace=~"openshift-etcd"}[2m]))
+  metricName: mgmt-99thEtcdDiskWalFsyncDurationSeconds
+
+- query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{namespace=~"openshift-etcd"}[5m]))
+  metricName: mgmt-99thEtcdRoundTripTimeSeconds
+
+# ControlPlane Containers & pod metrics
+# These metrics would be available in MC CMO
+
+- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~".+{{.HCP_NAMESPACE}}"}[2m]) * 100) by (pod, container, namespace, node)) > 0
+  metricName: podCPU-Controlplane
+
+- query: sum(container_memory_rss{name!="",container!="POD",namespace=~".+{{.HCP_NAMESPACE}}"}) by (pod, container, namespace, node)
+  metricName: podMemory-Controlplane
+
+- query: sum(container_memory_cache{name!="",container!="POD",namespace=~".+{{.HCP_NAMESPACE}}"}) by (pod, container, namespace, node)
+  metricName: podMemoryCache-Controlplane
+
+- query: sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{namespace=~".+{{.HCP_NAMESPACE}}"}) by (pod,container,namespace)
+  metricName: podCPUReq
+  instant: true
+
+- query: sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{namespace=~".+{{.HCP_NAMESPACE}}"}) by (pod,container,namespace)
+  metricName: podMemoryReq
+  instant: true
+
+- query: cluster_version{}
+  metricName: mgmtClusterVersion
+  instant: true  
+

workloads/kube-burner-ocp-wrapper/run.sh

@@ -11,10 +11,12 @@ QPS=${QPS:-20}
 BURST=${BURST:-20}
 GC=${GC:-true}
 EXTRA_FLAGS=${EXTRA_FLAGS:-}
+UUID=$(uuidgen)
+KUBE_DIR=${KUBE_DIR:-/tmp}
 
 download_binary(){
   KUBE_BURNER_URL=https://github.com/cloud-bulldozer/kube-burner/releases/download/v${KUBE_BURNER_VERSION}/kube-burner-${KUBE_BURNER_VERSION}-Linux-x86_64.tar.gz
-  curl -sS -L ${KUBE_BURNER_URL} | tar -xzC /tmp/ kube-burner
+  curl -sS -L ${KUBE_BURNER_URL} | tar -xzC ${KUBE_DIR}/ kube-burner
 }
 
 hypershift(){
@@ -40,6 +42,18 @@ hypershift(){
   MC_PROMETHEUS_TOKEN=$(oc --kubeconfig=${MC_KUBECONFIG} sa new-token -n openshift-monitoring prometheus-k8s)
   HOSTED_PROMETHEUS=https://$(oc get route -n openshift-monitoring prometheus-k8s -o jsonpath="{.spec.host}")
   HOSTED_PROMETHEUS_TOKEN=$(oc sa new-token -n openshift-monitoring prometheus-k8s)
+
+  echo "Get all management worker nodes, excludes infra, obo, workload"
+  Q_NODES=""
+  for n in $(curl -H "Authorization: Bearer ${MC_PROMETHEUS_TOKEN}" -k --silent --globoff  ${MC_PROMETHEUS}/api/v1/query?query='sum(kube_node_role{role!~"master|infra|workload|obo"})by(node)&time='$(date +"%s")'' | jq -r '.data.result[].metric.node'); do
+    if [[ ${Q_NODES} == "" ]]; then
+      Q_NODES=${n}
+    else
+      Q_NODES=${Q_NODES}"|"${n};
+    fi
+  done
+  MGMT_WORKER_NODES=${Q_NODES}
+
   echo "Exporting required vars"
   cat << EOF
 MC_OBO: ${MC_OBO}
@@ -48,13 +62,14 @@ MC_PROMETHEUS_TOKEN: <truncated>
 HOSTED_PROMETHEUS: ${HOSTED_PROMETHEUS}
 HOSTED_PROMETHEUS_TOKEN: <truncated>
 HCP_NAMESPACE: ${HCP_NAMESPACE}
+MGMT_WORKER_NODES: ${MGMT_WORKER_NODES}
 
 EOF
-  export MC_OBO MC_PROMETHEUS MC_PROMETHEUS_TOKEN HOSTED_PROMETHEUS HOSTED_PROMETHEUS_TOKEN HCP_NAMESPACE
+  export MC_OBO MC_PROMETHEUS MC_PROMETHEUS_TOKEN HOSTED_PROMETHEUS HOSTED_PROMETHEUS_TOKEN HCP_NAMESPACE MGMT_WORKER_NODES
 }
 
 download_binary
-cmd="/tmp/kube-burner ocp ${WORKLOAD} --log-level=${LOG_LEVEL} --qps=${QPS} --burst=${BURST} --gc=${GC}"
+cmd="${KUBE_DIR}/kube-burner ocp ${WORKLOAD} --log-level=${LOG_LEVEL} --qps=${QPS} --burst=${BURST} --gc=${GC} --uuid ${UUID}"
 if [[ ${WORKLOAD} =~ "cluster-density" ]]; then
   ITERATIONS=${ITERATIONS:?}
   cmd+=" --iterations=${ITERATIONS} --churn=${CHURN}"
@@ -68,5 +83,18 @@ if [[ -n ${ES_SERVER} ]]; then
   cmd+=" --es-server=${ES_SERVER} --es-index=ripsaw-kube-burner"
 fi
 cmd+=" ${EXTRA_FLAGS}"
+
+echo "Indexing Management cluster stats before executing"
+METADATA=$(cat << EOF
+{
+"uuid" : "${UUID}",
+"mgmtClusterName": "$(oc get --kubeconfig=${MC_KUBECONFIG} infrastructure.config.openshift.io cluster -o json 2>/dev/null | jq -r .status.infrastructureName)",
+"hostedClusterName": "$(oc get infrastructure.config.openshift.io cluster -o json 2>/dev/null | jq -r .status.infrastructureName)",
+"timestamp": "$(date +%s%3N)"
+}
+EOF
+)
+curl -k -sS -X POST -H "Content-type: application/json" ${ES_SERVER}/ripsaw-kube-burner/_doc -d "${METADATA}" -o /dev/null
+
 echo $cmd
 exec $cmd