diff --git a/hack/update-toc.sh b/hack/update-toc.sh index 21cfa81a..a80edf96 100755 --- a/hack/update-toc.sh +++ b/hack/update-toc.sh @@ -50,4 +50,4 @@ GOBIN="${TMP_DIR}" ${GO} install "github.com/tallclair/mdtoc@${TOOL_VERSION}" export PATH="${TMP_DIR}:${PATH}" # Update tables of contents if necessary. -find docs -name '*.md' | xargs mdtoc --inplace +find docs snowflake -name '*.md' | xargs mdtoc --inplace diff --git a/hack/verify-docs-for-website.sh b/hack/verify-docs-for-website.sh index 23f59834..39e54270 100755 --- a/hack/verify-docs-for-website.sh +++ b/hack/verify-docs-for-website.sh @@ -42,7 +42,7 @@ function check_one_file { fi } -for f in $(find docs -name '*.md'); do +for f in $(find docs snowflake -name '*.md'); do check_one_file $f done for f in $(find . -maxdepth 1 -name '*.md'); do diff --git a/hack/verify-toc.sh b/hack/verify-toc.sh index 98d17042..12f86e46 100755 --- a/hack/verify-toc.sh +++ b/hack/verify-toc.sh @@ -51,4 +51,4 @@ export PATH="${TMP_DIR}:${PATH}" echo "Checking table of contents are up to date..." # Verify tables of contents are up-to-date -find docs -name '*.md' | xargs mdtoc --inplace --dryrun +find docs snowflake -name '*.md' | xargs mdtoc --inplace --dryrun diff --git a/snowflake/README.md b/snowflake/README.md index f3cb46fc..36f1b746 100644 --- a/snowflake/README.md +++ b/snowflake/README.md @@ -1,5 +1,29 @@ # Theia with Snowflake +## Table of Contents + + +- [Overview](#overview) +- [Getting started](#getting-started) + - [Prerequisites](#prerequisites) + - [Install the theia-sf CLI](#install-the-theia-sf-cli) + - [Configure AWS credentials](#configure-aws-credentials) + - [Configure Snowflake credentials](#configure-snowflake-credentials) + - [Create an S3 bucket to store infrastructure state](#create-an-s3-bucket-to-store-infrastructure-state) + - [Create a KMS key to encrypt infrastructure state](#create-a-kms-key-to-encrypt-infrastructure-state) + - [Provision all cloud resources](#provision-all-cloud-resources) + - [Configure the Flow Aggregator in your cluster(s)](#configure-the-flow-aggregator-in-your-clusters) +- [Clean up](#clean-up) +- [Running applications](#running-applications) +- [Network flow visibility with Grafana](#network-flow-visibility-with-grafana) + - [Configure datasource](#configure-datasource) + - [Deployments](#deployments) + - [Pre-built dashboards](#pre-built-dashboards) + - [View flow data from selected cluster(s)](#view-flow-data-from-selected-clusters) + + +## Overview + We are introducing the ability to use Snowflake as the storage and computing platform for Theia. When using Snowflake, it is no longer necessary to run ClickHouse DB (flow records are stored in a Snowflake database) or Spark (flow @@ -117,3 +141,85 @@ Snowflake credentials are required. We are in the process of adding support for applications to Snowflake-powered Theia, starting with NetworkPolicy recommendation. + +## Network flow visibility with Grafana + +We use Grafana as the tool to query data from Snowflake, and visualize the +networking flows in the cluster(s). + +### Configure datasource + +Export the following environment variables: + + Name | Description +------------------------- | ------------ + SNOWFLAKE_ACCOUNT | Specifies the full name of your account (provided by Snowflake). + SNOWFLAKE_USER | Specifies the login name of the user for the connection. + SNOWFLAKE_PASSWORD | Specifies the password for the specified user. + SNOWFLAKE_WAREHOUSE | Specifies the virtual warehouse to use once connected. + SNOWFLAKE_DATABASE | Specifies the default database to use once connected. + SNOWFLAKE_ROLE (Optional)| Specifies the default access control role to use in the Snowflake session initiated by Grafana. + +Database name can be found in the output of the [onboard](#getting-started) +command. + +### Deployments + +We suggest running Grafana using the official Docker image. + +Before running Grafana, we need to download the Snowflake datasource plugin. +We suggest creating your own plugin directory. + +```bash +mkdir your-plugin-path && cd your-plugin-path +wget https://github.com/michelin/snowflake-grafana-datasource/releases/download/v1.2.0/snowflake-grafana-datasource.zip +unzip snowflake-grafana-datasource.zip +export GF_PLUGINS=$(pwd) +``` + +Then run Grafana with Docker: + +```bash +cd theia/snowflake + +docker run -d \ +-p 3000:3000 \ +-v "${GF_PLUGINS}":/var/lib/grafana/plugins \ +-v "$(pwd)"/grafana/provisioning:/etc/grafana/provisioning \ +--name=grafana \ +-e "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=michelin-snowflake-datasource" \ +-e "GF_INSTALL_PLUGINS=https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-sankey-plugin-1.0.2.zip;theia-grafana-sankey-plugin,https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-chord-plugin-1.0.1.zip;theia-grafana-chord-plugin" \ +-e "GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboards/homepage.json" \ +-e "SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ACCOUNT}" \ +-e "SNOWFLAKE_USER=${SNOWFLAKE_USER}" \ +-e "SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD}" \ +-e "SNOWFLAKE_WAREHOUSE=${SNOWFLAKE_WAREHOUSE}" \ +-e "SNOWFLAKE_DATABASE=${SNOWFLAKE_DATABASE}" \ +-e "SNOWFLAKE_ROLE=${SNOWFLAKE_ROLE}" \ +grafana/grafana:9.1.6 +``` + +Open your web browser and go to `http://localhost:3000/`, login with: + +- username: admin +- password: admin + +### Pre-built dashboards + +You will see the home dashboard after login. It provides an overview +of the monitored Kubernetes cluster, short introduction and links +to the other pre-built dashboards for more specific flow visualization. +Detailed introduction of these dashboards can be found at +[Pre-built Dashboards](https://github.com/antrea-io/theia/blob/main/docs/network-flow-visibility.md#pre-built-dashboards). + +Currently, with Snowflake datasource, we have built the Home Dashboard, Flow +Records Dashboard, Pod-to-Pod Flows Dashboard, and Network-Policy Flows +Dashboard. + +#### View flow data from selected cluster(s) + +The dashboards allow you to select and view flow data from one or more +clusters. You will be able to find a dropdown menu on the top left corner of +each dashboard. The filter will affect all the panels in the dashboard. + +Filtering on clusterID diff --git a/snowflake/grafana/provisioning/dashboards/dashboard_provider.yaml b/snowflake/grafana/provisioning/dashboards/dashboard_provider.yaml new file mode 100644 index 00000000..556267e0 --- /dev/null +++ b/snowflake/grafana/provisioning/dashboards/dashboard_provider.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +providers: + - name: grafana-dashboards + folder: '' + type: file + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/snowflake/grafana/provisioning/dashboards/flows_records_dashboard.json b/snowflake/grafana/provisioning/dashboards/flows_records_dashboard.json new file mode 100644 index 00000000..f2940933 --- /dev/null +++ b/snowflake/grafana/provisioning/dashboards/flows_records_dashboard.json @@ -0,0 +1,1109 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT count(*) as count\nFROM flows\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Flow Records Count", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 19, + "x": 5, + "y": 0 + }, + "id": 6, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 60, 'SECOND', 'START') as time, \ncount(*) as count\nFROM flows\nWHERE $__timeFilter(time)\nAND clusterUUID in (${clusterID:raw})\nGROUP BY time\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Flow Records Count", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "flowType" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "index": 0, + "text": "Intra-Node" + }, + "2": { + "index": 1, + "text": "Inter-Node" + }, + "3": { + "index": 2, + "text": "To External" + }, + "4": { + "index": 3, + "text": "From External" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sourcePodNamespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 199 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationPodName" + }, + "properties": [ + { + "id": "custom.width", + "value": 183 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationPodNamespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 214 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationNodeName" + }, + "properties": [ + { + "id": "custom.width", + "value": 183 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sourceNodeName" + }, + "properties": [ + { + "id": "custom.width", + "value": 161 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sourcePodName" + }, + "properties": [ + { + "id": "custom.width", + "value": 149 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationServicePort" + }, + "properties": [ + { + "id": "custom.width", + "value": 184 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationServicePortName" + }, + "properties": [ + { + "id": "custom.width", + "value": 215 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyName" + }, + "properties": [ + { + "id": "custom.width", + "value": 213 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyNamespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 256 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyRuleName" + }, + "properties": [ + { + "id": "custom.width", + "value": 243 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyRuleAction" + }, + "properties": [ + { + "id": "custom.width", + "value": 249 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "No Action" + }, + "1": { + "index": 1, + "text": "Allow" + }, + "2": { + "index": 2, + "text": "Drop" + }, + "3": { + "index": 3, + "text": "Reject" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ingressNetworkPolicyType" + }, + "properties": [ + { + "id": "custom.width", + "value": 209 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 3, + "text": "N/A" + }, + "1": { + "index": 0, + "text": "K8s NetworkPolicy" + }, + "2": { + "index": 1, + "text": "Antrea NetworkPolicy" + }, + "3": { + "index": 2, + "text": "Antrea ClusterNetworkPolicy" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyName" + }, + "properties": [ + { + "id": "custom.width", + "value": 215 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyRuleName" + }, + "properties": [ + { + "id": "custom.width", + "value": 238 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyNamespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 248 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyType" + }, + "properties": [ + { + "id": "custom.width", + "value": 207 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 3, + "text": "N/A" + }, + "1": { + "index": 0, + "text": "K8s NetworkPolicy" + }, + "2": { + "index": 1, + "text": "Antrea NetworkPolicy" + }, + "3": { + "index": 2, + "text": "Antrea ClusterNetworkPolicy" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationPodLabels" + }, + "properties": [ + { + "id": "custom.width", + "value": 173 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "throughputFromSourceNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 230 + }, + { + "id": "displayName", + "value": "throughput reported by source" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "throughputFromDestinationNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + }, + { + "id": "displayName", + "value": "throughput reported by destination" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseThroughputFromSourceNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 280 + }, + { + "id": "displayName", + "value": "reverse throughput reported by source" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseThroughputFromDestinationNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 310 + }, + { + "id": "displayName", + "value": "reverse throughput reported by destination" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseOctetDeltaCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 186 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reversePacketDeltaCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 204 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseOctetTotalCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 191 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reversePacketTotalCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 199 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowEndSecondsFromDestinationNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 290 + }, + { + "id": "displayName", + "value": "end time of flow reported by destination" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowEndSecondsFromSourceNode" + }, + "properties": [ + { + "id": "custom.width", + "value": 260 + }, + { + "id": "displayName", + "value": "end time of flow reported by source" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "protocolIdentifier" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "6": { + "index": 0, + "text": "TCP" + }, + "17": { + "index": 1, + "text": "UDP" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sourceTransportPort" + }, + "properties": [ + { + "id": "custom.width", + "value": 170 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationTransportPort" + }, + "properties": [ + { + "id": "custom.width", + "value": 196 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "destinationClusterIP" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "egressNetworkPolicyRuleAction" + }, + "properties": [ + { + "id": "custom.width", + "value": 244 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "No Action" + }, + "1": { + "index": 1, + "text": "Allow" + }, + "2": { + "index": 2, + "text": "Drop" + }, + "3": { + "index": 3, + "text": "Reject" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "reverseThroughput" + }, + "properties": [ + { + "id": "custom.width", + "value": 161 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowEndSeconds" + }, + "properties": [ + { + "id": "custom.width", + "value": 185 + }, + { + "id": "displayName", + "value": "end time of flow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowStartSeconds" + }, + "properties": [ + { + "id": "custom.width", + "value": 185 + }, + { + "id": "displayName", + "value": "start time of flow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timeInserted" + }, + "properties": [ + { + "id": "custom.width", + "value": 187 + }, + { + "id": "displayName", + "value": "insert time of flow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowEndReason" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "index": 0, + "text": "Idle Timeout" + }, + "2": { + "index": 1, + "text": "Active Timeout" + }, + "3": { + "index": 2, + "text": "End Of Flow" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.width", + "value": 142 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "octetDeltaCount" + }, + "properties": [ + { + "id": "displayName", + "value": "delta bytes from the last record" + }, + { + "id": "custom.width", + "value": 235 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "packetDeltaCount" + }, + "properties": [ + { + "id": "custom.width", + "value": 250 + }, + { + "id": "displayName", + "value": "delta packets from the last record" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowStartSeconds" + }, + "properties": [ + { + "id": "custom.width", + "value": 153 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "flowEndReason" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "Idle Timeout" + }, + "1": { + "index": 1, + "text": "Active Timeout" + }, + "2": { + "index": 2, + "text": "End of Flow" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT * FROM flows\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nORDER BY flowEndSeconds DESC\nLIMIT 1000", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Flow Records Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "CLUSTERUUID": "clusterUUID", + "DESTINATIONCLUSTERIP": "destinationClusterIP", + "DESTINATIONIP": "destinationIP", + "DESTINATIONNODENAME": "destinationNodeName", + "DESTINATIONPODLABELS": "destinationPodLabels", + "DESTINATIONPODNAME": "destinationPodName", + "DESTINATIONPODNAMESPACE": "destinationPodNamespace", + "DESTINATIONSERVICEPORT": "destinationServicePort", + "DESTINATIONSERVICEPORTNAME": "destinationServicePortName", + "DESTINATIONTRANSPORTPORT": "destinationTransportPort", + "EGRESSNETWORKPOLICYNAME": "egressNetworkPolicyName", + "EGRESSNETWORKPOLICYNAMESPACE": "egressNetworkPolicyNamespace", + "EGRESSNETWORKPOLICYRULEACTION": "egressNetworkPolicyRuleAction", + "EGRESSNETWORKPOLICYRULENAME": "egressNetworkPolicyRuleName", + "EGRESSNETWORKPOLICYTYPE": "egressNetworkPolicyType", + "FLOWENDREASON": "flowEndReason", + "FLOWENDSECONDS": "flowEndSeconds", + "FLOWENDSECONDSFROMDESTINATIONNODE": "flowEndSecondsFromDestinationNode", + "FLOWENDSECONDSFROMSOURCENODE": "flowEndSecondsFromSourceNode", + "FLOWSTARTSECONDS": "flowStartSeconds", + "FLOWTYPE": "flowType", + "INGRESSNETWORKPOLICYNAME": "ingressNetworkPolicyName", + "INGRESSNETWORKPOLICYNAMESPACE": "ingressNetworkPolicyNamespace", + "INGRESSNETWORKPOLICYRULEACTION": "ingressNetworkPolicyRuleAction", + "INGRESSNETWORKPOLICYRULENAME": "ingressNetworkPolicyRuleName", + "INGRESSNETWORKPOLICYTYPE": "ingressNetworkPolicyType", + "OCTETDELTACOUNT": "octetDeltaCount", + "OCTETTOTALCOUNT": "octetTotalCount", + "PACKETDELTACOUNT": "packetDeltaCount", + "PACKETTOTALCOUNT": "packetTotalCount", + "PROTOCOLIDENTIFIER": "protocolIdentifier", + "REVERSEOCTETDELTACOUNT": "reverseOctetDeltaCount", + "REVERSEOCTETTOTALCOUNT": "reverseOctetTotalCount", + "REVERSEPACKETDELTACOUNT": "reversePacketDeltaCount", + "REVERSEPACKETTOTALCOUNT": "reversePacketTotalCount", + "REVERSETHROUGHPUT": "reverseThroughput", + "REVERSETHROUGHPUTFROMDESTINATIONNODE": "reverseThroughputFromDestinationNode", + "REVERSETHROUGHPUTFROMSOURCENODE": "reverseThroughputFromSourceNode", + "SOURCEIP": "sourceIP", + "SOURCENODENAME": "sourceNodeName", + "SOURCEPODLABELS": "sourcePodLabels", + "SOURCEPODNAME": "sourcePodName", + "SOURCEPODNAMESPACE": "sourcePodNamespace", + "SOURCETRANSPORTPORT": "sourceTransportPort", + "TCPSTATE": "tcpState", + "THROUGHPUT": "throughput", + "THROUGHPUTFROMDESTINATIONNODE": "throughputFromDestinationNode", + "THROUGHPUTFROMSOURCENODE": "throughputFromSourceNode", + "TIMEINSERTED": "timeInserted" + } + } + } + ], + "transparent": true, + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "definition": "select distinct concat('''',clusterUUID,'''') from flows", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "clusterID", + "options": [], + "query": "select distinct concat('''',clusterUUID,'''') from flows", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "flow_records_dashboard", + "uid": "dbxiHWV4k", + "version": 20, + "weekStart": "" +} diff --git a/snowflake/grafana/provisioning/dashboards/homepage.json b/snowflake/grafana/provisioning/dashboards/homepage.json new file mode 100644 index 00000000..a981a4a9 --- /dev/null +++ b/snowflake/grafana/provisioning/dashboards/homepage.json @@ -0,0 +1,1145 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "_UhIbRV4k" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 32, + "panels": [], + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "_UhIbRV4k" + }, + "refId": "A" + } + ], + "title": "Cluster Overview", + "type": "row" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "displayName": "Number of Pods", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 12, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT COUNT(derivedtable.pod) as Number_of_Pods\nFROM\n(\n SELECT DISTINCT CONCAT(sourcePodName, sourcePodNamespace) AS pod FROM flows WHERE pod != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom() AND clusterUUID in (${clusterID:raw})\n UNION ALL\n SELECT DISTINCT CONCAT(destinationPodName, destinationPodNamespace) AS pod FROM flows WHERE pod != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\n) derivedtable\nWHERE derivedtable.pod != ''", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "transformations": [ + { + "id": "labelsToFields", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "displayName": "Number of Services", + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 14, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT COUNT(DISTINCT destinationServicePortName) as Number_of_Services \nFROM flows \nWHERE destinationServicePortName != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "displayName": "Number of Nodes", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 13, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT COUNT(DISTINCT derivedtable.node) as Number_of_Nodes\nFROM\n(\n SELECT DISTINCT sourceNodeName AS node FROM flows WHERE node != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom() AND clusterUUID in (${clusterID:raw})\n UNION ALL\n SELECT DISTINCT destinationNodeName AS node FROM flows WHERE node != '' AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\n) derivedtable\nWHERE derivedtable.node != ''", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "content": "
\n \n

Theia is a network observability and\nanalytics platform for Kubernetes, built on top of\nAntrea. It supports network flow\nvisualization and monitoring with Grafana. Follow this\ndocument\nto know more about the Grafana Flow Collector and network flow visualization\nfunctionality of Theia.

\n
\n\n", + "mode": "html" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "refId": "A" + } + ], + "title": "Overview of Project Theia", + "type": "text" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "displayName": "Number of Active Connections", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 15, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value_and_name" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_Active_Connections\nfrom flows\nWHERE flowEndReason = 2 AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "displayName": "Number of Terminated Connections", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 16, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_Stopped_Connections\nfrom flows WHERE flowEndReason != 2 AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "displayName": "Number of Denied Connections", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 23, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_Denied_Connections\nfrom flows\nWHERE (ingressNetworkPolicyRuleAction in (2,3) OR egressNetworkPolicyRuleAction in (2,3))\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 27, + "options": { + "content": "The following dashboards are pre-built and are recommended for network flow\nvisualization.\n\nFlow Records Dashboard displays the flow records being captured in the selected\ntime range.\n\nPod-to-Pod Flows Dashboard shows cumulative bytes and throughput of Pod-to-Pod\ntraffic.\n\nNetwork-Policy Flows Dashboard visualizes both the traffic with NetworkPolicies enforced,\nand unprotected traffic.", + "mode": "markdown" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "refId": "A" + } + ], + "title": "Introduction of Pre-built Dashboards", + "type": "text" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "displayName": "Data Transmitted", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 21, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount) as Data_Transmitted\nfrom pods WHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "displayName": "Overall Throughput", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 22, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT (SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount))/60 as Overall_Throughput\nfrom pods WHERE DATEDIFF(SECOND, CURRENT_TIMESTAMP(), flowEndSeconds) < 60\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "displayName": "Number of NetworkPolicies", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 26, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value_and_name" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "format": 1, + "queryText": "SELECT (COUNT(DISTINCT CONCAT(ingressNetworkPolicyNamespace, ingressNetworkPolicyName)) + COUNT(DISTINCT CONCAT(egressNetworkPolicyNamespace, egressNetworkPolicyName))) as Number_of_NetworkPolicies\nfrom flows\nWHERE CONCAT(ingressNetworkPolicyNamespace, ingressNetworkPolicyName, egressNetworkPolicyNamespace, egressNetworkPolicyName) != ''\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "displayName": "Data Transmitted with External", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 28, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount) as Data_Transmitted_With_External\nFROM pods\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND flowType = 3\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "displayName": "Overall Throughput with External", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 13 + }, + "id": 29, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT (SUM(octetDeltaCount)+SUM(reverseOctetDeltaCount))/60 as Overall_Throughput_With_External\nfrom pods WHERE DATEDIFF(SECOND, CURRENT_TIMESTAMP(), flowEndSeconds) < 60\nAND flowType = 3\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "displayName": "Number of ToExternal Connections", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 13 + }, + "id": 33, + "interval": "2s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "format": 1, + "queryText": "SELECT COUNT(DISTINCT CONCAT(sourceIP, destinationIP)) as Number_of_ToExternal_Connections\nfrom flows\nWHERE flowType = 3\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 18, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showUnfilled": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT CONCAT(sourcePodNamespace, '/', sourcePodName) as pod,\nSUM(octetDeltaCount) as bytes\nFROM pods\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND pod != '/'\nAND clusterUUID in (${clusterID:raw})\nGROUP BY pod\nORDER BY bytes DESC LIMIT 10", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Top 10 Active Source Pods", + "type": "bargauge" + }, + { + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "gridPos": { + "h": 18, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 4, + "links": [], + "options": { + "folderId": 0, + "maxItems": 10, + "query": "", + "showHeadings": true, + "showRecentlyViewed": true, + "showSearch": true, + "showStarred": true, + "tags": [] + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "list", + "refId": "A" + } + ], + "title": "Dashboard Links", + "type": "dashlist" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 25, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 60, 'SECOND', 'START') as time,\ncount(*) as count\nFROM pods\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY time\nORDER BY time", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Number of Flow Records Per Minute", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "definition": "select distinct concat('''',clusterUUID,'''') from flows", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "clusterID", + "options": [], + "query": "select distinct concat('''',clusterUUID,'''') from flows", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "homepage", + "uid": "F_PWNZ44z", + "version": 6, + "weekStart": "" +} diff --git a/snowflake/grafana/provisioning/dashboards/networkpolicy_dashboard.json b/snowflake/grafana/provisioning/dashboards/networkpolicy_dashboard.json new file mode 100644 index 00000000..1146f736 --- /dev/null +++ b/snowflake/grafana/provisioning/dashboards/networkpolicy_dashboard.json @@ -0,0 +1,767 @@ +{ + "__inputs": [ + { + "name": "DS_SNOWFLAKE", + "label": "Snowflake", + "description": "", + "type": "datasource", + "pluginId": "michelin-snowflake-datasource", + "pluginName": "Snowflake" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "michelin-snowflake-datasource", + "name": "Snowflake", + "version": "1.2.0" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "theia-grafana-chord-plugin", + "name": "grafana-chord-plugin", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "gridPos": { + "h": 20, + "w": 15, + "x": 0, + "y": 0 + }, + "id": 2, + "interval": "1s", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT CONCAT(sourcePodNamespace, '/', sourcePodName) as srcPod,\nCONCAT(destinationPodNamespace, '/', destinationPodName) as dstPod,\nsourceTransportPort as srcPort,\ndestinationTransportPort as dstPort,\ndestinationServicePort as dstSvcPort,\ndestinationServicePortName as dstSvc,\ndestinationIP as dstIP,\nSUM(octetDeltaCount) as bytes,\nSUM(reverseOctetDeltaCount) as revBytes,\negressNetworkPolicyName,\negressNetworkPolicyRuleAction,\ningressNetworkPolicyName,\ningressNetworkPolicyRuleAction\nfrom policies\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\n AND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\n AND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY srcPod, dstPod, srcPort, dstPort, dstSvcPort, dstSvc, dstIP, egressNetworkPolicyName, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyRuleAction\nHAVING bytes > 0\norder by bytes DESC\nLIMIT 25", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Flows with NetworkPolicy Information", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "BYTES": "bytes", + "DSTIP": "dstIP", + "DSTPOD": "dstPod", + "DSTPORT": "dstPort", + "DSTSVC": "dstSvc", + "DSTSVCPORT": "dstSvcPort", + "EGRESSNETWORKPOLICYNAME": "egressNetworkPolicyName", + "EGRESSNETWORKPOLICYRULEACTION": "egressNetworkPolicyRuleAction", + "INGRESSNETWORKPOLICYNAME": "ingressNetworkPolicyName", + "INGRESSNETWORKPOLICYRULEACTION": "ingressNetworkPolicyRuleAction", + "REVBYTES": "revBytes", + "SRCPOD": "srcPod", + "SRCPORT": "srcPort" + } + } + } + ], + "transparent": true, + "type": "theia-grafana-chord-plugin" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 25, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes, \nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace, '/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np\nFROM policies\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyName != ''\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY np\nHAVING SUM(octetDeltaCount) != 0\nORDER BY bytes DESC", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Ingress Network Policy", + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 9, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 25, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace, '/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND AS np\nFROM policies\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyName != ''\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY np\nHAVING SUM(octetDeltaCount) > 0\nORDER BY bytes DESC", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Egress Network Policy", + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 4, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 1, 'SECOND', 'START') as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort = 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort = 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND AS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM policies\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction = 1\nAND egressNetworkPolicyRuleAction NOT IN (2,\n3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time\nLIMIT 1000 //", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Ingress Allow NetworkPolicy", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 5, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 1, 'SECOND', 'START') as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort = 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort = 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM policies\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction = 1\nAND ingressNetworkPolicyRuleAction not in (2,3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time\nLIMIT 1000 //", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Egress Allow NetworkPolicy", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 6, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 1, 'SECOND', 'START') as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort = 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort = 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM policies\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction in (2,3)\nGROUP BY time, src, dst, np\nHAVING SUM(octetDeltaCount) > 0\nORDER BY time\nLIMIT 1000 //", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Ingress Deny NetworkPolicy", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 7, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 1, 'SECOND', 'START') as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort = 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort = 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM policies\nWHERE CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction in (2,\n3)\nGROUP BY time, src, dst, np\nHAVING SUM(octetDeltaCount) > 0\nORDER BY time\nLIMIT 1000 //", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Egress Deny NetworkPolicy", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "definition": "select distinct concat('''',clusterUUID,'''') from flows", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "clusterID", + "options": [], + "query": "select distinct concat('''',clusterUUID,'''') from flows", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "networkpolicy_dashboard", + "uid": "LvtkNZ4Vz", + "version": 9, + "weekStart": "" +} diff --git a/snowflake/grafana/provisioning/dashboards/pod_to_pod_dashboard.json b/snowflake/grafana/provisioning/dashboards/pod_to_pod_dashboard.json new file mode 100644 index 00000000..3faf1467 --- /dev/null +++ b/snowflake/grafana/provisioning/dashboards/pod_to_pod_dashboard.json @@ -0,0 +1,846 @@ +{ + "__inputs": [ + { + "name": "DS_SNOWFLAKE", + "label": "Snowflake", + "description": "", + "type": "datasource", + "pluginId": "michelin-snowflake-datasource", + "pluginName": "Snowflake" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "michelin-snowflake-datasource", + "name": "Snowflake", + "version": "1.2.0" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "theia-grafana-sankey-plugin", + "name": "grafana-sankey-plugin", + "version": "1.0.1" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "gridPos": { + "h": 18, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 13, + "options": { + "seriesCountSize": "sm", + "showSeriesCount": false, + "text": "Default value of text input option" + }, + "pluginVersion": "7.5.2", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes, \nsource,\ndestination\nFROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY source, destination\nORDER BY bytes DESC\nLIMIT 50", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Pod-to-Pod", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "BYTES": "bytes", + "DESTINATION": "destination", + "SOURCE": "source" + } + } + } + ], + "transparent": true, + "type": "theia-grafana-sankey-plugin" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "gridPos": { + "h": 18, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 12, + "options": { + "seriesCountSize": "sm", + "showSeriesCount": false, + "text": "Default value of text input option" + }, + "pluginVersion": "7.5.2", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT SUM(reverseOctetDeltaCount) as bytes, \nsource,\ndestination\nFROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY source, destination\nORDER BY bytes DESC\nLIMIT 50", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Reverse Bytes of Pod-to-Pod", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "BYTES": "bytes", + "DESTINATION": "destination", + "SOURCE": "source" + } + } + } + ], + "transparent": true, + "type": "theia-grafana-sankey-plugin" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 21, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 1, 'SECOND', 'START') as time,\nCONCAT(source, ' -> ', destination) as pair,\nAVG(throughput)\nFROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY time, pair\nHAVING AVG(throughput) > 0\nORDER BY time\nLIMIT 1000 //", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Pod-to-Pod", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns", + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 22, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 1, 'SECOND', 'START') as time,\nCONCAT(source, ' -> ', destination) as pair,\nAVG(reverseThroughput)\nFROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY time, pair\nHAVING AVG(throughput) > 0\nORDER BY time\nLIMIT 1000 //", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Reverse Throughput of Pod-to-Pod", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "PAIR" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 15, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 60, 'SECOND', 'START') as time,\nsource as src,\nSUM(octetDeltaCount)/60 as tp FROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY time, src\nHAVING tp > 0\nORDER BY time\nLIMIT 1000 //", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Pod as Source", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "SRC" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 25, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes, \nsourcePodNamespace\nFrom pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY sourcePodNamespace\nHAVING bytes > 0\nORDER BY bytes DESC", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Source Pod Namespace", + "transformations": [ + { + "id": "labelsToFields", + "options": {} + } + ], + "transparent": true, + "type": "piechart" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 90000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 16, + "interval": "60s", + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT TIME_SLICE(TO_TIMESTAMP(CONVERT_TIMEZONE('UTC', flowEndSeconds)), 60, 'SECOND', 'START') as time,\ndestination as dst,\nSUM(octetDeltaCount)/60 as tp FROM pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY time, dst\nHAVING tp > 0\nORDER BY time\nLIMIT 1000 //", + "queryType": "time series", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Throughput of Pod as Destination", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "valueLabel": "DST" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "string", + "targetField": "Value" + } + ], + "fields": {} + } + } + ], + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "destinationPodNamespace" + }, + "properties": [ + { + "id": "noValue", + "value": "N/A" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 25, + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "queryText": "SELECT SUM(octetDeltaCount) as bytes, \ndestinationPodNamespace\nFrom pods\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND CONVERT_TIMEZONE('UTC', flowEndSeconds) < $__timeTo() AND CONVERT_TIMEZONE('UTC', flowEndSeconds) > $__timeFrom()\nAND clusterUUID in (${clusterID:raw})\nGROUP BY destinationPodNamespace\nHAVING bytes > 0\nORDER BY bytes DESC", + "queryType": "table", + "refId": "A", + "timeColumns": [ + "time" + ] + } + ], + "title": "Cumulative Bytes of Destination Pod Namespace", + "transparent": true, + "type": "piechart" + } + ], + "refresh": "", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "michelin-snowflake-datasource", + "uid": "P1DBD59F661D68B90" + }, + "definition": "select distinct concat('''',clusterUUID,'''') from flows", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "clusterID", + "options": [], + "query": "select distinct concat('''',clusterUUID,'''') from flows", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "pod_to_pod_dashboard", + "uid": "B1t6SZVVz", + "version": 14, + "weekStart": "" +} diff --git a/snowflake/grafana/provisioning/datasources/datasource_provider.yaml b/snowflake/grafana/provisioning/datasources/datasource_provider.yaml new file mode 100644 index 00000000..73faec8f --- /dev/null +++ b/snowflake/grafana/provisioning/datasources/datasource_provider.yaml @@ -0,0 +1,15 @@ +apiVersion: 1 +datasources: + - name: Snowflake + type: michelin-snowflake-datasource + access: proxy + editable: true + jsonData: + account: ${SNOWFLAKE_ACCOUNT} + role: ${SNOWFLAKE_ROLE} + warehouse: ${SNOWFLAKE_WAREHOUSE} + database: ${SNOWFLAKE_DATABASE} + schema: THEIA + username: ${SNOWFLAKE_USER} + secureJsonData: + password: ${SNOWFLAKE_PASSWORD}