From f49a6d48b099b782c8fbd0a37df0e933f7d0d285 Mon Sep 17 00:00:00 2001 From: Yanjun Zhou Date: Wed, 11 May 2022 11:30:46 -0700 Subject: [PATCH] Add Helm support for Theia with ClickHouse PV Signed-off-by: Yanjun Zhou --- .github/workflows/go.yml | 9 + Makefile | 1 - build/charts/Makefile | 6 + build/charts/theia/.helmignore | 23 + build/charts/theia/Chart.yaml | 19 + build/charts/theia/README.md | 50 ++ .../clickhouse-operator-install-bundle.yaml} | 0 .../dashboards/dashboard_provider.yaml} | 0 .../dashboards/flow_records_dashboard.json | 0 .../dashboards/networkpolicy_dashboard.json | 14 +- .../dashboards/node_to_node_dashboard.json | 36 +- .../dashboards/pod_to_external_dashboard.json | 24 +- .../dashboards/pod_to_pod_dashboard.json | 40 +- .../dashboards/pod_to_service_dashboard.json | 36 +- .../provisioning/datasources/create_table.sh | 26 +- .../datasources/datasource_provider.yaml} | 6 +- build/charts/theia/templates/NOTES.txt | 1 + build/charts/theia/templates/_helpers.tpl | 0 .../clickhouse/clickhouseinstallation.yaml | 108 +++++ .../theia/templates/clickhouse/configmap.yaml | 8 + .../clickhouse/local-persistentvolume.yaml | 27 ++ .../clickhouse/nfs-persistentvolume.yaml | 16 + .../theia/templates/clickhouse/secret.yaml | 9 + .../templates/clickhouse/storageclass.yaml | 10 + .../grafana/dashboard-configmap.yaml | 10 + .../grafana/dashboard-provider-configmap.yaml | 8 + .../datasource-provider-configmap.yaml | 8 + .../theia/templates/grafana/deployment.yaml} | 104 +---- .../templates/grafana/persistentvolume.yaml | 12 + .../grafana/persistentvolumeclaim.yaml | 12 + .../charts/theia/templates/grafana/role.yaml | 16 + .../theia/templates/grafana/rolebinding.yaml | 15 + .../theia/templates/grafana/secret.yaml | 9 + .../theia/templates/grafana/service.yaml | 14 + .../templates/grafana/serviceaccount.yaml | 5 + .../theia/templates/grafana/storageclass.yaml | 8 + build/charts/theia/values.yaml | 103 ++++ build/yamls/base/clickhouse.yml | 76 --- build/yamls/base/kustomization.yaml | 7 + build/yamls/base/kustomization.yml | 41 -- build/yamls/base/kustomize-config.yml | 5 - build/yamls/base/namespace.yaml | 4 + build/yamls/flow-visibility.yml | 34 +- build/yamls/patches/dev/imagePullPolicy.yml | 3 - build/yamls/patches/release/.gitignore | 1 - ci/kind/test-e2e-kind.sh | 4 +- docs/network-flow-visibility.md | 440 ++++++++++++++---- hack/generate-manifest.sh | 80 ++-- hack/verify-helm.sh | 73 +++ plugins/clickhouse-monitor/main.go | 118 ++++- plugins/clickhouse-monitor/main_test.go | 20 +- test/e2e/framework.go | 2 +- 52 files changed, 1216 insertions(+), 485 deletions(-) create mode 100644 build/charts/Makefile create mode 100644 build/charts/theia/.helmignore create mode 100644 build/charts/theia/Chart.yaml create mode 100644 build/charts/theia/README.md rename build/{yamls/clickhouse-operator-install-bundle.yml => charts/theia/crds/clickhouse-operator-install-bundle.yaml} (100%) rename build/{yamls/base/provisioning/dashboards/dashboard_provider.yml => charts/theia/provisioning/dashboards/dashboard_provider.yaml} (100%) rename build/{yamls/base => charts/theia}/provisioning/dashboards/flow_records_dashboard.json (100%) rename build/{yamls/base => charts/theia}/provisioning/dashboards/networkpolicy_dashboard.json (91%) rename build/{yamls/base => charts/theia}/provisioning/dashboards/node_to_node_dashboard.json (88%) rename build/{yamls/base => charts/theia}/provisioning/dashboards/pod_to_external_dashboard.json (91%) rename build/{yamls/base => charts/theia}/provisioning/dashboards/pod_to_pod_dashboard.json (87%) rename build/{yamls/base => charts/theia}/provisioning/dashboards/pod_to_service_dashboard.json (88%) rename build/{yamls/base => charts/theia}/provisioning/datasources/create_table.sh (90%) rename build/{yamls/base/provisioning/datasources/datasource_provider.yml => charts/theia/provisioning/datasources/datasource_provider.yaml} (50%) create mode 100644 build/charts/theia/templates/NOTES.txt create mode 100644 build/charts/theia/templates/_helpers.tpl create mode 100644 build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml create mode 100644 build/charts/theia/templates/clickhouse/configmap.yaml create mode 100644 build/charts/theia/templates/clickhouse/local-persistentvolume.yaml create mode 100644 build/charts/theia/templates/clickhouse/nfs-persistentvolume.yaml create mode 100644 build/charts/theia/templates/clickhouse/secret.yaml create mode 100644 build/charts/theia/templates/clickhouse/storageclass.yaml create mode 100644 build/charts/theia/templates/grafana/dashboard-configmap.yaml create mode 100644 build/charts/theia/templates/grafana/dashboard-provider-configmap.yaml create mode 100644 build/charts/theia/templates/grafana/datasource-provider-configmap.yaml rename build/{yamls/base/grafana.yml => charts/theia/templates/grafana/deployment.yaml} (58%) create mode 100644 build/charts/theia/templates/grafana/persistentvolume.yaml create mode 100644 build/charts/theia/templates/grafana/persistentvolumeclaim.yaml create mode 100644 build/charts/theia/templates/grafana/role.yaml create mode 100644 build/charts/theia/templates/grafana/rolebinding.yaml create mode 100644 build/charts/theia/templates/grafana/secret.yaml create mode 100644 build/charts/theia/templates/grafana/service.yaml create mode 100644 build/charts/theia/templates/grafana/serviceaccount.yaml create mode 100644 build/charts/theia/templates/grafana/storageclass.yaml create mode 100644 build/charts/theia/values.yaml delete mode 100644 build/yamls/base/clickhouse.yml create mode 100644 build/yamls/base/kustomization.yaml delete mode 100644 build/yamls/base/kustomization.yml delete mode 100644 build/yamls/base/kustomize-config.yml create mode 100644 build/yamls/base/namespace.yaml delete mode 100644 build/yamls/patches/dev/imagePullPolicy.yml delete mode 100644 build/yamls/patches/release/.gitignore create mode 100644 hack/verify-helm.sh diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 887b1971..73271645 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -136,3 +136,12 @@ jobs: run: | sudo npm install -g markdownlint-cli@0.31.1 make markdownlint + - name: Checking whether autogenerated Helm chart documentation is up-to-date + working-directory: build/charts/ + run: | + make helm-docs + DIFF=$(git diff .) + if [ -n "$DIFF" ]; then + echo "The Helm chart documentation is out-of-date; please run 'make helm-docs' in 'build/charts/' and commit the changes" + exit 1 + fi diff --git a/Makefile b/Makefile index 9aa7ae3e..41ae1994 100644 --- a/Makefile +++ b/Makefile @@ -157,4 +157,3 @@ clickhouse-monitor: clickhouse-monitor-plugin: @mkdir -p $(BINDIR) GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/theia/plugins/clickhouse-monitor - \ No newline at end of file diff --git a/build/charts/Makefile b/build/charts/Makefile new file mode 100644 index 00000000..608a7eaf --- /dev/null +++ b/build/charts/Makefile @@ -0,0 +1,6 @@ +USERID := $(shell id -u) +GRPID := $(shell id -g) + +.PHONY: helm-docs +helm-docs: + docker run --rm --volume "$(CURDIR):/helm-docs" --user=$(USERID):$(GRPID) jnorwood/helm-docs:v1.7.0 diff --git a/build/charts/theia/.helmignore b/build/charts/theia/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/build/charts/theia/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/build/charts/theia/Chart.yaml b/build/charts/theia/Chart.yaml new file mode 100644 index 00000000..1ca32cea --- /dev/null +++ b/build/charts/theia/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: theia +type: application +displayName: Theia +home: https://antrea.io/ +version: 0.1.0-dev +appVersion: 0.1.0-dev +kubeVersion: ">= 1.16.0-0" +icon: https://raw.githubusercontent.com/antrea-io/antrea/main/docs/assets/logo/antrea_logo.svg +description: Antrea Network Flow Visibility +keywords: + - Kubernetes + - CNCF + - Networking + - CNI + - Security + - Flow visibility +sources: + - https://github.com/antrea-io/theia diff --git a/build/charts/theia/README.md b/build/charts/theia/README.md new file mode 100644 index 00000000..8a8d2a9d --- /dev/null +++ b/build/charts/theia/README.md @@ -0,0 +1,50 @@ +# theia + +![Version: 0.1.0-dev](https://img.shields.io/badge/Version-0.1.0--dev-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0-dev](https://img.shields.io/badge/AppVersion-0.1.0--dev-informational?style=flat-square) + +Antrea Network Flow Visibility + +**Homepage:** + +## Source Code + +* + +## Requirements + +Kubernetes: `>= 1.16.0-0` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| clickhouse.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-clickhouse-server","tag":"21.11"}` | Container image to use for ClickHouse. | +| clickhouse.monitor.deletePercentage | float | `0.5` | The percentage of records in ClickHouse that will be deleted when the storage grows above threshold. Vary from 0 to 1. | +| clickhouse.monitor.enable | bool | `true` | Determine whether to run a monitor to periodically check the ClickHouse memory usage and clean data. | +| clickhouse.monitor.execInterval | string | `"1m"` | The time interval between two round of monitoring. Can be a plain integer using one of these unit suffixes ns, us (or µs), ms, s, m, h. | +| clickhouse.monitor.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-clickhouse-monitor","tag":"latest"}` | Container image to use for the ClickHouse Monitor. | +| clickhouse.monitor.skipRoundsNum | int | `3` | The number of rounds for the monitor to stop after a deletion to wait for the ClickHouse MergeTree Engine to release memory. | +| clickhouse.monitor.threshold | float | `0.5` | The storage percentage at which the monitor starts to delete old records. Vary from 0 to 1. | +| clickhouse.secret.password | string | `"clickhouse_operator_password"` | ClickHouse password. It will be stored in a secret. | +| clickhouse.secret.username | string | `"clickhouse_operator"` | ClickHouse username. It will be stored in a secret. | +| clickhouse.service.httpPort | int | `8123` | TCP port number for the ClickHouse service. | +| clickhouse.service.tcpPort | int | `9000` | TCP port number for the ClickHouse service. | +| clickhouse.service.type | string | `"ClusterIP"` | The type of Service exposing ClickHouse. It can be one of ClusterIP, NodePort or LoadBalancer. | +| clickhouse.storage.createPersistentVolume.local.affinity | object | `{}` | Affinity for the Local Persistent Volume. By default it requires to label the Node used to store the ClickHouse data with "antrea.io/clickhouse-data-node=" | +| clickhouse.storage.createPersistentVolume.local.path | string | `"/data"` | The local path. Required when type is "Local". | +| clickhouse.storage.createPersistentVolume.nfs.host | string | `""` | The NFS server hostname or IP address. Required when type is "NFS". | +| clickhouse.storage.createPersistentVolume.nfs.path | string | `""` | The path exported on the NFS server. Required when type is "NFS". | +| clickhouse.storage.createPersistentVolume.type | string | `""` | Type of PersistentVolume. Can be set to "Local" or "NFS". Please set this value to use a PersistentVolume created by Theia. | +| clickhouse.storage.persistentVolumeClaimSpec | string | `nil` | Specification for PersistentVolumeClaim. This is ignored if createPersistentVolume.type is non-empty. To use a custom PersistentVolume, please set storageClassName: "" volumeName: "" To dynamically provision a PersistentVolume, please set storageClassName: "" Memory storage is used if both createPersistentVolume.type and persistentVolumeClaimSpec are empty. | +| clickhouse.storage.size | string | `"8Gi"` | ClickHouse storage size. Can be a plain integer or as a fixed-point number using one of these quantity suffixes: E, P, T, G, M, K. Or the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. | +| clickhouse.ttl | string | `"1 HOUR"` | Time to live for data in the ClickHouse. Can be a plain integer using one of these unit suffixes SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, YEAR. | +| grafana.dashboards | list | `["flow_records_dashboard.json","pod_to_pod_dashboard.json","pod_to_service_dashboard.json","pod_to_external_dashboard.json","node_to_node_dashboard.json","networkpolicy_dashboard.json"]` | The dashboards to be displayed in Grafana UI. The files must be put under provisioning/dashboards | +| grafana.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-grafana","tag":"8.3.3"}` | Container image to use for Grafana. | +| grafana.installPlugins | list | `["https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-sankey-plugin-1.0.0.zip;theia-grafana-sankey-plugin","https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-chord-plugin-1.0.0.zip;theia-grafana-chord-plugin","grafana-clickhouse-datasource 1.0.1"]` | Grafana plugins to install. | +| grafana.secret.password | string | `"admin"` | Grafana password. It will be stored in a secret. | +| grafana.secret.username | string | `"admin"` | Grafana username. It will be stored in a secret. | +| grafana.service.tcpPort | int | `3000` | TCP port number for the Grafana service. | +| grafana.service.type | string | `"NodePort"` | The type of Service exposing Grafana. It must be one of NodePort or LoadBalancer. | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.7.0](https://github.com/norwoodj/helm-docs/releases/v1.7.0) diff --git a/build/yamls/clickhouse-operator-install-bundle.yml b/build/charts/theia/crds/clickhouse-operator-install-bundle.yaml similarity index 100% rename from build/yamls/clickhouse-operator-install-bundle.yml rename to build/charts/theia/crds/clickhouse-operator-install-bundle.yaml diff --git a/build/yamls/base/provisioning/dashboards/dashboard_provider.yml b/build/charts/theia/provisioning/dashboards/dashboard_provider.yaml similarity index 100% rename from build/yamls/base/provisioning/dashboards/dashboard_provider.yml rename to build/charts/theia/provisioning/dashboards/dashboard_provider.yaml diff --git a/build/yamls/base/provisioning/dashboards/flow_records_dashboard.json b/build/charts/theia/provisioning/dashboards/flow_records_dashboard.json similarity index 100% rename from build/yamls/base/provisioning/dashboards/flow_records_dashboard.json rename to build/charts/theia/provisioning/dashboards/flow_records_dashboard.json diff --git a/build/yamls/base/provisioning/dashboards/networkpolicy_dashboard.json b/build/charts/theia/provisioning/dashboards/networkpolicy_dashboard.json similarity index 91% rename from build/yamls/base/provisioning/dashboards/networkpolicy_dashboard.json rename to build/charts/theia/provisioning/dashboards/networkpolicy_dashboard.json index a1199101..131e7dfc 100644 --- a/build/yamls/base/provisioning/dashboards/networkpolicy_dashboard.json +++ b/build/charts/theia/provisioning/dashboards/networkpolicy_dashboard.json @@ -55,7 +55,7 @@ } }, "queryType": "sql", - "rawSql": "SELECT CONCAT(sourcePodNamespace, '/', sourcePodName) as srcPod,\nCONCAT(destinationPodNamespace, '/', destinationPodName) as dstPod,\nsourceTransportPort as srcPort,\ndestinationTransportPort as dstPort,\ndestinationServicePort as dstSvcPort,\ndestinationServicePortName as dstSvc,\ndestinationIP as dstIP,\nSUM(octetDeltaCount) as bytes,\nSUM(reverseOctetDeltaCount) as revBytes,\negressNetworkPolicyName,\negressNetworkPolicyRuleAction,\ningressNetworkPolicyName,\ningressNetworkPolicyRuleAction\nfrom flows_policy_view\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\n AND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\n AND $__timeFilter(flowEndSeconds)\nGROUP BY srcPod, dstPod, srcPort, dstPort, dstSvcPort, dstSvc, dstIP, egressNetworkPolicyName, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyRuleAction\nHAVING bytes > 0\norder by srcPod", + "rawSql": "SELECT CONCAT(sourcePodNamespace, '/', sourcePodName) as srcPod,\nCONCAT(destinationPodNamespace, '/', destinationPodName) as dstPod,\nsourceTransportPort as srcPort,\ndestinationTransportPort as dstPort,\ndestinationServicePort as dstSvcPort,\ndestinationServicePortName as dstSvc,\ndestinationIP as dstIP,\nSUM(octetDeltaCount) as bytes,\nSUM(reverseOctetDeltaCount) as revBytes,\negressNetworkPolicyName,\negressNetworkPolicyRuleAction,\ningressNetworkPolicyName,\ningressNetworkPolicyRuleAction\nfrom flows_policy_view\nWHERE sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\n AND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\n AND $__timeFilter(flowEndSeconds)\nGROUP BY srcPod, dstPod, srcPort, dstPort, dstSvcPort, dstSvc, dstIP, egressNetworkPolicyName, egressNetworkPolicyRuleAction, ingressNetworkPolicyName, ingressNetworkPolicyRuleAction\nHAVING bytes > 0\norder by srcPod", "refId": "A" } ], @@ -120,7 +120,7 @@ "uid": "PDEE91DDB90597936" }, "format": 1, - "rawSql": "SELECT SUM(octetDeltaCount), \nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace, '/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np\nFROM flows_policy_view\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY np\nHAVING SUM(octetDeltaCount) != 0", + "rawSql": "SELECT SUM(octetDeltaCount), \nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace, '/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np\nFROM flows_policy_view\nWHERE sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND ingressNetworkPolicyName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY np\nHAVING SUM(octetDeltaCount) != 0", "refId": "A" } ], @@ -186,7 +186,7 @@ "uid": "PDEE91DDB90597936" }, "format": 1, - "rawSql": "SELECT SUM(octetDeltaCount),\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace, '/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND AS np\nFROM flows_policy_view\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY np\nHAVING SUM(octetDeltaCount) > 0", + "rawSql": "SELECT SUM(octetDeltaCount),\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace, '/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND AS np\nFROM flows_policy_view\nWHERE sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND egressNetworkPolicyName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY np\nHAVING SUM(octetDeltaCount) > 0", "refId": "A" } ], @@ -280,7 +280,7 @@ } }, "queryType": "sql", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND AS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction == 1\nAND egressNetworkPolicyRuleAction NOT IN (2, 3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND AS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND AS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction == 1\nAND egressNetworkPolicyRuleAction NOT IN (2, 3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", "refId": "A" } ], @@ -382,7 +382,7 @@ } }, "queryType": "sql", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction == 1\nAND ingressNetworkPolicyRuleAction not in (2, 3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction == 1\nAND ingressNetworkPolicyRuleAction not in (2, 3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", "refId": "A" } ], @@ -485,7 +485,7 @@ } }, "queryType": "sql", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction in (2,3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN ingressNetworkPolicyNamespace != '' THEN CONCAT(ingressNetworkPolicyNamespace,'/', ingressNetworkPolicyName)\nELSE ingressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND ingressNetworkPolicyRuleAction in (2,3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", "refId": "A" } ], @@ -587,7 +587,7 @@ } }, "queryType": "sql", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction in (2,3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time,\nCASE WHEN sourceTransportPort != 0 THEN CONCAT(sourcePodNamespace, '/', sourcePodName, ':', CAST(sourceTransportPort as VARCHAR))\nELSE CONCAT(sourcePodNamespace, '/', sourcePodName)\nEND AS src,\nCASE WHEN destinationServicePortName != '' AND destinationServicePort != 0 THEN CONCAT(destinationServicePortName, ':', CAST(destinationServicePort as VARCHAR))\nWHEN destinationServicePortName != '' AND destinationServicePort == 0 THEN destinationServicePortName\nWHEN destinationPodName != '' AND destinationTransportPort != 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName, ':', CAST(destinationTransportPort as VARCHAR))\nWHEN destinationPodName != '' AND destinationTransportPort == 0 THEN CONCAT(destinationPodNamespace,'/', destinationPodName)\nELSE destinationIP\nEND\nAS dst,\nCASE WHEN egressNetworkPolicyNamespace != '' THEN CONCAT(egressNetworkPolicyNamespace,'/', egressNetworkPolicyName)\nELSE egressNetworkPolicyName\nEND\nAS np,\nCONCAT(src, ' -> ', dst, ' : ', np) as pair,\nAVG(throughput)\nFROM flows_policy_view\nWHERE $__timeFilter(flowEndSeconds)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND egressNetworkPolicyRuleAction in (2,3)\nGROUP BY time, src, dst, np\nHAVING AVG(throughput) > 0\nORDER BY time", "refId": "A" } ], diff --git a/build/yamls/base/provisioning/dashboards/node_to_node_dashboard.json b/build/charts/theia/provisioning/dashboards/node_to_node_dashboard.json similarity index 88% rename from build/yamls/base/provisioning/dashboards/node_to_node_dashboard.json rename to build/charts/theia/provisioning/dashboards/node_to_node_dashboard.json index 3bc9f506..ac5dbf3c 100644 --- a/build/yamls/base/provisioning/dashboards/node_to_node_dashboard.json +++ b/build/charts/theia/provisioning/dashboards/node_to_node_dashboard.json @@ -61,10 +61,10 @@ "format": 1, "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT SUM(octetDeltaCount), (sourceNodeName, destinationNodeName) as pair\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair\n", + "query": "SELECT SUM(octetDeltaCount), (sourceNodeName, destinationNodeName) as pair\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair\n", "queryType": "sql", "rawQuery": false, - "rawSql": "select SUM(octetDeltaCount) as bytes, sourceNodeName as source, destinationNodeName as destination\nFrom flows_node_view\nWHERE source != '' AND destination != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination", + "rawSql": "select SUM(octetDeltaCount) as bytes, sourceNodeName as source, destinationNodeName as destination\nFrom flows_node_view\nWHERE source != '' AND destination != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination", "refId": "A", "round": "0s", "skip_comments": true, @@ -117,10 +117,10 @@ "format": 1, "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT SUM(reverseOctetDeltaCount), (sourceNodeName, destinationNodeName) as pair\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair\n", + "query": "SELECT SUM(reverseOctetDeltaCount), (sourceNodeName, destinationNodeName) as pair\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair\n", "queryType": "randomWalk", "rawQuery": false, - "rawSql": "select SUM(reverseOctetDeltaCount) as bytes, sourceNodeName as source, destinationNodeName as destination\nFrom flows_node_view\nWHERE source != '' AND destination != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination", + "rawSql": "select SUM(reverseOctetDeltaCount) as bytes, sourceNodeName as source, destinationNodeName as destination\nFrom flows_node_view\nWHERE source != '' AND destination != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination", "refId": "A", "round": "0s", "skip_comments": true, @@ -226,10 +226,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM default.flows_node_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642533454) AND flowEndSecondsFromDestinationNode <= toDateTime(1642535254)\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourceNodeName, '->', destinationNodeName) as pair, SUM(throughput) as Node\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM default.flows_node_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642533454) AND flowEndSecondsFromDestinationNode <= toDateTime(1642535254)\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourceNodeName, '->', destinationNodeName) as pair, SUM(throughput) as Node\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -354,10 +354,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t\n", "queryType": "randomWalk", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM default.flows_node_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642533454) AND flowEndSecondsFromDestinationNode <= toDateTime(1642535254)\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourceNodeName, '->', destinationNodeName) as pair, SUM(reverseThroughput) as Node\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM default.flows_node_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642533454) AND flowEndSecondsFromDestinationNode <= toDateTime(1642535254)\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourceNodeName, '->', destinationNodeName) as pair, SUM(reverseThroughput) as Node\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -482,10 +482,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM default.flows_node_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642533454) AND flowEndSecondsFromDestinationNode <= toDateTime(1642535254)\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSecondsFromSourceNode) as time, sourceNodeName, SUM(throughputFromSourceNode)\nFROM flows_node_view\nWHERE sourceNodeName != '' \nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, sourceNodeName\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM default.flows_node_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642533454) AND flowEndSecondsFromDestinationNode <= toDateTime(1642535254)\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSecondsFromSourceNode) as time, sourceNodeName, SUM(throughputFromSourceNode)\nFROM flows_node_view\nWHERE sourceNodeName != '' \nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, sourceNodeName\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -575,7 +575,7 @@ "uid": "PDEE91DDB90597936" }, "format": 1, - "rawSql": "SELECT SUM(octetDeltaCount), sourceNodeName\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY sourceNodeName", + "rawSql": "SELECT SUM(octetDeltaCount), sourceNodeName\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY sourceNodeName", "refId": "A" } ], @@ -677,10 +677,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM $table\nWHERE $timeFilter\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM default.flows_node_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642533454) AND flowEndSecondsFromDestinationNode <= toDateTime(1642535254)\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSecondsFromDestinationNode) as time, destinationNodeName, SUM(throughputFromDestinationNode)\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, destinationNodeName\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationNodeName\nFROM default.flows_node_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642533454) AND flowEndSecondsFromDestinationNode <= toDateTime(1642535254)\nAND sourceNodeName != ''\nAND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationNodeName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSecondsFromDestinationNode) as time, destinationNodeName, SUM(throughputFromDestinationNode)\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, destinationNodeName\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -768,7 +768,7 @@ "uid": "PDEE91DDB90597936" }, "format": 1, - "rawSql": "SELECT SUM(octetDeltaCount), destinationNodeName\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY destinationNodeName", + "rawSql": "SELECT SUM(octetDeltaCount), destinationNodeName\nFROM flows_node_view\nWHERE sourceNodeName != '' AND destinationNodeName != ''\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY destinationNodeName", "refId": "A" } ], diff --git a/build/yamls/base/provisioning/dashboards/pod_to_external_dashboard.json b/build/charts/theia/provisioning/dashboards/pod_to_external_dashboard.json similarity index 91% rename from build/yamls/base/provisioning/dashboards/pod_to_external_dashboard.json rename to build/charts/theia/provisioning/dashboards/pod_to_external_dashboard.json index 57be5d8e..99af0cc4 100644 --- a/build/yamls/base/provisioning/dashboards/pod_to_external_dashboard.json +++ b/build/charts/theia/provisioning/dashboards/pod_to_external_dashboard.json @@ -60,10 +60,10 @@ "format": 1, "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationIP) AS pair\nFROM $table\nWHERE $timeFilter\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-aggregator', 'flow-visibility')\nGROUP BY pair\n", + "query": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationIP) AS pair\nFROM $table\nWHERE $timeFilter\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-aggregator', '{{ .Release.Namespace }}')\nGROUP BY pair\n", "queryType": "randomWalk", - "rawQuery": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationIP) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642534343) AND flowEndSeconds <= toDateTime(1642536143)\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-aggregator', 'flow-visibility')\nGROUP BY pair", - "rawSql": "select SUM(octetDeltaCount) as bytes, sourcePodName as source, destinationIP as destination\nFrom flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nHAVING bytes != 0", + "rawQuery": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationIP) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642534343) AND flowEndSeconds <= toDateTime(1642536143)\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-aggregator', '{{ .Release.Namespace }}')\nGROUP BY pair", + "rawSql": "select SUM(octetDeltaCount) as bytes, sourcePodName as source, destinationIP as destination\nFrom flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nHAVING bytes != 0", "refId": "A", "round": "0s", "skip_comments": true, @@ -109,10 +109,10 @@ "format": 1, "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationIP) AS pair\nFROM $table\nWHERE $timeFilter\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-aggregator', 'flow-visibility')\nGROUP BY pair\n", + "query": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationIP) AS pair\nFROM $table\nWHERE $timeFilter\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-aggregator', '{{ .Release.Namespace }}')\nGROUP BY pair\n", "queryType": "randomWalk", - "rawQuery": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationIP) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642534382) AND flowEndSeconds <= toDateTime(1642536182)\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-aggregator', 'flow-visibility')\nGROUP BY pair", - "rawSql": "select SUM(reverseOctetDeltaCount) as bytes, sourcePodName as source, destinationIP as destination\nFrom flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nHAVING bytes != 0", + "rawQuery": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationIP) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642534382) AND flowEndSeconds <= toDateTime(1642536182)\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-aggregator', '{{ .Release.Namespace }}')\nGROUP BY pair", + "rawSql": "select SUM(reverseOctetDeltaCount) as bytes, sourcePodName as source, destinationIP as destination\nFrom flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination\nHAVING bytes != 0", "refId": "A", "round": "0s", "skip_comments": true, @@ -217,10 +217,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(octetDeltaCount), (sourcePodName, destinationIP) as pair\nFROM $table\nWHERE $timeFilter\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair,t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(octetDeltaCount), (sourcePodName, destinationIP) as pair\nFROM $table\nWHERE $timeFilter\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair,t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(octetDeltaCount), (sourcePodName, destinationIP) as pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642534150) AND flowEndSeconds <= toDateTime(1642535950)\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair,t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationIP) as pair, SUM(throughput)\nFROM flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING SUM(throughput) != 0\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(octetDeltaCount), (sourcePodName, destinationIP) as pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642534150) AND flowEndSeconds <= toDateTime(1642535950)\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair,t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationIP) as pair, SUM(throughput)\nFROM flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING SUM(throughput) != 0\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -341,10 +341,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(reverseOctetDeltaCount), (sourcePodName, destinationIP) as pair\nFROM $table\nWHERE $timeFilter\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair, t\nORDER BY t", + "query": "SELECT $timeSeries as t, SUM(reverseOctetDeltaCount), (sourcePodName, destinationIP) as pair\nFROM $table\nWHERE $timeFilter\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair, t\nORDER BY t", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(reverseOctetDeltaCount), (sourcePodName, destinationIP) as pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642534246) AND flowEndSeconds <= toDateTime(1642536046)\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationIP) as pair, SUM(reverseThroughput)\nFROM flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING SUM(reverseThroughput) != 0\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(reverseOctetDeltaCount), (sourcePodName, destinationIP) as pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642534246) AND flowEndSeconds <= toDateTime(1642536046)\nAND flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationIP) as pair, SUM(reverseThroughput)\nFROM flows_pod_view\nWHERE flowType == 3\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING SUM(reverseThroughput) != 0\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, diff --git a/build/yamls/base/provisioning/dashboards/pod_to_pod_dashboard.json b/build/charts/theia/provisioning/dashboards/pod_to_pod_dashboard.json similarity index 87% rename from build/yamls/base/provisioning/dashboards/pod_to_pod_dashboard.json rename to build/charts/theia/provisioning/dashboards/pod_to_pod_dashboard.json index dac3c9ba..761a372e 100644 --- a/build/yamls/base/provisioning/dashboards/pod_to_pod_dashboard.json +++ b/build/charts/theia/provisioning/dashboards/pod_to_pod_dashboard.json @@ -60,10 +60,10 @@ "format": 1, "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationPodName, destinationIP) AS pair\nFROM $table\nWHERE $timeFilter\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair\n", + "query": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationPodName, destinationIP) AS pair\nFROM $table\nWHERE $timeFilter\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair\n", "queryType": "sql", - "rawQuery": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationPodName, destinationIP) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642531723) AND flowEndSeconds <= toDateTime(1642533523)\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair", - "rawSql": "select SUM(octetDeltaCount) as bytes, sourcePodName as source, destinationPodName as destination, destinationIP\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination, destinationIP", + "rawQuery": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationPodName, destinationIP) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642531723) AND flowEndSeconds <= toDateTime(1642533523)\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair", + "rawSql": "select SUM(octetDeltaCount) as bytes, sourcePodName as source, destinationPodName as destination, destinationIP\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination, destinationIP", "refId": "A", "round": "0s", "skip_comments": true, @@ -109,10 +109,10 @@ "format": 1, "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationPodName, destinationIP) AS pair\nFROM $table\nWHERE $timeFilter\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair\n", + "query": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationPodName, destinationIP) AS pair\nFROM $table\nWHERE $timeFilter\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair\n", "queryType": "randomWalk", - "rawQuery": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationPodName, destinationIP) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642531743) AND flowEndSeconds <= toDateTime(1642533543)\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair", - "rawSql": "select SUM(reverseOctetDeltaCount) as bytes, sourcePodName as source, destinationPodName as destination, destinationIP\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination, destinationIP", + "rawQuery": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationPodName, destinationIP) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642531743) AND flowEndSeconds <= toDateTime(1642533543)\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair", + "rawSql": "select SUM(reverseOctetDeltaCount) as bytes, sourcePodName as source, destinationPodName as destination, destinationIP\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination, destinationIP", "refId": "A", "round": "0s", "skip_comments": true, @@ -218,10 +218,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationPodName) as pair, SUM(throughput)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING SUM(throughput) > 0\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationPodName) as pair, SUM(throughput)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING SUM(throughput) > 0\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -346,10 +346,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationPodName) as pair, SUM(reverseThroughput)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING SUM(reverseThroughput) > 0\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationPodName) as pair, SUM(reverseThroughput)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, pair\nHAVING SUM(reverseThroughput) > 0\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -474,10 +474,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSecondsFromSourceNode) as time, sourcePodName, SUM(throughputFromSourceNode)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, sourcePodName\nHAVING SUM(throughputFromSourceNode) > 0\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSecondsFromSourceNode) as time, sourcePodName, SUM(throughputFromSourceNode)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, sourcePodName\nHAVING SUM(throughputFromSourceNode) > 0\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -566,7 +566,7 @@ }, "format": 1, "queryType": "sql", - "rawSql": "select SUM(octetDeltaCount) as bytes, sourcePodNamespace\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY sourcePodNamespace\nHAVING bytes > 0", + "rawSql": "select SUM(octetDeltaCount) as bytes, sourcePodNamespace\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY sourcePodNamespace\nHAVING bytes > 0", "refId": "A" } ], @@ -668,10 +668,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationPodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationPodName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromDestinationNode), destinationPodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationPodName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationPodName\nFROM default.flows_pod_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642532702) AND flowEndSecondsFromDestinationNode <= toDateTime(1642534502) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY destinationPodName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSecondsFromDestinationNode) as time, destinationPodName, SUM(throughputFromDestinationNode)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, destinationPodName\nHAVING SUM(throughputFromDestinationNode) > 0\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSecondsFromDestinationNode), 60) * 60) * 1000 as t, SUM(throughputFromDestinationNode), destinationPodName\nFROM default.flows_pod_view\nWHERE flowEndSecondsFromDestinationNode >= toDateTime(1642532702) AND flowEndSecondsFromDestinationNode <= toDateTime(1642534502) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY destinationPodName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSecondsFromDestinationNode) as time, destinationPodName, SUM(throughputFromDestinationNode)\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(time)\nGROUP BY time, destinationPodName\nHAVING SUM(throughputFromDestinationNode) > 0\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -772,7 +772,7 @@ "uid": "PDEE91DDB90597936" }, "format": 1, - "rawSql": "select SUM(octetDeltaCount) as bytes, destinationPodNamespace\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY destinationPodNamespace", + "rawSql": "select SUM(octetDeltaCount) as bytes, destinationPodNamespace\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND $__timeFilter(flowEndSeconds)\nGROUP BY destinationPodNamespace", "refId": "A" } ], diff --git a/build/yamls/base/provisioning/dashboards/pod_to_service_dashboard.json b/build/charts/theia/provisioning/dashboards/pod_to_service_dashboard.json similarity index 88% rename from build/yamls/base/provisioning/dashboards/pod_to_service_dashboard.json rename to build/charts/theia/provisioning/dashboards/pod_to_service_dashboard.json index e24d8425..06bf914c 100644 --- a/build/yamls/base/provisioning/dashboards/pod_to_service_dashboard.json +++ b/build/charts/theia/provisioning/dashboards/pod_to_service_dashboard.json @@ -60,10 +60,10 @@ "format": 1, "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationServicePortName) AS pair\nFROM $table\nWHERE $timeFilter\nAND destinationServicePortName != ''\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair\n", + "query": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationServicePortName) AS pair\nFROM $table\nWHERE $timeFilter\nAND destinationServicePortName != ''\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair\n", "queryType": "randomWalk", - "rawQuery": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationServicePortName) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642193285) AND flowEndSeconds <= toDateTime(1642195085)\nAND destinationServicePortName != ''\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair", - "rawSql": "select SUM(octetDeltaCount) as bytes, sourcePodName as source, destinationServicePortName as destination\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination", + "rawQuery": "SELECT SUM(octetDeltaCount), (sourcePodName, destinationServicePortName) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642193285) AND flowEndSeconds <= toDateTime(1642195085)\nAND destinationServicePortName != ''\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair", + "rawSql": "select SUM(octetDeltaCount) as bytes, sourcePodName as source, destinationServicePortName as destination\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination", "refId": "A", "round": "0s", "skip_comments": true, @@ -109,10 +109,10 @@ "format": 1, "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "intervalFactor": 1, - "query": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationServicePortName) AS pair\nFROM $table\nWHERE $timeFilter\nAND destinationServicePortName != ''\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair\n", + "query": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationServicePortName) AS pair\nFROM $table\nWHERE $timeFilter\nAND destinationServicePortName != ''\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair\n", "queryType": "randomWalk", - "rawQuery": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationServicePortName) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642193431) AND flowEndSeconds <= toDateTime(1642195231)\nAND destinationServicePortName != ''\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY pair", - "rawSql": "select SUM(reverseOctetDeltaCount) as bytes, sourcePodName as source, destinationServicePortName as destination\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination", + "rawQuery": "SELECT SUM(reverseOctetDeltaCount), (sourcePodName, destinationServicePortName) AS pair\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642193431) AND flowEndSeconds <= toDateTime(1642195231)\nAND destinationServicePortName != ''\nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY pair", + "rawSql": "select SUM(reverseOctetDeltaCount) as bytes, sourcePodName as source, destinationServicePortName as destination\nFrom flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY source, destination", "refId": "A", "round": "0s", "skip_comments": true, @@ -218,10 +218,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationServicePortName) as pair, SUM(throughput) as Pod\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY time, pair\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationServicePortName) as pair, SUM(throughput) as Pod\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(flowEndSeconds)\nGROUP BY time, pair\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -346,10 +346,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationServicePortName) as pair, SUM(reverseThroughput) as Pod\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(time)\nGROUP BY time, pair\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSeconds) as time, CONCAT(sourcePodName, '->', destinationServicePortName) as pair, SUM(reverseThroughput) as Pod\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(time)\nGROUP BY time, pair\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -474,10 +474,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSecondsFromSourceNode) as time, sourcePodName, SUM(throughputFromSourceNode) as Pod\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(time)\nGROUP BY time, sourcePodName\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSecondsFromSourceNode) as time, sourcePodName, SUM(throughputFromSourceNode) as Pod\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(time)\nGROUP BY time, sourcePodName\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, @@ -602,10 +602,10 @@ "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", "hide": false, "intervalFactor": 1, - "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", + "query": "SELECT $timeSeries as t, SUM(throughputFromSourceNode), sourcePodName\nFROM $table\nWHERE $timeFilter \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t\n", "queryType": "sql", - "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", - "rawSql": "SELECT $__timeInterval(flowEndSecondsFromDestinationNode) as time, destinationServicePortName, SUM(throughputFromDestinationNode) as Service\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(time)\nGROUP BY time, destinationServicePortName\nORDER BY time", + "rawQuery": "SELECT (intDiv(toUInt32(flowEndSeconds), 60) * 60) * 1000 as t, SUM(throughputFromSourceNode), sourcePodName\nFROM default.flows_pod_view\nWHERE flowEndSeconds >= toDateTime(1642532448) AND flowEndSeconds <= toDateTime(1642534248) \nAND flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nGROUP BY sourcePodName, t\nORDER BY t", + "rawSql": "SELECT $__timeInterval(flowEndSecondsFromDestinationNode) as time, destinationServicePortName, SUM(throughputFromDestinationNode) as Service\nFROM flows_pod_view\nWHERE flowType IN (1, 2)\nAND sourcePodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', '{{ .Release.Namespace }}', 'flow-aggregator')\nAND destinationServicePortName != ''\nAND $__timeFilter(time)\nGROUP BY time, destinationServicePortName\nORDER BY time", "refId": "A", "round": "0s", "skip_comments": true, diff --git a/build/yamls/base/provisioning/datasources/create_table.sh b/build/charts/theia/provisioning/datasources/create_table.sh similarity index 90% rename from build/yamls/base/provisioning/datasources/create_table.sh rename to build/charts/theia/provisioning/datasources/create_table.sh index 17253821..92f566dc 100644 --- a/build/yamls/base/provisioning/datasources/create_table.sh +++ b/build/charts/theia/provisioning/datasources/create_table.sh @@ -14,6 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- $ttl := split " " .Values.clickhouse.ttl }} +{{- $ttlTimeout := 14400 }} +{{- if eq $ttl._1 "SECOND" }} +{{- $ttlTimeout = min $ttl._0 $ttlTimeout }} +{{- else if eq $ttl._1 "MINUTE" }} +{{- $ttlTimeout = min (mul $ttl._0 60) $ttlTimeout }} +{{- else if eq $ttl._1 "HOUR" }} +{{- $ttlTimeout = min (mul $ttl._0 60 60) $ttlTimeout }} +{{- end }} + set -e clickhouse client -n -h 127.0.0.1 <<-EOSQL @@ -69,8 +79,8 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL trusted UInt8 DEFAULT 0 ) engine=MergeTree ORDER BY (timeInserted, flowEndSeconds) - TTL timeInserted + INTERVAL 1 HOUR - SETTINGS merge_with_ttl_timeout = 3600; + TTL timeInserted + INTERVAL {{ .Values.clickhouse.ttl }} + SETTINGS merge_with_ttl_timeout = {{ $ttlTimeout }}; CREATE MATERIALIZED VIEW IF NOT EXISTS flows_pod_view ENGINE = SummingMergeTree @@ -86,8 +96,8 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL flowType, sourcePodNamespace, destinationPodNamespace) - TTL timeInserted + INTERVAL 1 HOUR - SETTINGS merge_with_ttl_timeout = 3600 + TTL timeInserted + INTERVAL {{ .Values.clickhouse.ttl }} + SETTINGS merge_with_ttl_timeout = {{ $ttlTimeout }} POPULATE AS SELECT timeInserted, @@ -132,8 +142,8 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL destinationNodeName, sourcePodNamespace, destinationPodNamespace) - TTL timeInserted + INTERVAL 1 HOUR - SETTINGS merge_with_ttl_timeout = 3600 + TTL timeInserted + INTERVAL {{ .Values.clickhouse.ttl }} + SETTINGS merge_with_ttl_timeout = {{ $ttlTimeout }} POPULATE AS SELECT timeInserted, @@ -185,8 +195,8 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL destinationServicePort, destinationServicePortName, destinationIP) - TTL timeInserted + INTERVAL 1 HOUR - SETTINGS merge_with_ttl_timeout = 3600 + TTL timeInserted + INTERVAL {{ .Values.clickhouse.ttl }} + SETTINGS merge_with_ttl_timeout = {{ $ttlTimeout }} POPULATE AS SELECT timeInserted, diff --git a/build/yamls/base/provisioning/datasources/datasource_provider.yml b/build/charts/theia/provisioning/datasources/datasource_provider.yaml similarity index 50% rename from build/yamls/base/provisioning/datasources/datasource_provider.yml rename to build/charts/theia/provisioning/datasources/datasource_provider.yaml index 31858cba..683894f0 100644 --- a/build/yamls/base/provisioning/datasources/datasource_provider.yml +++ b/build/charts/theia/provisioning/datasources/datasource_provider.yaml @@ -3,11 +3,11 @@ datasources: - name: ClickHouse type: grafana-clickhouse-datasource access: proxy - url: http://clickhouse-clickhouse.flow-visibility.svc:8123 + url: http://clickhouse-clickhouse.{{ .Release.Namespace }}.svc:{{ .Values.clickhouse.service.httpPort }} editable: true jsonData: - server: clickhouse-clickhouse.flow-visibility.svc - port: 9000 + server: clickhouse-clickhouse.{{ .Release.Namespace }}.svc + port: {{ .Values.clickhouse.service.tcpPort }} username: $CLICKHOUSE_USERNAME secureJsonData: password: $CLICKHOUSE_PASSWORD diff --git a/build/charts/theia/templates/NOTES.txt b/build/charts/theia/templates/NOTES.txt new file mode 100644 index 00000000..f1ce4535 --- /dev/null +++ b/build/charts/theia/templates/NOTES.txt @@ -0,0 +1 @@ +Theia has been successfully installed. diff --git a/build/charts/theia/templates/_helpers.tpl b/build/charts/theia/templates/_helpers.tpl new file mode 100644 index 00000000..e69de29b diff --git a/build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml b/build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml new file mode 100644 index 00000000..41bcaf07 --- /dev/null +++ b/build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml @@ -0,0 +1,108 @@ +{{- $enablePV := or .Values.clickhouse.storage.createPersistentVolume.type .Values.clickhouse.storage.persistentVolumeClaimSpec }} +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: clickhouse + labels: + app: clickhouse + namespace: {{ .Release.Namespace }} +spec: + configuration: + users: + {{ .Values.clickhouse.secret.username }}/k8s_secret_password: {{ .Release.Namespace }}/clickhouse-secret/password + {{ .Values.clickhouse.secret.username }}/networks/ip: "::/0" + clusters: + - name: "clickhouse" + layout: + shardsCount: 1 + replicasCount: 1 + defaults: + templates: + podTemplate: pod-template + serviceTemplate: service-template + {{- if $enablePV }} + dataVolumeClaimTemplate: clickhouse-storage-template + {{- end }} + templates: + serviceTemplates: + - name: service-template + spec: + type: {{ .Values.clickhouse.service.type }} + ports: + - name: http + port: {{ .Values.clickhouse.service.httpPort }} + - name: tcp + port: {{ .Values.clickhouse.service.tcpPort }} + podTemplates: + - name: pod-template + spec: + containers: + - name: clickhouse + image: {{ .Values.clickhouse.image.repository }}:{{ .Values.clickhouse.image.tag }} + imagePullPolicy: {{ .Values.clickhouse.image.pullPolicy }} + volumeMounts: + - name: clickhouse-configmap-volume + mountPath: /docker-entrypoint-initdb.d + {{- if not $enablePV }} + - name: clickhouse-storage-volume + mountPath: /var/lib/clickhouse + {{- end }} + {{- if .Values.clickhouse.monitor.enable}} + - name: clickhouse-monitor + image: {{ .Values.clickhouse.monitor.image.repository }}:{{ .Values.clickhouse.monitor.image.tag }} + imagePullPolicy: {{ .Values.clickhouse.monitor.image.pullPolicy }} + env: + - name: CLICKHOUSE_USERNAME + valueFrom: + secretKeyRef: + name: clickhouse-secret + key: username + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: clickhouse-secret + key: password + - name: DB_URL + value: "tcp://localhost:9000" + - name: TABLE_NAME + value: "default.flows" + - name: MV_NAMES + value: "default.flows_pod_view default.flows_node_view default.flows_policy_view" + - name: STORAGE_SIZE + value: {{ .Values.clickhouse.storage.size | quote }} + - name: THRESHOLD + value: {{ .Values.clickhouse.monitor.threshold | quote }} + - name: DELETE_PERCENTAGE + value: {{ .Values.clickhouse.monitor.deletePercentage | quote }} + - name: EXEC_INTERVAL + value: {{ .Values.clickhouse.monitor.execInterval }} + - name: SKIP_ROUNDS_NUM + value: {{ .Values.clickhouse.monitor.skipRoundsNum | quote }} + {{- end}} + volumes: + - name: clickhouse-configmap-volume + configMap: + name: clickhouse-mounted-configmap + {{- if not $enablePV }} + - name: clickhouse-storage-volume + emptyDir: + medium: Memory + sizeLimit: {{ .Values.clickhouse.storage.size }} + {{- end }} + {{- if $enablePV }} + volumeClaimTemplates: + - name: clickhouse-storage-template + spec: + {{- if .Values.clickhouse.storage.createPersistentVolume.type }} + storageClassName: clickhouse-storage + {{- else }} + {{- with .Values.clickhouse.storage.persistentVolumeClaimSpec }} + {{- toYaml . | trim | nindent 10 }} + {{- end }} + {{- end }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.clickhouse.storage.size }} + {{- end }} diff --git a/build/charts/theia/templates/clickhouse/configmap.yaml b/build/charts/theia/templates/clickhouse/configmap.yaml new file mode 100644 index 00000000..573bfd63 --- /dev/null +++ b/build/charts/theia/templates/clickhouse/configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: clickhouse-mounted-configmap + namespace: {{ .Release.Namespace }} +data: + create_table.sh: |- +{{ tpl (.Files.Get "provisioning/datasources/create_table.sh") . | indent 4}} diff --git a/build/charts/theia/templates/clickhouse/local-persistentvolume.yaml b/build/charts/theia/templates/clickhouse/local-persistentvolume.yaml new file mode 100644 index 00000000..588713e0 --- /dev/null +++ b/build/charts/theia/templates/clickhouse/local-persistentvolume.yaml @@ -0,0 +1,27 @@ +{{- if eq .Values.clickhouse.storage.createPersistentVolume.type "Local" }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clickhouse-pv +spec: + storageClassName: clickhouse-storage + capacity: + storage: {{ .Values.clickhouse.storage.size }} + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + local: + path: {{ .Values.clickhouse.storage.createPersistentVolume.local.path }} + nodeAffinity: + {{- if .Values.clickhouse.storage.createPersistentVolume.local.affinity }} + {{- with .Values.clickhouse.storage.createPersistentVolume.local.affinity }} + {{- toYaml . | trim | nindent 4 }} + {{- end }} + {{- else }} + required: + nodeSelectorTerms: + - matchExpressions: + - key: antrea.io/clickhouse-data-node + operator: Exists + {{- end }} +{{- end }} diff --git a/build/charts/theia/templates/clickhouse/nfs-persistentvolume.yaml b/build/charts/theia/templates/clickhouse/nfs-persistentvolume.yaml new file mode 100644 index 00000000..23881183 --- /dev/null +++ b/build/charts/theia/templates/clickhouse/nfs-persistentvolume.yaml @@ -0,0 +1,16 @@ +{{- if eq .Values.clickhouse.storage.createPersistentVolume.type "NFS" }} +apiVersion: v1 +kind: PersistentVolume +metadata: + name: clickhouse-pv +spec: + storageClassName: clickhouse-storage + capacity: + storage: {{ .Values.clickhouse.storage.size }} + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + nfs: + path: {{ .Values.clickhouse.storage.createPersistentVolume.nfs.path }} + server: {{ .Values.clickhouse.storage.createPersistentVolume.nfs.host }} +{{- end }} diff --git a/build/charts/theia/templates/clickhouse/secret.yaml b/build/charts/theia/templates/clickhouse/secret.yaml new file mode 100644 index 00000000..8a38773b --- /dev/null +++ b/build/charts/theia/templates/clickhouse/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: clickhouse-secret + namespace: {{ .Release.Namespace }} +type: Opaque +stringData: + username: {{ .Values.clickhouse.secret.username }} + password: {{ .Values.clickhouse.secret.password }} diff --git a/build/charts/theia/templates/clickhouse/storageclass.yaml b/build/charts/theia/templates/clickhouse/storageclass.yaml new file mode 100644 index 00000000..10196cd9 --- /dev/null +++ b/build/charts/theia/templates/clickhouse/storageclass.yaml @@ -0,0 +1,10 @@ +{{- if .Values.clickhouse.storage.createPersistentVolume.type }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: clickhouse-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Retain +allowVolumeExpansion: True +{{- end }} diff --git a/build/charts/theia/templates/grafana/dashboard-configmap.yaml b/build/charts/theia/templates/grafana/dashboard-configmap.yaml new file mode 100644 index 00000000..5b92a559 --- /dev/null +++ b/build/charts/theia/templates/grafana/dashboard-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-config + namespace: {{ .Release.Namespace }} +data: +{{ $files := .Files }} +{{- range $fileName := .Values.grafana.dashboards }} +{{ tpl ($files.Glob (printf "provisioning/dashboards/%s" $fileName)).AsConfig $ | indent 2}} +{{- end }} diff --git a/build/charts/theia/templates/grafana/dashboard-provider-configmap.yaml b/build/charts/theia/templates/grafana/dashboard-provider-configmap.yaml new file mode 100644 index 00000000..121073e0 --- /dev/null +++ b/build/charts/theia/templates/grafana/dashboard-provider-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-provider + namespace: {{ .Release.Namespace }} +data: + dashboard_provider.yaml: |- +{{ .Files.Get "provisioning/dashboards/dashboard_provider.yaml" | indent 4}} diff --git a/build/charts/theia/templates/grafana/datasource-provider-configmap.yaml b/build/charts/theia/templates/grafana/datasource-provider-configmap.yaml new file mode 100644 index 00000000..fd6a2e1c --- /dev/null +++ b/build/charts/theia/templates/grafana/datasource-provider-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasource-provider + namespace: {{ .Release.Namespace }} +data: + datasource_provider.yaml: |- +{{ tpl (.Files.Get "provisioning/datasources/datasource_provider.yaml") . | indent 4}} diff --git a/build/yamls/base/grafana.yml b/build/charts/theia/templates/grafana/deployment.yaml similarity index 58% rename from build/yamls/base/grafana.yml rename to build/charts/theia/templates/grafana/deployment.yaml index c14406a5..da19407b 100644 --- a/build/yamls/base/grafana.yml +++ b/build/charts/theia/templates/grafana/deployment.yaml @@ -1,92 +1,10 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: flow-visibility ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: grafana ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app: flow-visibility - name: grafana-role -rules: - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app: flow-visibility - name: grafana-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: grafana-role -subjects: - - kind: ServiceAccount - name: grafana ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: grafana-storage -provisioner: kubernetes.io/no-provisioner -volumeBindingMode: WaitForFirstConsumer -reclaimPolicy: Delete -allowVolumeExpansion: True ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: grafana-pvc -spec: - storageClassName: grafana-storage - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: grafana-pv -spec: - storageClassName: grafana-storage - capacity: - storage: 2Gi - accessModes: - - ReadWriteOnce - hostPath: - path: "/data/grafana" ---- -apiVersion: v1 -kind: Secret -metadata: - name: grafana-secret -type: Opaque -stringData: - admin-username: admin - admin-password: admin ---- apiVersion: apps/v1 kind: Deployment metadata: labels: app: grafana name: grafana + namespace: {{ .Release.Namespace }} spec: selector: matchLabels: @@ -103,11 +21,11 @@ spec: - 0 containers: - name: grafana - image: projects.registry.vmware.com/antrea/theia-grafana:8.3.3 - imagePullPolicy: IfNotPresent + image: {{ .Values.grafana.image.repository }}:{{ .Values.grafana.image.tag }} + imagePullPolicy: {{ .Values.grafana.image.pullPolicy }} env: - name: GF_INSTALL_PLUGINS - value: "https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-sankey-plugin-1.0.0.zip;theia-grafana-sankey-plugin,https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-chord-plugin-1.0.0.zip;theia-grafana-chord-plugin,grafana-clickhouse-datasource 1.0.1" + value: {{ join "," .Values.grafana.installPlugins }} - name: CLICKHOUSE_USERNAME valueFrom: secretKeyRef: @@ -180,17 +98,3 @@ spec: - name: grafana-dashboard-config configMap: name: grafana-dashboard-config ---- -apiVersion: v1 -kind: Service -metadata: - name: grafana -spec: - ports: - - port: 3000 - protocol: TCP - targetPort: http-grafana - selector: - app: grafana - sessionAffinity: None - type: NodePort diff --git a/build/charts/theia/templates/grafana/persistentvolume.yaml b/build/charts/theia/templates/grafana/persistentvolume.yaml new file mode 100644 index 00000000..6d585cda --- /dev/null +++ b/build/charts/theia/templates/grafana/persistentvolume.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: grafana-pv +spec: + storageClassName: grafana-storage + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/data/grafana" diff --git a/build/charts/theia/templates/grafana/persistentvolumeclaim.yaml b/build/charts/theia/templates/grafana/persistentvolumeclaim.yaml new file mode 100644 index 00000000..1ddf22a0 --- /dev/null +++ b/build/charts/theia/templates/grafana/persistentvolumeclaim.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana-pvc + namespace: {{ .Release.Namespace }} +spec: + storageClassName: grafana-storage + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/build/charts/theia/templates/grafana/role.yaml b/build/charts/theia/templates/grafana/role.yaml new file mode 100644 index 00000000..8761ada0 --- /dev/null +++ b/build/charts/theia/templates/grafana/role.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: {{ .Release.Namespace }} + name: grafana-role + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch diff --git a/build/charts/theia/templates/grafana/rolebinding.yaml b/build/charts/theia/templates/grafana/rolebinding.yaml new file mode 100644 index 00000000..c3d8591a --- /dev/null +++ b/build/charts/theia/templates/grafana/rolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: {{ .Release.Namespace }} + name: grafana-role-binding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: grafana-role +subjects: + - kind: ServiceAccount + name: grafana + namespace: {{ .Release.Namespace }} diff --git a/build/charts/theia/templates/grafana/secret.yaml b/build/charts/theia/templates/grafana/secret.yaml new file mode 100644 index 00000000..bfa7d70c --- /dev/null +++ b/build/charts/theia/templates/grafana/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: grafana-secret + namespace: {{ .Release.Namespace }} +type: Opaque +stringData: + admin-username: {{ .Values.grafana.secret.username }} + admin-password: {{ .Values.grafana.secret.password }} diff --git a/build/charts/theia/templates/grafana/service.yaml b/build/charts/theia/templates/grafana/service.yaml new file mode 100644 index 00000000..259f65f3 --- /dev/null +++ b/build/charts/theia/templates/grafana/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: {{ .Values.grafana.service.tcpPort }} + protocol: TCP + targetPort: http-grafana + selector: + app: grafana + sessionAffinity: None + type: {{ .Values.grafana.service.type }} diff --git a/build/charts/theia/templates/grafana/serviceaccount.yaml b/build/charts/theia/templates/grafana/serviceaccount.yaml new file mode 100644 index 00000000..9593ee43 --- /dev/null +++ b/build/charts/theia/templates/grafana/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana + namespace: {{ .Release.Namespace }} diff --git a/build/charts/theia/templates/grafana/storageclass.yaml b/build/charts/theia/templates/grafana/storageclass.yaml new file mode 100644 index 00000000..d6d06fba --- /dev/null +++ b/build/charts/theia/templates/grafana/storageclass.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: grafana-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +allowVolumeExpansion: True diff --git a/build/charts/theia/values.yaml b/build/charts/theia/values.yaml new file mode 100644 index 00000000..31d5af4f --- /dev/null +++ b/build/charts/theia/values.yaml @@ -0,0 +1,103 @@ +clickhouse: + # -- Container image to use for ClickHouse. + image: + repository: "projects.registry.vmware.com/antrea/theia-clickhouse-server" + pullPolicy: "IfNotPresent" + tag: "21.11" + monitor: + # -- Determine whether to run a monitor to periodically check the ClickHouse + # memory usage and clean data. + enable: true + # -- The storage percentage at which the monitor starts to delete old records. + # Vary from 0 to 1. + threshold: 0.5 + # -- The percentage of records in ClickHouse that will be deleted when the + # storage grows above threshold. Vary from 0 to 1. + deletePercentage: 0.5 + # -- The time interval between two round of monitoring. Can be a plain integer using + # one of these unit suffixes ns, us (or µs), ms, s, m, h. + execInterval: "1m" + # -- The number of rounds for the monitor to stop after a deletion to wait for + # the ClickHouse MergeTree Engine to release memory. + skipRoundsNum: 3 + # -- Container image to use for the ClickHouse Monitor. + image: + repository: "projects.registry.vmware.com/antrea/theia-clickhouse-monitor" + pullPolicy: "IfNotPresent" + tag: "latest" + secret: + # -- ClickHouse username. It will be stored in a secret. + username: "clickhouse_operator" + # -- ClickHouse password. It will be stored in a secret. + password: "clickhouse_operator_password" + service: + # -- The type of Service exposing ClickHouse. It can be one of ClusterIP, NodePort or LoadBalancer. + type: ClusterIP + # -- TCP port number for the ClickHouse service. + tcpPort: 9000 + # -- TCP port number for the ClickHouse service. + httpPort: 8123 + # -- Time to live for data in the ClickHouse. Can be a plain integer using + # one of these unit suffixes SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, + # YEAR. + ttl: 1 HOUR + storage: + # -- ClickHouse storage size. Can be a plain integer or as a fixed-point + # number using one of these quantity suffixes: E, P, T, G, M, K. Or the + # power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + size: "8Gi" + createPersistentVolume: + # -- Type of PersistentVolume. Can be set to "Local" or "NFS". Please set this value + # to use a PersistentVolume created by Theia. + type: "" + local: + # -- The local path. Required when type is "Local". + path: "/data" + # -- Affinity for the Local Persistent Volume. By default it requires to label the + # Node used to store the ClickHouse data with "antrea.io/clickhouse-data-node=" + affinity: {} + nfs: + # -- The NFS server hostname or IP address. Required when type is "NFS". + host: "" + # -- The path exported on the NFS server. Required when type is "NFS". + path: "" + # -- Specification for PersistentVolumeClaim. This is ignored if createPersistentVolume.type is non-empty. + # To use a custom PersistentVolume, please set + # storageClassName: "" + # volumeName: "" + # To dynamically provision a PersistentVolume, please set + # storageClassName: "" + # Memory storage is used if both createPersistentVolume.type and persistentVolumeClaimSpec are empty. + persistentVolumeClaimSpec: + # storageClassName: "" + # volumeName: "" +grafana: + # -- Container image to use for Grafana. + image: + repository: "projects.registry.vmware.com/antrea/theia-grafana" + pullPolicy: "IfNotPresent" + tag: "8.3.3" + secret: + # -- Grafana username. It will be stored in a secret. + username: "admin" + # -- Grafana password. It will be stored in a secret. + password: "admin" + service: + # -- The type of Service exposing Grafana. It must be one of NodePort or LoadBalancer. + type: NodePort + # -- TCP port number for the Grafana service. + tcpPort: 3000 + # -- Grafana plugins to install. + installPlugins: + - https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-sankey-plugin-1.0.0.zip;theia-grafana-sankey-plugin + - https://downloads.antrea.io/artifacts/grafana-custom-plugins/theia-grafana-chord-plugin-1.0.0.zip;theia-grafana-chord-plugin + - grafana-clickhouse-datasource 1.0.1 + # -- The dashboards to be displayed in Grafana UI. The files must be put under + # provisioning/dashboards + dashboards: + - flow_records_dashboard.json + - pod_to_pod_dashboard.json + - pod_to_service_dashboard.json + - pod_to_external_dashboard.json + - node_to_node_dashboard.json + - networkpolicy_dashboard.json diff --git a/build/yamls/base/clickhouse.yml b/build/yamls/base/clickhouse.yml deleted file mode 100644 index 70add9d9..00000000 --- a/build/yamls/base/clickhouse.yml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: clickhouse-secret -type: Opaque -stringData: - username: clickhouse_operator - password: clickhouse_operator_password ---- -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallation" -metadata: - name: clickhouse - labels: - app: clickhouse -spec: - configuration: - users: - clickhouse_operator/k8s_secret_password: flow-visibility/clickhouse-secret/password - clickhouse_operator/networks/ip: "::/0" - clusters: - - name: "clickhouse" - layout: - shardsCount: 1 - replicasCount: 1 - defaults: - templates: - podTemplate: pod-template - serviceTemplate: service-template - templates: - serviceTemplates: - - name: service-template - spec: - ports: - - name: http - port: 8123 - - name: tcp - port: 9000 - podTemplates: - - name: pod-template - spec: - containers: - - name: clickhouse - image: projects.registry.vmware.com/antrea/theia-clickhouse-server:21.11 - volumeMounts: - - name: clickhouse-configmap-volume - mountPath: /docker-entrypoint-initdb.d - - name: clickhouse-storage-volume - mountPath: /var/lib/clickhouse - - name: clickhouse-monitor - image: clickhouse-monitor - env: - - name: CLICKHOUSE_USERNAME - valueFrom: - secretKeyRef: - name: clickhouse-secret - key: username - - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: clickhouse-secret - key: password - - name: DB_URL - value: "tcp://localhost:9000" - - name: TABLE_NAME - value: "default.flows" - - name: MV_NAMES - value: "default.flows_pod_view default.flows_node_view default.flows_policy_view" - volumes: - - name: clickhouse-configmap-volume - configMap: - name: $(CLICKHOUSE_CONFIG_MAP_NAME) - - name: clickhouse-storage-volume - emptyDir: - medium: Memory - sizeLimit: 8Gi diff --git a/build/yamls/base/kustomization.yaml b/build/yamls/base/kustomization.yaml new file mode 100644 index 00000000..fa6ed17a --- /dev/null +++ b/build/yamls/base/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: flow-visibility + +resources: +- namespace.yaml +- manifest.yaml diff --git a/build/yamls/base/kustomization.yml b/build/yamls/base/kustomization.yml deleted file mode 100644 index 57b50369..00000000 --- a/build/yamls/base/kustomization.yml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: flow-visibility - -resources: -- clickhouse.yml -- grafana.yml - -configMapGenerator: -- name: grafana-datasource-provider - files: - - provisioning/datasources/datasource_provider.yml -- name: grafana-dashboard-provider - files: - - provisioning/dashboards/dashboard_provider.yml -- name: clickhouse-mounted-configmap - namespace: flow-visibility - files: - - provisioning/datasources/create_table.sh -- name: grafana-dashboard-config - files: - - provisioning/dashboards/flow_records_dashboard.json - - provisioning/dashboards/pod_to_pod_dashboard.json - - provisioning/dashboards/pod_to_service_dashboard.json - - provisioning/dashboards/pod_to_external_dashboard.json - - provisioning/dashboards/node_to_node_dashboard.json - - provisioning/dashboards/networkpolicy_dashboard.json - -# CLICKHOUSE_CONFIG_MAP_NAME exports the value in `metadata.name` from `ConfigMap` named `clickhouse-mounted-configmap`, -# which is used for inserting the value to a CRD for an object of kind `ClickHouseInstallation` -vars: -- name: CLICKHOUSE_CONFIG_MAP_NAME - objref: - kind: ConfigMap - name: clickhouse-mounted-configmap - apiVersion: v1 - fieldref: - fieldpath: metadata.name - -configurations: -- kustomize-config.yml diff --git a/build/yamls/base/kustomize-config.yml b/build/yamls/base/kustomize-config.yml deleted file mode 100644 index 3e103590..00000000 --- a/build/yamls/base/kustomize-config.yml +++ /dev/null @@ -1,5 +0,0 @@ -# These are the "extra" (non built-in) paths where Kustomize needs to look for variable -# substitutions if needed. The value of the variable comes from the ConfigMap declaration. -varReference: -- path: spec/templates/podTemplates/spec/volumes/configMap/name - kind: ClickHouseInstallation diff --git a/build/yamls/base/namespace.yaml b/build/yamls/base/namespace.yaml new file mode 100644 index 00000000..f85217da --- /dev/null +++ b/build/yamls/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flow-visibility diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml index 5ed734f5..b7134dba 100644 --- a/build/yamls/flow-visibility.yml +++ b/build/yamls/flow-visibility.yml @@ -152,10 +152,10 @@ data: \ destinationServicePort,\n destinationServicePortName,\n destinationIP;\n\n \ CREATE TABLE IF NOT EXISTS recommendations (\n id String,\n type String,\n timeCreated DateTime,\n yamls String\n ) engine=MergeTree\n - \ ORDER BY (timeCreated);\n \nEOSQL\n" + \ ORDER BY (timeCreated);\n \nEOSQL" kind: ConfigMap metadata: - name: clickhouse-mounted-configmap-cc8984h57b + name: clickhouse-mounted-configmap namespace: flow-visibility --- apiVersion: v1 @@ -4492,12 +4492,12 @@ data: } kind: ConfigMap metadata: - name: grafana-dashboard-config-ct9c8f55cb + name: grafana-dashboard-config namespace: flow-visibility --- apiVersion: v1 data: - dashboard_provider.yml: | + dashboard_provider.yaml: |- apiVersion: 1 providers: - name: grafana-dashboards @@ -4508,12 +4508,12 @@ data: path: /var/lib/grafana/dashboards kind: ConfigMap metadata: - name: grafana-dashboard-provider-m7d5kfmmc6 + name: grafana-dashboard-provider namespace: flow-visibility --- apiVersion: v1 data: - datasource_provider.yml: | + datasource_provider.yaml: |- apiVersion: 1 datasources: - name: ClickHouse @@ -4529,7 +4529,7 @@ data: password: $CLICKHOUSE_PASSWORD kind: ConfigMap metadata: - name: grafana-datasource-provider-h868k56k95 + name: grafana-datasource-provider namespace: flow-visibility --- apiVersion: v1 @@ -4686,13 +4686,13 @@ spec: persistentVolumeClaim: claimName: grafana-pvc - configMap: - name: grafana-datasource-provider-h868k56k95 + name: grafana-datasource-provider name: grafana-datasource-provider - configMap: - name: grafana-dashboard-provider-m7d5kfmmc6 + name: grafana-dashboard-provider name: grafana-dashboard-provider - configMap: - name: grafana-dashboard-config-ct9c8f55cb + name: grafana-dashboard-config name: grafana-dashboard-config --- apiVersion: clickhouse.altinity.com/v1 @@ -4722,6 +4722,7 @@ spec: spec: containers: - image: projects.registry.vmware.com/antrea/theia-clickhouse-server:21.11 + imagePullPolicy: IfNotPresent name: clickhouse volumeMounts: - mountPath: /docker-entrypoint-initdb.d @@ -4745,12 +4746,22 @@ spec: value: default.flows - name: MV_NAMES value: default.flows_pod_view default.flows_node_view default.flows_policy_view + - name: STORAGE_SIZE + value: 8Gi + - name: THRESHOLD + value: "0.5" + - name: DELETE_PERCENTAGE + value: "0.5" + - name: EXEC_INTERVAL + value: 1m + - name: SKIP_ROUNDS_NUM + value: "3" image: projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest imagePullPolicy: IfNotPresent name: clickhouse-monitor volumes: - configMap: - name: clickhouse-mounted-configmap-cc8984h57b + name: clickhouse-mounted-configmap name: clickhouse-configmap-volume - emptyDir: medium: Memory @@ -4764,3 +4775,4 @@ spec: port: 8123 - name: tcp port: 9000 + type: ClusterIP diff --git a/build/yamls/patches/dev/imagePullPolicy.yml b/build/yamls/patches/dev/imagePullPolicy.yml deleted file mode 100644 index 1045d54a..00000000 --- a/build/yamls/patches/dev/imagePullPolicy.yml +++ /dev/null @@ -1,3 +0,0 @@ -- op: add - path: /spec/templates/podTemplates/0/spec/containers/1/imagePullPolicy - value: IfNotPresent diff --git a/build/yamls/patches/release/.gitignore b/build/yamls/patches/release/.gitignore deleted file mode 100644 index fdffa2a0..00000000 --- a/build/yamls/patches/release/.gitignore +++ /dev/null @@ -1 +0,0 @@ -# placeholder diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index 51c8cf04..8c94315e 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -39,7 +39,7 @@ function print_usage { TESTBED_CMD=$(dirname $0)"/kind-setup.sh" YML_DIR=$(dirname $0)"/../../build/yamls" FLOW_VISIBILITY_CMD=$(dirname $0)"/../../hack/generate-manifest.sh" -CH_OPERATOR_YML=$(dirname $0)"/../../build/yamls/clickhouse-operator-install-bundle.yml" +CH_OPERATOR_YML=$(dirname $0)"/../../build/charts/theia/crds/clickhouse-operator-install-bundle.yaml" function quit { result=$? @@ -151,7 +151,7 @@ function run_test { docker exec -i kind-control-plane dd of=/root/antrea.yml < $TMP_DIR/antrea.yml docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml < $TMP_DIR/flow-aggregator.yml - docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yml < $CH_OPERATOR_YML + docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yaml < $CH_OPERATOR_YML $FLOW_VISIBILITY_CMD | docker exec -i kind-control-plane dd of=/root/flow-visibility.yml rm -rf $TMP_DIR sleep 1 diff --git a/docs/network-flow-visibility.md b/docs/network-flow-visibility.md index 2d2b25fc..4fa6e3b2 100644 --- a/docs/network-flow-visibility.md +++ b/docs/network-flow-visibility.md @@ -8,8 +8,17 @@ - [Purpose](#purpose) - [About Grafana and ClickHouse](#about-grafana-and-clickhouse) - [Deployment Steps](#deployment-steps) - - [Credentials Configuration](#credentials-configuration) - - [ClickHouse Configuration](#clickhouse-configuration) + - [Configuration](#configuration) + - [With Helm](#with-helm) + - [With Standalone Manifest](#with-standalone-manifest) + - [Grafana Configuration](#grafana-configuration) + - [Service Customization](#service-customization) + - [Credentials Configuration](#credentials-configuration) + - [ClickHouse Configuration](#clickhouse-configuration) + - [Credentials Configuration](#credentials-configuration-1) + - [Service Customization](#service-customization-1) + - [Performance Configuration](#performance-configuration) + - [Persistent Volumes](#persistent-volumes) - [Pre-built Dashboards](#pre-built-dashboards) - [Flow Records Dashboard](#flow-records-dashboard) - [Pod-to-Pod Flows Dashboard](#pod-to-pod-flows-dashboard) @@ -56,14 +65,30 @@ ClickHouse as the data storage, and use Grafana as the data visualization and mo ### Deployment Steps -To deploy the Grafana Flow Collector, the first step is to install the ClickHouse -Operator, which creates, configures and manages ClickHouse clusters. Check the [homepage](https://github.com/Altinity/clickhouse-operator) -for more information about the ClickHouse Operator. Current checked-in yaml is based on their -[v0.18.2](https://github.com/Altinity/clickhouse-operator/blob/refs/tags/0.18.2/deploy/operator/clickhouse-operator-install-bundle.yaml) released version. Running the following command -will install ClickHouse Operator into `kube-system` Namespace. +We support deploying the Grafana Flow Collector with Helm. Here is the +[Helm chart](../build/charts/theia/) for the Grafana Flow Collector. Please follow +the instructions from the Helm chart [README](../build/charts/theia/README.md) +to customize the installation. + +You can clone the repository and run the following command to install the Grafana +Flow Collector into Namespace `flow-visibility`. See [helm install](https://helm.sh/docs/helm/helm_install/) +for command documentation. + +```bash +helm install -f values.yaml theia build/charts/theia -n flow-visibility --create-namespace +``` + +We recommend using Helm to deploy the Grafana Flow Collector. But if you prefer +not to clone the repository, you can mannually deploy it. The first step is to +install the ClickHouse Operator, which creates, configures and manages ClickHouse +clusters. Check the [homepage](https://github.com/Altinity/clickhouse-operator) +for more information about the ClickHouse Operator. Current checked-in yaml is +based on their [v0.18.2](https://github.com/Altinity/clickhouse-operator/blob/refs/tags/0.18.2/deploy/operator/clickhouse-operator-install-bundle.yaml) +released version. Running the following command will install ClickHouse Operator +into `kube-system` Namespace. ```bash -kubectl apply -f https://raw.githubusercontent.com/antrea-io/theia/main/build/yamls/clickhouse-operator-install-bundle.yml +kubectl apply -f https://raw.githubusercontent.com/antrea-io/theia/main/build/charts/theia/crds/clickhouse-operator-install-bundle.yaml ``` To deploy a released version of the Grafana Flow Collector, find a deployment manifest @@ -81,31 +106,6 @@ use the checked-in [deployment yaml](/build/yamls/flow-visibility.yml): kubectl apply -f https://raw.githubusercontent.com/antrea-io/theia/main/build/yamls/flow-visibility.yml ``` -Grafana is exposed through a NodePort Service by default in `flow-visibility.yml`. -If the given K8s cluster supports LoadBalancer Services, Grafana can be exposed -through a LoadBalancer Service by changing the `grafana` Service type in the manifest -like below. - -```yaml -apiVersion: v1 -kind: Service -metadata: - name: grafana - namespace: flow-visibility -spec: - ports: - - port: 3000 - protocol: TCP - targetPort: http-grafana - selector: - app: grafana - sessionAffinity: None - type: LoadBalancer -``` - -Please refer to the [Flow Aggregator Configuration](https://github.com/antrea-io/antrea/blob/main/docs/network-flow-visibility.md#configuration-1) -to learn about the ClickHouse configuration options. - Run the following command to check if ClickHouse and Grafana are deployed properly: ```bash @@ -151,50 +151,134 @@ You should be able to see a Grafana login page. Login credentials: - username: admin - password: admin -To stop the Grafana Flow Collector, run the following commands: +To stop the Grafana Flow Collector, run the following commands if you deploy it +by Helm: + +```bash +helm uninstall theia -n flow-visibility +kubectl delete namespace flow-visibility +kubectl delete -f https://raw.githubusercontent.com/antrea-io/theia/main/build/charts/theia/crds/clickhouse-operator-install-bundle.yaml -n kube-system +``` + +Run the following commands if you deploy it by the generated manifest available +online: -```shell +```bash kubectl delete -f flow-visibility.yml -kubectl delete -f https://raw.githubusercontent.com/antrea-io/theia/main/build/yamls/clickhouse-operator-install-bundle.yml -n kube-system +kubectl delete -f https://raw.githubusercontent.com/antrea-io/theia/main/build/charts/theia/crds/clickhouse-operator-install-bundle.yaml -n kube-system ``` -#### Credentials Configuration +### Configuration + +#### With Helm -ClickHouse credentials are specified in `flow-visibility.yml` as a Secret named -`clickhouse-secret`. +If you install the Grafana Flow Collector using the Helm command, please refer +to Helm chart [README](../build/charts/theia/README.md) or run the following +commands to see configurable options with descriptions and default values. + +```bash +helm show values build/charts/theia +``` + +You can override any of these settings in a YAML formatted file, and then pass +that file during installation or upgrading. + +```bash +helm install -f values.yaml theia build/charts/theia -n flow-visibility --create-namespace +helm upgrade -f values.yaml theia build/charts/theia -n flow-visibility +``` + +Or you can specify each parameter using the `--set key=value[,key=value]` +argument and run the following commands. + +```bash +helm install theia build/charts/theia -n flow-visibility --create-namespace \ + --set=clickhouse.storageSize="2Gi" +``` + +The ClickHouse TCP service is used by the Flow Aggregator. If you have changed the TCP port, +please update the `databaseURL` following +[Flow Aggregator Configuration](https://github.com/antrea-io/antrea/blob/main/docs/network-flow-visibility.md#configuration-1). + +The ClickHouse credentials are used in the Flow Aggregator. Them are also specified +in `flow-aggregator.yml` as a Secret named `clickhouse-secret` as shown below. +Please make the corresponding changes if you change the ClickHouse credentials. ```yaml apiVersion: v1 kind: Secret metadata: + labels: + app: flow-aggregator name: clickhouse-secret - namespace: flow-visibility + namespace: flow-aggregator stringData: password: clickhouse_operator_password username: clickhouse_operator type: Opaque ``` -If the username `clickhouse_operator` has been changed, please -update the following section accordingly. +Please refer to the [Flow Aggregator Configuration](https://github.com/antrea-io/antrea/blob/main/docs/network-flow-visibility.md#configuration-1) +to learn about more ClickHouse configuration options in Flow Aggregator. + +#### With Standalone Manifest + +If you deploy the Grafana Flow Collector with `flow-visibility.yml`, please +follow the instructions below to customize the configurations. + +##### Grafana Configuration + +###### Service Customization + +Grafana is exposed through a NodePort Service by default in `flow-visibility.yml`. +If the given K8s cluster supports LoadBalancer Services, Grafana can be exposed +through a LoadBalancer Service by changing the `grafana` Service type in the manifest +like below. ```yaml -apiVersion: "clickhouse.altinity.com/v1" -kind: "ClickHouseInstallation" +apiVersion: v1 +kind: Service metadata: - name: clickhouse - labels: - app: clickhouse + name: grafana + namespace: flow-visibility spec: - configuration: - users: - # replace clickhouse_operator by [new_username] - clickhouse_operator/k8s_secret_password: flow-visibility/clickhouse-secret/password - clickhouse_operator/networks/ip: "::/0" + ports: + - port: 3000 + protocol: TCP + targetPort: http-grafana + selector: + app: grafana + sessionAffinity: None + type: LoadBalancer ``` +###### Credentials Configuration + +Grafana login credentials are specified in `flow-visibility.yml` as a Secret +named `grafana-secret`. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana-secret + namespace: flow-visibility +stringData: + admin-password: admin + admin-username: admin +type: Opaque +``` + +We recommend changing the credentials if you are going to run the Flow Collector +in production. + +##### ClickHouse Configuration + +###### Credentials Configuration + ClickHouse credentials are also specified in `flow-aggregator.yml` as a Secret -named `clickhouse-secret` as shown below. Please also make the corresponding changes. +named `clickhouse-secret` as shown below. Please also make the corresponding +changes. ```yaml apiVersion: v1 @@ -210,30 +294,37 @@ stringData: type: Opaque ``` -Grafana login credentials are specified in `flow-visibility.yml` as a Secret named -`grafana-secret`. +If the username `clickhouse_operator` has been changed in `flow-visibility.yml`, +please update the following section accordingly. ```yaml -apiVersion: v1 -kind: Secret +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" metadata: - name: grafana-secret - namespace: flow-visibility -stringData: - admin-password: admin - admin-username: admin -type: Opaque + name: clickhouse + labels: + app: clickhouse +spec: + configuration: + users: + # replace clickhouse_operator by [new_username] + clickhouse_operator/k8s_secret_password: flow-visibility/clickhouse-secret/password + clickhouse_operator/networks/ip: "::/0" ``` -We recommend changing all the credentials above if you are going to run the Flow -Collector in production. +Please refer to [the section above](#with-helm) on how to make corresponding +changes of the Clickhouse credentials in the Flow Aggregator. + +We recommend changing the credentials if you are going to run the Flow Collector +in production. -#### ClickHouse Configuration +###### Service Customization -The ClickHouse database can be accessed through the Service `clickhouse-clickhouse`. -The Pod exposes HTTP port at 8123 and TCP port at 9000 by default. The ports are -specified in `flow-visibility.yml` as `serviceTemplates` of a `ClickHouseInstallation` -resource. To use other ports, please update the following section. +The ClickHouse database is exposed by a ClusterIP Service by default in +`flow-visibility.yml`. The Pod exposes HTTP port at 8123 and TCP port at 9000 +by default. The Service type and ports are specified in `flow-visibility.yml` +as `serviceTemplates` of a `ClickHouseInstallation` resource. To use other +Service type and ports, please update the following section. ```yaml serviceTemplates: @@ -244,6 +335,7 @@ serviceTemplates: port: 8123 - name: tcp port: 9000 + type: ClusterIP ``` This Service is used by the Flow Aggregator and Grafana. @@ -252,13 +344,13 @@ This Service is used by the Flow Aggregator and Grafana. `grafana-datasource-provider` in `flow-visibility.yml`. - If you have changed the TCP port, please update the `databaseURL` following -[Flow Aggregator Configuration](https://github.com/antrea-io/antrea/blob/main/docs/network-flow-visibility.md#configuration-1), -and also update the `jsonData.port` of the `grafana-datasource-provider` ConfigMap. +[Flow Aggregator Configuration](#configuration-1), and also update the `jsonData.port` +of the `grafana-datasource-provider` ConfigMap. ```yaml apiVersion: v1 data: - datasource_provider.yml: | + datasource_provider.yaml: |- apiVersion: 1 datasources: - name: ClickHouse @@ -274,10 +366,12 @@ data: password: $CLICKHOUSE_PASSWORD kind: ConfigMap metadata: - name: grafana-datasource-provider-h868k56k95 + name: grafana-datasource-provider namespace: flow-visibility ``` +###### Performance Configuration + The ClickHouse throughput depends on two factors - the storage size of the ClickHouse and the time interval between the batch commits to the ClickHouse. Larger storage size and longer commit interval provide higher throughput. @@ -285,10 +379,12 @@ size and longer commit interval provide higher throughput. Grafana flow collector supports the ClickHouse in-memory deployment with limited storage size. This is specified in `flow-visibility.yml` under the `clickhouse` resource of kind: `ClickHouseInstallation`. The default value of storage size for -the ClickHouse server is 8 GiB. Users can expect a linear growth in the ClickHouse -throughput when they enlarge the storage size. For development or testing environment, -you can decrease the storage size to 2GiB. To deploy the ClickHouse with a different -storage size, please modify the `sizeLimit` in the following section. +the ClickHouse server is 8 GiB. The ClickHouse insertion rate is at around 4,000 +records per seconds with this default storage size. Users can expect a linear +growth in the ClickHouse throughput when they enlarge the storage size. For +development or testing environment, you can decrease the storage size to 2 GiB. +To deploy the ClickHouse with a different storage size, please modify the +`sizeLimit` in the following section. ```yaml - emptyDir: @@ -297,11 +393,160 @@ storage size, please modify the `sizeLimit` in the following section. name: clickhouse-storage-volume ``` +To deploy ClickHouse with Persistent Volumes and limited storage size, please refer +to [Persistent Volumes](#persistent-volumes). + The time interval between the batch commits to the ClickHouse is specified in the [Flow Aggregator Configuration](https://github.com/antrea-io/antrea/blob/main/docs/network-flow-visibility.md#configuration-1) as `commitInterval`. The ClickHouse throughput grows sightly when the commit interval grows from 1s to 8s. A commit interval larger than 8s provides little improvement on the throughput. +##### Persistent Volumes + +By default, ClickHouse is deployed in memory. We support deploying ClickHouse with +Persistent Volumes. + +[PersistentVolume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) +(PV) is a piece of storage in the K8s cluster, which requires to be manually +provisioned by an administrator or dynamically provisioned using Storage Classes. +A PersistentVolumeClaim (PVC) is a request for storage which consumes PV. As +ClickHouse is deployed as a StatefulSet, the volume can be claimed using +`volumeClaimTemplate`. + +Please follow the steps below to deploy the ClickHouse with Persistent Volumes: + +1. Provision the PersistentVolume. K8s supports a great number of +[PersistentVolume types](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#types-of-persistent-volumes). +You can provision your own PersistentVolume per your requirements. Here are +two simple examples for your reference. + + Before creating the PV manually, you need to create a `StorageClass` shown + in the section below. + + ```yaml + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: clickhouse-storage + provisioner: kubernetes.io/no-provisioner + volumeBindingMode: WaitForFirstConsumer + reclaimPolicy: Retain + allowVolumeExpansion: True + ``` + + After the `StorageClass` is created, you can create a Local PV or a NFS PV + by following the steps below. + + - Local PV allows you to store the ClickHouse data at a pre-defined path on + a specific Node. Refer to the section below to create the PV. Please replace + `LOCAL_PATH` with the path to store the ClickHouse data and label the Node + used to store the ClickHouse data with `antrea.io/clickhouse-data-node=`. + + ```yaml + apiVersion: v1 + kind: PersistentVolume + metadata: + name: clickhouse-pv + spec: + storageClassName: clickhouse-storage + capacity: + storage: 8Gi + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + local: + path: LOCAL_PATH + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: antrea.io/clickhouse-data-node + operator: Exists + ``` + + - NFS PV allows you to store the ClickHouse data on an existing NFS server. + Refer to the section below to create the PV. Please replace `NFS_SERVER_ADDRESS` + with the host name of the NFS server and `NFS_SERVER_PATH` with the exported + path on the NFS server. + + ```yaml + apiVersion: v1 + kind: PersistentVolume + metadata: + name: clickhouse-pv + spec: + storageClassName: clickhouse-storage + capacity: + storage: 8Gi + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + nfs: + path: NFS_SERVER_PATH + server: NFS_SERVER_ADDRESS + ``` + + In both examples, you can set `.spec.capacity.storage` in PersistentVolume + to your storage size. This value is for informative purpose as K8s does not + enforce the capacity of PVs. If you want to limit the storage usage, you need + to ask for your storage system to enforce that. For example, you can create + a Local PV on a partition with the limited size. We recommend using a dedicated + saving space for the ClickHouse if you are going to run the Flow Collector in + production. + + As these examples do not use any dynamic provisioner, the reclaim policy + for the PVs is `Retain` by default. After stopping the Grafana Flow Collector, + if you no long need the data for future use, you may need to manually clean + up the data on the local disk or NFS server. + +1. Request the PV for ClickHouse. Please add a `volumeClaimTemplate` section +under `.spec.templates` for the resource `ClickHouseInstallation` in +`flow-visibility.yml` as shown in the example below. `storageClassName` should +be set to your own `StorageClass` name, and `.resources.requests.storage` +should be set to your storage size. + + ```yaml + volumeClaimTemplates: + - name: clickhouse-storage-template + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + storageClassName: clickhouse-storage + ``` + + Then add this template as `dataVolumeClaimTemplate` to the section below. + + ```yaml + defaults: + templates: + dataVolumeClaimTemplate: clickhouse-storage-template + podTemplate: pod-template + serviceTemplate: service-template + ``` + +1. Remove the in-memory related deployment options, by removing the appropriate +`volume` and `volumeMount` for the `ClickHouseInstallation` resource in +`flow-visibility.yml`. + + The `volumeMounts` entry to be removed is the following one: + + ```yaml + - mountPath: /var/lib/clickhouse + name: clickhouse-storage-volume + ``` + + The `volumes` entry to be removed is the following one: + + ```yaml + - emptyDir: + medium: Memory + sizeLimit: 8Gi + name: clickhouse-storage-volume + ``` + ### Pre-built Dashboards The following dashboards are pre-built and are recommended for Antrea flow @@ -438,26 +683,31 @@ are two ways to import the dashboard depending on your needs: like our pre-built dashboards, generate a deployment manifest with the changes by following the steps below: -1. Clone the repository. Exported dashboard JSON files should be placed under `antrea/build/yamls/base/provisioning/dashboards`. -1. If a new dashboard is added, edit [kustomization.yml][flow_visibility_kustomization_yaml] -by adding the file in the following section: +1. Clone the repository. Exported dashboard JSON files should be placed under +`theia/build/charts/theia/provisioning/dashboards`. + +1. If a new dashboard is added, add the file to `grafana.dashboard` in `values.yaml`. +You can also remove a dashboard by deleting the file in this section. ```yaml - - name: grafana-dashboard-config - files: - - provisioning/dashboards/flow_records_dashboard.json - - provisioning/dashboards/pod_to_pod_dashboard.json - - provisioning/dashboards/pod_to_service_dashboard.json - - provisioning/dashboards/pod_to_external_dashboard.json - - provisioning/dashboards/node_to_node_dashboard.json - - provisioning/dashboards/networkpolicy_dashboard.json - - provisioning/dashboards/[new_dashboard_name].json + dashboards: + - flow_records_dashboard.json + - pod_to_pod_dashboard.json + - pod_to_service_dashboard.json + - pod_to_external_dashboard.json + - node_to_node_dashboard.json + - networkpolicy_dashboard.json + - [new_dashboard_name].json ``` -1. Generate the new YAML manifest by running: +1. Deploy the Grafana Flow Collector with Helm by running -```bash -./hack/generate-manifest.sh > build/yamls/flow-visibility.yml -``` + ```bash + helm install -f theia build/charts/theia -n flow-visibility --create-namespace + ``` -[flow_visibility_kustomization_yaml]: ../build/yamls/base/kustomization.yml + Or generate the new YAML manifest by running: + + ```bash + ./hack/generate-manifest.sh > build/yamls/flow-visibility.yml + ``` diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh index 48662c95..d1a241c2 100755 --- a/hack/generate-manifest.sh +++ b/hack/generate-manifest.sh @@ -21,15 +21,14 @@ function echoerr { } _usage="Usage: $0 [--mode (dev|release)] [--keep] [--help|-h] -Generate a YAML manifest for the Clickhouse-Grafana Flow-visibility Solution, using Kustomize, and -print it to stdout. +Generate a YAML manifest for the Clickhouse-Grafana Flow-visibility Solution, using Helm and +Kustomize, and print it to stdout. --mode (dev|release) Choose the configuration variant that you need (default is 'dev') - --keep Debug flag which will preserve the generated kustomization.yml - -This tool uses kustomize (https://github.com/kubernetes-sigs/kustomize) to generate manifests for -Clickhouse-Grafana Flow-visibility Solution. You can set the KUSTOMIZE environment variable to the -path of the kustomize binary you want us to use. Otherwise we will look for kustomize in your PATH -and your GOPATH. If we cannot find kustomize there, we will try to install it." +This tool uses Helm 3 (https://helm.sh/) and Kustomize (https://github.com/kubernetes-sigs/kustomize) +to generate manifests for Antrea. You can set the HELM and KUSTOMIZE environment variable to +the path of the helm and kustomize binary you want us to use. Otherwise we will download the +appropriate version of the helm and kustomize binary and use it (this is the recommended +approach since different versions of helm and kustomize may create different output YAMLs)." function print_usage { echoerr "$_usage" @@ -40,7 +39,6 @@ function print_help { } MODE="dev" -KEEP=false while [[ $# -gt 0 ]] do @@ -51,10 +49,6 @@ case $key in MODE="$2" shift 2 ;; - --keep) - KEEP=true - shift - ;; -h|--help) print_usage exit 0 @@ -86,6 +80,16 @@ fi THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +source $THIS_DIR/verify-helm.sh + +if [ -z "$HELM" ]; then + HELM="$(verify_helm)" +elif ! $HELM version > /dev/null 2>&1; then + echoerr "$HELM does not appear to be a valid helm binary" + print_help + exit 1 +fi + source $THIS_DIR/verify-kustomize.sh if [ -z "$KUSTOMIZE" ]; then @@ -96,36 +100,32 @@ elif ! $KUSTOMIZE version > /dev/null 2>&1; then exit 1 fi -KUSTOMIZATION_DIR=$THIS_DIR/../build/yamls - -TMP_DIR=$(mktemp -d $KUSTOMIZATION_DIR/overlays.XXXXXXXX) - -pushd $TMP_DIR > /dev/null - -BASE=../../base - -mkdir $MODE && cd $MODE -touch kustomization.yml -$KUSTOMIZE edit add base $BASE -# ../../patches/$MODE may be empty so we use find and not simply cp -find ../../patches/$MODE -name \*.yml -exec cp {} . \; - -if [ "$MODE" == "dev" ]; then - $KUSTOMIZE edit set image clickhouse-monitor=projects.registry.vmware.com/antrea/theia-clickhouse-monitor:latest - $KUSTOMIZE edit add patch --path imagePullPolicy.yml --group clickhouse.altinity.com --version v1 --kind ClickHouseInstallation --name clickhouse -fi +HELM_TMP_DIR=$(mktemp -d $THIS_DIR/../build/yamls/chart-values.XXXXXXXX) +EXTRA_VALUES="" if [ "$MODE" == "release" ]; then - $KUSTOMIZE edit set image clickhouse-monitor=$IMG_NAME:$IMG_TAG + EXTRA_VALUES="--set clickhouse.monitorImage.repository=$IMG_NAME,clickhouse.monitorImage.tag=$IMG_TAG" fi +THEIA_CHART=$THIS_DIR/../build/charts/theia +KUSTOMIZATION_DIR=$THIS_DIR/../build/yamls +# intermediate manifest +MANIFEST=$KUSTOMIZATION_DIR/base/manifest.yaml +# Suppress potential Helm warnings about invalid permissions for Kubeconfig file +# by throwing away related warnings. +$HELM template \ + --namespace flow-visibility \ + $EXTRA_VALUES \ + "$THEIA_CHART"\ + 2> >(grep -v 'This is insecure' >&2)\ + > $MANIFEST + +# Add flow-visibility Namespace resource by Kustomize +KUSTOMIZE_TMP_DIR=$(mktemp -d $KUSTOMIZATION_DIR/overlays.XXXXXXXX) +cd $KUSTOMIZATION_DIR/base $KUSTOMIZE build -popd > /dev/null - - -if $KEEP; then - echoerr "Kustomization file is at $TMP_DIR/$MODE/kustomization.yml" -else - rm -rf $TMP_DIR -fi +# clean +rm -rf $MANIFEST +rm -rf $HELM_TMP_DIR +rm -rf $KUSTOMIZE_TMP_DIR diff --git a/hack/verify-helm.sh b/hack/verify-helm.sh new file mode 100644 index 00000000..40f2929b --- /dev/null +++ b/hack/verify-helm.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +# Copyright 2022 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +_BINDIR="$THIS_DIR/.bin" +# Must be an exact match, as the generated YAMLs may not be consistent across +# versions +_HELM_VERSION="v3.8.1" + +# Ensure the helm tool exists and is the correct version, or install it +verify_helm() { + # Check if there is already a helm binary in $_BINDIR and if yes, check if + # the version matches the expected one. + local helm="$(PATH=$_BINDIR command -v helm)" + if [ -x "$helm" ]; then + # Verify version if helm was already installed. + local helm_version="$($helm version --short 2> >(grep -v 'This is insecure' >&2))" + # Should work with: + # - v3.8.1 + # - v3.8.1+g5cb9af4 + helm_version="${helm_version%+*}" + if [ "${helm_version}" == "${_HELM_VERSION}" ]; then + # If version is exact match, stop here. + echo "$helm" + return 0 + fi + >&2 echo "Detected helm version ($helm_version) does not match expected one ($_HELM_VERSION), installing correct version" + fi + local ostype="" + if [[ "$OSTYPE" == "linux-gnu" ]]; then + ostype="linux" + elif [[ "$OSTYPE" == "darwin"* ]]; then + ostype="darwin" + else + >&2 echo "Unsupported OS type $OSTYPE" + return 1 + fi + rc=0 + local unameArch="$(uname -m)" || rc=$? + if [ $rc -ne 0 ]; then + >&2 echo "Cannot detect architecture type, uname not available?" + return 1 + fi + local arch="" + case "$unameArch" in + x86_64) arch="amd64";; + arm64) arch="arm64";; + *) >&2 echo "Unsupported architecture type $unameArch"; return 1;; + esac + + >&2 echo "Installing helm" + local helm_url="https://get.helm.sh/helm-${_HELM_VERSION}-${ostype}-${arch}.tar.gz" + curl -sLo helm.tar.gz "${helm_url}" || return 1 + mkdir -p "$_BINDIR" || return 1 + tar -xzf helm.tar.gz -C "$_BINDIR" --strip-components=1 "${ostype}-${arch}/helm" || return 1 + rm -f helm.tar.gz + helm="$_BINDIR/helm" + echo "$helm" + return 0 +} diff --git a/plugins/clickhouse-monitor/main.go b/plugins/clickhouse-monitor/main.go index cc1f6c61..a6115603 100644 --- a/plugins/clickhouse-monitor/main.go +++ b/plugins/clickhouse-monitor/main.go @@ -18,21 +18,17 @@ import ( "database/sql" "fmt" "os" + "strconv" "strings" "time" "github.com/ClickHouse/clickhouse-go" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) const ( - // The storage percentage at which the monitor starts to delete old records. By default, if the storage usage is larger than 50%, it starts to delete the old records. - threshold = 0.5 - // The percentage of records in ClickHouse that will be deleted when the storage grows above threshold. - deletePercentage = 0.5 - // The monitor stops for 3 intervals after a deletion to wait for the ClickHouse MergeTree Engine to release memory. - skipRoundsNum = 3 // Connection to ClickHouse times out if it fails for 1 minute. connTimeout = time.Minute // Retry connection to ClickHouse every 10 seconds if it fails. @@ -43,23 +39,65 @@ const ( queryRetryInterval = 1 * time.Second // Time format for timeInserted timeFormat = "2006-01-02 15:04:05" - // The monitor runs every minute. - monitorExecInterval = 1 * time.Minute ) var ( + // Storage size allocated for the ClickHouse in number of bytes + allocatedSpace uint64 // The name of the table to store the flow records tableName = os.Getenv("TABLE_NAME") // The names of the materialized views mvNames = strings.Split(os.Getenv("MV_NAMES"), " ") // The remaining number of rounds to be skipped remainingRoundsNum = 0 + // The storage percentage at which the monitor starts to delete old records. + threshold float64 + // The percentage of records in ClickHouse that will be deleted when the storage grows above threshold. + deletePercentage float64 + // The number of rounds for the monitor to stop after a deletion to wait for the ClickHouse MergeTree Engine to release memory. + skipRoundsNum int + // The time interval between two round of monitoring. + monitorExecInterval time.Duration ) func main() { // Check environment variables - if len(tableName) == 0 || len(mvNames) == 0 { - klog.ErrorS(nil, "Unable to load environment variables, TABLE_NAME and MV_NAMES must be defined") + allocatedSpaceStr := os.Getenv("STORAGE_SIZE") + thresholdStr := os.Getenv("THRESHOLD") + deletePercentageStr := os.Getenv("DELETE_PERCENTAGE") + skipRoundsNumStr := os.Getenv("SKIP_ROUNDS_NUM") + monitorExecIntervalStr := os.Getenv("EXEC_INTERVAL") + + if len(tableName) == 0 || len(mvNames) == 0 || len(allocatedSpaceStr) == 0 || len(thresholdStr) == 0 || len(deletePercentageStr) == 0 || len(skipRoundsNumStr) == 0 || len(monitorExecIntervalStr) == 0 { + klog.ErrorS(nil, "Unable to load environment variables, TABLE_NAME, MV_NAMES, STORAGE_SIZE, THRESHOLD, DELETE_PERCENTAGE, SKIP_ROUNDS_NUM, and EXEC_INTERVAL must be defined") + return + } + var err error + quantity, err := resource.ParseQuantity(allocatedSpaceStr) + if err != nil { + klog.ErrorS(err, "Error when parsing STORAGE_SIZE") + return + } + allocatedSpace = uint64(quantity.Value()) + + threshold, err = strconv.ParseFloat(thresholdStr, 64) + if err != nil { + klog.ErrorS(err, "Error when parsing THRESHOLD") + return + } + deletePercentage, err = strconv.ParseFloat(deletePercentageStr, 64) + if err != nil { + klog.ErrorS(err, "Error when parsing DELETE_PERCENTAGE") + return + } + skipRoundsNum, err = strconv.Atoi(skipRoundsNumStr) + if err != nil { + klog.ErrorS(err, "Error when parsing SKIP_ROUNDS_NUM") + return + } + monitorExecInterval, err = time.ParseDuration(monitorExecIntervalStr) + if err != nil { + klog.ErrorS(err, "Error when parsing EXEC_INTERVAL") return } @@ -68,6 +106,7 @@ func main() { klog.ErrorS(err, "Error when connecting to ClickHouse") os.Exit(1) } + checkStorageCondition(connect) wait.Forever(func() { // The monitor stops working for several rounds after a deletion // as the release of memory space by the ClickHouse MergeTree engine requires time @@ -118,28 +157,69 @@ func connectLoop() (*sql.DB, error) { return connect, nil } -// Checks the memory usage in the ClickHouse, and deletes records when it exceeds the threshold. -func monitorMemory(connect *sql.DB) { +// Check if ClickHouse shares storage space with other software +func checkStorageCondition(connect *sql.DB) { var ( freeSpace uint64 + usedSpace uint64 totalSpace uint64 ) - // Get memory usage from ClickHouse system table + getDiskUsage(connect, &freeSpace, &totalSpace) + getClickHouseUsage(connect, &usedSpace) + availablePercentage := float64(freeSpace+usedSpace) / float64(totalSpace) + klog.InfoS("Low available percentage implies ClickHouse does not save data on a dedicated disk", "availablePercentage", availablePercentage) +} + +func getDiskUsage(connect *sql.DB, freeSpace *uint64, totalSpace *uint64) { + // Get free space from ClickHouse system table + if err := wait.PollImmediate(queryRetryInterval, queryTimeout, func() (bool, error) { + if err := connect.QueryRow("SELECT free_space, total_space FROM system.disks").Scan(freeSpace, totalSpace); err != nil { + klog.ErrorS(err, "Failed to get the disk usage") + return false, nil + } else { + return true, nil + } + }); err != nil { + klog.ErrorS(err, "Failed to get the disk usage", "timeout", queryTimeout) + return + } +} + +func getClickHouseUsage(connect *sql.DB, usedSpace *uint64) { + // Get space usage from ClickHouse system table if err := wait.PollImmediate(queryRetryInterval, queryTimeout, func() (bool, error) { - if err := connect.QueryRow("SELECT free_space, total_space FROM system.disks").Scan(&freeSpace, &totalSpace); err != nil { - klog.ErrorS(err, "Failed to get memory usage for ClickHouse") + if err := connect.QueryRow("SELECT SUM(bytes) FROM system.parts").Scan(usedSpace); err != nil { + klog.ErrorS(err, "Failed to get the used space size by the ClickHouse") return false, nil } else { return true, nil } }); err != nil { - klog.ErrorS(err, "Failed to get memory usage for ClickHouse", "timeout", queryTimeout) + klog.ErrorS(err, "Failed to get the used space size by the ClickHouse", "timeout", queryTimeout) return } +} + +// Checks the memory usage in the ClickHouse, and deletes records when it exceeds the threshold. +func monitorMemory(connect *sql.DB) { + var ( + freeSpace uint64 + usedSpace uint64 + totalSpace uint64 + ) + getDiskUsage(connect, &freeSpace, &totalSpace) + getClickHouseUsage(connect, &usedSpace) + + // Total space for ClickHouse is the smaller one of the user allocated space size and the actual space size on the disk + if (freeSpace + usedSpace) < allocatedSpace { + totalSpace = freeSpace + usedSpace + } else { + totalSpace = allocatedSpace + } // Calculate the memory usage - usagePercentage := float64(totalSpace-freeSpace) / float64(totalSpace) - klog.InfoS("Memory usage", "total", totalSpace, "used", totalSpace-freeSpace, "percentage", usagePercentage) + usagePercentage := float64(usedSpace) / float64(totalSpace) + klog.InfoS("Memory usage", "total", totalSpace, "used", usedSpace, "percentage", usagePercentage) // Delete records when memory usage is larger than threshold if usagePercentage > threshold { timeBoundary, err := getTimeBoundary(connect) @@ -169,7 +249,7 @@ func getTimeBoundary(connect *sql.DB) (time.Time, error) { if err != nil { return timeBoundary, err } - command := fmt.Sprintf("SELECT timeInserted FROM %s LIMIT 1 OFFSET %d", tableName, deleteRowNum) + command := fmt.Sprintf("SELECT timeInserted FROM %s LIMIT 1 OFFSET %d", tableName, deleteRowNum-1) if err := wait.PollImmediate(queryRetryInterval, queryTimeout, func() (bool, error) { if err := connect.QueryRow(command).Scan(&timeBoundary); err != nil { klog.ErrorS(err, "Failed to get timeInserted boundary", "table name", tableName) diff --git a/plugins/clickhouse-monitor/main_test.go b/plugins/clickhouse-monitor/main_test.go index 2de36cbf..069770ca 100644 --- a/plugins/clickhouse-monitor/main_test.go +++ b/plugins/clickhouse-monitor/main_test.go @@ -30,6 +30,7 @@ func TestMonitor(t *testing.T) { t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) } defer db.Close() + initEnv() t.Run("testMonitorMemoryWithDeletion", func(t *testing.T) { testMonitorMemoryWithDeletion(t, db, mock) @@ -42,21 +43,31 @@ func TestMonitor(t *testing.T) { }) } +func initEnv() { + tableName = "flows" + mvNames = []string{"flows_pod_view", "flows_node_view", "flows_policy_view"} + allocatedSpace = 10 + threshold = 0.5 + deletePercentage = 0.5 + skipRoundsNum = 3 + monitorExecInterval = 1 * time.Minute +} + func testMonitorMemoryWithDeletion(t *testing.T, db *sql.DB, mock sqlmock.Sqlmock) { baseTime := time.Now() diskRow := sqlmock.NewRows([]string{"free_space", "total_space"}).AddRow(4, 10) + partsRow := sqlmock.NewRows([]string{"SUM(bytes)"}).AddRow(5) countRow := sqlmock.NewRows([]string{"count"}).AddRow(10) timeRow := sqlmock.NewRows([]string{"timeInserted"}).AddRow(baseTime.Add(5 * time.Second)) mock.ExpectQuery("SELECT free_space, total_space FROM system.disks").WillReturnRows(diskRow) + mock.ExpectQuery("SELECT SUM(bytes) FROM system.parts").WillReturnRows(partsRow) mock.ExpectQuery("SELECT COUNT() FROM flows").WillReturnRows(countRow) - mock.ExpectQuery("SELECT timeInserted FROM flows LIMIT 1 OFFSET 5").WillReturnRows(timeRow) + mock.ExpectQuery("SELECT timeInserted FROM flows LIMIT 1 OFFSET 4").WillReturnRows(timeRow) for _, table := range []string{"flows", "flows_pod_view", "flows_node_view", "flows_policy_view"} { command := fmt.Sprintf("ALTER TABLE %s DELETE WHERE timeInserted < toDateTime('%v')", table, baseTime.Add(5*time.Second).Format(timeFormat)) mock.ExpectExec(command).WillReturnResult(sqlmock.NewResult(0, 5)) } - tableName = "flows" - mvNames = []string{"flows_pod_view", "flows_node_view", "flows_policy_view"} monitorMemory(db) if err := mock.ExpectationsWereMet(); err != nil { @@ -67,7 +78,9 @@ func testMonitorMemoryWithDeletion(t *testing.T, db *sql.DB, mock sqlmock.Sqlmoc func testMonitorMemoryWithoutDeletion(t *testing.T, db *sql.DB, mock sqlmock.Sqlmock) { diskRow := sqlmock.NewRows([]string{"free_space", "total_space"}).AddRow(6, 10) + partsRow := sqlmock.NewRows([]string{"SUM(bytes)"}).AddRow(5) mock.ExpectQuery("SELECT free_space, total_space FROM system.disks").WillReturnRows(diskRow) + mock.ExpectQuery("SELECT SUM(bytes) FROM system.parts").WillReturnRows(partsRow) monitorMemory(db) @@ -80,7 +93,6 @@ func testGetDeleteRowNum(t *testing.T, db *sql.DB, mock sqlmock.Sqlmock) { countRow := sqlmock.NewRows([]string{"count"}).AddRow(10) mock.ExpectQuery("SELECT COUNT() FROM flows").WillReturnRows(countRow) - tableName = "flows" deleteRowNumber, err := getDeleteRowNum(db) assert.Equalf(t, uint64(5), deleteRowNumber, "Got deleteRowNumber %d, expect %d", deleteRowNumber, 5) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 2b3495f3..78750835 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -76,7 +76,7 @@ const ( flowAggregatorDeployment string = "flow-aggregator" flowAggregatorYML string = "flow-aggregator.yml" flowVisibilityYML string = "flow-visibility.yml" - chOperatorYML string = "clickhouse-operator-install-bundle.yml" + chOperatorYML string = "clickhouse-operator-install-bundle.yaml" flowVisibilityCHPodName string = "chi-clickhouse-clickhouse-0-0-0" agnhostImage = "k8s.gcr.io/e2e-test-images/agnhost:2.29"