Merge pull request #690 from alexanderldavis/enable_metrics_by_namespace

feat: add new opt-in metric of reloads by namespace
stakater · Jun 21, 2024 · 7f5ea8c · 7f5ea8c
2 parents b73b04d + 6cd458b
commit 7f5ea8c
Show file tree

Hide file tree

Showing 8 changed files with 306 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -338,6 +338,7 @@ helm uninstall {{RELEASE_NAME}} -n {{NAMESPACE}}
 | `reloader.readOnlyRootFileSystem` | Enforce readOnlyRootFilesystem                                                                                                                      | boolean     | `false`   |
 | `reloader.legacy.rbac`            |                                                                                                                                                     | boolean     | `false`   |
 | `reloader.matchLabels`            | Pod labels to match                                                                                                                                 | map         | `{}`      |
+| `reloader.enableMetricsByNamespace`            | Expose an additional Prometheus counter of reloads by namespace (this metric may have high cardinality in clusters with many namespaces) | boolean         | `false`      |
 
 #### Deployment Reloader Parameters
 

diff --git a/deployments/kubernetes/chart/reloader/templates/deployment.yaml b/deployments/kubernetes/chart/reloader/templates/deployment.yaml
@@ -76,7 +76,7 @@ spec:
       - image: "{{ .Values.reloader.deployment.image.name }}:{{ .Values.reloader.deployment.image.tag }}"
         imagePullPolicy: {{ .Values.reloader.deployment.image.pullPolicy }}
         name: {{ template "reloader-fullname" . }}
-      {{- if or (.Values.reloader.deployment.env.open) (.Values.reloader.deployment.env.secret) (.Values.reloader.deployment.env.field) (.Values.reloader.deployment.env.existing) (eq .Values.reloader.watchGlobally false) (.Values.reloader.enableHA)}}
+      {{- if or (.Values.reloader.deployment.env.open) (.Values.reloader.deployment.env.secret) (.Values.reloader.deployment.env.field) (.Values.reloader.deployment.env.existing) (eq .Values.reloader.watchGlobally false) (.Values.reloader.enableHA) (.Values.reloader.enableMetricsByNamespace)}}
         env:
       {{- range $name, $value := .Values.reloader.deployment.env.open }}
       {{- if not (empty $value) }}
@@ -129,6 +129,10 @@ spec:
             fieldRef:
               fieldPath: metadata.namespace
       {{- end }}
+      {{- if .Values.reloader.enableMetricsByNamespace }}
+        - name: METRICS_COUNT_BY_NAMESPACE
+          value: enabled
+      {{- end }}
       {{- end }}
 
         ports:

diff --git a/deployments/kubernetes/chart/reloader/values.yaml b/deployments/kubernetes/chart/reloader/values.yaml
@@ -32,6 +32,8 @@ reloader:
   legacy:
     rbac: false
   matchLabels: {}
+  # Set to true to expose a prometheus counter of reloads by namespace (this metric may have high cardinality in clusters with many namespaces)
+  enableMetricsByNamespace: false
   deployment:
     # If you wish to run multiple replicas set reloader.enableHA = true
     replicas: 1

diff --git a/deployments/kubernetes/templates/chart/values.yaml.tmpl b/deployments/kubernetes/templates/chart/values.yaml.tmpl
@@ -23,6 +23,8 @@ reloader:
   legacy:
     rbac: false
   matchLabels: {}
+  # Set to true to expose a prometheus counter of reloads by namespace (this metric may have high cardinality in clusters with many namespaces)
+  enableMetricsByNamespace: false
   deployment:
     replicas: 1
     nodeSelector:

diff --git a/docs/Verify-Reloader-Working.md b/docs/Verify-Reloader-Working.md
@@ -60,3 +60,16 @@ When Reloader is unable to reload, `reloader_reload_executed_total{success="fals
 reloader_reload_executed_total{success="false"} 15
 reloader_reload_executed_total{success="true"} 12
 ```
+
+### Reloads by Namespace
+
+Reloader can also export a metric to show the number of reloads by namespace. This feature is disabled by default, as it can lead to high cardinality in clusters with many namespaces.
+
+The metric will have both `success` and `namespace` as attributes:
+
+```text
+reloader_reload_executed_total{success="false", namespace="some-namespace"} 2
+reloader_reload_executed_total{success="true", namespace="some-namespace"} 1
+```
+
+To opt in, set the environment variable `METRICS_COUNT_BY_NAMESPACE` to `enabled` or set the Helm value `reloader.enableMetricsByNamespace` to `true`.
diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go
@@ -245,6 +245,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 				message := fmt.Sprintf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err)
 				logrus.Errorf(message)
 				collectors.Reloaded.With(prometheus.Labels{"success": "false"}).Inc()
+				collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "false", "namespace": config.Namespace}).Inc()
 				if recorder != nil {
 					recorder.Event(i, v1.EventTypeWarning, "ReloadFail", message)
 				}
@@ -254,6 +255,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 				message += fmt.Sprintf(", Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace)
 				logrus.Infof(message)
 				collectors.Reloaded.With(prometheus.Labels{"success": "true"}).Inc()
+				collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": config.Namespace}).Inc()
 				alert_on_reload, ok := os.LookupEnv("ALERT_ON_RELOAD")
 				if recorder != nil {
 					recorder.Event(i, v1.EventTypeNormal, "Reloaded", message)