[kube-prometheus-stack] fixes thanos configs (prometheus-community#3918)

* break: kube-prom-stack thanos-ruler config secrets Signed-off-by: ThameezBo <thameezbo@double.cloud> * fix: linting Signed-off-by: ThameezBo <thameezbo@double.cloud> * fix: update README Signed-off-by: ThameezBo <thameezbo@double.cloud> * fix: linting Signed-off-by: ThameezBo <thameezbo@double.cloud> * fix: comments Signed-off-by: ThameezBo <thameezbo@double.cloud> * Update charts/kube-prometheus-stack/values.yaml Co-authored-by: Gabriel Martinez <19713226+GMartinez-Sisti@users.noreply.github.com> Signed-off-by: thameezb <thameezbo@double.cloud> * fix: cater for prom.promSpec.thanos Signed-off-by: ThameezBo <thameezbo@double.cloud> * fix: linting Signed-off-by: ThameezBo <thameezbo@double.cloud> * fix: cater for all three permutations Signed-off-by: ThameezBo <thameezbo@double.cloud> --------- Signed-off-by: ThameezBo <thameezbo@double.cloud> Signed-off-by: thameezb <thameezbo@double.cloud> Signed-off-by: thameezbo <thameezbo@double.cloud> Co-authored-by: Gabriel Martinez <19713226+GMartinez-Sisti@users.noreply.github.com>
giffgaff · Mar 20, 2024 · dd09a22 · dd09a22
1 parent c139d07
commit dd09a22
Show file tree

Hide file tree

Showing 7 changed files with 198 additions and 34 deletions.
diff --git a/charts/kube-prometheus-stack/Chart.yaml b/charts/kube-prometheus-stack/Chart.yaml
@@ -21,7 +21,7 @@ name: kube-prometheus-stack
 sources:
   - https://github.com/prometheus-community/helm-charts
   - https://github.com/prometheus-operator/kube-prometheus
-version: 51.10.0
+version: 52.0.0
 appVersion: v0.68.0
 kubeVersion: ">=1.19.0-0"
 home: https://github.com/prometheus-operator/kube-prometheus

diff --git a/charts/kube-prometheus-stack/README.md b/charts/kube-prometheus-stack/README.md
@@ -82,6 +82,75 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen
 
 A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions.
 
+### From 51.x to 52.x
+
+This includes the ability to select between using existing secrets or create new secret objects for various thanos config. The defaults have not changed but if you were setting:
+
+- `thanosRuler.thanosRulerSpec.alertmanagersConfig` or
+- `thanosRuler.thanosRulerSpec.objectStorageConfig` or
+- `thanosRuler.thanosRulerSpec.queryConfig` or
+- `prometheus.prometheusSpec.thanos.objectStorageConfig`
+
+you will have to need to set `existingSecret` or `secret` based on your requirement
+
+For instance, the `thanosRuler.thanosRulerSpec.alertmanagersConfig` used to be configured as follow:
+
+```yaml
+thanosRuler:
+  thanosRulerSpec:
+    alertmanagersConfig:
+      alertmanagers:
+        - api_version: v2
+          http_config:
+            basic_auth:
+              username: some_user
+              password: some_pass
+          static_configs:
+            - alertmanager.thanos.io
+          scheme: http
+          timeout: 10s
+```
+
+But it now moved to:
+
+```yaml
+thanosRuler:
+  thanosRulerSpec:
+    alertmanagersConfig:
+      secret:
+        alertmanagers:
+          - api_version: v2
+            http_config:
+              basic_auth:
+                username: some_user
+                password: some_pass
+            static_configs:
+              - alertmanager.thanos.io
+            scheme: http
+            timeout: 10s
+```
+
+or the `thanosRuler.thanosRulerSpec.objectStorageConfig` used to be configured as follow:
+
+```yaml
+thanosRuler:
+  thanosRulerSpec:
+    objectStorageConfig:
+      name: existing-secret-not-created-by-this-chart
+      key: object-storage-configs.yaml
+```
+
+But it now moved to:
+
+```yaml
+thanosRuler:
+  thanosRulerSpec:
+    objectStorageConfig:
+      existingSecret:
+        name: existing-secret-not-created-by-this-chart
+        key: object-storage-configs.yaml
+```
+
 ### From 50.x to 51.x
 
 This version upgrades Prometheus-Operator to v0.68.0, Prometheus to 2.47.0 and Thanos to v0.32.2

diff --git a/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml
@@ -356,7 +356,18 @@ spec:
 {{- if not .Values.prometheus.agentMode }}
 {{- if .Values.prometheus.prometheusSpec.thanos }}
   thanos:
-{{ toYaml .Values.prometheus.prometheusSpec.thanos | indent 4 }}
+{{- with (omit .Values.prometheus.prometheusSpec.thanos "objectStorageConfig")}}
+{{ toYaml . | indent 4 }}
+{{- end }}
+{{- if .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret }}
+    objectStorageConfig:
+      key: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.key }}"
+      name: "{{.Values.prometheus.prometheusSpec.thanos.objectStorageConfig.existingSecret.name }}"
+{{- else if .Values.prometheus.prometheusSpec.thanos.objectStorageConfig.secret}}
+    objectStorageConfig:
+      key: object-storage-configs.yaml
+      name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus
+{{- end }}
 {{- end }}
 {{- if .Values.prometheus.prometheusSpec.disableCompaction }}
   disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }}

diff --git a/charts/kube-prometheus-stack/templates/prometheus/secret.yaml b/charts/kube-prometheus-stack/templates/prometheus/secret.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.prometheus.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus
+  namespace: {{ template "kube-prometheus-stack.namespace" . }}
+  labels:
+    app: {{ template "kube-prometheus-stack.name" . }}-prometheus
+    app.kubernetes.io/component: prometheus
+{{ include "kube-prometheus-stack.labels" . | indent 4 }}
+data:
+  {{- with .Values.prometheus.prometheusSpec.thanos.objectStorageConfig }}
+  {{- if and .secret (not .existingSecret) }}
+  object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }}
+  {{- end }}
+  {{- end }}
+{{- end }}
diff --git a/charts/kube-prometheus-stack/templates/thanos-ruler/ruler.yaml b/charts/kube-prometheus-stack/templates/thanos-ruler/ruler.yaml
@@ -71,7 +71,11 @@ spec:
   alertmanagersUrl:
 {{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersUrl | indent 4 }}
 {{- end }}
-{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }}
+{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret }}
+  alertmanagersConfig:
+    key: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.key }}"
+    name: "{{.Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.existingSecret.name }}"
+{{- else if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig.secret }}
   alertmanagersConfig:
     key: alertmanager-configs.yaml
     name: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
@@ -80,9 +84,14 @@ spec:
   queryEndpoints:
 {{ toYaml .Values.thanosRuler.thanosRulerSpec.queryEndpoints | indent 4 }}
 {{- end }}
-{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig }}
+{{- if .Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret }}
+  queryConfig:
+    key: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.key }}"
+    name: "{{.Values.thanosRuler.thanosRulerSpec.queryConfig.existingSecret.name }}"
+{{- else if .Values.thanosRuler.thanosRulerSpec.queryConfig.secret }}
   queryConfig:
-{{ toYaml .Values.thanosRuler.thanosRulerSpec.queryConfig | indent 4 }}
+    key: query-configs.yaml
+    name: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
 {{- end }}
 {{- if .Values.thanosRuler.thanosRulerSpec.resources }}
   resources:
@@ -99,7 +108,11 @@ spec:
   storage:
 {{ toYaml .Values.thanosRuler.thanosRulerSpec.storage | indent 4 }}
 {{- end }}
-{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }}
+{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret }}
+  objectStorageConfig:
+    key: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.key }}"
+    name: "{{.Values.thanosRuler.thanosRulerSpec.objectStorageConfig.existingSecret.name }}"
+{{- else if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig.secret }}
   objectStorageConfig:
     key: object-storage-configs.yaml
     name: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
@@ -108,9 +121,6 @@ spec:
   labels:
 {{ toYaml .Values.thanosRuler.thanosRulerSpec.labels | indent 4 }}
 {{- end }}
-{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfigFile }}
-  objectStorageConfigFile: {{ .Values.thanosRuler.thanosRulerSpec.objectStorageConfigFile }}
-{{- end }}
 {{- if .Values.thanosRuler.thanosRulerSpec.podMetadata }}
   podMetadata:
 {{ toYaml .Values.thanosRuler.thanosRulerSpec.podMetadata | indent 4 }}

diff --git a/charts/kube-prometheus-stack/templates/thanos-ruler/secret.yaml b/charts/kube-prometheus-stack/templates/thanos-ruler/secret.yaml
@@ -8,10 +8,19 @@ metadata:
     app: {{ include "kube-prometheus-stack.thanosRuler.name" . }}
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 data:
-{{- if .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }}
-  alertmanager-configs.yaml: {{ toYaml .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig | b64enc | quote }}
-{{- end }}
-{{- if .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }}
-  object-storage-configs.yaml: {{ toYaml .Values.thanosRuler.thanosRulerSpec.objectStorageConfig | b64enc | quote }}
-{{- end }}
+  {{- with .Values.thanosRuler.thanosRulerSpec.alertmanagersConfig }}
+  {{- if and .secret (not .existingSecret) }}
+  alertmanager-configs.yaml: {{ toYaml .secret | b64enc | quote }}
+  {{- end }}
+  {{- end }}
+  {{- with .Values.thanosRuler.thanosRulerSpec.objectStorageConfig }}
+  {{- if and .secret (not .existingSecret) }}
+  object-storage-configs.yaml: {{ toYaml .secret | b64enc | quote }}
+  {{- end }}
+  {{- end }}
+  {{- with .Values.thanosRuler.thanosRulerSpec.queryConfig }}
+  {{- if and .secret (not .existingSecret) }}
+  query-configs.yaml: {{ toYaml .secret | b64enc | quote }}
+  {{- end }}
+  {{- end }}
 {{- end }}
diff --git a/charts/kube-prometheus-stack/values.yaml b/charts/kube-prometheus-stack/values.yaml
@@ -3438,7 +3438,23 @@ prometheus:
       #     secrets: |
       #       - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest"
       #         fileName: "objstore.yaml"
-      # objectStorageConfigFile: /var/secrets/object-store.yaml
+      ## ObjectStorageConfig configures object storage in Thanos.
+      # objectStorageConfig:
+      #   # use existing secret, if configured, objectStorageConfig.secret will not be used
+      #   existingSecret: {}
+      #     # name: ""
+      #     # key: ""
+      #   # will render objectStorageConfig secret data and configure it to be used by Thanos custom resource,
+      #   # ignored when prometheusspec.thanos.objectStorageConfig.existingSecret is set
+      #   # https://thanos.io/tip/thanos/storage.md/#s3
+      #   secret: {}
+      #     # type: S3
+      #     # config:
+      #     #   bucket: ""
+      #     #   endpoint: ""
+      #     #   region: ""
+      #     #   access_key: ""
+      #     #   secret_key: ""
 
     ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod.
     ## if using proxy extraContainer update targetPort with proxy container port
@@ -3977,16 +3993,24 @@ thanosRuler:
 
     ## AlertmanagerConfig define configuration for connecting to alertmanager.
     ## Only available with Thanos v0.10.0 and higher. Maps to the alertmanagers.config Thanos Ruler arg.
-    alertmanagersConfig: {}
-    #   - api_version: v2
-    #     http_config:
-    #       basic_auth:
-    #         username: some_user
-    #         password: some_pass
-    #     static_configs:
-    #       - alertmanager.thanos.io
-    #     scheme: http
-    #     timeout: 10s
+    alertmanagersConfig:
+      # use existing secret, if configured, alertmanagersConfig.secret will not be used
+      existingSecret: {}
+        # name: ""
+        # key: ""
+      # will render render alertmanagersConfig secret data and configure it to be used by Thanos Ruler custom resource, ignored when alertmanagersConfig.existingSecret is set
+      # https://thanos.io/tip/components/rule.md/#alertmanager
+      secret: {}
+        # alertmanagers:
+        # - api_version: v2
+        #   http_config:
+        #     basic_auth:
+        #       username: some_user
+        #       password: some_pass
+        #   static_configs:
+        #     - alertmanager.thanos.io
+        #   scheme: http
+        #   timeout: 10s
 
     ## DEPRECATED. Define URLs to send alerts to Alertmanager. For Thanos v0.10.0 and higher, alertmanagersConfig should be used instead.
     ## Note: this field will be ignored if alertmanagersConfig is specified. Maps to the alertmanagers.url Thanos Ruler arg.
@@ -4001,21 +4025,45 @@ thanosRuler:
     ##
     routePrefix: /
 
-    ## ObjectStorageConfig configures object storage in Thanos. Alternative to
-    ## ObjectStorageConfigFile, and lower order priority.
-    objectStorageConfig: {}
-
-    ## ObjectStorageConfigFile specifies the path of the object storage configuration file.
-    ## When used alongside with ObjectStorageConfig, ObjectStorageConfigFile takes precedence.
-    objectStorageConfigFile: ""
+    ## ObjectStorageConfig configures object storage in Thanos
+    objectStorageConfig:
+      # use existing secret, if configured, objectStorageConfig.secret will not be used
+      existingSecret: {}
+        # name: ""
+        # key: ""
+      # will render objectStorageConfig secret data and configure it to be used by Thanos Ruler custom resource, ignored when objectStorageConfig.existingSecret is set
+      # https://thanos.io/tip/thanos/storage.md/#s3
+      secret: {}
+        # type: S3
+        # config:
+        #   bucket: ""
+        #   endpoint: ""
+        #   region: ""
+        #   access_key: ""
+        #   secret_key: ""
 
     ## QueryEndpoints defines Thanos querier endpoints from which to query metrics.
     ## Maps to the --query flag of thanos ruler.
     queryEndpoints: []
 
     ## Define configuration for connecting to thanos query instances. If this is defined, the queryEndpoints field will be ignored.
     ## Maps to the query.config CLI argument. Only available with thanos v0.11.0 and higher.
-    queryConfig: {}
+    queryConfig:
+      # use existing secret, if configured, queryConfig.secret will not be used
+      existingSecret: {}
+        # name: ""
+        # key: ""
+      # render queryConfig secret data and configure it to be used by Thanos Ruler custom resource, ignored when queryConfig.existingSecret is set
+      # https://thanos.io/tip/components/rule.md/#query-api
+      secret: {}
+        # - http_config:
+        #     basic_auth:
+        #       username: some_user
+        #       password: some_pass
+        #   static_configs:
+        #     - URL
+        #   scheme: http
+        #   timeout: 10s
 
     ## Labels configure the external label pairs to ThanosRuler. A default replica
     ## label `thanos_ruler_replica` will be always added as a label with the value