Add support for securing the operator metrics endpoint with RBAC and TLS

cmd/manager/main.go

@@ -286,7 +286,12 @@ func Command() *cobra.Command {
 	cmd.Flags().Int(
 		operator.MetricsPortFlag,
 		DefaultMetricPort,
-		"Port to use for exposing metrics in the Prometheus format (set 0 to disable)",
+		"(Deprecated) Port to use for exposing metrics in the Prometheus format. (set 0 to disable. Use --metrics-bind-address instead)",
+	)
+	cmd.Flags().String(
+		operator.MetricsBindAddressFlag,
+		"",
+		fmt.Sprintf("The address which the operator should listen on to serve metrics in the Prometheus format. Cannot be combined with %s. (set to empty to disable)", operator.MetricsPortFlag),
 	)
 	cmd.Flags().StringSlice(
 		operator.NamespacesFlag,

deploy/eck-operator/templates/auth-proxy-service.yaml

@@ -0,0 +1,27 @@
+{{- if .Values.config.enableSecureMetrics }}
+{{ $metricsPort := 0 }}
+{{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
+{{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}
+{{- else if (gt (int .Values.config.metricsPort) 0) }}
+{{ $metricsPort = int .Values.config.metricsPort }}
+{{- end }}
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: {{ include "eck-operator.name" . }}-metrics-service
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    helm.sh/chart: {{ include "eck-operator.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  name: "{{ include "eck-operator.fullname" . }}-metrics-service"
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+  - name: https
+    port: {{ $metricsPort }}
+    protocol: TCP
+    targetPort: metrics
+  selector:
+    {{- include "eck-operator.selectorLabels" . | nindent 4 }}
+{{- end }}

deploy/eck-operator/templates/cluster-roles.yaml

@@ -1,3 +1,6 @@
+{{- if and (not .Values.createClusterScopedResources) (.Values.config.enableSecureMetrics) -}}
+{{ fail "createClusterScopedResources is required to enable secure metrics" }}
+{{- end }}
 {{- if .Values.createClusterScopedResources -}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -93,4 +96,26 @@ rules:
   - apiGroups: ["logstash.k8s.elastic.co"]
     resources: ["logstashes"]
     verbs: ["create", "delete", "deletecollection", "patch", "update"]
+{{- if and .Values.config.enableSecureMetrics }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    {{- include "eck-operator.labels" . | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+{{- end }}
 {{- end -}}

deploy/eck-operator/templates/configmap.yaml

@@ -9,7 +9,22 @@ metadata:
 data:
   eck.yaml: |-
     log-verbosity: {{ int .Values.config.logVerbosity }}
-    metrics-port: {{ int .Values.config.metricsPort }}
+    {{- if (and .Values.config.enableSecureMetrics (eq (int .Values.config.metricsPort) 0) (or (eq .Values.config.metricsBindAddress "") (not (hasKey .Values.config "metricsBindAddress")))) }}
+    {{ fail "metricsPort or metricsBindAddress is required when enableSecureMetrics is true" }}
+    {{- end }}
+    {{- if (and .Values.config.enableSecureMetrics (gt (int .Values.config.metricsPort) 0) (or (eq .Values.config.metricsBindAddress "") (not (hasKey .Values.config "metricsBindAddress")))) }}
+    metrics-port: {{ add (int .Values.config.metricsPort) 1 }}
+    {{- else }}
+    metrics-port: {{ int (.Values.config.metricsBindAddress | splitList ":" | last) }}
+    {{- end }}
+    {{- if (and .Values.config.enableSecureMetrics (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress ""))) }}
+    {{- if .Values.config.enableSecureMetrics }}
+    {{ $host := .Values.config.metricsBindAddress | splitList ":" | first }}
+    metrics-bind-address: {{ $host }}:{{ add (int (.Values.config.metricsBindAddress | splitList ":" | last)) 1 }}
+    {{- else }}
+    metrics-bind-address: {{ int .Values.config.metricsBindAddress | splitList ":" | last }}
+    {{- end }}
+    {{- end }} 
     container-registry: {{ .Values.config.containerRegistry }}
     {{- with .Values.config.containerSuffix }}
     container-suffix: {{ . }}

deploy/eck-operator/templates/operator-network-policy.yaml

@@ -1,6 +1,5 @@
 {{- if .Values.softMultiTenancy.enabled -}}
 {{- $kubeAPIServerIP := (required "kubeAPIServerIP is required" .Values.kubeAPIServerIP) -}}
-{{- $metricsPort := int .Values.config.metricsPort -}}
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
@@ -40,7 +39,13 @@ spec:
           podSelector:
             matchLabels:
               common.k8s.elastic.co/type: "elasticsearch"
-{{- if or .Values.webhook.enabled (gt $metricsPort 0) }}
+{{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort ) 0) (and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")))}}
+{{ $metricsPort := 0 }}
+{{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
+{{ $metricsPort = int .Values.config.metricsBindAddress | splitList ":" | last }}
+{{- else if (gt .Values.config.metricsPort 0) }}
+{{ $metricsPort = int .Values.config.metricsPort }}
+{{- end }}
   ingress:
 {{- if .Values.webhook.enabled }}
     - ports:

deploy/eck-operator/templates/podMonitor.yaml

@@ -1,5 +1,7 @@
-{{- $metricsPort := int .Values.config.metricsPort -}}
-{{- if and .Values.podMonitor.enabled (gt $metricsPort 0) }}
+{{- if and .Values.podMonitor.enabled .Values.config.enableSecureMetrics}}
+{{ fail "podMonitor.enabled and enableSecureMetrics are mutually exclusive" }}
+{{- end }}
+{{- if and .Values.podMonitor.enabled (or (gt (int .Values.config.metricsPort) 0) (and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")))) }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
@@ -33,4 +35,4 @@ spec:
       - {{ .Release.Namespace }}
   selector:
     matchLabels: {{- include "eck-operator.selectorLabels" . | nindent 6 }}
-{{- end }}
+{{- end }}

deploy/eck-operator/templates/role-bindings.yaml

@@ -1,6 +1,7 @@
 {{- $operatorNSIsManaged := has .Release.Namespace .Values.managedNamespaces -}}
 {{- $fullName := include "eck-operator.fullname" . -}}
 {{- $svcAccount := include "eck-operator.serviceAccountName" . }}
+{{- $enableSecureMetrics := and .Values.config.enableSecureMetrics -}}
 
 {{- if not .Values.createClusterScopedResources }}
 {{- range .Values.managedNamespaces }}
@@ -77,4 +78,21 @@ subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
+{{- if $enableSecureMetrics }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    {{- include "eck-operator.labels" $ | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-proxy-rolebinding"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
+subjects:
+- kind: ServiceAccount
+  name: {{ $svcAccount }}
+  namespace: {{ $.Release.Namespace }}
+{{- end }}
 {{- end }}

deploy/eck-operator/templates/serviceMonitor.yaml

@@ -0,0 +1,24 @@
+{{- if and .Values.config.enableSecureMetrics }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "eck-operator.fullname" . }}
+  namespace: {{ ternary .Values.serviceMonitor.namespace .Release.Namespace (not (empty .Values.serviceMonitor.namespace)) }}
+  labels: {{- include "eck-operator.labels" . | nindent 4 }}
+spec:
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "eck-operator.name" . }}-metrics-service
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  endpoints:
+  - port: https
+    path: /metrics
+    scheme: https
+    interval: 30s
+    tlsConfig:
+      insecureSkipVerify: true
+    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+{{- end }}

deploy/eck-operator/templates/statefulset.yaml

@@ -1,4 +1,3 @@
-{{- $metricsPort := int .Values.config.metricsPort -}}
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -79,9 +78,15 @@ spec:
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          {{- if or (gt $metricsPort 0) .Values.webhook.enabled }}
+          {{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort) 0) (and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")))}}
+          {{ $metricsPort := 0 }}
+          {{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
+          {{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}
+          {{- else if (gt (int .Values.config.metricsPort) 0) }}
+          {{ $metricsPort = int .Values.config.metricsPort }}
+          {{- end }}
           ports:
-            {{- if (gt $metricsPort 0) }}
+            {{- if and (gt $metricsPort 0) (not .Values.config.enableSecureMetrics) }}
             - containerPort: {{ .Values.config.metricsPort }}
               name: metrics
               protocol: TCP
@@ -104,6 +109,37 @@ spec:
             {{- with .Values.volumeMounts }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
+        {{- if .Values.config.enableSecureMetrics }}
+        {{ $metricsPort := 0 }}
+        {{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
+        {{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}
+        {{- else if (gt (int .Values.config.metricsPort) 0) }}
+        {{ $metricsPort = int .Values.config.metricsPort }}
+        {{- end }}
+        - name: kube-rbac-proxy
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "ALL"
+          image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
+          args:
+          - "--secure-listen-address=0.0.0.0:{{ $metricsPort }}"
+          - "--upstream=http://127.0.0.1:{{ add $metricsPort 1 }}/"
+          - "--logtostderr=true"
+          - "--v=0"
+          ports:
+          - containerPort: {{ $metricsPort }}
+            protocol: TCP
+            name: metrics
+          resources:
+            limits:
+              cpu: 500m
+              memory: 128Mi
+            requests:
+              cpu: 5m
+              memory: 64Mi
+        {{- end }}
       volumes:
         - name: conf
           configMap:

deploy/eck-operator/values.yaml

@@ -159,9 +159,43 @@ config:
   #  number greater than 0: Errors, warnings, information, and debug details.
   logVerbosity: "0"
 
-  # metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting.
+  # metricsBindAddress defines the address which the operator should listen on to serve metrics in the Prometheus format.
+  #
+  # *Note* If this option is combined with the enableSecureMetrics option then only the port is used, the bind address is ignored
+  #        as the operator will only listen on the localhost (127.0.0.1) interface and kube-rbac-proxy will intercept traffic
+  #        to the metrics port.
+  #
+  # Examples follow:
+  # - :8080 (equivalent to 0.0.0.0:8080)
+  # - 0.0.0.0:8080
+  # - localhost:8080
+  metricsBindAddress: ""
+
+  # (Deprecated; Will be removed in v2.14.0) metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting. Use `metricsBindAddress` instead.
   metricsPort: "0"
 
+  # enableSecureMetrics specifies whether to enable RBAC and TLS/HTTPs for the metrics endpoint. (Will be enabled by default in v2.14.0)
+  # * This option requires using a ServiceMonitor to scrape the metrics and as such is mutually exclusive with the podMonitor.enabled option.
+  # * This option also requires using cluster scoped resources (ClusterRole, ClusterRoleBinding) to
+  #   grant access to the /metrics endpoint. (createClusterScopedResources: true is required)
+  #
+  # This option requires the following settings within Prometheus to function:
+  # 1. RBAC settings for the Prometheus instance to access the metrics endpoint.
+  #
+  # - nonResourceURLs:
+  #   - /metrics
+  #   verbs:
+  #   - get
+  #
+  # 2. If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the operator you will need
+  #    the Prometheus Operator configured with the following Helm values:
+  #
+  #   prometheus:
+  #     prometheusSpec:
+  #       serviceMonitorNamespaceSelector: {}
+  #       serviceMonitorSelectorNilUsesHelmValues: false
+  enableSecureMetrics: false
+
   # containerRegistry to use for pulling Elasticsearch and other application container images.
   containerRegistry: docker.elastic.co
 

docs/advanced-topics/advanced-topics.asciidoc

@@ -18,6 +18,7 @@ endif::[]
 - <<{p}-webhook-namespace-selectors>>
 - <<{p}-stack-monitoring>>
 - <<{p}-fips>>
+- <<{p}-secure-metrics>>
 --
 
 include::openshift.asciidoc[leveloffset=+1]
@@ -29,3 +30,4 @@ include::network-policies.asciidoc[leveloffset=+1]
 include::webhook-namespace-selectors.asciidoc[leveloffset=+1]
 include::stack-monitoring.asciidoc[leveloffset=+1]
 include::fips.asciidoc[leveloffset=+1]
+include::secure-metrics.asciidoc[leveloffset=+1]