From c7dbb50cbd259006280bf37f13780725e177b10a Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Wed, 14 Feb 2024 14:36:23 -0600
Subject: [PATCH 01/29] WIP

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../templates/auth-proxy-service.yaml         | 17 ++++++
 .../eck-operator/templates/cluster-roles.yaml | 34 +++++++++++
 .../eck-operator/templates/role-bindings.yaml | 57 +++++++++++++++++++
 .../eck-operator/templates/statefulset.yaml   | 27 ++++++++-
 deploy/eck-operator/values.yaml               |  3 +
 5 files changed, 137 insertions(+), 1 deletion(-)
 create mode 100644 deploy/eck-operator/templates/auth-proxy-service.yaml

diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml
new file mode 100644
index 0000000000..77a8feb07b
--- /dev/null
+++ b/deploy/eck-operator/templates/auth-proxy-service.yaml
@@ -0,0 +1,17 @@
+{{- if and .Values.config.enableMetricsAuth (gt .Values.config.metricsPort 0) }}
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- include "eck-operator.labels" . | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-metrics-service"
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+  - name: https
+    port: {{ .Values.config.metricsPort }}
+    protocol: TCP
+    targetPort: metrics
+  selector:
+    {{- "eck-operator.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/deploy/eck-operator/templates/cluster-roles.yaml b/deploy/eck-operator/templates/cluster-roles.yaml
index 1b623f37fe..009b39fe7f 100644
--- a/deploy/eck-operator/templates/cluster-roles.yaml
+++ b/deploy/eck-operator/templates/cluster-roles.yaml
@@ -93,4 +93,38 @@ rules:
   - apiGroups: ["logstash.k8s.elastic.co"]
     resources: ["logstashes"]
     verbs: ["create", "delete", "deletecollection", "patch", "update"]
+{{- if and .Values.config.enableMetricsAuth (gt .Values.config.metricsPort 0) }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    {{- include "eck-operator.labels" . | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    {{- include "eck-operator.labels" . | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-metrics-reader"
+rules:
+- nonResourceURLs:
+  - "/metrics"
+  verbs:
+  - get
+{{- end }}
 {{- end -}}
diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml
index ca6cf2ba57..a0d4e44e04 100644
--- a/deploy/eck-operator/templates/role-bindings.yaml
+++ b/deploy/eck-operator/templates/role-bindings.yaml
@@ -31,6 +31,45 @@ subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
+{{- if and .Values.config.enableMetricsAuth (gt .Values.config.metricsPort 0) }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    {{- include "eck-operator.labels" . | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
+  namespace: {{ $namespace }}
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    {{- include "eck-operator.labels" . | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-proxy-rolebinding"
+  namespace: {{ $namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
+subjects:
+- kind: ServiceAccount
+  name: {{ $svcAccount }}
+  namespace: {{ $.Release.Namespace }}
+{{- end }} {{- /* end of if enableMetricsAuth check */}}
 {{- end }} {{- /* end of range over managed namespaces */}}
 {{- /* If createClusterScopedResources is false and operator namespace is not in the managed namespaces list, create additional role binding */}}
 {{- if not $operatorNSIsManaged }}
@@ -74,7 +113,25 @@ roleRef:
   kind: ClusterRole
   name: {{ $fullName }}
 subjects:
+- kind: ServiceAccount
+  name: {{ $svcAccount }}
+  namespace: {{ $.Release.Namespace }}
+{{- if and .Values.config.enableMetricsAuth (gt .Values.config.metricsPort 0) }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    {{- include "eck-operator.labels" $ | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-proxy-rolebinding"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
+subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
 {{- end }}
+{{- end }}
+
diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index e72ae30eed..10bfbb93c6 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -81,7 +81,7 @@ spec:
           {{- end }}
           {{- if or (gt $metricsPort 0) .Values.webhook.enabled }}
           ports:
-            {{- if (gt $metricsPort 0) }}
+            {{- if and (gt $metricsPort 0) (not .Values.config.enableMetricsAuth) }}
             - containerPort: {{ .Values.config.metricsPort }}
               name: metrics
               protocol: TCP
@@ -104,6 +104,31 @@ spec:
             {{- with .Values.volumeMounts }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
+        {{- if (and .Values.config.enableMetricsAuth (gt $metricsPort 0)) }}
+        - name: kube-rbac-proxy
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "ALL"
+          image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0
+          args:
+          - "--secure-listen-address=0.0.0.0:{{ $metricsPort }}"
+          - "--upstream=http://127.0.0.1:{{ add $metricsPort 1 }}/"
+          - "--logtostderr=true"
+          - "--v=0"
+          ports:
+          - containerPort: {{ $metricsPort }}
+            protocol: TCP
+            name: metrics
+          resources:
+            limits:
+              cpu: 500m
+              memory: 128Mi
+            requests:
+              cpu: 5m
+              memory: 64Mi
+        {{- end }}
       volumes:
         - name: conf
           configMap:
diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index 3c8a82116a..0029c07556 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -162,6 +162,9 @@ config:
   # metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting.
   metricsPort: "0"
 
+  # enableMetricsAuth specifies whether to enable basic authentication for the metrics endpoint. (Will be enabled by default in v2.14.0)
+  enableMetricsAuth: false
+
   # containerRegistry to use for pulling Elasticsearch and other application container images.
   containerRegistry: docker.elastic.co
 

From 2420f8dd2bf00545f39f16005dc00b5b22dd6fb7 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Thu, 15 Feb 2024 17:07:54 -0600
Subject: [PATCH 02/29] Final settings enabling authentication for the metrics
 endpoint.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../templates/auth-proxy-service.yaml         | 11 +++++++---
 .../eck-operator/templates/cluster-roles.yaml |  3 ++-
 deploy/eck-operator/templates/configmap.yaml  |  7 ++++++-
 deploy/eck-operator/templates/podMonitor.yaml |  7 +++++--
 .../eck-operator/templates/role-bindings.yaml |  5 +++--
 .../eck-operator/templates/statefulset.yaml   |  4 ++--
 deploy/eck-operator/values.yaml               | 21 ++++++++++++++++++-
 7 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml
index 77a8feb07b..8664d1c45b 100644
--- a/deploy/eck-operator/templates/auth-proxy-service.yaml
+++ b/deploy/eck-operator/templates/auth-proxy-service.yaml
@@ -1,9 +1,14 @@
-{{- if and .Values.config.enableMetricsAuth (gt .Values.config.metricsPort 0) }}
+{{- $metricsPort := int .Values.config.metricsPort -}}
+{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
 apiVersion: v1
 kind: Service
 metadata:
   labels:
-    {{- include "eck-operator.labels" . | nindent 4 }}
+    app.kubernetes.io/name: {{ include "eck-operator.name" . }}-metrics-service
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    helm.sh/chart: {{ include "eck-operator.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
   name: "{{ include "eck-operator.fullname" . }}-metrics-service"
   namespace: {{ .Release.Namespace }}
 spec:
@@ -13,5 +18,5 @@ spec:
     protocol: TCP
     targetPort: metrics
   selector:
-    {{- "eck-operator.selectorLabels" . | nindent 4 }}
+    {{- include "eck-operator.selectorLabels" . | nindent 4 }}
 {{- end }}
diff --git a/deploy/eck-operator/templates/cluster-roles.yaml b/deploy/eck-operator/templates/cluster-roles.yaml
index 009b39fe7f..afa64dfa74 100644
--- a/deploy/eck-operator/templates/cluster-roles.yaml
+++ b/deploy/eck-operator/templates/cluster-roles.yaml
@@ -1,3 +1,4 @@
+{{- $metricsPort := int .Values.config.metricsPort -}}
 {{- if .Values.createClusterScopedResources -}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -93,7 +94,7 @@ rules:
   - apiGroups: ["logstash.k8s.elastic.co"]
     resources: ["logstashes"]
     verbs: ["create", "delete", "deletecollection", "patch", "update"]
-{{- if and .Values.config.enableMetricsAuth (gt .Values.config.metricsPort 0) }}
+{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml
index eec71de366..6484e6454d 100644
--- a/deploy/eck-operator/templates/configmap.yaml
+++ b/deploy/eck-operator/templates/configmap.yaml
@@ -1,3 +1,4 @@
+{{- $metricsPort := int .Values.config.metricsPort -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -9,7 +10,11 @@ metadata:
 data:
   eck.yaml: |-
     log-verbosity: {{ int .Values.config.logVerbosity }}
-    metrics-port: {{ int .Values.config.metricsPort }}
+    {{- if (and .Values.config.enableMetricsAuth (gt $metricsPort 0)) }}
+    metrics-port: {{ add $metricsPort 1 }}
+    {{- else }}
+    metrics-port: {{ $metricsPort }}
+    {{- end }}
     container-registry: {{ .Values.config.containerRegistry }}
     {{- with .Values.config.containerSuffix }}
     container-suffix: {{ . }}
diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml
index c269cb7be7..9d7f379dd8 100644
--- a/deploy/eck-operator/templates/podMonitor.yaml
+++ b/deploy/eck-operator/templates/podMonitor.yaml
@@ -1,5 +1,8 @@
 {{- $metricsPort := int .Values.config.metricsPort -}}
-{{- if and .Values.podMonitor.enabled (gt $metricsPort 0) }}
+{{- if and .Values.podMonitor.enabled .Values.config.enableMetricsAuth}}
+{{ fail "podMonitor.enabled and enableMetricsAuth are mutually exclusive" }}
+{{- end }}
+{{- if and .Values.podMonitor.enabled (gt $metricsPort 0)}}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
@@ -33,4 +36,4 @@ spec:
       - {{ .Release.Namespace }}
   selector:
     matchLabels: {{- include "eck-operator.selectorLabels" . | nindent 6 }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml
index a0d4e44e04..1946e4daef 100644
--- a/deploy/eck-operator/templates/role-bindings.yaml
+++ b/deploy/eck-operator/templates/role-bindings.yaml
@@ -1,6 +1,7 @@
 {{- $operatorNSIsManaged := has .Release.Namespace .Values.managedNamespaces -}}
 {{- $fullName := include "eck-operator.fullname" . -}}
 {{- $svcAccount := include "eck-operator.serviceAccountName" . }}
+{{- $metricsPort := int .Values.config.metricsPort -}}
 
 {{- if not .Values.createClusterScopedResources }}
 {{- range .Values.managedNamespaces }}
@@ -31,7 +32,7 @@ subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
-{{- if and .Values.config.enableMetricsAuth (gt .Values.config.metricsPort 0) }}
+{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -116,7 +117,7 @@ subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
-{{- if and .Values.config.enableMetricsAuth (gt .Values.config.metricsPort 0) }}
+{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index 10bfbb93c6..05003e3ac9 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -111,12 +111,12 @@ spec:
             capabilities:
               drop:
                 - "ALL"
-          image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0
+          image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
           args:
           - "--secure-listen-address=0.0.0.0:{{ $metricsPort }}"
           - "--upstream=http://127.0.0.1:{{ add $metricsPort 1 }}/"
           - "--logtostderr=true"
-          - "--v=0"
+          - "--v=10"
           ports:
           - containerPort: {{ $metricsPort }}
             protocol: TCP
diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index 0029c07556..f1157662e9 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -162,7 +162,26 @@ config:
   # metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting.
   metricsPort: "0"
 
-  # enableMetricsAuth specifies whether to enable basic authentication for the metrics endpoint. (Will be enabled by default in v2.14.0)
+  # enableMetricsAuth specifies whether to enable RBAC authentication for the metrics endpoint. (Will be enabled by default in v2.14.0)
+  # * This option requires using a ServiceMonitor to scrape the metrics and as such is mutually exclusive with the podMonitor.enabled option.
+  # * This option also enable tls/HTTPs metrics by default.
+  #
+  # This option requires the following settings within Prometheus to function:
+  # 1. RBAC settings for the Prometheus instance to access the metrics endpoint.
+  #
+  # - nonResourceURLs:
+  #   - /metrics
+  #   - /metrics/cadvisor
+  #   verbs:
+  #   - get
+  #
+  # 2. If using the Prometheus Operator and your Prometheus instance are not in the same namespace as the operator you will need
+  #    the Prometheus Operator configured with the following Helm values:
+  #
+  #   prometheus:
+  #     prometheusSpec:
+  #       serviceMonitorNamespaceSelector: {}
+  #       serviceMonitorSelectorNilUsesHelmValues: false
   enableMetricsAuth: false
 
   # containerRegistry to use for pulling Elasticsearch and other application container images.

From 60b5b903bf77357e3ea54dd57cdd95cd7341cf68 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Thu, 15 Feb 2024 17:10:52 -0600
Subject: [PATCH 03/29] add servicemonitor template.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../templates/serviceMonitor.yaml             | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 deploy/eck-operator/templates/serviceMonitor.yaml

diff --git a/deploy/eck-operator/templates/serviceMonitor.yaml b/deploy/eck-operator/templates/serviceMonitor.yaml
new file mode 100644
index 0000000000..15b622ef8f
--- /dev/null
+++ b/deploy/eck-operator/templates/serviceMonitor.yaml
@@ -0,0 +1,25 @@
+{{- $metricsPort := int .Values.config.metricsPort -}}
+{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "eck-operator.fullname" . }}
+  namespace: {{ ternary .Values.serviceMonitor.namespace .Release.Namespace (not (empty .Values.serviceMonitor.namespace)) }}
+  labels: {{- include "eck-operator.labels" . | nindent 4 }}
+spec:
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "eck-operator.name" . }}-metrics-service
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  endpoints:
+  - port: https
+    path: /metrics
+    scheme: https
+    interval: 30s
+    tlsConfig:
+      insecureSkipVerify: true
+    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+{{- end }}

From bdf972dc47bd1cc4141236ed4aded248125d0577 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 20 Feb 2024 09:54:07 -0600
Subject: [PATCH 04/29] Move non-cluster related roles to proper location.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../eck-operator/templates/role-bindings.yaml | 67 ++++++++++---------
 1 file changed, 34 insertions(+), 33 deletions(-)

diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml
index 1946e4daef..476b2cebca 100644
--- a/deploy/eck-operator/templates/role-bindings.yaml
+++ b/deploy/eck-operator/templates/role-bindings.yaml
@@ -2,6 +2,7 @@
 {{- $fullName := include "eck-operator.fullname" . -}}
 {{- $svcAccount := include "eck-operator.serviceAccountName" . }}
 {{- $metricsPort := int .Values.config.metricsPort -}}
+{{- $enableMetricsAuth := and .Values.config.enableMetricsAuth (gt $metricsPort 0) -}}
 
 {{- if not .Values.createClusterScopedResources }}
 {{- range .Values.managedNamespaces }}
@@ -32,74 +33,74 @@ subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
-{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
+{{- end }} {{- /* end of range over managed namespaces */}}
+{{- /* If createClusterScopedResources is false and operator namespace is not in the managed namespaces list, create additional role binding */}}
+{{- if not $operatorNSIsManaged }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
+  name: {{ $fullName }}
+  namespace: {{ $.Release.Namespace }}
   labels:
-    {{- include "eck-operator.labels" . | nindent 4 }}
-  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
-  namespace: {{ $namespace }}
+    {{- include "eck-operator.labels" $ | nindent 4 }}
 rules:
-- apiGroups:
-  - authentication.k8s.io
-  resources:
-  - tokenreviews
-  verbs:
-  - create
-- apiGroups:
-  - authorization.k8s.io
-  resources:
-  - subjectaccessreviews
-  verbs:
-  - create
+{{ template "eck-operator.rbacRules" $ | toYaml | indent 2 }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
+  name: "{{ $fullName }}"
+  namespace: {{ $.Release.Namespace }}
   labels:
-    {{- include "eck-operator.labels" . | nindent 4 }}
-  name: "{{ include "eck-operator.fullname" . }}-proxy-rolebinding"
-  namespace: {{ $namespace }}
+    {{- include "eck-operator.labels" $ | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
+  name: "{{ $fullName }}"
 subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
-{{- end }} {{- /* end of if enableMetricsAuth check */}}
-{{- end }} {{- /* end of range over managed namespaces */}}
-{{- /* If createClusterScopedResources is false and operator namespace is not in the managed namespaces list, create additional role binding */}}
-{{- if not $operatorNSIsManaged }}
+{{- if $enableMetricsAuth }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: {{ $fullName }}
-  namespace: {{ $.Release.Namespace }}
   labels:
-    {{- include "eck-operator.labels" $ | nindent 4 }}
+    {{- include "eck-operator.labels" . | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
+  namespace: {{ $.Release.Namespace }}
 rules:
-{{ template "eck-operator.rbacRules" $ | toYaml | indent 2 }}
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: "{{ $fullName }}"
-  namespace: {{ $.Release.Namespace }}
   labels:
-    {{- include "eck-operator.labels" $ | nindent 4 }}
+    {{- include "eck-operator.labels" . | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-proxy-rolebinding"
+  namespace: {{ $.Release.Namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: "{{ $fullName }}"
+  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
 subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
+{{- end }} {{- /* end of if enableMetricsAuth check */}}
 {{- end }} {{- /* end of operator role binding if operator namespace is not managed */}}
 {{- else }} {{- /* we can create cluster-scoped resources so just create a cluster role binding */}}
 ---
@@ -117,7 +118,7 @@ subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
-{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
+{{- if $enableMetricsAuth }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

From 2d9f8859a8f7dbcfbfe848ea1609672484e17f77 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 20 Feb 2024 14:13:51 -0600
Subject: [PATCH 05/29] Remove non-cluster scoped option.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../eck-operator/templates/cluster-roles.yaml |  3 +
 .../eck-operator/templates/role-bindings.yaml | 56 -------------------
 deploy/eck-operator/values.yaml               |  5 +-
 3 files changed, 6 insertions(+), 58 deletions(-)

diff --git a/deploy/eck-operator/templates/cluster-roles.yaml b/deploy/eck-operator/templates/cluster-roles.yaml
index afa64dfa74..8d4f5878bc 100644
--- a/deploy/eck-operator/templates/cluster-roles.yaml
+++ b/deploy/eck-operator/templates/cluster-roles.yaml
@@ -1,4 +1,7 @@
 {{- $metricsPort := int .Values.config.metricsPort -}}
+{{- if and (not .Values.createClusterScopedResources) (.Values.config.enableMetricsAuth) (gt $metricsPort 0) -}}
+{{ fail "createClusterScopedResources is required to enable metrics authentication" }}
+{{- end }}
 {{- if .Values.createClusterScopedResources -}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml
index 476b2cebca..93547e4114 100644
--- a/deploy/eck-operator/templates/role-bindings.yaml
+++ b/deploy/eck-operator/templates/role-bindings.yaml
@@ -62,45 +62,6 @@ subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
-{{- if $enableMetricsAuth }}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  labels:
-    {{- include "eck-operator.labels" . | nindent 4 }}
-  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
-  namespace: {{ $.Release.Namespace }}
-rules:
-- apiGroups:
-  - authentication.k8s.io
-  resources:
-  - tokenreviews
-  verbs:
-  - create
-- apiGroups:
-  - authorization.k8s.io
-  resources:
-  - subjectaccessreviews
-  verbs:
-  - create
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  labels:
-    {{- include "eck-operator.labels" . | nindent 4 }}
-  name: "{{ include "eck-operator.fullname" . }}-proxy-rolebinding"
-  namespace: {{ $.Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
-subjects:
-- kind: ServiceAccount
-  name: {{ $svcAccount }}
-  namespace: {{ $.Release.Namespace }}
-{{- end }} {{- /* end of if enableMetricsAuth check */}}
 {{- end }} {{- /* end of operator role binding if operator namespace is not managed */}}
 {{- else }} {{- /* we can create cluster-scoped resources so just create a cluster role binding */}}
 ---
@@ -118,22 +79,5 @@ subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
-{{- if $enableMetricsAuth }}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    {{- include "eck-operator.labels" $ | nindent 4 }}
-  name: "{{ include "eck-operator.fullname" . }}-proxy-rolebinding"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
-subjects:
-- kind: ServiceAccount
-  name: {{ $svcAccount }}
-  namespace: {{ $.Release.Namespace }}
-{{- end }}
 {{- end }}
 
diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index f1157662e9..a1b5b75b30 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -164,14 +164,15 @@ config:
 
   # enableMetricsAuth specifies whether to enable RBAC authentication for the metrics endpoint. (Will be enabled by default in v2.14.0)
   # * This option requires using a ServiceMonitor to scrape the metrics and as such is mutually exclusive with the podMonitor.enabled option.
-  # * This option also enable tls/HTTPs metrics by default.
+  # * This option also requires using cluster scoped resources (ClusterRole, ClusterRoleBinding) to
+  #   grant access to the /metrics endpoint. (createClusterScopedResources: true is required)
+  # * This option also enables tls/HTTPs metrics by default.
   #
   # This option requires the following settings within Prometheus to function:
   # 1. RBAC settings for the Prometheus instance to access the metrics endpoint.
   #
   # - nonResourceURLs:
   #   - /metrics
-  #   - /metrics/cadvisor
   #   verbs:
   #   - get
   #

From 7bf352ff43089728e07ad5af05c864d7cad8cfd2 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 20 Feb 2024 14:58:38 -0600
Subject: [PATCH 06/29] Add back missing cluster role binding.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../eck-operator/templates/role-bindings.yaml  | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml
index 93547e4114..f06ea503f9 100644
--- a/deploy/eck-operator/templates/role-bindings.yaml
+++ b/deploy/eck-operator/templates/role-bindings.yaml
@@ -76,8 +76,24 @@ roleRef:
   kind: ClusterRole
   name: {{ $fullName }}
 subjects:
+- kind: ServiceAccount
+  name: {{ $svcAccount }}
+  namespace: {{ $.Release.Namespace }}
+{{- if $enableMetricsAuth }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    {{- include "eck-operator.labels" $ | nindent 4 }}
+  name: "{{ include "eck-operator.fullname" . }}-proxy-rolebinding"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: "{{ include "eck-operator.fullname" . }}-proxy-role"
+subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
 {{- end }}
-
+{{- end }}

From 4c6eabdb3a7b2f422a77fd7efb630cb4f1a9f106 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Thu, 22 Feb 2024 11:16:20 -0600
Subject: [PATCH 07/29] Docs for secure metrics. Update for metricsBindAddress.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 cmd/manager/main.go                           |   7 +-
 deploy/eck-operator/templates/_helpers.tpl    |   9 ++
 .../templates/auth-proxy-service.yaml         |   5 +-
 .../eck-operator/templates/cluster-roles.yaml |  17 +--
 deploy/eck-operator/templates/configmap.yaml  |  12 +-
 .../templates/operator-network-policy.yaml    |   1 -
 deploy/eck-operator/templates/podMonitor.yaml |   5 +-
 .../eck-operator/templates/role-bindings.yaml |   5 +-
 .../templates/serviceMonitor.yaml             |   3 +-
 .../eck-operator/templates/statefulset.yaml   |   5 +-
 deploy/eck-operator/values.yaml               |  21 +++-
 docs/advanced-topics/advanced-topics.asciidoc |   2 +
 docs/advanced-topics/secure-metrics.asciidoc  | 103 ++++++++++++++++++
 pkg/controller/common/operator/flags.go       |   1 +
 14 files changed, 158 insertions(+), 38 deletions(-)
 create mode 100644 docs/advanced-topics/secure-metrics.asciidoc

diff --git a/cmd/manager/main.go b/cmd/manager/main.go
index 64cb692989..4dcdf510c4 100644
--- a/cmd/manager/main.go
+++ b/cmd/manager/main.go
@@ -286,7 +286,12 @@ func Command() *cobra.Command {
 	cmd.Flags().Int(
 		operator.MetricsPortFlag,
 		DefaultMetricPort,
-		"Port to use for exposing metrics in the Prometheus format (set 0 to disable)",
+		"(Deprecated) Port to use for exposing metrics in the Prometheus format. (set 0 to disable. Use --metrics-bind-address instead)",
+	)
+	cmd.Flags().String(
+		operator.MetricsBindAddressFlag,
+		"",
+		fmt.Sprintf("The address which the operator should listen on to serve metrics in the Prometheus format. Cannot be combined with %s. (set to empty to disable)", operator.MetricsPortFlag),
 	)
 	cmd.Flags().StringSlice(
 		operator.NamespacesFlag,
diff --git a/deploy/eck-operator/templates/_helpers.tpl b/deploy/eck-operator/templates/_helpers.tpl
index 8c421f7b55..a6001942f1 100644
--- a/deploy/eck-operator/templates/_helpers.tpl
+++ b/deploy/eck-operator/templates/_helpers.tpl
@@ -114,6 +114,15 @@ elastic-webhook-server
 {{- end -}}
 {{- end -}}
 
+{{/*
+Determine the metrics port
+*/}}
+{{- if and (has .Values.config.metricsBindAddress) (not (eq .Values.config.metricsBindAddress "")) }}
+{{ $metricsPort := int .Values.config.metricsBindAddress | split ":" | last }}
+{{- else if (gt .Values.config.metricsPort 0) }}
+{{ $metricsPort := int .Values.config.metricsPort }}
+{{- end }}
+
 {{/*
 RBAC permissions
 NOTE - any changes made to RBAC permissions below require
diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml
index 8664d1c45b..64c69a72e7 100644
--- a/deploy/eck-operator/templates/auth-proxy-service.yaml
+++ b/deploy/eck-operator/templates/auth-proxy-service.yaml
@@ -1,5 +1,4 @@
-{{- $metricsPort := int .Values.config.metricsPort -}}
-{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
+{{- if .Values.config.enableSecureMetrics }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -14,7 +13,7 @@ metadata:
 spec:
   ports:
   - name: https
-    port: {{ .Values.config.metricsPort }}
+    port: {{ $metricsPort }}
     protocol: TCP
     targetPort: metrics
   selector:
diff --git a/deploy/eck-operator/templates/cluster-roles.yaml b/deploy/eck-operator/templates/cluster-roles.yaml
index 8d4f5878bc..45f21fe120 100644
--- a/deploy/eck-operator/templates/cluster-roles.yaml
+++ b/deploy/eck-operator/templates/cluster-roles.yaml
@@ -1,5 +1,4 @@
-{{- $metricsPort := int .Values.config.metricsPort -}}
-{{- if and (not .Values.createClusterScopedResources) (.Values.config.enableMetricsAuth) (gt $metricsPort 0) -}}
+{{- if and (not .Values.createClusterScopedResources) (.Values.config.enableSecureMetrics) -}}
 {{ fail "createClusterScopedResources is required to enable metrics authentication" }}
 {{- end }}
 {{- if .Values.createClusterScopedResources -}}
@@ -97,7 +96,7 @@ rules:
   - apiGroups: ["logstash.k8s.elastic.co"]
     resources: ["logstashes"]
     verbs: ["create", "delete", "deletecollection", "patch", "update"]
-{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
+{{- if and .Values.config.enableSecureMetrics }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -118,17 +117,5 @@ rules:
   - subjectaccessreviews
   verbs:
   - create
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    {{- include "eck-operator.labels" . | nindent 4 }}
-  name: "{{ include "eck-operator.fullname" . }}-metrics-reader"
-rules:
-- nonResourceURLs:
-  - "/metrics"
-  verbs:
-  - get
 {{- end }}
 {{- end -}}
diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml
index 6484e6454d..b28b48b6c7 100644
--- a/deploy/eck-operator/templates/configmap.yaml
+++ b/deploy/eck-operator/templates/configmap.yaml
@@ -1,4 +1,3 @@
-{{- $metricsPort := int .Values.config.metricsPort -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -10,11 +9,20 @@ metadata:
 data:
   eck.yaml: |-
     log-verbosity: {{ int .Values.config.logVerbosity }}
-    {{- if (and .Values.config.enableMetricsAuth (gt $metricsPort 0)) }}
+    {{- if (and .Values.config.enableSecureMetrics (eq $metricsPort 0) (not (has .Values.metricsBindAddress))) }}
+    {{ fail "metricsPort or metricsBindAddress is required when enableSecureMetrics is true" }}
+    {{- end }}
+    {{- if (and .Values.config.enableSecureMetrics (gt $metricsPort 0) (or (eq .Values.metricsBindAddress "") (not (has .Values.metricsBindAddress)))) }}
     metrics-port: {{ add $metricsPort 1 }}
     {{- else }}
     metrics-port: {{ $metricsPort }}
     {{- end }}
+    {{- if (and .Values.config.enableSecureMetrics (has .Values.metricsBindAddress)) }}
+    {{ $host := int .Values.metricsBindAddress | split ":" | first }}
+    metrics-bind-address: {{ $host }}:{{ add $metricsPort 1 }}
+    {{- else }}
+    metrics-bind-address: {{ .Values.metricsBindAddress }}
+    {{- end }}
     container-registry: {{ .Values.config.containerRegistry }}
     {{- with .Values.config.containerSuffix }}
     container-suffix: {{ . }}
diff --git a/deploy/eck-operator/templates/operator-network-policy.yaml b/deploy/eck-operator/templates/operator-network-policy.yaml
index 10aaa56654..c0f60fd1a8 100644
--- a/deploy/eck-operator/templates/operator-network-policy.yaml
+++ b/deploy/eck-operator/templates/operator-network-policy.yaml
@@ -1,6 +1,5 @@
 {{- if .Values.softMultiTenancy.enabled -}}
 {{- $kubeAPIServerIP := (required "kubeAPIServerIP is required" .Values.kubeAPIServerIP) -}}
-{{- $metricsPort := int .Values.config.metricsPort -}}
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml
index 9d7f379dd8..f26e60cf5d 100644
--- a/deploy/eck-operator/templates/podMonitor.yaml
+++ b/deploy/eck-operator/templates/podMonitor.yaml
@@ -1,6 +1,5 @@
-{{- $metricsPort := int .Values.config.metricsPort -}}
-{{- if and .Values.podMonitor.enabled .Values.config.enableMetricsAuth}}
-{{ fail "podMonitor.enabled and enableMetricsAuth are mutually exclusive" }}
+{{- if and .Values.podMonitor.enabled .Values.config.enableSecureMetrics}}
+{{ fail "podMonitor.enabled and enableSecureMetrics are mutually exclusive" }}
 {{- end }}
 {{- if and .Values.podMonitor.enabled (gt $metricsPort 0)}}
 apiVersion: monitoring.coreos.com/v1
diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml
index f06ea503f9..0cf7db3548 100644
--- a/deploy/eck-operator/templates/role-bindings.yaml
+++ b/deploy/eck-operator/templates/role-bindings.yaml
@@ -1,8 +1,7 @@
 {{- $operatorNSIsManaged := has .Release.Namespace .Values.managedNamespaces -}}
 {{- $fullName := include "eck-operator.fullname" . -}}
 {{- $svcAccount := include "eck-operator.serviceAccountName" . }}
-{{- $metricsPort := int .Values.config.metricsPort -}}
-{{- $enableMetricsAuth := and .Values.config.enableMetricsAuth (gt $metricsPort 0) -}}
+{{- $enableSecureMetrics := and .Values.config.enableSecureMetrics -}}
 
 {{- if not .Values.createClusterScopedResources }}
 {{- range .Values.managedNamespaces }}
@@ -79,7 +78,7 @@ subjects:
 - kind: ServiceAccount
   name: {{ $svcAccount }}
   namespace: {{ $.Release.Namespace }}
-{{- if $enableMetricsAuth }}
+{{- if $enableSecureMetrics }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
diff --git a/deploy/eck-operator/templates/serviceMonitor.yaml b/deploy/eck-operator/templates/serviceMonitor.yaml
index 15b622ef8f..7215a98801 100644
--- a/deploy/eck-operator/templates/serviceMonitor.yaml
+++ b/deploy/eck-operator/templates/serviceMonitor.yaml
@@ -1,5 +1,4 @@
-{{- $metricsPort := int .Values.config.metricsPort -}}
-{{- if and .Values.config.enableMetricsAuth (gt $metricsPort 0) }}
+{{- if and .Values.config.enableSecureMetrics }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index 05003e3ac9..66b1a947e9 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -1,4 +1,3 @@
-{{- $metricsPort := int .Values.config.metricsPort -}}
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -81,7 +80,7 @@ spec:
           {{- end }}
           {{- if or (gt $metricsPort 0) .Values.webhook.enabled }}
           ports:
-            {{- if and (gt $metricsPort 0) (not .Values.config.enableMetricsAuth) }}
+            {{- if and (gt $metricsPort 0) (not .Values.config.enableSecureMetrics) }}
             - containerPort: {{ .Values.config.metricsPort }}
               name: metrics
               protocol: TCP
@@ -104,7 +103,7 @@ spec:
             {{- with .Values.volumeMounts }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
-        {{- if (and .Values.config.enableMetricsAuth (gt $metricsPort 0)) }}
+        {{- if (and .Values.config.enableSecureMetrics ) }}
         - name: kube-rbac-proxy
           securityContext:
             allowPrivilegeEscalation: false
diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index a1b5b75b30..3eed1873ba 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -159,14 +159,25 @@ config:
   #  number greater than 0: Errors, warnings, information, and debug details.
   logVerbosity: "0"
 
-  # metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting.
+  # metricsBindAddress defines the address which the operator should listen on to serve metrics in the Prometheus format.
+  #
+  # *Note* If this option is combined with the enableSecureMetrics option then only the port is used, the bind address is ignored
+  #        as the operator will only listen on the localhost (127.0.0.1) interface and kube-rbac-proxy will intercept traffic
+  #        to the metrics port.
+  #
+  # Examples follow:
+  # - :8080 (equivalent to 0.0.0.0:8080)
+  # - 0.0.0.0:8080
+  # - localhost:8080
+  metricsBindAddress: ""
+
+  # (Deprecated; Will be removed in v2.14.0) metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting. Use `metricsBindAddress` instead.
   metricsPort: "0"
 
-  # enableMetricsAuth specifies whether to enable RBAC authentication for the metrics endpoint. (Will be enabled by default in v2.14.0)
+  # enableSecureMetrics specifies whether to enable RBAC and TLS/HTTPs for the metrics endpoint. (Will be enabled by default in v2.14.0)
   # * This option requires using a ServiceMonitor to scrape the metrics and as such is mutually exclusive with the podMonitor.enabled option.
   # * This option also requires using cluster scoped resources (ClusterRole, ClusterRoleBinding) to
   #   grant access to the /metrics endpoint. (createClusterScopedResources: true is required)
-  # * This option also enables tls/HTTPs metrics by default.
   #
   # This option requires the following settings within Prometheus to function:
   # 1. RBAC settings for the Prometheus instance to access the metrics endpoint.
@@ -176,14 +187,14 @@ config:
   #   verbs:
   #   - get
   #
-  # 2. If using the Prometheus Operator and your Prometheus instance are not in the same namespace as the operator you will need
+  # 2. If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the operator you will need
   #    the Prometheus Operator configured with the following Helm values:
   #
   #   prometheus:
   #     prometheusSpec:
   #       serviceMonitorNamespaceSelector: {}
   #       serviceMonitorSelectorNilUsesHelmValues: false
-  enableMetricsAuth: false
+  enableSecureMetrics: false
 
   # containerRegistry to use for pulling Elasticsearch and other application container images.
   containerRegistry: docker.elastic.co
diff --git a/docs/advanced-topics/advanced-topics.asciidoc b/docs/advanced-topics/advanced-topics.asciidoc
index 4128581220..d4d64727b6 100644
--- a/docs/advanced-topics/advanced-topics.asciidoc
+++ b/docs/advanced-topics/advanced-topics.asciidoc
@@ -18,6 +18,7 @@ endif::[]
 - <<{p}-webhook-namespace-selectors>>
 - <<{p}-stack-monitoring>>
 - <<{p}-fips>>
+- <<{p}-secure-metrics>>
 --
 
 include::openshift.asciidoc[leveloffset=+1]
@@ -29,3 +30,4 @@ include::network-policies.asciidoc[leveloffset=+1]
 include::webhook-namespace-selectors.asciidoc[leveloffset=+1]
 include::stack-monitoring.asciidoc[leveloffset=+1]
 include::fips.asciidoc[leveloffset=+1]
+include::secure-metrics.asciidoc[leveloffset=+1]
diff --git a/docs/advanced-topics/secure-metrics.asciidoc b/docs/advanced-topics/secure-metrics.asciidoc
new file mode 100644
index 0000000000..20a6cc2c47
--- /dev/null
+++ b/docs/advanced-topics/secure-metrics.asciidoc
@@ -0,0 +1,103 @@
+:page_id: secure-metrics
+ifdef::env-github[]
+****
+link:https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-{page_id}.html[View this document on the Elastic website]
+****
+endif::[]
+
+[id="{p}-{page_id}"]
+= Securing the ECK Operator metrics endpoint
+
+== Using the ECK Operator Helm Chart
+
+If you install ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `enableSecureMetrics` and `metricsBindAddress` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
+
+== Using the ECK Operator manifests
+
+If you install ECK through using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable secure metrics.
+
+1. `ClusterRole` and `ClusterRoleBinding` for the ECK Operator.
+
+[source,yaml,subs="attributes"]
+----
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: elastic-operator-proxy-role
+rules:
+- apiGroups:
+  - authentication.k8s.io
+  resources:
+  - tokenreviews
+  verbs:
+  - create
+- apiGroups:
+  - authorization.k8s.io
+  resources:
+  - subjectaccessreviews
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: elastic-operator-proxy-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: elastic-operator-proxy-role
+subjects:
+- kind: ServiceAccount
+  name: elastic-operator
+  namespace: elastic-system
+----
+
+2. ServiceMonitor to allow scraping of the metrics endpoint by Prometheus.
+
+[source,yaml,subs="attributes"]
+----
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: elastic-operator
+  namespace: elastic-system
+spec:
+  namespaceSelector:
+    matchNames:
+      - elastic-system
+  selector:
+    matchLabels:
+      control-plane: elastic-operator
+  endpoints:
+  - port: https
+    path: /metrics
+    scheme: https
+    interval: 30s
+    tlsConfig:
+      insecureSkipVerify: true
+    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+----
+
+== Prometheus requirements
+
+This option requires the following settings within Prometheus to function:
+
+1. RBAC settings for the Prometheus instance to access the metrics endpoint.
+
+[source,yaml,subs="attributes"]
+----
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get
+----
+
+NOTE: If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the operator you will need the Prometheus Operator configured with the following Helm values:
+
+[source,yaml,subs="attributes"]
+----
+prometheus:
+  prometheusSpec:
+    serviceMonitorNamespaceSelector: {}
+    serviceMonitorSelectorNilUsesHelmValues: false
+----
diff --git a/pkg/controller/common/operator/flags.go b/pkg/controller/common/operator/flags.go
index f7df112bc1..70fcd95b4f 100644
--- a/pkg/controller/common/operator/flags.go
+++ b/pkg/controller/common/operator/flags.go
@@ -33,6 +33,7 @@ const (
 	ManageWebhookCertsFlag               = "manage-webhook-certs"
 	MaxConcurrentReconcilesFlag          = "max-concurrent-reconciles"
 	MetricsPortFlag                      = "metrics-port"
+	MetricsBindAddressFlag               = "metrics-bind-address"
 	NamespacesFlag                       = "namespaces"
 	OperatorNamespaceFlag                = "operator-namespace"
 	SetDefaultSecurityContextFlag        = "set-default-security-context"

From b0e8e658267b361f0148c7cbe0ca1b153258c187 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Thu, 22 Feb 2024 14:54:17 -0600
Subject: [PATCH 08/29] Update docs for enabling secure metrics. Remove
 debugging of kube-rbac-proxy

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../eck-operator/templates/statefulset.yaml   |  2 +-
 docs/advanced-topics/secure-metrics.asciidoc  | 86 +++++++++++++++++--
 2 files changed, 82 insertions(+), 6 deletions(-)

diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index 66b1a947e9..b70c96a7b6 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -115,7 +115,7 @@ spec:
           - "--secure-listen-address=0.0.0.0:{{ $metricsPort }}"
           - "--upstream=http://127.0.0.1:{{ add $metricsPort 1 }}/"
           - "--logtostderr=true"
-          - "--v=10"
+          - "--v=0"
           ports:
           - containerPort: {{ $metricsPort }}
             protocol: TCP
diff --git a/docs/advanced-topics/secure-metrics.asciidoc b/docs/advanced-topics/secure-metrics.asciidoc
index 20a6cc2c47..acee8bac64 100644
--- a/docs/advanced-topics/secure-metrics.asciidoc
+++ b/docs/advanced-topics/secure-metrics.asciidoc
@@ -16,10 +16,18 @@ If you install ECK through the Helm chart commands listed in <<{p}-install-helm>
 
 If you install ECK through using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable secure metrics.
 
-1. `ClusterRole` and `ClusterRoleBinding` for the ECK Operator.
+* Enable the metrics endpoint in the `ConfigMap`.
 
-[source,yaml,subs="attributes"]
+[source,sh]
+----
+kubectl get cm elastic-operator -n elastic-system -o yaml | sed "s|metrics-port: 0|metrics-port: 8081|" | kubectl apply -f -
+----
+
+* Add `ClusterRole` and `ClusterRoleBinding` for the ECK Operator.
+
+[source,shell,subs="attributes,+macros"]
 ----
+cat $$<<$$EOF | kubectl apply -f -
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -50,12 +58,73 @@ subjects:
 - kind: ServiceAccount
   name: elastic-operator
   namespace: elastic-system
+EOF
 ----
 
-2. ServiceMonitor to allow scraping of the metrics endpoint by Prometheus.
+* Add a `Service` to expose the metrics endpoint.
 
-[source,yaml,subs="attributes"]
+[source,shell,subs="attributes,+macros"]
 ----
+cat $$<<$$EOF | kubectl apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    control-plane: elastic-operator
+    app.kubernetes.io/component: metrics
+  name: elastic-operator-metrics
+  namespace: elastic-system
+spec:
+  ports:
+  - name: https
+    port: 8080
+    protocol: TCP
+    targetPort: metrics
+  selector:
+    control-plane: elastic-operator
+EOF
+----
+
+* Patch the `StatefulSet`
+
+[source,shell,subs="attributes,+macros"]
+----
+kubectl patch sts -n elastic-system elastic-operator --patch-file=/dev/stdin <<-EOF
+spec:
+  template:
+    spec:
+      containers:
+        - name: kube-rbac-proxy
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "ALL"
+          image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
+          args:
+          - "--secure-listen-address=0.0.0.0:8080"
+          - "--upstream=http://127.0.0.1:8081"
+          - "--logtostderr=true"
+          - "--v=0"
+          ports:
+          - containerPort: 8080
+            protocol: TCP
+            name: metrics
+          resources:
+            limits:
+              cpu: 500m
+              memory: 128Mi
+            requests:
+              cpu: 5m
+              memory: 64Mi
+EOF
+----
+
+* Add a `ServiceMonitor` to allow scraping of the metrics endpoint by Prometheus.
+
+[source,shell,subs="attributes,+macros"]
+----
+cat $$<<$$EOF | kubectl apply -f -
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -68,6 +137,7 @@ spec:
   selector:
     matchLabels:
       control-plane: elastic-operator
+      app.kubernetes.io/component: metrics
   endpoints:
   - port: https
     path: /metrics
@@ -76,16 +146,22 @@ spec:
     tlsConfig:
       insecureSkipVerify: true
     bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+EOF
 ----
 
 == Prometheus requirements
 
 This option requires the following settings within Prometheus to function:
 
-1. RBAC settings for the Prometheus instance to access the metrics endpoint.
+1. RBAC settings for the Prometheus instance to access the metrics endpoint similar to the following:
 
 [source,yaml,subs="attributes"]
 ----
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus
+rules:
 - nonResourceURLs:
   - /metrics
   verbs:

From 21cf39054cdebaa19173a8214d50977491fc5f79 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Thu, 22 Feb 2024 15:51:15 -0600
Subject: [PATCH 09/29] Fix all helm templating issues

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/templates/_helpers.tpl     |  9 ---------
 .../templates/auth-proxy-service.yaml          |  6 ++++++
 deploy/eck-operator/templates/configmap.yaml   | 18 ++++++++++--------
 .../templates/operator-network-policy.yaml     |  8 +++++++-
 deploy/eck-operator/templates/podMonitor.yaml  |  2 +-
 deploy/eck-operator/templates/statefulset.yaml | 14 +++++++++++++-
 6 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/deploy/eck-operator/templates/_helpers.tpl b/deploy/eck-operator/templates/_helpers.tpl
index a6001942f1..8c421f7b55 100644
--- a/deploy/eck-operator/templates/_helpers.tpl
+++ b/deploy/eck-operator/templates/_helpers.tpl
@@ -114,15 +114,6 @@ elastic-webhook-server
 {{- end -}}
 {{- end -}}
 
-{{/*
-Determine the metrics port
-*/}}
-{{- if and (has .Values.config.metricsBindAddress) (not (eq .Values.config.metricsBindAddress "")) }}
-{{ $metricsPort := int .Values.config.metricsBindAddress | split ":" | last }}
-{{- else if (gt .Values.config.metricsPort 0) }}
-{{ $metricsPort := int .Values.config.metricsPort }}
-{{- end }}
-
 {{/*
 RBAC permissions
 NOTE - any changes made to RBAC permissions below require
diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml
index 64c69a72e7..232b4ff0f6 100644
--- a/deploy/eck-operator/templates/auth-proxy-service.yaml
+++ b/deploy/eck-operator/templates/auth-proxy-service.yaml
@@ -1,4 +1,10 @@
 {{- if .Values.config.enableSecureMetrics }}
+{{ $metricsPort := 0 }}
+{{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
+{{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}
+{{- else if (gt (int .Values.config.metricsPort) 0) }}
+{{ $metricsPort = int .Values.config.metricsPort }}
+{{- end }}
 apiVersion: v1
 kind: Service
 metadata:
diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml
index b28b48b6c7..363ebf7a33 100644
--- a/deploy/eck-operator/templates/configmap.yaml
+++ b/deploy/eck-operator/templates/configmap.yaml
@@ -9,20 +9,22 @@ metadata:
 data:
   eck.yaml: |-
     log-verbosity: {{ int .Values.config.logVerbosity }}
-    {{- if (and .Values.config.enableSecureMetrics (eq $metricsPort 0) (not (has .Values.metricsBindAddress))) }}
+    {{- if (and .Values.config.enableSecureMetrics (eq (int .Values.config.metricsPort) 0) (not (hasKey .Values.config "metricsBindAddress"))) }}
     {{ fail "metricsPort or metricsBindAddress is required when enableSecureMetrics is true" }}
     {{- end }}
-    {{- if (and .Values.config.enableSecureMetrics (gt $metricsPort 0) (or (eq .Values.metricsBindAddress "") (not (has .Values.metricsBindAddress)))) }}
-    metrics-port: {{ add $metricsPort 1 }}
+    {{- if (and .Values.config.enableSecureMetrics (gt (int .Values.config.metricsPort) 0) (or (eq .Values.config.metricsBindAddress "") (not (hasKey .Values.config "metricsBindAddress")))) }}
+    metrics-port: {{ add (int .Values.config.metricsPort) 1 }}
     {{- else }}
-    metrics-port: {{ $metricsPort }}
+    metrics-port: {{ int (.Values.config.metricsBindAddress | splitList ":" | last) }}
     {{- end }}
-    {{- if (and .Values.config.enableSecureMetrics (has .Values.metricsBindAddress)) }}
-    {{ $host := int .Values.metricsBindAddress | split ":" | first }}
-    metrics-bind-address: {{ $host }}:{{ add $metricsPort 1 }}
+    {{- if (and .Values.config.enableSecureMetrics (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress ""))) }}
+    {{- if .Values.config.enableSecureMetrics }}
+    {{ $host := .Values.config.metricsBindAddress | splitList ":" | first }}
+    metrics-bind-address: {{ $host }}:{{ add (int (.Values.config.metricsBindAddress | splitList ":" | last)) 1 }}
     {{- else }}
-    metrics-bind-address: {{ .Values.metricsBindAddress }}
+    metrics-bind-address: {{ int .Values.config.metricsBindAddress | splitList ":" | last }}
     {{- end }}
+    {{- end }} 
     container-registry: {{ .Values.config.containerRegistry }}
     {{- with .Values.config.containerSuffix }}
     container-suffix: {{ . }}
diff --git a/deploy/eck-operator/templates/operator-network-policy.yaml b/deploy/eck-operator/templates/operator-network-policy.yaml
index c0f60fd1a8..af0db05fb2 100644
--- a/deploy/eck-operator/templates/operator-network-policy.yaml
+++ b/deploy/eck-operator/templates/operator-network-policy.yaml
@@ -39,7 +39,13 @@ spec:
           podSelector:
             matchLabels:
               common.k8s.elastic.co/type: "elasticsearch"
-{{- if or .Values.webhook.enabled (gt $metricsPort 0) }}
+{{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort ) 0) (and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")))}}
+{{ $metricsPort := 0 }}
+{{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
+{{ $metricsPort = int .Values.config.metricsBindAddress | splitList ":" | last }}
+{{- else if (gt .Values.config.metricsPort 0) }}
+{{ $metricsPort = int .Values.config.metricsPort }}
+{{- end }}
   ingress:
 {{- if .Values.webhook.enabled }}
     - ports:
diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml
index f26e60cf5d..11ed377044 100644
--- a/deploy/eck-operator/templates/podMonitor.yaml
+++ b/deploy/eck-operator/templates/podMonitor.yaml
@@ -1,7 +1,7 @@
 {{- if and .Values.podMonitor.enabled .Values.config.enableSecureMetrics}}
 {{ fail "podMonitor.enabled and enableSecureMetrics are mutually exclusive" }}
 {{- end }}
-{{- if and .Values.podMonitor.enabled (gt $metricsPort 0)}}
+{{- if and .Values.podMonitor.enabled (or (gt (int .Values.config.metricsPort) 0) (and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")))) }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index b70c96a7b6..3e68adf049 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -78,7 +78,13 @@ spec:
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          {{- if or (gt $metricsPort 0) .Values.webhook.enabled }}
+          {{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort) 0) (and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")))}}
+          {{ $metricsPort := 0 }}
+          {{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
+          {{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}
+          {{- else if (gt (int .Values.config.metricsPort) 0) }}
+          {{ $metricsPort = int .Values.config.metricsPort }}
+          {{- end }}
           ports:
             {{- if and (gt $metricsPort 0) (not .Values.config.enableSecureMetrics) }}
             - containerPort: {{ .Values.config.metricsPort }}
@@ -104,6 +110,12 @@ spec:
               {{- toYaml . | nindent 12 }}
             {{- end }}
         {{- if (and .Values.config.enableSecureMetrics ) }}
+        {{ $metricsPort := 0 }}
+        {{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
+        {{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}
+        {{- else if (gt (int .Values.config.metricsPort) 0) }}
+        {{ $metricsPort = int .Values.config.metricsPort }}
+        {{- end }}
         - name: kube-rbac-proxy
           securityContext:
             allowPrivilegeEscalation: false

From 62336437b2f34ad7e520e1b7d608e66c26b4cb71 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Thu, 22 Feb 2024 19:43:40 -0600
Subject: [PATCH 10/29] Expand fail directive

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/templates/configmap.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml
index 363ebf7a33..7f2a04555f 100644
--- a/deploy/eck-operator/templates/configmap.yaml
+++ b/deploy/eck-operator/templates/configmap.yaml
@@ -9,7 +9,7 @@ metadata:
 data:
   eck.yaml: |-
     log-verbosity: {{ int .Values.config.logVerbosity }}
-    {{- if (and .Values.config.enableSecureMetrics (eq (int .Values.config.metricsPort) 0) (not (hasKey .Values.config "metricsBindAddress"))) }}
+    {{- if (and .Values.config.enableSecureMetrics (eq (int .Values.config.metricsPort) 0) (or (eq .Values.config.metricsBindAddress "") (not (hasKey .Values.config "metricsBindAddress")))) }}
     {{ fail "metricsPort or metricsBindAddress is required when enableSecureMetrics is true" }}
     {{- end }}
     {{- if (and .Values.config.enableSecureMetrics (gt (int .Values.config.metricsPort) 0) (or (eq .Values.config.metricsBindAddress "") (not (hasKey .Values.config "metricsBindAddress")))) }}

From 473924980c46f654c134f6a796edeb1d89ae7346 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Thu, 22 Feb 2024 19:46:40 -0600
Subject: [PATCH 11/29] update fail directive #2

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/templates/cluster-roles.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/eck-operator/templates/cluster-roles.yaml b/deploy/eck-operator/templates/cluster-roles.yaml
index 45f21fe120..cb1d499255 100644
--- a/deploy/eck-operator/templates/cluster-roles.yaml
+++ b/deploy/eck-operator/templates/cluster-roles.yaml
@@ -1,5 +1,5 @@
 {{- if and (not .Values.createClusterScopedResources) (.Values.config.enableSecureMetrics) -}}
-{{ fail "createClusterScopedResources is required to enable metrics authentication" }}
+{{ fail "createClusterScopedResources is required to enable secure metrics" }}
 {{- end }}
 {{- if .Values.createClusterScopedResources -}}
 ---

From 1db91457980288b19cbfd2763d8473b7f9fbb350 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Thu, 22 Feb 2024 19:55:38 -0600
Subject: [PATCH 12/29] fix helm if statement

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/templates/statefulset.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index 3e68adf049..400f6bd374 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -109,7 +109,7 @@ spec:
             {{- with .Values.volumeMounts }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
-        {{- if (and .Values.config.enableSecureMetrics ) }}
+        {{- if .Values.config.enableSecureMetrics }}
         {{ $metricsPort := 0 }}
         {{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
         {{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}

From 664cffb42fcd888a827d027bc01bd3ff4b73091f Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Mon, 26 Feb 2024 12:07:30 +0100
Subject: [PATCH 13/29] Move to using 'metricsHost' instead of
 'metricsBindAddress'

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 cmd/manager/main.go                            | 11 ++++++-----
 config/eck.yaml                                |  1 +
 .../templates/auth-proxy-service.yaml          |  7 +------
 deploy/eck-operator/templates/configmap.yaml   | 17 +++++------------
 .../templates/operator-network-policy.yaml     |  7 +------
 deploy/eck-operator/templates/podMonitor.yaml  |  2 +-
 .../eck-operator/templates/role-bindings.yaml  |  2 +-
 deploy/eck-operator/templates/statefulset.yaml | 18 ++++--------------
 deploy/eck-operator/values.yaml                | 14 +-------------
 docs/advanced-topics/secure-metrics.asciidoc   |  4 ++--
 pkg/controller/common/operator/flags.go        |  2 +-
 11 files changed, 24 insertions(+), 61 deletions(-)

diff --git a/cmd/manager/main.go b/cmd/manager/main.go
index 4dcdf510c4..1ec2cc8f64 100644
--- a/cmd/manager/main.go
+++ b/cmd/manager/main.go
@@ -289,9 +289,9 @@ func Command() *cobra.Command {
 		"(Deprecated) Port to use for exposing metrics in the Prometheus format. (set 0 to disable. Use --metrics-bind-address instead)",
 	)
 	cmd.Flags().String(
-		operator.MetricsBindAddressFlag,
-		"",
-		fmt.Sprintf("The address which the operator should listen on to serve metrics in the Prometheus format. Cannot be combined with %s. (set to empty to disable)", operator.MetricsPortFlag),
+		operator.MetricsHostFlag,
+		"0.0.0.0",
+		fmt.Sprintf("The host to which the operator should bind to serve metrics in the Prometheus format. Will be combined with %s.", operator.MetricsPortFlag),
 	)
 	cmd.Flags().StringSlice(
 		operator.NamespacesFlag,
@@ -582,11 +582,12 @@ func startOperator(ctx context.Context) error {
 
 	// only expose prometheus metrics if provided a non-zero port
 	metricsPort := viper.GetInt(operator.MetricsPortFlag)
+	metricsHost := viper.GetString(operator.MetricsHostFlag)
 	if metricsPort != 0 {
-		log.Info("Exposing Prometheus metrics on /metrics", "port", metricsPort)
+		log.Info("Exposing Prometheus metrics on /metrics", "bindAddress", fmt.Sprintf("%s:%d", metricsHost, metricsPort))
 	}
 	opts.Metrics = metricsserver.Options{
-		BindAddress: fmt.Sprintf(":%d", metricsPort), // 0 to disable
+		BindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), // 0 to disable
 	}
 
 	webhookPort := viper.GetInt(operator.WebhookPortFlag)
diff --git a/config/eck.yaml b/config/eck.yaml
index 549435a4ed..1d0f19e4c3 100644
--- a/config/eck.yaml
+++ b/config/eck.yaml
@@ -1,5 +1,6 @@
 log-verbosity: 0
 metrics-port: 0
+metrics-host: 0.0.0.0
 container-registry: docker.elastic.co
 max-concurrent-reconciles: 3
 ca-cert-validity: 8760h
diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml
index 232b4ff0f6..60bf99378b 100644
--- a/deploy/eck-operator/templates/auth-proxy-service.yaml
+++ b/deploy/eck-operator/templates/auth-proxy-service.yaml
@@ -1,10 +1,5 @@
 {{- if .Values.config.enableSecureMetrics }}
-{{ $metricsPort := 0 }}
-{{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
-{{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}
-{{- else if (gt (int .Values.config.metricsPort) 0) }}
-{{ $metricsPort = int .Values.config.metricsPort }}
-{{- end }}
+{{ $metricsPort := int .Values.config.metricsPort }}
 apiVersion: v1
 kind: Service
 metadata:
diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml
index 7f2a04555f..f8a098d67c 100644
--- a/deploy/eck-operator/templates/configmap.yaml
+++ b/deploy/eck-operator/templates/configmap.yaml
@@ -9,22 +9,15 @@ metadata:
 data:
   eck.yaml: |-
     log-verbosity: {{ int .Values.config.logVerbosity }}
-    {{- if (and .Values.config.enableSecureMetrics (eq (int .Values.config.metricsPort) 0) (or (eq .Values.config.metricsBindAddress "") (not (hasKey .Values.config "metricsBindAddress")))) }}
-    {{ fail "metricsPort or metricsBindAddress is required when enableSecureMetrics is true" }}
+    {{- if and .Values.config.enableSecureMetrics (eq (int .Values.config.metricsPort) 0) }}
+    {{- fail "metricsPort must be greater than 0 when enableSecureMetrics is true" }}
     {{- end }}
-    {{- if (and .Values.config.enableSecureMetrics (gt (int .Values.config.metricsPort) 0) (or (eq .Values.config.metricsBindAddress "") (not (hasKey .Values.config "metricsBindAddress")))) }}
-    metrics-port: {{ add (int .Values.config.metricsPort) 1 }}
-    {{- else }}
-    metrics-port: {{ int (.Values.config.metricsBindAddress | splitList ":" | last) }}
-    {{- end }}
-    {{- if (and .Values.config.enableSecureMetrics (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress ""))) }}
+    {{- $metricsPort := int .Values.config.metricsPort }}
     {{- if .Values.config.enableSecureMetrics }}
-    {{ $host := .Values.config.metricsBindAddress | splitList ":" | first }}
-    metrics-bind-address: {{ $host }}:{{ add (int (.Values.config.metricsBindAddress | splitList ":" | last)) 1 }}
+    metrics-port: {{ add $metricsPort 1 }}
     {{- else }}
-    metrics-bind-address: {{ int .Values.config.metricsBindAddress | splitList ":" | last }}
+    metrics-port: {{ $metricsPort }}
     {{- end }}
-    {{- end }} 
     container-registry: {{ .Values.config.containerRegistry }}
     {{- with .Values.config.containerSuffix }}
     container-suffix: {{ . }}
diff --git a/deploy/eck-operator/templates/operator-network-policy.yaml b/deploy/eck-operator/templates/operator-network-policy.yaml
index af0db05fb2..67fda92f42 100644
--- a/deploy/eck-operator/templates/operator-network-policy.yaml
+++ b/deploy/eck-operator/templates/operator-network-policy.yaml
@@ -39,13 +39,8 @@ spec:
           podSelector:
             matchLabels:
               common.k8s.elastic.co/type: "elasticsearch"
-{{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort ) 0) (and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")))}}
-{{ $metricsPort := 0 }}
-{{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
-{{ $metricsPort = int .Values.config.metricsBindAddress | splitList ":" | last }}
-{{- else if (gt .Values.config.metricsPort 0) }}
+{{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort ) 0) }}
 {{ $metricsPort = int .Values.config.metricsPort }}
-{{- end }}
   ingress:
 {{- if .Values.webhook.enabled }}
     - ports:
diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml
index 11ed377044..190ca87ca5 100644
--- a/deploy/eck-operator/templates/podMonitor.yaml
+++ b/deploy/eck-operator/templates/podMonitor.yaml
@@ -1,7 +1,7 @@
 {{- if and .Values.podMonitor.enabled .Values.config.enableSecureMetrics}}
 {{ fail "podMonitor.enabled and enableSecureMetrics are mutually exclusive" }}
 {{- end }}
-{{- if and .Values.podMonitor.enabled (or (gt (int .Values.config.metricsPort) 0) (and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")))) }}
+{{- if and .Values.podMonitor.enabled (gt (int .Values.config.metricsPort) 0) }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml
index 0cf7db3548..2cf8d7c202 100644
--- a/deploy/eck-operator/templates/role-bindings.yaml
+++ b/deploy/eck-operator/templates/role-bindings.yaml
@@ -1,7 +1,7 @@
 {{- $operatorNSIsManaged := has .Release.Namespace .Values.managedNamespaces -}}
 {{- $fullName := include "eck-operator.fullname" . -}}
 {{- $svcAccount := include "eck-operator.serviceAccountName" . }}
-{{- $enableSecureMetrics := and .Values.config.enableSecureMetrics -}}
+{{- $enableSecureMetrics := .Values.config.enableSecureMetrics -}}
 
 {{- if not .Values.createClusterScopedResources }}
 {{- range .Values.managedNamespaces }}
diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index 400f6bd374..4017c0696a 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -78,16 +78,11 @@ spec:
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          {{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort) 0) (and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")))}}
-          {{ $metricsPort := 0 }}
-          {{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
-          {{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}
-          {{- else if (gt (int .Values.config.metricsPort) 0) }}
-          {{ $metricsPort = int .Values.config.metricsPort }}
-          {{- end }}
+          {{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort) 0) }}
+          {{ $metricsPort := int .Values.config.metricsPort }}
           ports:
             {{- if and (gt $metricsPort 0) (not .Values.config.enableSecureMetrics) }}
-            - containerPort: {{ .Values.config.metricsPort }}
+            - containerPort: {{ $metricsPort }}
               name: metrics
               protocol: TCP
             {{- end }}
@@ -110,12 +105,7 @@ spec:
               {{- toYaml . | nindent 12 }}
             {{- end }}
         {{- if .Values.config.enableSecureMetrics }}
-        {{ $metricsPort := 0 }}
-        {{- if and (hasKey .Values.config "metricsBindAddress") (not (eq .Values.config.metricsBindAddress "")) }}
-        {{ $metricsPort = int (.Values.config.metricsBindAddress | splitList ":" | last) }}
-        {{- else if (gt (int .Values.config.metricsPort) 0) }}
-        {{ $metricsPort = int .Values.config.metricsPort }}
-        {{- end }}
+        {{ $metricsPort := int .Values.config.metricsPort }}
         - name: kube-rbac-proxy
           securityContext:
             allowPrivilegeEscalation: false
diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index 3eed1873ba..d169c53317 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -159,19 +159,7 @@ config:
   #  number greater than 0: Errors, warnings, information, and debug details.
   logVerbosity: "0"
 
-  # metricsBindAddress defines the address which the operator should listen on to serve metrics in the Prometheus format.
-  #
-  # *Note* If this option is combined with the enableSecureMetrics option then only the port is used, the bind address is ignored
-  #        as the operator will only listen on the localhost (127.0.0.1) interface and kube-rbac-proxy will intercept traffic
-  #        to the metrics port.
-  #
-  # Examples follow:
-  # - :8080 (equivalent to 0.0.0.0:8080)
-  # - 0.0.0.0:8080
-  # - localhost:8080
-  metricsBindAddress: ""
-
-  # (Deprecated; Will be removed in v2.14.0) metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting. Use `metricsBindAddress` instead.
+  # metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting. Use `metricsBindAddress` instead.
   metricsPort: "0"
 
   # enableSecureMetrics specifies whether to enable RBAC and TLS/HTTPs for the metrics endpoint. (Will be enabled by default in v2.14.0)
diff --git a/docs/advanced-topics/secure-metrics.asciidoc b/docs/advanced-topics/secure-metrics.asciidoc
index acee8bac64..96a1ec8105 100644
--- a/docs/advanced-topics/secure-metrics.asciidoc
+++ b/docs/advanced-topics/secure-metrics.asciidoc
@@ -10,7 +10,7 @@ endif::[]
 
 == Using the ECK Operator Helm Chart
 
-If you install ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `enableSecureMetrics` and `metricsBindAddress` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
+If you install ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `enableSecureMetrics` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
 
 == Using the ECK Operator manifests
 
@@ -20,7 +20,7 @@ If you install ECK through using the manifests using the commands listed in <<{p
 
 [source,sh]
 ----
-kubectl get cm elastic-operator -n elastic-system -o yaml | sed "s|metrics-port: 0|metrics-port: 8081|" | kubectl apply -f -
+kubectl get cm elastic-operator -n elastic-system -o yaml | sed "s|metrics-port: 0|metrics-port: 8081|" | sed "s|metrics-host: 0.0.0.0|metrics-host: 127.0.0.1|" | kubectl apply -f -
 ----
 
 * Add `ClusterRole` and `ClusterRoleBinding` for the ECK Operator.
diff --git a/pkg/controller/common/operator/flags.go b/pkg/controller/common/operator/flags.go
index 70fcd95b4f..c965d81a8d 100644
--- a/pkg/controller/common/operator/flags.go
+++ b/pkg/controller/common/operator/flags.go
@@ -33,7 +33,7 @@ const (
 	ManageWebhookCertsFlag               = "manage-webhook-certs"
 	MaxConcurrentReconcilesFlag          = "max-concurrent-reconciles"
 	MetricsPortFlag                      = "metrics-port"
-	MetricsBindAddressFlag               = "metrics-bind-address"
+	MetricsHostFlag                      = "metrics-host"
 	NamespacesFlag                       = "namespaces"
 	OperatorNamespaceFlag                = "operator-namespace"
 	SetDefaultSecurityContextFlag        = "set-default-security-context"

From 6916389aaf78be2a4ad263e26fc390f237f2d01e Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Mon, 26 Feb 2024 12:12:47 +0100
Subject: [PATCH 14/29] Add serviceMonitor to default values.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/values.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index d169c53317..2947f842d2 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -280,6 +280,15 @@ podMonitor:
   podMetricsEndpointConfig: {}
   # honorTimestamps: true
 
+# Prometheus ServiceMonitor configuration
+# Only used when config.enableSecureMetrics is true
+# Reference: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#servicemonitor
+serviceMonitor: {}
+
+  # namespace determines in which namespace the servicedMonitor will be deployed.
+  # If not set the servicedMonitor will be created in the namespace where the Helm release is installed into
+  # namespace: monitoring
+
 # Globals meant for internal use only
 global:
   # manifestGen specifies whether the chart is running under manifest generator.

From 361ca674e223032456421dede8511122b08743be Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Mon, 26 Feb 2024 17:28:39 +0100
Subject: [PATCH 15/29] WIP

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../eck-operator/templates/statefulset.yaml   | 31 ++++++++++++++
 deploy/eck-operator/values.yaml               |  8 ++++
 .../configure-operator-metrics.asciidoc}      | 40 ++++++++++++++++++-
 3 files changed, 77 insertions(+), 2 deletions(-)
 rename docs/{advanced-topics/secure-metrics.asciidoc => operating-eck/configure-operator-metrics.asciidoc} (70%)

diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index 4017c0696a..b8b40ac519 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -118,6 +118,25 @@ spec:
           - "--upstream=http://127.0.0.1:{{ add $metricsPort 1 }}/"
           - "--logtostderr=true"
           - "--v=0"
+          {{- if .Values.config.secureMetricsTLSCertSecret }}
+          - "--tls-cert-file=/tls/cert/tls.crt"
+          {{- end }}
+          {{- if .Values.config.secureMetricsTLSPrivateKeySecret }}
+          - "--tls-private-key-file=/tls/key/tls.key"
+          {{- end }}
+          {{- if or .Values.config.secureMetricsTLSCertSecret .Values.config.secureMetricsTLSPrivateKeySecret }}
+          volumeMounts:
+          {{- if .Values.config.secureMetricsTLSCertSecret }}
+          - mountPath: "/tls/cert"
+            name: tls-certificate
+            readOnly: true
+          {{- end }}
+          {{- if .Values.config.secureMetricsTLSPrivateKeySecret }}
+          - mountPath: "/tls/key"
+            name: tls-secret-key
+            readOnly: true
+          {{- end }}
+          {{- end }}
           ports:
           - containerPort: {{ $metricsPort }}
             protocol: TCP
@@ -140,6 +159,18 @@ spec:
             defaultMode: 420
             secretName: {{ include "eck-operator.webhookSecretName" . }}
         {{- end }}
+        {{- if .Values.config.secureMetricsTLSCertSecret }}
+        - name: tls-certificate
+          secret:
+            defaultMode: 420
+            secretName: {{ .Values.config.secureMetricsTLSCertSecret }}
+        {{- end }}
+        {{- if .Values.config.secureMetricsTLSPrivateKeySecret }}
+        - name: tls-secret-key
+          secret:
+            defaultMode: 420
+            secretName: {{ .Values.config.secureMetricsTLSPrivateKeySecret }}
+        {{- end }}
         {{- with .Values.volumes }}
           {{- toYaml . | nindent 8 }}
         {{- end }}
diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index 2947f842d2..2099f6a6b0 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -184,6 +184,14 @@ config:
   #       serviceMonitorSelectorNilUsesHelmValues: false
   enableSecureMetrics: false
 
+  # secureMetricsTLSCertSecret is the name of the secret containing the custom TLS certificate for the secure metrics endpoint.
+  # Key must be named tls.crt.
+  secureMetricsTLSCertSecret: ""
+
+  # secureMetricsTLSPrivateKeySecret is the name of the secret containing the custom TLS private key for the secure metrics endpoint.
+  # Key must be named tls.key.
+  secureMetricsTLSPrivateKeySecret: ""
+
   # containerRegistry to use for pulling Elasticsearch and other application container images.
   containerRegistry: docker.elastic.co
 
diff --git a/docs/advanced-topics/secure-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc
similarity index 70%
rename from docs/advanced-topics/secure-metrics.asciidoc
rename to docs/operating-eck/configure-operator-metrics.asciidoc
index 96a1ec8105..ed9a059a72 100644
--- a/docs/advanced-topics/secure-metrics.asciidoc
+++ b/docs/operating-eck/configure-operator-metrics.asciidoc
@@ -1,4 +1,4 @@
-:page_id: secure-metrics
+:page_id: configure-operator-metrics
 ifdef::env-github[]
 ****
 link:https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-{page_id}.html[View this document on the Elastic website]
@@ -6,7 +6,39 @@ link:https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-{page_id}.html[View
 endif::[]
 
 [id="{p}-{page_id}"]
-= Securing the ECK Operator metrics endpoint
+= Configuring the ECK Operator metrics endpoint
+
+== Enabling the ECK Operator metrics endpoint
+
+The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled. To enable the metrics endpoint simply adjust the `metricsPort` to a value greater than 0.
+
+== Using the ECK Operator Helm Chart
+
+If you install ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `metricsPort` to a value greater than 0 and the metrics endpoint will be enabled.
+
+== Using the ECK Operator manifests
+
+If you install ECK through using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable the metrics endpoint.
+
+* Enable the metrics endpoint in the `ConfigMap`.
+
+[source,sh]
+----
+kubectl get cm elastic-operator -n elastic-system -o yaml | sed "s|metrics-port: 0|metrics-port: 8081|" | kubectl apply -f -
+----
+
+* Restart the ECK Operator
+
+[source,sh]
+----
+kubectl delete pod -n elastic-system elastic-operator-0
+----
+
+== Securing the ECK Operator metrics endpoint
+
+NOTE: The ECK Operator metrics endpoint will be secured by default in future releases (which????)
+
+The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. To enable the metrics endpoint follow the previous instructions. To enable TLS/HTTPs and RBAC on the metrics endpoint, you can set the `enableSecureMetrics` option to `true` in the ECK Operator configuration. When enabled, the metrics endpoint is secured with RBAC and TLS/HTTPs.
 
 == Using the ECK Operator Helm Chart
 
@@ -177,3 +209,7 @@ prometheus:
     serviceMonitorNamespaceSelector: {}
     serviceMonitorSelectorNilUsesHelmValues: false
 ----
+
+== Bringing your own TLS certificate to secure the metrics endpoint
+
+TODO: 

From bd6f90db3a55c8e55a284c42f88554f49f84f5f8 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 27 Feb 2024 15:49:12 +0100
Subject: [PATCH 16/29] Add additional information to docs.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/templates/_helpers.tpl    |  13 ++
 .../templates/auth-proxy-service.yaml         |   4 +-
 .../eck-operator/templates/cluster-roles.yaml |   4 +-
 deploy/eck-operator/templates/configmap.yaml  |   6 +-
 .../templates/operator-network-policy.yaml    |   4 +-
 deploy/eck-operator/templates/podMonitor.yaml |   6 +-
 .../eck-operator/templates/role-bindings.yaml |   2 +-
 .../templates/serviceMonitor.yaml             |   2 +-
 .../eck-operator/templates/statefulset.yaml   |  27 ++-
 deploy/eck-operator/values.yaml               |  81 ++++----
 .../configure-operator-metrics.asciidoc       | 181 ++++++++++++++++--
 11 files changed, 251 insertions(+), 79 deletions(-)

diff --git a/deploy/eck-operator/templates/_helpers.tpl b/deploy/eck-operator/templates/_helpers.tpl
index 8c421f7b55..218f5742d0 100644
--- a/deploy/eck-operator/templates/_helpers.tpl
+++ b/deploy/eck-operator/templates/_helpers.tpl
@@ -114,6 +114,19 @@ elastic-webhook-server
 {{- end -}}
 {{- end -}}
 
+{{/*
+Determine the metrics port
+*/}}
+{{- define "eck-operator.metrics.port" -}}
+{{- if .Values.config.metrics.port -}}
+{{- .Values.config.metrics.port -}}
+{{- else if .Values.config.metricsPort -}}
+{{- .Values.config.metricsPort -}}
+{{- else -}}
+0
+{{- end -}}
+{{- end -}}
+
 {{/*
 RBAC permissions
 NOTE - any changes made to RBAC permissions below require
diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml
index 60bf99378b..81d1a5ee60 100644
--- a/deploy/eck-operator/templates/auth-proxy-service.yaml
+++ b/deploy/eck-operator/templates/auth-proxy-service.yaml
@@ -1,5 +1,5 @@
-{{- if .Values.config.enableSecureMetrics }}
-{{ $metricsPort := int .Values.config.metricsPort }}
+{{- if .Values.config.metrics.secureMode.enable }}
+{{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
 apiVersion: v1
 kind: Service
 metadata:
diff --git a/deploy/eck-operator/templates/cluster-roles.yaml b/deploy/eck-operator/templates/cluster-roles.yaml
index cb1d499255..13c3987f3f 100644
--- a/deploy/eck-operator/templates/cluster-roles.yaml
+++ b/deploy/eck-operator/templates/cluster-roles.yaml
@@ -1,4 +1,4 @@
-{{- if and (not .Values.createClusterScopedResources) (.Values.config.enableSecureMetrics) -}}
+{{- if and (not .Values.createClusterScopedResources) (.Values.config.metrics.secureMode.enable) -}}
 {{ fail "createClusterScopedResources is required to enable secure metrics" }}
 {{- end }}
 {{- if .Values.createClusterScopedResources -}}
@@ -96,7 +96,7 @@ rules:
   - apiGroups: ["logstash.k8s.elastic.co"]
     resources: ["logstashes"]
     verbs: ["create", "delete", "deletecollection", "patch", "update"]
-{{- if and .Values.config.enableSecureMetrics }}
+{{- if and .Values.config.metrics.secureMode.enable }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml
index f8a098d67c..eb56f878a1 100644
--- a/deploy/eck-operator/templates/configmap.yaml
+++ b/deploy/eck-operator/templates/configmap.yaml
@@ -8,12 +8,12 @@ metadata:
     {{- include "eck-operator.labels" . | nindent 4 }}
 data:
   eck.yaml: |-
+    {{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
     log-verbosity: {{ int .Values.config.logVerbosity }}
-    {{- if and .Values.config.enableSecureMetrics (eq (int .Values.config.metricsPort) 0) }}
+    {{- if and .Values.config.metrics.secureMode.enable (eq $metricsPort 0) }}
     {{- fail "metricsPort must be greater than 0 when enableSecureMetrics is true" }}
     {{- end }}
-    {{- $metricsPort := int .Values.config.metricsPort }}
-    {{- if .Values.config.enableSecureMetrics }}
+    {{- if .Values.config.metrics.secureMode.enable }}
     metrics-port: {{ add $metricsPort 1 }}
     {{- else }}
     metrics-port: {{ $metricsPort }}
diff --git a/deploy/eck-operator/templates/operator-network-policy.yaml b/deploy/eck-operator/templates/operator-network-policy.yaml
index 67fda92f42..9407f3227a 100644
--- a/deploy/eck-operator/templates/operator-network-policy.yaml
+++ b/deploy/eck-operator/templates/operator-network-policy.yaml
@@ -1,5 +1,6 @@
 {{- if .Values.softMultiTenancy.enabled -}}
 {{- $kubeAPIServerIP := (required "kubeAPIServerIP is required" .Values.kubeAPIServerIP) -}}
+{{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
@@ -39,8 +40,7 @@ spec:
           podSelector:
             matchLabels:
               common.k8s.elastic.co/type: "elasticsearch"
-{{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort ) 0) }}
-{{ $metricsPort = int .Values.config.metricsPort }}
+{{- if or .Values.webhook.enabled (gt $metricsPort 0) }}
   ingress:
 {{- if .Values.webhook.enabled }}
     - ports:
diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml
index 190ca87ca5..e445d2bd5d 100644
--- a/deploy/eck-operator/templates/podMonitor.yaml
+++ b/deploy/eck-operator/templates/podMonitor.yaml
@@ -1,7 +1,5 @@
-{{- if and .Values.podMonitor.enabled .Values.config.enableSecureMetrics}}
-{{ fail "podMonitor.enabled and enableSecureMetrics are mutually exclusive" }}
-{{- end }}
-{{- if and .Values.podMonitor.enabled (gt (int .Values.config.metricsPort) 0) }}
+{{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
+{{- if and .Values.podMonitor.enabled (gt $metricsPort 0) }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml
index 2cf8d7c202..fc155c7921 100644
--- a/deploy/eck-operator/templates/role-bindings.yaml
+++ b/deploy/eck-operator/templates/role-bindings.yaml
@@ -1,7 +1,7 @@
 {{- $operatorNSIsManaged := has .Release.Namespace .Values.managedNamespaces -}}
 {{- $fullName := include "eck-operator.fullname" . -}}
 {{- $svcAccount := include "eck-operator.serviceAccountName" . }}
-{{- $enableSecureMetrics := .Values.config.enableSecureMetrics -}}
+{{- $enableSecureMetrics := .Values.config.metrics.secureMode.enable -}}
 
 {{- if not .Values.createClusterScopedResources }}
 {{- range .Values.managedNamespaces }}
diff --git a/deploy/eck-operator/templates/serviceMonitor.yaml b/deploy/eck-operator/templates/serviceMonitor.yaml
index 7215a98801..7b5380b994 100644
--- a/deploy/eck-operator/templates/serviceMonitor.yaml
+++ b/deploy/eck-operator/templates/serviceMonitor.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.config.enableSecureMetrics }}
+{{- if and .Values.config.metrics.secureMode.enable }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index b8b40ac519..c151d57c35 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -1,4 +1,5 @@
 ---
+{{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -78,10 +79,9 @@ spec:
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          {{- if or .Values.webhook.enabled (gt (int .Values.config.metricsPort) 0) }}
-          {{ $metricsPort := int .Values.config.metricsPort }}
+          {{- if or .Values.webhook.enabled (gt $metricsPort 0) }}
           ports:
-            {{- if and (gt $metricsPort 0) (not .Values.config.enableSecureMetrics) }}
+            {{- if and (gt $metricsPort 0) (not .Values.config.metrics.secureMode.enable) }}
             - containerPort: {{ $metricsPort }}
               name: metrics
               protocol: TCP
@@ -104,8 +104,7 @@ spec:
             {{- with .Values.volumeMounts }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
-        {{- if .Values.config.enableSecureMetrics }}
-        {{ $metricsPort := int .Values.config.metricsPort }}
+        {{- if .Values.config.metrics.secureMode.enable }}
         - name: kube-rbac-proxy
           securityContext:
             allowPrivilegeEscalation: false
@@ -118,20 +117,20 @@ spec:
           - "--upstream=http://127.0.0.1:{{ add $metricsPort 1 }}/"
           - "--logtostderr=true"
           - "--v=0"
-          {{- if .Values.config.secureMetricsTLSCertSecret }}
+          {{- if .Values.config.metrics.secureMode.tlsCertSecret }}
           - "--tls-cert-file=/tls/cert/tls.crt"
           {{- end }}
-          {{- if .Values.config.secureMetricsTLSPrivateKeySecret }}
+          {{- if .Values.config.metrics.secureMode.tlsPrivateKeySecret }}
           - "--tls-private-key-file=/tls/key/tls.key"
           {{- end }}
-          {{- if or .Values.config.secureMetricsTLSCertSecret .Values.config.secureMetricsTLSPrivateKeySecret }}
+          {{- if or .Values.config.metrics.secureMode.tlsCertSecret .Values.config.metrics.secureMode.tlsPrivateKeySecret }}
           volumeMounts:
-          {{- if .Values.config.secureMetricsTLSCertSecret }}
+          {{- if .Values.config.metrics.secureMode.tlsCertSecret }}
           - mountPath: "/tls/cert"
             name: tls-certificate
             readOnly: true
           {{- end }}
-          {{- if .Values.config.secureMetricsTLSPrivateKeySecret }}
+          {{- if .Values.config.metrics.secureMode.tlsPrivateKeySecret }}
           - mountPath: "/tls/key"
             name: tls-secret-key
             readOnly: true
@@ -159,17 +158,17 @@ spec:
             defaultMode: 420
             secretName: {{ include "eck-operator.webhookSecretName" . }}
         {{- end }}
-        {{- if .Values.config.secureMetricsTLSCertSecret }}
+        {{- if .Values.config.metrics.secureMode.tlsCertSecret }}
         - name: tls-certificate
           secret:
             defaultMode: 420
-            secretName: {{ .Values.config.secureMetricsTLSCertSecret }}
+            secretName: {{ .Values.config.metrics.secureMode.tlsCertSecret  }}
         {{- end }}
-        {{- if .Values.config.secureMetricsTLSPrivateKeySecret }}
+        {{- if .Values.config.metrics.secureMode.tlsPrivateKeySecret  }}
         - name: tls-secret-key
           secret:
             defaultMode: 420
-            secretName: {{ .Values.config.secureMetricsTLSPrivateKeySecret }}
+            secretName: {{ .Values.config.metrics.secureMode.tlsPrivateKeySecret }}
         {{- end }}
         {{- with .Values.volumes }}
           {{- toYaml . | nindent 8 }}
diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index 2099f6a6b0..ee9aa76c4f 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -159,38 +159,55 @@ config:
   #  number greater than 0: Errors, warnings, information, and debug details.
   logVerbosity: "0"
 
-  # metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting. Use `metricsBindAddress` instead.
-  metricsPort: "0"
-
-  # enableSecureMetrics specifies whether to enable RBAC and TLS/HTTPs for the metrics endpoint. (Will be enabled by default in v2.14.0)
-  # * This option requires using a ServiceMonitor to scrape the metrics and as such is mutually exclusive with the podMonitor.enabled option.
-  # * This option also requires using cluster scoped resources (ClusterRole, ClusterRoleBinding) to
-  #   grant access to the /metrics endpoint. (createClusterScopedResources: true is required)
-  #
-  # This option requires the following settings within Prometheus to function:
-  # 1. RBAC settings for the Prometheus instance to access the metrics endpoint.
-  #
-  # - nonResourceURLs:
-  #   - /metrics
-  #   verbs:
-  #   - get
-  #
-  # 2. If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the operator you will need
-  #    the Prometheus Operator configured with the following Helm values:
-  #
-  #   prometheus:
-  #     prometheusSpec:
-  #       serviceMonitorNamespaceSelector: {}
-  #       serviceMonitorSelectorNilUsesHelmValues: false
-  enableSecureMetrics: false
-
-  # secureMetricsTLSCertSecret is the name of the secret containing the custom TLS certificate for the secure metrics endpoint.
-  # Key must be named tls.crt.
-  secureMetricsTLSCertSecret: ""
-
-  # secureMetricsTLSPrivateKeySecret is the name of the secret containing the custom TLS private key for the secure metrics endpoint.
-  # Key must be named tls.key.
-  secureMetricsTLSPrivateKeySecret: ""
+  # (Deprecated: use metrics.port: will be removed in v2.14.0) metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting.
+  metricsPort: 0
+
+  metrics:
+    # port defines the port to expose operator metrics. Set to 0 to disable metrics reporting.
+    port: "0"
+    # secureMode contains the options for enabling and configuring RBAC and TLS/HTTPs for the metrics endpoint.
+    secureMode:
+      # secureMode.enabled specifies whether to enable RBAC and TLS/HTTPs for the metrics endpoint. (Will be enabled by default in v2.14.0)
+      # * This option requires using a ServiceMonitor to scrape the metrics and as such is mutually exclusive with the podMonitor.enabled option.
+      # * This option also requires using cluster scoped resources (ClusterRole, ClusterRoleBinding) to
+      #   grant access to the /metrics endpoint. (createClusterScopedResources: true is required)
+      #
+      # This option requires the following settings within Prometheus to function:
+      # 1. RBAC settings for the Prometheus instance to access the metrics endpoint.
+      #
+      # - nonResourceURLs:
+      #   - /metrics
+      #   verbs:
+      #   - get
+      #
+      # 2. If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the operator you will need
+      #    the Prometheus Operator configured with the following Helm values:
+      #
+      #   prometheus:
+      #     prometheusSpec:
+      #       serviceMonitorNamespaceSelector: {}
+      #       serviceMonitorSelectorNilUsesHelmValues: false
+      enabled: false
+      # tlsCertSecret is the name of the secret containing the custom TLS certificate for the secure metrics endpoint.
+      #
+      # * This is an optional setting and is only required if you are using a custom TLS certificate. A self-signed certificate will be generated by default.
+      # * tlsConfig.insecureSkipVerify within the ServiceMonitor will be set to true even when this is set (this may change in the future).
+      # * A PEM-encoded CA certificate can be concatenated after the TLS certificate in the same secret.
+      # * Key must be named tls.crt.
+      # * Is assumed to be in the same namespace as the ServiceMonitor.
+      #
+      # example: kubectl create secret tls my-tls-secret -n elastic-system \
+      #            --cert=/path/to/tls.crt --key=/path/to/tls.key
+      tlsCertSecret: ""
+      # tlsPrivateKeySecret is the name of the secret containing the custom TLS private key for the secure metrics endpoint.
+      #
+      # * This is an optional setting and is only required if you have bundled the CA certificate with the TLS certificate.
+      # * Key must be named tls.key.
+      # * Is assumed to be in the same namespace as the ServiceMonitor.
+      #
+      # example: kubectl create secret tls my-tls-secret -n elastic-system \
+      #            --cert=/path/to/tls.crt --key=/path/to/tls.key
+      tlsPrivateKeySecret: ""
 
   # containerRegistry to use for pulling Elasticsearch and other application container images.
   containerRegistry: docker.elastic.co
diff --git a/docs/operating-eck/configure-operator-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc
index ed9a059a72..b2fa10d4ed 100644
--- a/docs/operating-eck/configure-operator-metrics.asciidoc
+++ b/docs/operating-eck/configure-operator-metrics.asciidoc
@@ -10,52 +10,116 @@ endif::[]
 
 == Enabling the ECK Operator metrics endpoint
 
-The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled. To enable the metrics endpoint simply adjust the `metricsPort` to a value greater than 0.
+The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled. To enable the metrics endpoint, follow the instructions in the following sections depending on whether you installed ECK through the Helm chart or the manifests.
 
-== Using the ECK Operator Helm Chart
+=== Using the ECK Operator Helm Chart
 
-If you install ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `metricsPort` to a value greater than 0 and the metrics endpoint will be enabled.
+If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set  `config.metrics.port` to a value greater than 0 in your values file and the metrics endpoint will be enabled.
 
-== Using the ECK Operator manifests
+=== Using the ECK Operator manifests
 
-If you install ECK through using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable the metrics endpoint.
+If you installed ECK using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable the metrics endpoint.
 
 * Enable the metrics endpoint in the `ConfigMap`.
 
 [source,sh]
 ----
-kubectl get cm elastic-operator -n elastic-system -o yaml | sed "s|metrics-port: 0|metrics-port: 8081|" | kubectl apply -f -
+kubectl get cm elastic-operator -n elastic-system -o yaml | sed "s|metrics-port: 0|metrics-port: 8080|" | kubectl apply -f -
 ----
 
-* Restart the ECK Operator
+* Patch the `StatefulSet` to expose the metrics port.
+
+[source,shell,subs="attributes,+macros"]
+----
+kubectl patch sts -n elastic-system elastic-operator --patch-file=/dev/stdin <<-EOF
+spec:
+  template:
+    spec:
+      containers:
+        - name: manager
+          ports:
+          - containerPort: 9443
+            name: https-webhook
+            protocol: TCP
+          - containerPort: 8080
+            protocol: TCP
+            name: metrics
+EOF
+----
+
+* Restart the ECK Operator.
 
 [source,sh]
 ----
 kubectl delete pod -n elastic-system elastic-operator-0
 ----
 
+* Install the `PodMonitor` to allow scraping of the metrics endpoint by Prometheus.
+
+[source,shell,subs="attributes,+macros"]
+----
+cat $$<<$$EOF | kubectl apply -f -
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: elastic-operator
+  namespace: elastic-system
+  labels:
+    control-plane: elastic-operator
+    app.kubernetes.io/component: metrics
+spec:
+  podMetricsEndpoints:
+    - port: metrics
+      path: /metrics
+      interval: 1m
+      scrapeTimeout: 30s
+  namespaceSelector:
+    matchNames:
+      - elastic-system
+  selector:
+    matchLabels:
+      control-plane: elastic-operator
+EOF
+----
+
 == Securing the ECK Operator metrics endpoint
 
-NOTE: The ECK Operator metrics endpoint will be secured by default in future releases (which????)
+NOTE: The ECK Operator metrics endpoint will be secured by default beginning in version 2.14.0
 
-The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. To enable the metrics endpoint follow the previous instructions. To enable TLS/HTTPs and RBAC on the metrics endpoint, you can set the `enableSecureMetrics` option to `true` in the ECK Operator configuration. When enabled, the metrics endpoint is secured with RBAC and TLS/HTTPs.
+The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. To enable the metrics endpoint follow the previous instructions. To enable TLS/HTTPs and RBAC on the metrics endpoint, follow the instructions in the following sections depending on whether you installed ECK through the Helm chart or the manifests.
 
-== Using the ECK Operator Helm Chart
+=== Using the ECK Operator Helm Chart
 
-If you install ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `enableSecureMetrics` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
+If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `config.metrics.secureMode.enable` to `true` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
 
-== Using the ECK Operator manifests
+==== Using your own TLS certificate for the metrics endpoint when using the Helm Chart
 
-If you install ECK through using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable secure metrics.
+By default a self-signed certificate will be generated for use by the metrics endpoint. If you want to use your own TLS certificate for the metrics endpoint you can provide the `config.metrics.secureMode.tlsCertSecret` to the Helm chart. The `tlsCertSecret` should be the name of a Kubernetes `Secret` that contains the TLS certificate. The same secret can also contain an optional TLS key, and if so provide the same secret name to `config.metrics.secureMode.tlsPrivateKeySecret`. The following keys are supported within the secret:
 
-* Enable the metrics endpoint in the `ConfigMap`.
+* `tls.crt` - The PEM-encoded TLS certificate
+* `tls.key` - (Optional) The PEM-encoded TLS private key
+
+Using your own TLS certificate will still enable `tlsConfig.insecureSkipVerify` within the `ServiceMonitor` to allow Prometheus to scrape the metrics endpoint. This setting may be revisited in the future.
+
+The easiest way to create this secret is to use the `kubectl create secret tls` command. For example:
+
+[source,sh]
+----
+kubectl create secret tls my-tls-secret -n elastic-system --cert=/Path/to/tls.crt --key=/Path/to/tls.key
+----
+
+=== Using the ECK Operator manifests
+
+If you installed ECK through using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable secure metrics.
+
+* Enable the metrics port in the `ConfigMap`, and set the metrics host to `127.0.0.1` to force communication through `kube-rbac-proxy`.
 
 [source,sh]
 ----
 kubectl get cm elastic-operator -n elastic-system -o yaml | sed "s|metrics-port: 0|metrics-port: 8081|" | sed "s|metrics-host: 0.0.0.0|metrics-host: 127.0.0.1|" | kubectl apply -f -
 ----
 
-* Add `ClusterRole` and `ClusterRoleBinding` for the ECK Operator.
+* Add an additional `ClusterRole` and `ClusterRoleBinding` for the ECK Operator.
 
 [source,shell,subs="attributes,+macros"]
 ----
@@ -117,7 +181,7 @@ spec:
 EOF
 ----
 
-* Patch the `StatefulSet`
+* Patch the `StatefulSet` to include a sidecar container for `kube-rbac-proxy` to secure the metrics endpoint.
 
 [source,shell,subs="attributes,+macros"]
 ----
@@ -180,12 +244,91 @@ spec:
     bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
 EOF
 ----
+==== Using your own TLS certificate for the metrics endpoint when using the manifests
+
+By default a self-signed certificate will be generated for use by the metrics endpoint. If you want to use your own TLS certificate for the metrics endpoint you will need to follow the previous instructions to enable secure metrics as well as the following steps:
+
+* Create a `Secret` containing the TLS certificate and optional key. The following keys are supported within the secret:
+
+  * `tls.crt` - The PEM-encoded TLS certificate
+  * `tls.key` - (Optional) The PEM-encoded TLS private key
+
+Using your own TLS certificate will still enable `tlsConfig.insecureSkipVerify` within the `ServiceMonitor` to allow Prometheus to scrape the metrics endpoint. This setting may be revisited in the future.
+
+The easiest way to create this secret is to use the `kubectl create secret tls` command. For example:
+
+[source,sh]
+----
+kubectl create secret tls my-tls-secret -n elastic-system --cert=/Path/to/tls.crt --key=/Path/to/tls.key
+----
+
+* Patch the `StatefulSet` to include the `tls.crt` and `tls.key` as a volume and mount it into the `kube-rbac-proxy` container.
+
+[source,shell,subs="attributes,+macros"]
+----
+kubectl patch sts -n elastic-system elastic-operator --patch-file=/dev/stdin <<-EOF
+spec:
+  template:
+    spec:
+      containers:
+        - name: kube-rbac-proxy
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "ALL"
+          image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0
+          args:
+          - "--secure-listen-address=0.0.0.0:8080"
+          - "--upstream=http://127.0.0.1:8081"
+          - "--logtostderr=true"
+          - "--v=0"
+          - "--tls-cert-file=/tls/cert/tls.crt"
+          # Optional
+          - "--tls-private-key-file=/tls/key/tls.key"
+          volumeMounts:
+          - mountPath: "/tls/cert"
+            name: tls-certificate
+            readOnly: true
+          # Optional
+          - mountPath: "/tls/key"
+            name: tls-secret-key
+            readOnly: true
+          ports:
+          - containerPort: 8080
+            protocol: TCP
+            name: metrics
+          resources:
+            limits:
+              cpu: 500m
+              memory: 128Mi
+            requests:
+              cpu: 5m
+              memory: 64Mi
+      volumes:
+      - name: conf
+        configMap:
+          name: elastic-operator
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: elastic-webhook-server-cert
+      - name: tls-certificate
+        secret:
+          defaultMode: 420
+          secretName: my-tls-secret
+      - name: tls-secret-key
+        secret:
+          defaultMode: 420
+          secretName: my-tls-secret
+EOF
+----
 
 == Prometheus requirements
 
 This option requires the following settings within Prometheus to function:
 
-1. RBAC settings for the Prometheus instance to access the metrics endpoint similar to the following:
+* RBAC settings for the Prometheus instance to access the metrics endpoint similar to the following:
 
 [source,yaml,subs="attributes"]
 ----
@@ -200,12 +343,14 @@ rules:
   - get
 ----
 
-NOTE: If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the operator you will need the Prometheus Operator configured with the following Helm values:
+* If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the operator you will need the Prometheus Operator configured with the following Helm values:
 
 [source,yaml,subs="attributes"]
 ----
 prometheus:
   prometheusSpec:
+    podMonitorNamespaceSelector: {}
+    podMonitorSelectorNilUsesHelmValues: false
     serviceMonitorNamespaceSelector: {}
     serviceMonitorSelectorNilUsesHelmValues: false
 ----

From 566f8f0701d3959d11960b353509f995151af033 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 27 Feb 2024 16:16:15 +0100
Subject: [PATCH 17/29] Add preceeding '-'

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/templates/operator-network-policy.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/eck-operator/templates/operator-network-policy.yaml b/deploy/eck-operator/templates/operator-network-policy.yaml
index 9407f3227a..ad74156d50 100644
--- a/deploy/eck-operator/templates/operator-network-policy.yaml
+++ b/deploy/eck-operator/templates/operator-network-policy.yaml
@@ -1,6 +1,6 @@
 {{- if .Values.softMultiTenancy.enabled -}}
 {{- $kubeAPIServerIP := (required "kubeAPIServerIP is required" .Values.kubeAPIServerIP) -}}
-{{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
+{{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy

From e45f691dfb489de4509a07683b681ec3c9ab318a Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 27 Feb 2024 16:17:16 +0100
Subject: [PATCH 18/29] Fix all metricsports missing '-'

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/templates/auth-proxy-service.yaml | 2 +-
 deploy/eck-operator/templates/configmap.yaml          | 2 +-
 deploy/eck-operator/templates/podMonitor.yaml         | 2 +-
 deploy/eck-operator/templates/statefulset.yaml        | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml
index 81d1a5ee60..ccedc32732 100644
--- a/deploy/eck-operator/templates/auth-proxy-service.yaml
+++ b/deploy/eck-operator/templates/auth-proxy-service.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.config.metrics.secureMode.enable }}
-{{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
+{{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
 apiVersion: v1
 kind: Service
 metadata:
diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml
index eb56f878a1..02f3908bf6 100644
--- a/deploy/eck-operator/templates/configmap.yaml
+++ b/deploy/eck-operator/templates/configmap.yaml
@@ -8,7 +8,7 @@ metadata:
     {{- include "eck-operator.labels" . | nindent 4 }}
 data:
   eck.yaml: |-
-    {{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
+    {{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
     log-verbosity: {{ int .Values.config.logVerbosity }}
     {{- if and .Values.config.metrics.secureMode.enable (eq $metricsPort 0) }}
     {{- fail "metricsPort must be greater than 0 when enableSecureMetrics is true" }}
diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml
index e445d2bd5d..792ac948fb 100644
--- a/deploy/eck-operator/templates/podMonitor.yaml
+++ b/deploy/eck-operator/templates/podMonitor.yaml
@@ -1,4 +1,4 @@
-{{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
+{{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
 {{- if and .Values.podMonitor.enabled (gt $metricsPort 0) }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index c151d57c35..1b1b3acea0 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -1,5 +1,5 @@
 ---
-{{ $metricsPort := int (include "eck-operator.metrics.port" .)}}
+{{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:

From 7c41f51bc6556d85d29779221b4c2537626f4b9b Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 27 Feb 2024 17:12:23 +0100
Subject: [PATCH 19/29] securemode.enabled, not enable. Fix having nil
 pod/servicemonitor causing crash.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/templates/auth-proxy-service.yaml | 2 +-
 deploy/eck-operator/templates/cluster-roles.yaml      | 4 ++--
 deploy/eck-operator/templates/configmap.yaml          | 4 ++--
 deploy/eck-operator/templates/podMonitor.yaml         | 2 +-
 deploy/eck-operator/templates/role-bindings.yaml      | 2 +-
 deploy/eck-operator/templates/serviceMonitor.yaml     | 4 ++--
 deploy/eck-operator/templates/statefulset.yaml        | 4 ++--
 7 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml
index ccedc32732..a7030fbbd0 100644
--- a/deploy/eck-operator/templates/auth-proxy-service.yaml
+++ b/deploy/eck-operator/templates/auth-proxy-service.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.config.metrics.secureMode.enable }}
+{{- if .Values.config.metrics.secureMode.enabled }}
 {{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
 apiVersion: v1
 kind: Service
diff --git a/deploy/eck-operator/templates/cluster-roles.yaml b/deploy/eck-operator/templates/cluster-roles.yaml
index 13c3987f3f..085fc335a6 100644
--- a/deploy/eck-operator/templates/cluster-roles.yaml
+++ b/deploy/eck-operator/templates/cluster-roles.yaml
@@ -1,4 +1,4 @@
-{{- if and (not .Values.createClusterScopedResources) (.Values.config.metrics.secureMode.enable) -}}
+{{- if and (not .Values.createClusterScopedResources) (.Values.config.metrics.secureMode.enabled) -}}
 {{ fail "createClusterScopedResources is required to enable secure metrics" }}
 {{- end }}
 {{- if .Values.createClusterScopedResources -}}
@@ -96,7 +96,7 @@ rules:
   - apiGroups: ["logstash.k8s.elastic.co"]
     resources: ["logstashes"]
     verbs: ["create", "delete", "deletecollection", "patch", "update"]
-{{- if and .Values.config.metrics.secureMode.enable }}
+{{- if and .Values.config.metrics.secureMode.enabled }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml
index 02f3908bf6..a6d581b9b3 100644
--- a/deploy/eck-operator/templates/configmap.yaml
+++ b/deploy/eck-operator/templates/configmap.yaml
@@ -10,10 +10,10 @@ data:
   eck.yaml: |-
     {{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
     log-verbosity: {{ int .Values.config.logVerbosity }}
-    {{- if and .Values.config.metrics.secureMode.enable (eq $metricsPort 0) }}
+    {{- if and .Values.config.metrics.secureMode.enabled (eq $metricsPort 0) }}
     {{- fail "metricsPort must be greater than 0 when enableSecureMetrics is true" }}
     {{- end }}
-    {{- if .Values.config.metrics.secureMode.enable }}
+    {{- if .Values.config.metrics.secureMode.enabled }}
     metrics-port: {{ add $metricsPort 1 }}
     {{- else }}
     metrics-port: {{ $metricsPort }}
diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml
index 792ac948fb..e7368a754a 100644
--- a/deploy/eck-operator/templates/podMonitor.yaml
+++ b/deploy/eck-operator/templates/podMonitor.yaml
@@ -4,7 +4,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
   name: {{ include "eck-operator.fullname" . }}
-  namespace: {{ ternary .Values.podMonitor.namespace .Release.Namespace (not (empty .Values.podMonitor.namespace)) }}
+  namespace: {{ ternary .Values.podMonitor.namespace .Release.Namespace (not (and (.Values.podMonitor) (empty .Values.podMonitor.namespace))) }}
   labels: {{- include "eck-operator.labels" . | nindent 4 }}
   {{- with .Values.podMonitor.labels }}
     {{- toYaml . | nindent 4 }}
diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml
index fc155c7921..4b57a3f008 100644
--- a/deploy/eck-operator/templates/role-bindings.yaml
+++ b/deploy/eck-operator/templates/role-bindings.yaml
@@ -1,7 +1,7 @@
 {{- $operatorNSIsManaged := has .Release.Namespace .Values.managedNamespaces -}}
 {{- $fullName := include "eck-operator.fullname" . -}}
 {{- $svcAccount := include "eck-operator.serviceAccountName" . }}
-{{- $enableSecureMetrics := .Values.config.metrics.secureMode.enable -}}
+{{- $enableSecureMetrics := .Values.config.metrics.secureMode.enabled -}}
 
 {{- if not .Values.createClusterScopedResources }}
 {{- range .Values.managedNamespaces }}
diff --git a/deploy/eck-operator/templates/serviceMonitor.yaml b/deploy/eck-operator/templates/serviceMonitor.yaml
index 7b5380b994..1d968fd4ac 100644
--- a/deploy/eck-operator/templates/serviceMonitor.yaml
+++ b/deploy/eck-operator/templates/serviceMonitor.yaml
@@ -1,9 +1,9 @@
-{{- if and .Values.config.metrics.secureMode.enable }}
+{{- if and .Values.config.metrics.secureMode.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: {{ include "eck-operator.fullname" . }}
-  namespace: {{ ternary .Values.serviceMonitor.namespace .Release.Namespace (not (empty .Values.serviceMonitor.namespace)) }}
+  namespace: {{ ternary .Values.serviceMonitor.namespace .Release.Namespace (not (and (.Values.serviceMonitor) (empty .Values.serviceMonitor.namespace))) }}
   labels: {{- include "eck-operator.labels" . | nindent 4 }}
 spec:
   namespaceSelector:
diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index 1b1b3acea0..b689248f02 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -81,7 +81,7 @@ spec:
           {{- end }}
           {{- if or .Values.webhook.enabled (gt $metricsPort 0) }}
           ports:
-            {{- if and (gt $metricsPort 0) (not .Values.config.metrics.secureMode.enable) }}
+            {{- if and (gt $metricsPort 0) (not .Values.config.metrics.secureMode.enabled) }}
             - containerPort: {{ $metricsPort }}
               name: metrics
               protocol: TCP
@@ -104,7 +104,7 @@ spec:
             {{- with .Values.volumeMounts }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
-        {{- if .Values.config.metrics.secureMode.enable }}
+        {{- if .Values.config.metrics.secureMode.enabled }}
         - name: kube-rbac-proxy
           securityContext:
             allowPrivilegeEscalation: false

From 1f1bacf8afa8b2771b3f7bb329b06d29e56349dd Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 27 Feb 2024 17:14:54 +0100
Subject: [PATCH 20/29] remove todo item.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 docs/operating-eck/configure-operator-metrics.asciidoc | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/operating-eck/configure-operator-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc
index b2fa10d4ed..46cf741b69 100644
--- a/docs/operating-eck/configure-operator-metrics.asciidoc
+++ b/docs/operating-eck/configure-operator-metrics.asciidoc
@@ -354,7 +354,3 @@ prometheus:
     serviceMonitorNamespaceSelector: {}
     serviceMonitorSelectorNilUsesHelmValues: false
 ----
-
-== Bringing your own TLS certificate to secure the metrics endpoint
-
-TODO: 

From 4348ce813a91cca77fca52973e4ee542e760eb4b Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 27 Feb 2024 17:21:37 +0100
Subject: [PATCH 21/29] remove metrics-host from helm values, as it's not
 needed.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 config/eck.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/config/eck.yaml b/config/eck.yaml
index 1d0f19e4c3..549435a4ed 100644
--- a/config/eck.yaml
+++ b/config/eck.yaml
@@ -1,6 +1,5 @@
 log-verbosity: 0
 metrics-port: 0
-metrics-host: 0.0.0.0
 container-registry: docker.elastic.co
 max-concurrent-reconciles: 3
 ca-cert-validity: 8760h

From d8622230bb719d19cec819062fddaef8aba22c99 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 27 Feb 2024 20:27:48 +0100
Subject: [PATCH 22/29] remove secure-metrics link. Add configure operator
 metrics link.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 docs/advanced-topics/advanced-topics.asciidoc | 2 --
 docs/operating-eck/operating-eck.asciidoc     | 2 ++
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/advanced-topics/advanced-topics.asciidoc b/docs/advanced-topics/advanced-topics.asciidoc
index d4d64727b6..4128581220 100644
--- a/docs/advanced-topics/advanced-topics.asciidoc
+++ b/docs/advanced-topics/advanced-topics.asciidoc
@@ -18,7 +18,6 @@ endif::[]
 - <<{p}-webhook-namespace-selectors>>
 - <<{p}-stack-monitoring>>
 - <<{p}-fips>>
-- <<{p}-secure-metrics>>
 --
 
 include::openshift.asciidoc[leveloffset=+1]
@@ -30,4 +29,3 @@ include::network-policies.asciidoc[leveloffset=+1]
 include::webhook-namespace-selectors.asciidoc[leveloffset=+1]
 include::stack-monitoring.asciidoc[leveloffset=+1]
 include::fips.asciidoc[leveloffset=+1]
-include::secure-metrics.asciidoc[leveloffset=+1]
diff --git a/docs/operating-eck/operating-eck.asciidoc b/docs/operating-eck/operating-eck.asciidoc
index 1b6ac63edb..397e37434f 100644
--- a/docs/operating-eck/operating-eck.asciidoc
+++ b/docs/operating-eck/operating-eck.asciidoc
@@ -10,6 +10,7 @@ endif::[]
 [partintro]
 --
 - <<{p}-operator-config>>
+- <<{p}-configure-operator-metrics>>
 - <<{p}-eck-permissions>>
 - <<{p}-webhook>>
 - <<{p}-restrict-cross-namespace-associations>>
@@ -22,6 +23,7 @@ endif::[]
 --
 
 include::operator-config.asciidoc[leveloffset=+1]
+include::configure-operator-metrics.asciidoc[leveloffset=+1]
 include::eck-permissions.asciidoc[leveloffset=+1]
 include::webhook.asciidoc[leveloffset=+1]
 include::restrict-cross-namespace-associations.asciidoc[leveloffset=+1]

From 11c17427e99c2a68f3950409daa65c41a6e08155 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Fri, 1 Mar 2024 12:42:31 +0100
Subject: [PATCH 23/29] Updated documentation with full instructions on using
 custom tls certificate.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 cmd/manager/main.go                           |  2 +-
 .../templates/auth-proxy-service.yaml         |  2 +-
 deploy/eck-operator/templates/podMonitor.yaml |  3 +
 .../templates/serviceMonitor.yaml             |  9 +-
 .../eck-operator/templates/statefulset.yaml   | 29 ++----
 deploy/eck-operator/values.yaml               | 52 ++++++----
 .../configure-operator-metrics.asciidoc       | 98 +++++++++++++------
 7 files changed, 122 insertions(+), 73 deletions(-)

diff --git a/cmd/manager/main.go b/cmd/manager/main.go
index 1ec2cc8f64..b79e667b16 100644
--- a/cmd/manager/main.go
+++ b/cmd/manager/main.go
@@ -286,7 +286,7 @@ func Command() *cobra.Command {
 	cmd.Flags().Int(
 		operator.MetricsPortFlag,
 		DefaultMetricPort,
-		"(Deprecated) Port to use for exposing metrics in the Prometheus format. (set 0 to disable. Use --metrics-bind-address instead)",
+		"Port to use for exposing metrics in the Prometheus format. (set 0 to disable)",
 	)
 	cmd.Flags().String(
 		operator.MetricsHostFlag,
diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml
index a7030fbbd0..53bdc02b75 100644
--- a/deploy/eck-operator/templates/auth-proxy-service.yaml
+++ b/deploy/eck-operator/templates/auth-proxy-service.yaml
@@ -9,7 +9,7 @@ metadata:
     app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
     helm.sh/chart: {{ include "eck-operator.chart" . }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
-  name: "{{ include "eck-operator.fullname" . }}-metrics-service"
+  name: "{{ include "eck-operator.fullname" . }}-metrics"
   namespace: {{ .Release.Namespace }}
 spec:
   ports:
diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml
index e7368a754a..f643ea0eaf 100644
--- a/deploy/eck-operator/templates/podMonitor.yaml
+++ b/deploy/eck-operator/templates/podMonitor.yaml
@@ -1,5 +1,8 @@
 {{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
 {{- if and .Values.podMonitor.enabled (gt $metricsPort 0) }}
+{{- if and .Values.podMonitor.enabled .Values.config.metrics.secureMode.enabled }}
+{{- fail "podMonitor and secureMode are mutually exclusive" }}
+{{- end }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
diff --git a/deploy/eck-operator/templates/serviceMonitor.yaml b/deploy/eck-operator/templates/serviceMonitor.yaml
index 1d968fd4ac..8d4c4709b3 100644
--- a/deploy/eck-operator/templates/serviceMonitor.yaml
+++ b/deploy/eck-operator/templates/serviceMonitor.yaml
@@ -19,6 +19,13 @@ spec:
     scheme: https
     interval: 30s
     tlsConfig:
-      insecureSkipVerify: true
+      insecureSkipVerify: {{ .Values.config.metrics.secureMode.tls.insecureSkipVerify | default false }}
+      {{- if (not .Values.config.metrics.secureMode.tls.insecureSkipVerify) }}
+      {{- with .Values.config.metrics.secureMode.tls.caSecret }}
+      {{- $leading_path := trimSuffix "/" .Values.config.metrics.secureMode.tls.caMountDirectory }}
+      caFile: "{{ $leading_path }}/{{ . }}/ca.crt"
+      {{- end }}
+      serverName: "{{ include "eck-operator.fullname" . }}-metrics.{{ .Release.Namespace }}.svc"
+      {{- end }}
     bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
 {{- end }}
diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml
index b689248f02..7f6a8f716b 100644
--- a/deploy/eck-operator/templates/statefulset.yaml
+++ b/deploy/eck-operator/templates/statefulset.yaml
@@ -117,25 +117,16 @@ spec:
           - "--upstream=http://127.0.0.1:{{ add $metricsPort 1 }}/"
           - "--logtostderr=true"
           - "--v=0"
-          {{- if .Values.config.metrics.secureMode.tlsCertSecret }}
-          - "--tls-cert-file=/tls/cert/tls.crt"
+          {{- if .Values.config.metrics.secureMode.tls.certificateSecret }}
+          - "--tls-cert-file=/tls/tls.crt"
+          - "--tls-private-key-file=/tls/tls.key"
           {{- end }}
-          {{- if .Values.config.metrics.secureMode.tlsPrivateKeySecret }}
-          - "--tls-private-key-file=/tls/key/tls.key"
-          {{- end }}
-          {{- if or .Values.config.metrics.secureMode.tlsCertSecret .Values.config.metrics.secureMode.tlsPrivateKeySecret }}
+          {{- if .Values.config.metrics.secureMode.tls.certificateSecret }}
           volumeMounts:
-          {{- if .Values.config.metrics.secureMode.tlsCertSecret }}
-          - mountPath: "/tls/cert"
+          - mountPath: "/tls"
             name: tls-certificate
             readOnly: true
           {{- end }}
-          {{- if .Values.config.metrics.secureMode.tlsPrivateKeySecret }}
-          - mountPath: "/tls/key"
-            name: tls-secret-key
-            readOnly: true
-          {{- end }}
-          {{- end }}
           ports:
           - containerPort: {{ $metricsPort }}
             protocol: TCP
@@ -158,17 +149,11 @@ spec:
             defaultMode: 420
             secretName: {{ include "eck-operator.webhookSecretName" . }}
         {{- end }}
-        {{- if .Values.config.metrics.secureMode.tlsCertSecret }}
+        {{- if .Values.config.metrics.secureMode.tls.certificateSecret }}
         - name: tls-certificate
           secret:
             defaultMode: 420
-            secretName: {{ .Values.config.metrics.secureMode.tlsCertSecret  }}
-        {{- end }}
-        {{- if .Values.config.metrics.secureMode.tlsPrivateKeySecret  }}
-        - name: tls-secret-key
-          secret:
-            defaultMode: 420
-            secretName: {{ .Values.config.metrics.secureMode.tlsPrivateKeySecret }}
+            secretName: {{ .Values.config.metrics.secureMode.tls.certificateSecret  }}
         {{- end }}
         {{- with .Values.volumes }}
           {{- toYaml . | nindent 8 }}
diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index ee9aa76c4f..d8c7f3d95a 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -188,26 +188,38 @@ config:
       #       serviceMonitorNamespaceSelector: {}
       #       serviceMonitorSelectorNilUsesHelmValues: false
       enabled: false
-      # tlsCertSecret is the name of the secret containing the custom TLS certificate for the secure metrics endpoint.
-      #
-      # * This is an optional setting and is only required if you are using a custom TLS certificate. A self-signed certificate will be generated by default.
-      # * tlsConfig.insecureSkipVerify within the ServiceMonitor will be set to true even when this is set (this may change in the future).
-      # * A PEM-encoded CA certificate can be concatenated after the TLS certificate in the same secret.
-      # * Key must be named tls.crt.
-      # * Is assumed to be in the same namespace as the ServiceMonitor.
-      #
-      # example: kubectl create secret tls my-tls-secret -n elastic-system \
-      #            --cert=/path/to/tls.crt --key=/path/to/tls.key
-      tlsCertSecret: ""
-      # tlsPrivateKeySecret is the name of the secret containing the custom TLS private key for the secure metrics endpoint.
-      #
-      # * This is an optional setting and is only required if you have bundled the CA certificate with the TLS certificate.
-      # * Key must be named tls.key.
-      # * Is assumed to be in the same namespace as the ServiceMonitor.
-      #
-      # example: kubectl create secret tls my-tls-secret -n elastic-system \
-      #            --cert=/path/to/tls.crt --key=/path/to/tls.key
-      tlsPrivateKeySecret: ""
+      tls:
+        # certificateSecret is the name of the tls secret containing the custom TLS certificate and key for the secure metrics endpoint.
+        #
+        # * This is an optional setting and is only required if you are using a custom TLS certificate. A self-signed certificate will be generated by default.
+        # * TLS secret key must be named tls.crt.
+        # * TLS key's secret key must be named tls.key.
+        # * It is assumed to be in the same namespace as the ServiceMonitor.
+        #
+        # example: kubectl create secret tls eck-metrics-tls-certificate -n elastic-system \
+        #            --cert=/path/to/tls.crt --key=/path/to/tls.key
+        certificateSecret: ""
+        # caSecret is the name of the secret containing the custom CA certificate used to generate the custom TLS certificate for the secure metrics endpoint.
+        #
+        # * This *must* be the name of the secret containing the CA certificate used to sign the custom TLS certificate.
+        # * This secret *must* be in the same namespace as the Prometheus instance that will scrape the metrics.
+        # * If using the Prometheus operator this secret must be within the `spec.secrets` field of the `Prometheus` custom resource such that it is mounted into the Prometheus pod at `caMountDirectory`, which defaults to /etc/prometheus/secrets/{secret-name}.
+        # * This is an optional setting and is only required if you are using a custom TLS certificate.
+        # * Key must be named ca.crt.
+        #
+        # example: kubectl create secret generic eck-metrics-tls-ca -n monitoring \
+        #            --from-file=ca.crt=/path/to/ca.pem
+        caSecret: ""
+        # caMountDirectory is the directory at which the CA certificate is mounted within the Prometheus pod.
+        #
+        # * You should only need to adjust this if you are *not* using the Prometheus operator.
+        caMountDirectory: "/etc/prometheus/secrets/"
+        # insecureSkipVerify specifies whether to skip verification of the TLS certificate for the secure metrics endpoint.
+        #
+        # * If this setting is set to false, then the following settings are required:
+        #   - certificateSecret
+        #   - caSecret
+        insecureSkipVerify: true
 
   # containerRegistry to use for pulling Elasticsearch and other application container images.
   containerRegistry: docker.elastic.co
diff --git a/docs/operating-eck/configure-operator-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc
index 46cf741b69..fb77ac0eb1 100644
--- a/docs/operating-eck/configure-operator-metrics.asciidoc
+++ b/docs/operating-eck/configure-operator-metrics.asciidoc
@@ -10,7 +10,7 @@ endif::[]
 
 == Enabling the ECK Operator metrics endpoint
 
-The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled. To enable the metrics endpoint, follow the instructions in the following sections depending on whether you installed ECK through the Helm chart or the manifests.
+The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled. To enable the metrics endpoint, follow the instructions in the next sections depending on whether you installed ECK through the Helm chart or the manifests.
 
 === Using the ECK Operator Helm Chart
 
@@ -90,24 +90,33 @@ The ECK Operator provides a metrics endpoint that can be used to monitor the ope
 
 === Using the ECK Operator Helm Chart
 
-If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `config.metrics.secureMode.enable` to `true` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
+If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `config.metrics.secureMode.enabled` to `true` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
 
 ==== Using your own TLS certificate for the metrics endpoint when using the Helm Chart
 
-By default a self-signed certificate will be generated for use by the metrics endpoint. If you want to use your own TLS certificate for the metrics endpoint you can provide the `config.metrics.secureMode.tlsCertSecret` to the Helm chart. The `tlsCertSecret` should be the name of a Kubernetes `Secret` that contains the TLS certificate. The same secret can also contain an optional TLS key, and if so provide the same secret name to `config.metrics.secureMode.tlsPrivateKeySecret`. The following keys are supported within the secret:
+By default a self-signed certificate will be generated for use by the metrics endpoint. If you want to use your own TLS certificate for the metrics endpoint you can provide the `config.metrics.secureMode.tls.certificateSecret` to the Helm chart. The `certificateSecret` should be the name of an existing Kubernetes `Secret` that contains both the TLS certificate and the TLS private key. The following keys are supported within the secret:
 
 * `tls.crt` - The PEM-encoded TLS certificate
-* `tls.key` - (Optional) The PEM-encoded TLS private key
-
-Using your own TLS certificate will still enable `tlsConfig.insecureSkipVerify` within the `ServiceMonitor` to allow Prometheus to scrape the metrics endpoint. This setting may be revisited in the future.
+* `tls.key` - The PEM-encoded TLS private key
 
 The easiest way to create this secret is to use the `kubectl create secret tls` command. For example:
 
 [source,sh]
 ----
-kubectl create secret tls my-tls-secret -n elastic-system --cert=/Path/to/tls.crt --key=/Path/to/tls.key
+kubectl create secret tls eck-metrics-tls-certificate -n elastic-system --cert=/path/to/tls.crt --key=/path/to/tls.key
 ----
 
+Providing this secret is sufficient to use your own certificate if it is from a trusted Certificate Authority. If the certificate is not signed by a trusted CA you have 2 options:
+
+* Disable TLS verification.
+    * Set `config.metrics.secureMode.tls.insecureSkipVerify` to `true` to disable TLS validation.
+* Provide the Certificate Authority to Prometheus.
+    * Set `config.metrics.secureMode.tls.insecureSkipVerify` to `false` to enable TLS validation.
+    * Set `config.metrics.secureMode.tls.caSecret` to the name of an existing Kubernetes secret within the Prometheus namespace that contains the CA in PEM format.
+    * Set the `spec.secrets` field of the `Prometheus` custom resource such that the CA secret is mounted into the Prometheus pod at `config.metrics.secureMode.tls.caMountDirectory` (assuming you are using the Prometheus operator). See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm Chart values file] for more information.
+
+See the <<{p}-prometheus-requirements,prometheus requirements section>> for more information on creating the CA secret.
+
 === Using the ECK Operator manifests
 
 If you installed ECK through using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable secure metrics.
@@ -244,22 +253,21 @@ spec:
     bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
 EOF
 ----
+
 ==== Using your own TLS certificate for the metrics endpoint when using the manifests
 
 By default a self-signed certificate will be generated for use by the metrics endpoint. If you want to use your own TLS certificate for the metrics endpoint you will need to follow the previous instructions to enable secure metrics as well as the following steps:
 
-* Create a `Secret` containing the TLS certificate and optional key. The following keys are supported within the secret:
+* Create a `Secret` containing the TLS certificate and TLS private key. The following keys are supported within the secret:
 
   * `tls.crt` - The PEM-encoded TLS certificate
-  * `tls.key` - (Optional) The PEM-encoded TLS private key
-
-Using your own TLS certificate will still enable `tlsConfig.insecureSkipVerify` within the `ServiceMonitor` to allow Prometheus to scrape the metrics endpoint. This setting may be revisited in the future.
+  * `tls.key` - The PEM-encoded TLS private key
 
 The easiest way to create this secret is to use the `kubectl create secret tls` command. For example:
 
 [source,sh]
 ----
-kubectl create secret tls my-tls-secret -n elastic-system --cert=/Path/to/tls.crt --key=/Path/to/tls.key
+kubectl create secret tls my-tls-secret -n elastic-system --cert=/path/to/tls.crt --key=/path/to/tls.key
 ----
 
 * Patch the `StatefulSet` to include the `tls.crt` and `tls.key` as a volume and mount it into the `kube-rbac-proxy` container.
@@ -283,17 +291,12 @@ spec:
           - "--upstream=http://127.0.0.1:8081"
           - "--logtostderr=true"
           - "--v=0"
-          - "--tls-cert-file=/tls/cert/tls.crt"
-          # Optional
-          - "--tls-private-key-file=/tls/key/tls.key"
+          - "--tls-cert-file=/tls/tls.crt"
+          - "--tls-private-key-file=/tls/tls.key"
           volumeMounts:
-          - mountPath: "/tls/cert"
+          - mountPath: "/tls"
             name: tls-certificate
             readOnly: true
-          # Optional
-          - mountPath: "/tls/key"
-            name: tls-secret-key
-            readOnly: true
           ports:
           - containerPort: 8080
             protocol: TCP
@@ -316,19 +319,39 @@ spec:
       - name: tls-certificate
         secret:
           defaultMode: 420
-          secretName: my-tls-secret
-      - name: tls-secret-key
-        secret:
-          defaultMode: 420
-          secretName: my-tls-secret
+          secretName: eck-metrics-tls-certificate
 EOF
 ----
 
+* Potentially patch the `ServiceMonitor`. This will only need to be done if you are adjusting the `insecureSkipVerify` field to `false`.
+
+[source,shell,subs="attributes,+macros,callouts"]
+----
+kubectl patch servicemonitor -n elastic-system elastic-operator --patch-file=/dev/stdin <<-EOF
+spec:
+  endpoints:
+  - port: https
+    path: /metrics
+    scheme: https
+    interval: 30s
+    tlsConfig:
+      insecureSkipVerify: false
+      caFile: /etc/prometheus/secrets/{secret-name}/ca.crt <1>
+      serverName: elastic-operator-metrics.elastic-system.svc
+    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+EOF
+----
+
+<1> See the <<{p}-prometheus-requirements,prometheus requirements section>> for more information on creating the CA secret.
+
+[id="{p}-prometheus-requirements"]
 == Prometheus requirements
 
-This option requires the following settings within Prometheus to function:
+The previous options requires the following settings within Prometheus to function properly:
 
-* RBAC settings for the Prometheus instance to access the metrics endpoint similar to the following:
+=== RBAC settings for scraping the metrics
+
+Configure the RBAC settings for the Prometheus instance to access the metrics endpoint similar to the following: (These typically will be set automatically when using the Prometheus operator)
 
 [source,yaml,subs="attributes"]
 ----
@@ -343,7 +366,9 @@ rules:
   - get
 ----
 
-* If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the operator you will need the Prometheus Operator configured with the following Helm values:
+=== Optional Prometheus Operator Helm settings to allow reading PodMonitor and ServiceMonitor across namespaces
+
+* If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the ECK operator you will need the Prometheus Operator configured with the following Helm values:
 
 [source,yaml,subs="attributes"]
 ----
@@ -354,3 +379,20 @@ prometheus:
     serviceMonitorNamespaceSelector: {}
     serviceMonitorSelectorNilUsesHelmValues: false
 ----
+
+=== Optional settings to allow full TLS verification when using a custom TLS certificate
+
+If you are using a custom TLS certificate and you need to set `insecureSkipVerify` to `false` you will need to do the following:
+
+* Create a Kubernetes secret within the Prometheus namespace that contains the Certificate Authority in PEM format.
+
+The easiest way to create the CA secret within the Prometheus namespace is to use the `kubectl create secret generic` command. For example:
+
+[source,sh]
+----
+kubectl create secret generic eck-metrics-tls-ca -n monitoring --from-file=ca.crt=/path/to/ca.pem
+----
+
+* Ensure that the CA secret is mounted within the Prometheus Pod.
+
+This will vary between Prometheus installations, but if using the Prometheus Operator you can set the `spec.secrets` field of the `Prometheus` custom resource to the name of the previously created Kubernetes Secret. See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm Chart values file] for more information.

From 144e6f197307dfff9eaddda7c70ae391a81a26bd Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Mon, 4 Mar 2024 07:55:29 -0600
Subject: [PATCH 24/29] Update to include the whole configmap, not a script to
 patch it.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../configure-operator-metrics.asciidoc       | 72 +++++++++++++++++--
 1 file changed, 68 insertions(+), 4 deletions(-)

diff --git a/docs/operating-eck/configure-operator-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc
index fb77ac0eb1..d5f7cc93fb 100644
--- a/docs/operating-eck/configure-operator-metrics.asciidoc
+++ b/docs/operating-eck/configure-operator-metrics.asciidoc
@@ -22,9 +22,41 @@ If you installed ECK using the manifests using the commands listed in <<{p}-depl
 
 * Enable the metrics endpoint in the `ConfigMap`.
 
-[source,sh]
+[source,shell,subs="attributes,+macros"]
 ----
-kubectl get cm elastic-operator -n elastic-system -o yaml | sed "s|metrics-port: 0|metrics-port: 8080|" | kubectl apply -f -
+cat $$<<$$EOF | kubectl apply -f -
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: elastic-operator
+  namespace: elastic-system
+data:
+  eck.yaml: |-
+    log-verbosity: 0
+    metrics-port: 8080
+    metrics-host: 0.0.0.0
+    container-registry: docker.elastic.co
+    max-concurrent-reconciles: 3
+    ca-cert-validity: 8760h
+    ca-cert-rotate-before: 24h
+    cert-validity: 8760h
+    cert-rotate-before: 24h
+    disable-config-watch: false
+    exposed-node-labels: [topology.kubernetes.io/.*,failure-domain.beta.kubernetes.io/.*]
+    set-default-security-context: auto-detect
+    kube-client-timeout: 60s
+    elasticsearch-client-timeout: 180s
+    disable-telemetry: false
+    distribution-channel: all-in-one
+    validate-storage-class: true
+    enable-webhook: true
+    webhook-name: elastic-webhook.k8s.elastic.co
+    webhook-port: 9443
+    operator-namespace: elastic-system
+    enable-leader-election: true
+    elasticsearch-observation-interval: 10s
+    ubi-only: false
+EOF
 ----
 
 * Patch the `StatefulSet` to expose the metrics port.
@@ -123,9 +155,41 @@ If you installed ECK through using the manifests using the commands listed in <<
 
 * Enable the metrics port in the `ConfigMap`, and set the metrics host to `127.0.0.1` to force communication through `kube-rbac-proxy`.
 
-[source,sh]
+[source,shell,subs="attributes,+macros"]
 ----
-kubectl get cm elastic-operator -n elastic-system -o yaml | sed "s|metrics-port: 0|metrics-port: 8081|" | sed "s|metrics-host: 0.0.0.0|metrics-host: 127.0.0.1|" | kubectl apply -f -
+cat $$<<$$EOF | kubectl apply -f -
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: elastic-operator
+  namespace: elastic-system
+data:
+  eck.yaml: |-
+    log-verbosity: 0
+    metrics-port: 8081
+    metrics-host: 127.0.0.1
+    container-registry: docker.elastic.co
+    max-concurrent-reconciles: 3
+    ca-cert-validity: 8760h
+    ca-cert-rotate-before: 24h
+    cert-validity: 8760h
+    cert-rotate-before: 24h
+    disable-config-watch: false
+    exposed-node-labels: [topology.kubernetes.io/.*,failure-domain.beta.kubernetes.io/.*]
+    set-default-security-context: auto-detect
+    kube-client-timeout: 60s
+    elasticsearch-client-timeout: 180s
+    disable-telemetry: false
+    distribution-channel: all-in-one
+    validate-storage-class: true
+    enable-webhook: true
+    webhook-name: elastic-webhook.k8s.elastic.co
+    webhook-port: 9443
+    operator-namespace: elastic-system
+    enable-leader-election: true
+    elasticsearch-observation-interval: 10s
+    ubi-only: false
+EOF
 ----
 
 * Add an additional `ClusterRole` and `ClusterRoleBinding` for the ECK Operator.

From 119150effc31a7e2ca4a9351e5318eaa98649f2a Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Mon, 4 Mar 2024 21:10:51 -0600
Subject: [PATCH 25/29] Review comments

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../configure-operator-metrics.asciidoc       | 24 +++++++++----------
 docs/operating-eck/operating-eck.asciidoc     |  4 ++--
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/docs/operating-eck/configure-operator-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc
index d5f7cc93fb..b2bf5243f8 100644
--- a/docs/operating-eck/configure-operator-metrics.asciidoc
+++ b/docs/operating-eck/configure-operator-metrics.asciidoc
@@ -6,17 +6,17 @@ link:https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-{page_id}.html[View
 endif::[]
 
 [id="{p}-{page_id}"]
-= Configuring the ECK Operator metrics endpoint
+= Configure the metrics endpoint
 
-== Enabling the ECK Operator metrics endpoint
+== Enabling the metrics endpoint
 
-The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled. To enable the metrics endpoint, follow the instructions in the next sections depending on whether you installed ECK through the Helm chart or the manifests.
+The ECK operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled. To enable the metrics endpoint, follow the instructions in the next sections depending on whether you installed ECK through the Helm chart or the manifests.
 
-=== Using the ECK Operator Helm Chart
+=== Using the operator Helm Chart
 
 If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set  `config.metrics.port` to a value greater than 0 in your values file and the metrics endpoint will be enabled.
 
-=== Using the ECK Operator manifests
+=== Using the operator manifests
 
 If you installed ECK using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable the metrics endpoint.
 
@@ -79,7 +79,7 @@ spec:
 EOF
 ----
 
-* Restart the ECK Operator.
+* Restart the ECK operator.
 
 [source,sh]
 ----
@@ -114,13 +114,13 @@ spec:
 EOF
 ----
 
-== Securing the ECK Operator metrics endpoint
+== Securing the metrics endpoint
 
-NOTE: The ECK Operator metrics endpoint will be secured by default beginning in version 2.14.0
+NOTE: The ECK operator metrics endpoint will be secured by default beginning in version 2.14.0
 
-The ECK Operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. To enable the metrics endpoint follow the previous instructions. To enable TLS/HTTPs and RBAC on the metrics endpoint, follow the instructions in the following sections depending on whether you installed ECK through the Helm chart or the manifests.
+The ECK operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. To enable the metrics endpoint follow the previous instructions. To enable TLS/HTTPs and RBAC on the metrics endpoint, follow the instructions in the following sections depending on whether you installed ECK through the Helm chart or the manifests.
 
-=== Using the ECK Operator Helm Chart
+=== Using the operator Helm Chart
 
 If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `config.metrics.secureMode.enabled` to `true` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
 
@@ -149,7 +149,7 @@ Providing this secret is sufficient to use your own certificate if it is from a
 
 See the <<{p}-prometheus-requirements,prometheus requirements section>> for more information on creating the CA secret.
 
-=== Using the ECK Operator manifests
+=== Using the operator manifests
 
 If you installed ECK through using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable secure metrics.
 
@@ -192,7 +192,7 @@ data:
 EOF
 ----
 
-* Add an additional `ClusterRole` and `ClusterRoleBinding` for the ECK Operator.
+* Add an additional `ClusterRole` and `ClusterRoleBinding` for the ECK operator.
 
 [source,shell,subs="attributes,+macros"]
 ----
diff --git a/docs/operating-eck/operating-eck.asciidoc b/docs/operating-eck/operating-eck.asciidoc
index 397e37434f..ec60f1a4eb 100644
--- a/docs/operating-eck/operating-eck.asciidoc
+++ b/docs/operating-eck/operating-eck.asciidoc
@@ -10,9 +10,9 @@ endif::[]
 [partintro]
 --
 - <<{p}-operator-config>>
-- <<{p}-configure-operator-metrics>>
 - <<{p}-eck-permissions>>
 - <<{p}-webhook>>
+- <<{p}-configure-operator-metrics>>
 - <<{p}-restrict-cross-namespace-associations>>
 - <<{p}-licensing>>
 - <<{p}-troubleshooting>>
@@ -23,9 +23,9 @@ endif::[]
 --
 
 include::operator-config.asciidoc[leveloffset=+1]
-include::configure-operator-metrics.asciidoc[leveloffset=+1]
 include::eck-permissions.asciidoc[leveloffset=+1]
 include::webhook.asciidoc[leveloffset=+1]
+include::configure-operator-metrics.asciidoc[leveloffset=+1]
 include::restrict-cross-namespace-associations.asciidoc[leveloffset=+1]
 include::licensing.asciidoc[leveloffset=+1]
 include::troubleshooting.asciidoc[leveloffset=+1]

From 3e3572303bc874deeef139995736335f051de1a1 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Mon, 4 Mar 2024 21:17:44 -0600
Subject: [PATCH 26/29] Review comments

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/templates/cluster-roles.yaml  | 4 ++--
 deploy/eck-operator/templates/configmap.yaml      | 2 +-
 deploy/eck-operator/templates/podMonitor.yaml     | 5 ++++-
 deploy/eck-operator/templates/serviceMonitor.yaml | 2 +-
 deploy/eck-operator/values.yaml                   | 2 +-
 5 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/deploy/eck-operator/templates/cluster-roles.yaml b/deploy/eck-operator/templates/cluster-roles.yaml
index 085fc335a6..be7cdde5ef 100644
--- a/deploy/eck-operator/templates/cluster-roles.yaml
+++ b/deploy/eck-operator/templates/cluster-roles.yaml
@@ -1,5 +1,5 @@
 {{- if and (not .Values.createClusterScopedResources) (.Values.config.metrics.secureMode.enabled) -}}
-{{ fail "createClusterScopedResources is required to enable secure metrics" }}
+{{ fail "createClusterScopedResources is required to set config.metrics.secureMode.enabled to true" }}
 {{- end }}
 {{- if .Values.createClusterScopedResources -}}
 ---
@@ -96,7 +96,7 @@ rules:
   - apiGroups: ["logstash.k8s.elastic.co"]
     resources: ["logstashes"]
     verbs: ["create", "delete", "deletecollection", "patch", "update"]
-{{- if and .Values.config.metrics.secureMode.enabled }}
+{{- if .Values.config.metrics.secureMode.enabled }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml
index a6d581b9b3..bc13953a15 100644
--- a/deploy/eck-operator/templates/configmap.yaml
+++ b/deploy/eck-operator/templates/configmap.yaml
@@ -11,7 +11,7 @@ data:
     {{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
     log-verbosity: {{ int .Values.config.logVerbosity }}
     {{- if and .Values.config.metrics.secureMode.enabled (eq $metricsPort 0) }}
-    {{- fail "metricsPort must be greater than 0 when enableSecureMetrics is true" }}
+    {{- fail "config.metrics.port must be greater than 0 when config.metrics.secureMode.enabled is true" }}
     {{- end }}
     {{- if .Values.config.metrics.secureMode.enabled }}
     metrics-port: {{ add $metricsPort 1 }}
diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml
index f643ea0eaf..8e073cd3ad 100644
--- a/deploy/eck-operator/templates/podMonitor.yaml
+++ b/deploy/eck-operator/templates/podMonitor.yaml
@@ -1,7 +1,10 @@
 {{- $metricsPort := int (include "eck-operator.metrics.port" .)}}
+{{- if and .Values.config.metrics.secureMode.enabled (eq $metricsPort 0) }}
+{{- fail "config.metrics.port must be greater than 0 when config.metrics.secureMode.enabled is true" }}
+{{- end }}
 {{- if and .Values.podMonitor.enabled (gt $metricsPort 0) }}
 {{- if and .Values.podMonitor.enabled .Values.config.metrics.secureMode.enabled }}
-{{- fail "podMonitor and secureMode are mutually exclusive" }}
+{{- fail "podMonitor and config.metrics.secureMode are mutually exclusive" }}
 {{- end }}
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
diff --git a/deploy/eck-operator/templates/serviceMonitor.yaml b/deploy/eck-operator/templates/serviceMonitor.yaml
index 8d4c4709b3..2b080bb846 100644
--- a/deploy/eck-operator/templates/serviceMonitor.yaml
+++ b/deploy/eck-operator/templates/serviceMonitor.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.config.metrics.secureMode.enabled }}
+{{- if .Values.config.metrics.secureMode.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index d8c7f3d95a..53a9d342b8 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -291,7 +291,7 @@ config:
 podMonitor:
 
   # enabled determines whether a podMonitor should deployed to scrape the eck metrics.
-  # This requires the prometheus operator and the config.metricsPort not to be 0
+  # This requires the prometheus operator and the config.metrics.port not to be 0
   enabled: false
 
   # labels adds additional labels to the podMonitor

From 2f89b3bcfc8b06fc6099ba6303ced0e91fdd5684 Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 5 Mar 2024 07:13:09 -0600
Subject: [PATCH 27/29] Review comments.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 deploy/eck-operator/values.yaml                    |  4 ++--
 .../configure-operator-metrics.asciidoc            | 14 +++++++-------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml
index 53a9d342b8..6a12465082 100644
--- a/deploy/eck-operator/values.yaml
+++ b/deploy/eck-operator/values.yaml
@@ -322,8 +322,8 @@ podMonitor:
 # Reference: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#servicemonitor
 serviceMonitor: {}
 
-  # namespace determines in which namespace the servicedMonitor will be deployed.
-  # If not set the servicedMonitor will be created in the namespace where the Helm release is installed into
+  # namespace determines in which namespace the serviceMonitor will be deployed.
+  # If not set the serviceMonitor will be created in the namespace where the Helm release is installed into
   # namespace: monitoring
 
 # Globals meant for internal use only
diff --git a/docs/operating-eck/configure-operator-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc
index b2bf5243f8..db14a01cd1 100644
--- a/docs/operating-eck/configure-operator-metrics.asciidoc
+++ b/docs/operating-eck/configure-operator-metrics.asciidoc
@@ -116,11 +116,11 @@ EOF
 
 == Securing the metrics endpoint
 
-NOTE: The ECK operator metrics endpoint will be secured by default beginning in version 2.14.0
+NOTE: The ECK operator metrics endpoint will be secured by default beginning in version 2.14.0.
 
-The ECK operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. To enable the metrics endpoint follow the previous instructions. To enable TLS/HTTPs and RBAC on the metrics endpoint, follow the instructions in the following sections depending on whether you installed ECK through the Helm chart or the manifests.
+The ECK operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. To enable the metrics endpoint follow the previous instructions. To enable RBAC and TLS on the metrics endpoint, follow the instructions in the following sections depending on whether you installed ECK through the Helm chart or the manifests.
 
-=== Using the operator Helm Chart
+=== Using the operator Helm chart
 
 If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `config.metrics.secureMode.enabled` to `true` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
 
@@ -141,11 +141,11 @@ kubectl create secret tls eck-metrics-tls-certificate -n elastic-system --cert=/
 Providing this secret is sufficient to use your own certificate if it is from a trusted Certificate Authority. If the certificate is not signed by a trusted CA you have 2 options:
 
 * Disable TLS verification.
-    * Set `config.metrics.secureMode.tls.insecureSkipVerify` to `true` to disable TLS validation.
+    ** Set `config.metrics.secureMode.tls.insecureSkipVerify` to `true` to disable TLS validation.
 * Provide the Certificate Authority to Prometheus.
-    * Set `config.metrics.secureMode.tls.insecureSkipVerify` to `false` to enable TLS validation.
-    * Set `config.metrics.secureMode.tls.caSecret` to the name of an existing Kubernetes secret within the Prometheus namespace that contains the CA in PEM format.
-    * Set the `spec.secrets` field of the `Prometheus` custom resource such that the CA secret is mounted into the Prometheus pod at `config.metrics.secureMode.tls.caMountDirectory` (assuming you are using the Prometheus operator). See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm Chart values file] for more information.
+    ** Set `config.metrics.secureMode.tls.insecureSkipVerify` to `false` to enable TLS validation.
+    ** Set `config.metrics.secureMode.tls.caSecret` to the name of an existing Kubernetes secret within the Prometheus namespace that contains the CA in PEM format.
+    ** Set the `spec.secrets` field of the `Prometheus` custom resource such that the CA secret is mounted into the Prometheus pod at `config.metrics.secureMode.tls.caMountDirectory` (assuming you are using the Prometheus operator). See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm Chart values file] for more information.
 
 See the <<{p}-prometheus-requirements,prometheus requirements section>> for more information on creating the CA secret.
 

From 848e8ed986a8de32921f4d39f6bd849b056efabc Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Tue, 5 Mar 2024 15:35:09 -0600
Subject: [PATCH 28/29] Addressing review comments

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../configure-operator-metrics.asciidoc            | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docs/operating-eck/configure-operator-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc
index db14a01cd1..c7a8050c3a 100644
--- a/docs/operating-eck/configure-operator-metrics.asciidoc
+++ b/docs/operating-eck/configure-operator-metrics.asciidoc
@@ -8,9 +8,18 @@ endif::[]
 [id="{p}-{page_id}"]
 = Configure the metrics endpoint
 
+The ECK operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. The following sections describe how to enable it, secure it and the associated Prometheus requirements:
+
+* <<{p}-enabling-the-metrics-endpoint,Enabling the metrics endpoint>>
+* <<{p}-securing-the-metrics-endpoint,Securing the metrics endpoint>>
+* <<{p}-prometheus-requirements,Prometheus requirements>>
+
+NOTE: The ECK operator metrics endpoint will be secured by default beginning in version 2.14.0.
+
+[id="{p}-enabling-the-metrics-endpoint"]
 == Enabling the metrics endpoint
 
-The ECK operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled. To enable the metrics endpoint, follow the instructions in the next sections depending on whether you installed ECK through the Helm chart or the manifests.
+The metrics endpoint is not enabled by default. To enable the metrics endpoint, follow the instructions in the next sections depending on whether you installed ECK through the Helm chart or the manifests.
 
 === Using the operator Helm Chart
 
@@ -86,7 +95,7 @@ EOF
 kubectl delete pod -n elastic-system elastic-operator-0
 ----
 
-* Install the `PodMonitor` to allow scraping of the metrics endpoint by Prometheus.
+* If using the Prometheus Operator, install a `PodMonitor` to allow scraping of the metrics endpoint by Prometheus.
 
 [source,shell,subs="attributes,+macros"]
 ----
@@ -114,6 +123,7 @@ spec:
 EOF
 ----
 
+[id="{p}-securing-the-metrics-endpoint"]
 == Securing the metrics endpoint
 
 NOTE: The ECK operator metrics endpoint will be secured by default beginning in version 2.14.0.

From e8eccd1466118e383bb65b03504d02ecad5272fb Mon Sep 17 00:00:00 2001
From: Michael Montgomery <mmontg1@gmail.com>
Date: Wed, 6 Mar 2024 08:01:16 -0600
Subject: [PATCH 29/29] Lowercasing operator, and chart.

Signed-off-by: Michael Montgomery <mmontg1@gmail.com>
---
 .../configure-operator-metrics.asciidoc          | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/operating-eck/configure-operator-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc
index c7a8050c3a..6a699152d7 100644
--- a/docs/operating-eck/configure-operator-metrics.asciidoc
+++ b/docs/operating-eck/configure-operator-metrics.asciidoc
@@ -21,7 +21,7 @@ NOTE: The ECK operator metrics endpoint will be secured by default beginning in
 
 The metrics endpoint is not enabled by default. To enable the metrics endpoint, follow the instructions in the next sections depending on whether you installed ECK through the Helm chart or the manifests.
 
-=== Using the operator Helm Chart
+=== Using the operator Helm chart
 
 If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set  `config.metrics.port` to a value greater than 0 in your values file and the metrics endpoint will be enabled.
 
@@ -95,7 +95,7 @@ EOF
 kubectl delete pod -n elastic-system elastic-operator-0
 ----
 
-* If using the Prometheus Operator, install a `PodMonitor` to allow scraping of the metrics endpoint by Prometheus.
+* If using the Prometheus operator, install a `PodMonitor` to allow scraping of the metrics endpoint by Prometheus.
 
 [source,shell,subs="attributes,+macros"]
 ----
@@ -134,7 +134,7 @@ The ECK operator provides a metrics endpoint that can be used to monitor the ope
 
 If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `config.metrics.secureMode.enabled` to `true` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint.
 
-==== Using your own TLS certificate for the metrics endpoint when using the Helm Chart
+==== Using your own TLS certificate for the metrics endpoint when using the Helm chart
 
 By default a self-signed certificate will be generated for use by the metrics endpoint. If you want to use your own TLS certificate for the metrics endpoint you can provide the `config.metrics.secureMode.tls.certificateSecret` to the Helm chart. The `certificateSecret` should be the name of an existing Kubernetes `Secret` that contains both the TLS certificate and the TLS private key. The following keys are supported within the secret:
 
@@ -155,7 +155,7 @@ Providing this secret is sufficient to use your own certificate if it is from a
 * Provide the Certificate Authority to Prometheus.
     ** Set `config.metrics.secureMode.tls.insecureSkipVerify` to `false` to enable TLS validation.
     ** Set `config.metrics.secureMode.tls.caSecret` to the name of an existing Kubernetes secret within the Prometheus namespace that contains the CA in PEM format.
-    ** Set the `spec.secrets` field of the `Prometheus` custom resource such that the CA secret is mounted into the Prometheus pod at `config.metrics.secureMode.tls.caMountDirectory` (assuming you are using the Prometheus operator). See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm Chart values file] for more information.
+    ** Set the `spec.secrets` field of the `Prometheus` custom resource such that the CA secret is mounted into the Prometheus pod at `config.metrics.secureMode.tls.caMountDirectory` (assuming you are using the Prometheus operator). See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm chart values file] for more information.
 
 See the <<{p}-prometheus-requirements,prometheus requirements section>> for more information on creating the CA secret.
 
@@ -299,7 +299,7 @@ spec:
 EOF
 ----
 
-* Add a `ServiceMonitor` to allow scraping of the metrics endpoint by Prometheus.
+* If using the Prometheus operator, add a `ServiceMonitor` to allow scraping of the metrics endpoint by Prometheus.
 
 [source,shell,subs="attributes,+macros"]
 ----
@@ -440,9 +440,9 @@ rules:
   - get
 ----
 
-=== Optional Prometheus Operator Helm settings to allow reading PodMonitor and ServiceMonitor across namespaces
+=== Optional Prometheus operator Helm settings to allow reading PodMonitor and ServiceMonitor across namespaces
 
-* If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the ECK operator you will need the Prometheus Operator configured with the following Helm values:
+* If using the Prometheus operator and your Prometheus instance is not in the same namespace as the ECK operator you will need the Prometheus operator configured with the following Helm values:
 
 [source,yaml,subs="attributes"]
 ----
@@ -469,4 +469,4 @@ kubectl create secret generic eck-metrics-tls-ca -n monitoring --from-file=ca.cr
 
 * Ensure that the CA secret is mounted within the Prometheus Pod.
 
-This will vary between Prometheus installations, but if using the Prometheus Operator you can set the `spec.secrets` field of the `Prometheus` custom resource to the name of the previously created Kubernetes Secret. See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm Chart values file] for more information.
+This will vary between Prometheus installations, but if using the Prometheus operator you can set the `spec.secrets` field of the `Prometheus` custom resource to the name of the previously created Kubernetes Secret. See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm chart values file] for more information.