diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 64cb692989..b79e667b16 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -286,7 +286,12 @@ func Command() *cobra.Command { cmd.Flags().Int( operator.MetricsPortFlag, DefaultMetricPort, - "Port to use for exposing metrics in the Prometheus format (set 0 to disable)", + "Port to use for exposing metrics in the Prometheus format. (set 0 to disable)", + ) + cmd.Flags().String( + operator.MetricsHostFlag, + "0.0.0.0", + fmt.Sprintf("The host to which the operator should bind to serve metrics in the Prometheus format. Will be combined with %s.", operator.MetricsPortFlag), ) cmd.Flags().StringSlice( operator.NamespacesFlag, @@ -577,11 +582,12 @@ func startOperator(ctx context.Context) error { // only expose prometheus metrics if provided a non-zero port metricsPort := viper.GetInt(operator.MetricsPortFlag) + metricsHost := viper.GetString(operator.MetricsHostFlag) if metricsPort != 0 { - log.Info("Exposing Prometheus metrics on /metrics", "port", metricsPort) + log.Info("Exposing Prometheus metrics on /metrics", "bindAddress", fmt.Sprintf("%s:%d", metricsHost, metricsPort)) } opts.Metrics = metricsserver.Options{ - BindAddress: fmt.Sprintf(":%d", metricsPort), // 0 to disable + BindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), // 0 to disable } webhookPort := viper.GetInt(operator.WebhookPortFlag) diff --git a/deploy/eck-operator/templates/_helpers.tpl b/deploy/eck-operator/templates/_helpers.tpl index 8c421f7b55..218f5742d0 100644 --- a/deploy/eck-operator/templates/_helpers.tpl +++ b/deploy/eck-operator/templates/_helpers.tpl @@ -114,6 +114,19 @@ elastic-webhook-server {{- end -}} {{- end -}} +{{/* +Determine the metrics port +*/}} +{{- define "eck-operator.metrics.port" -}} +{{- if .Values.config.metrics.port -}} +{{- .Values.config.metrics.port -}} +{{- else if .Values.config.metricsPort -}} +{{- .Values.config.metricsPort -}} +{{- else -}} +0 +{{- end -}} +{{- end -}} + {{/* RBAC permissions NOTE - any changes made to RBAC permissions below require diff --git a/deploy/eck-operator/templates/auth-proxy-service.yaml b/deploy/eck-operator/templates/auth-proxy-service.yaml new file mode 100644 index 0000000000..53bdc02b75 --- /dev/null +++ b/deploy/eck-operator/templates/auth-proxy-service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.config.metrics.secureMode.enabled }} +{{- $metricsPort := int (include "eck-operator.metrics.port" .)}} +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: {{ include "eck-operator.name" . }}-metrics-service + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} + helm.sh/chart: {{ include "eck-operator.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + name: "{{ include "eck-operator.fullname" . }}-metrics" + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: https + port: {{ $metricsPort }} + protocol: TCP + targetPort: metrics + selector: + {{- include "eck-operator.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/deploy/eck-operator/templates/cluster-roles.yaml b/deploy/eck-operator/templates/cluster-roles.yaml index 1b623f37fe..be7cdde5ef 100644 --- a/deploy/eck-operator/templates/cluster-roles.yaml +++ b/deploy/eck-operator/templates/cluster-roles.yaml @@ -1,3 +1,6 @@ +{{- if and (not .Values.createClusterScopedResources) (.Values.config.metrics.secureMode.enabled) -}} +{{ fail "createClusterScopedResources is required to set config.metrics.secureMode.enabled to true" }} +{{- end }} {{- if .Values.createClusterScopedResources -}} --- apiVersion: rbac.authorization.k8s.io/v1 @@ -93,4 +96,26 @@ rules: - apiGroups: ["logstash.k8s.elastic.co"] resources: ["logstashes"] verbs: ["create", "delete", "deletecollection", "patch", "update"] +{{- if .Values.config.metrics.secureMode.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "eck-operator.labels" . | nindent 4 }} + name: "{{ include "eck-operator.fullname" . }}-proxy-role" +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +{{- end }} {{- end -}} diff --git a/deploy/eck-operator/templates/configmap.yaml b/deploy/eck-operator/templates/configmap.yaml index eec71de366..bc13953a15 100644 --- a/deploy/eck-operator/templates/configmap.yaml +++ b/deploy/eck-operator/templates/configmap.yaml @@ -8,8 +8,16 @@ metadata: {{- include "eck-operator.labels" . | nindent 4 }} data: eck.yaml: |- + {{- $metricsPort := int (include "eck-operator.metrics.port" .)}} log-verbosity: {{ int .Values.config.logVerbosity }} - metrics-port: {{ int .Values.config.metricsPort }} + {{- if and .Values.config.metrics.secureMode.enabled (eq $metricsPort 0) }} + {{- fail "config.metrics.port must be greater than 0 when config.metrics.secureMode.enabled is true" }} + {{- end }} + {{- if .Values.config.metrics.secureMode.enabled }} + metrics-port: {{ add $metricsPort 1 }} + {{- else }} + metrics-port: {{ $metricsPort }} + {{- end }} container-registry: {{ .Values.config.containerRegistry }} {{- with .Values.config.containerSuffix }} container-suffix: {{ . }} diff --git a/deploy/eck-operator/templates/operator-network-policy.yaml b/deploy/eck-operator/templates/operator-network-policy.yaml index 10aaa56654..ad74156d50 100644 --- a/deploy/eck-operator/templates/operator-network-policy.yaml +++ b/deploy/eck-operator/templates/operator-network-policy.yaml @@ -1,6 +1,6 @@ {{- if .Values.softMultiTenancy.enabled -}} {{- $kubeAPIServerIP := (required "kubeAPIServerIP is required" .Values.kubeAPIServerIP) -}} -{{- $metricsPort := int .Values.config.metricsPort -}} +{{- $metricsPort := int (include "eck-operator.metrics.port" .)}} --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy diff --git a/deploy/eck-operator/templates/podMonitor.yaml b/deploy/eck-operator/templates/podMonitor.yaml index c269cb7be7..8e073cd3ad 100644 --- a/deploy/eck-operator/templates/podMonitor.yaml +++ b/deploy/eck-operator/templates/podMonitor.yaml @@ -1,10 +1,16 @@ -{{- $metricsPort := int .Values.config.metricsPort -}} +{{- $metricsPort := int (include "eck-operator.metrics.port" .)}} +{{- if and .Values.config.metrics.secureMode.enabled (eq $metricsPort 0) }} +{{- fail "config.metrics.port must be greater than 0 when config.metrics.secureMode.enabled is true" }} +{{- end }} {{- if and .Values.podMonitor.enabled (gt $metricsPort 0) }} +{{- if and .Values.podMonitor.enabled .Values.config.metrics.secureMode.enabled }} +{{- fail "podMonitor and config.metrics.secureMode are mutually exclusive" }} +{{- end }} apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: {{ include "eck-operator.fullname" . }} - namespace: {{ ternary .Values.podMonitor.namespace .Release.Namespace (not (empty .Values.podMonitor.namespace)) }} + namespace: {{ ternary .Values.podMonitor.namespace .Release.Namespace (not (and (.Values.podMonitor) (empty .Values.podMonitor.namespace))) }} labels: {{- include "eck-operator.labels" . | nindent 4 }} {{- with .Values.podMonitor.labels }} {{- toYaml . | nindent 4 }} @@ -33,4 +39,4 @@ spec: - {{ .Release.Namespace }} selector: matchLabels: {{- include "eck-operator.selectorLabels" . | nindent 6 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/deploy/eck-operator/templates/role-bindings.yaml b/deploy/eck-operator/templates/role-bindings.yaml index ca6cf2ba57..4b57a3f008 100644 --- a/deploy/eck-operator/templates/role-bindings.yaml +++ b/deploy/eck-operator/templates/role-bindings.yaml @@ -1,6 +1,7 @@ {{- $operatorNSIsManaged := has .Release.Namespace .Values.managedNamespaces -}} {{- $fullName := include "eck-operator.fullname" . -}} {{- $svcAccount := include "eck-operator.serviceAccountName" . }} +{{- $enableSecureMetrics := .Values.config.metrics.secureMode.enabled -}} {{- if not .Values.createClusterScopedResources }} {{- range .Values.managedNamespaces }} @@ -77,4 +78,21 @@ subjects: - kind: ServiceAccount name: {{ $svcAccount }} namespace: {{ $.Release.Namespace }} +{{- if $enableSecureMetrics }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "eck-operator.labels" $ | nindent 4 }} + name: "{{ include "eck-operator.fullname" . }}-proxy-rolebinding" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: "{{ include "eck-operator.fullname" . }}-proxy-role" +subjects: +- kind: ServiceAccount + name: {{ $svcAccount }} + namespace: {{ $.Release.Namespace }} +{{- end }} {{- end }} diff --git a/deploy/eck-operator/templates/serviceMonitor.yaml b/deploy/eck-operator/templates/serviceMonitor.yaml new file mode 100644 index 0000000000..2b080bb846 --- /dev/null +++ b/deploy/eck-operator/templates/serviceMonitor.yaml @@ -0,0 +1,31 @@ +{{- if .Values.config.metrics.secureMode.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "eck-operator.fullname" . }} + namespace: {{ ternary .Values.serviceMonitor.namespace .Release.Namespace (not (and (.Values.serviceMonitor) (empty .Values.serviceMonitor.namespace))) }} + labels: {{- include "eck-operator.labels" . | nindent 4 }} +spec: + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "eck-operator.name" . }}-metrics-service + app.kubernetes.io/instance: {{ .Release.Name }} + endpoints: + - port: https + path: /metrics + scheme: https + interval: 30s + tlsConfig: + insecureSkipVerify: {{ .Values.config.metrics.secureMode.tls.insecureSkipVerify | default false }} + {{- if (not .Values.config.metrics.secureMode.tls.insecureSkipVerify) }} + {{- with .Values.config.metrics.secureMode.tls.caSecret }} + {{- $leading_path := trimSuffix "/" .Values.config.metrics.secureMode.tls.caMountDirectory }} + caFile: "{{ $leading_path }}/{{ . }}/ca.crt" + {{- end }} + serverName: "{{ include "eck-operator.fullname" . }}-metrics.{{ .Release.Namespace }}.svc" + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- end }} diff --git a/deploy/eck-operator/templates/statefulset.yaml b/deploy/eck-operator/templates/statefulset.yaml index e72ae30eed..7f6a8f716b 100644 --- a/deploy/eck-operator/templates/statefulset.yaml +++ b/deploy/eck-operator/templates/statefulset.yaml @@ -1,5 +1,5 @@ -{{- $metricsPort := int .Values.config.metricsPort -}} --- +{{- $metricsPort := int (include "eck-operator.metrics.port" .)}} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -79,10 +79,10 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - {{- if or (gt $metricsPort 0) .Values.webhook.enabled }} + {{- if or .Values.webhook.enabled (gt $metricsPort 0) }} ports: - {{- if (gt $metricsPort 0) }} - - containerPort: {{ .Values.config.metricsPort }} + {{- if and (gt $metricsPort 0) (not .Values.config.metrics.secureMode.enabled) }} + - containerPort: {{ $metricsPort }} name: metrics protocol: TCP {{- end }} @@ -104,6 +104,41 @@ spec: {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.config.metrics.secureMode.enabled }} + - name: kube-rbac-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 + args: + - "--secure-listen-address=0.0.0.0:{{ $metricsPort }}" + - "--upstream=http://127.0.0.1:{{ add $metricsPort 1 }}/" + - "--logtostderr=true" + - "--v=0" + {{- if .Values.config.metrics.secureMode.tls.certificateSecret }} + - "--tls-cert-file=/tls/tls.crt" + - "--tls-private-key-file=/tls/tls.key" + {{- end }} + {{- if .Values.config.metrics.secureMode.tls.certificateSecret }} + volumeMounts: + - mountPath: "/tls" + name: tls-certificate + readOnly: true + {{- end }} + ports: + - containerPort: {{ $metricsPort }} + protocol: TCP + name: metrics + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + {{- end }} volumes: - name: conf configMap: @@ -114,6 +149,12 @@ spec: defaultMode: 420 secretName: {{ include "eck-operator.webhookSecretName" . }} {{- end }} + {{- if .Values.config.metrics.secureMode.tls.certificateSecret }} + - name: tls-certificate + secret: + defaultMode: 420 + secretName: {{ .Values.config.metrics.secureMode.tls.certificateSecret }} + {{- end }} {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/deploy/eck-operator/values.yaml b/deploy/eck-operator/values.yaml index 3c8a82116a..6a12465082 100644 --- a/deploy/eck-operator/values.yaml +++ b/deploy/eck-operator/values.yaml @@ -159,8 +159,67 @@ config: # number greater than 0: Errors, warnings, information, and debug details. logVerbosity: "0" - # metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting. - metricsPort: "0" + # (Deprecated: use metrics.port: will be removed in v2.14.0) metricsPort defines the port to expose operator metrics. Set to 0 to disable metrics reporting. + metricsPort: 0 + + metrics: + # port defines the port to expose operator metrics. Set to 0 to disable metrics reporting. + port: "0" + # secureMode contains the options for enabling and configuring RBAC and TLS/HTTPs for the metrics endpoint. + secureMode: + # secureMode.enabled specifies whether to enable RBAC and TLS/HTTPs for the metrics endpoint. (Will be enabled by default in v2.14.0) + # * This option requires using a ServiceMonitor to scrape the metrics and as such is mutually exclusive with the podMonitor.enabled option. + # * This option also requires using cluster scoped resources (ClusterRole, ClusterRoleBinding) to + # grant access to the /metrics endpoint. (createClusterScopedResources: true is required) + # + # This option requires the following settings within Prometheus to function: + # 1. RBAC settings for the Prometheus instance to access the metrics endpoint. + # + # - nonResourceURLs: + # - /metrics + # verbs: + # - get + # + # 2. If using the Prometheus Operator and your Prometheus instance is not in the same namespace as the operator you will need + # the Prometheus Operator configured with the following Helm values: + # + # prometheus: + # prometheusSpec: + # serviceMonitorNamespaceSelector: {} + # serviceMonitorSelectorNilUsesHelmValues: false + enabled: false + tls: + # certificateSecret is the name of the tls secret containing the custom TLS certificate and key for the secure metrics endpoint. + # + # * This is an optional setting and is only required if you are using a custom TLS certificate. A self-signed certificate will be generated by default. + # * TLS secret key must be named tls.crt. + # * TLS key's secret key must be named tls.key. + # * It is assumed to be in the same namespace as the ServiceMonitor. + # + # example: kubectl create secret tls eck-metrics-tls-certificate -n elastic-system \ + # --cert=/path/to/tls.crt --key=/path/to/tls.key + certificateSecret: "" + # caSecret is the name of the secret containing the custom CA certificate used to generate the custom TLS certificate for the secure metrics endpoint. + # + # * This *must* be the name of the secret containing the CA certificate used to sign the custom TLS certificate. + # * This secret *must* be in the same namespace as the Prometheus instance that will scrape the metrics. + # * If using the Prometheus operator this secret must be within the `spec.secrets` field of the `Prometheus` custom resource such that it is mounted into the Prometheus pod at `caMountDirectory`, which defaults to /etc/prometheus/secrets/{secret-name}. + # * This is an optional setting and is only required if you are using a custom TLS certificate. + # * Key must be named ca.crt. + # + # example: kubectl create secret generic eck-metrics-tls-ca -n monitoring \ + # --from-file=ca.crt=/path/to/ca.pem + caSecret: "" + # caMountDirectory is the directory at which the CA certificate is mounted within the Prometheus pod. + # + # * You should only need to adjust this if you are *not* using the Prometheus operator. + caMountDirectory: "/etc/prometheus/secrets/" + # insecureSkipVerify specifies whether to skip verification of the TLS certificate for the secure metrics endpoint. + # + # * If this setting is set to false, then the following settings are required: + # - certificateSecret + # - caSecret + insecureSkipVerify: true # containerRegistry to use for pulling Elasticsearch and other application container images. containerRegistry: docker.elastic.co @@ -232,7 +291,7 @@ config: podMonitor: # enabled determines whether a podMonitor should deployed to scrape the eck metrics. - # This requires the prometheus operator and the config.metricsPort not to be 0 + # This requires the prometheus operator and the config.metrics.port not to be 0 enabled: false # labels adds additional labels to the podMonitor @@ -258,6 +317,15 @@ podMonitor: podMetricsEndpointConfig: {} # honorTimestamps: true +# Prometheus ServiceMonitor configuration +# Only used when config.enableSecureMetrics is true +# Reference: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#servicemonitor +serviceMonitor: {} + + # namespace determines in which namespace the serviceMonitor will be deployed. + # If not set the serviceMonitor will be created in the namespace where the Helm release is installed into + # namespace: monitoring + # Globals meant for internal use only global: # manifestGen specifies whether the chart is running under manifest generator. diff --git a/docs/operating-eck/configure-operator-metrics.asciidoc b/docs/operating-eck/configure-operator-metrics.asciidoc new file mode 100644 index 0000000000..6a699152d7 --- /dev/null +++ b/docs/operating-eck/configure-operator-metrics.asciidoc @@ -0,0 +1,472 @@ +:page_id: configure-operator-metrics +ifdef::env-github[] +**** +link:https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-{page_id}.html[View this document on the Elastic website] +**** +endif::[] + +[id="{p}-{page_id}"] += Configure the metrics endpoint + +The ECK operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. The following sections describe how to enable it, secure it and the associated Prometheus requirements: + +* <<{p}-enabling-the-metrics-endpoint,Enabling the metrics endpoint>> +* <<{p}-securing-the-metrics-endpoint,Securing the metrics endpoint>> +* <<{p}-prometheus-requirements,Prometheus requirements>> + +NOTE: The ECK operator metrics endpoint will be secured by default beginning in version 2.14.0. + +[id="{p}-enabling-the-metrics-endpoint"] +== Enabling the metrics endpoint + +The metrics endpoint is not enabled by default. To enable the metrics endpoint, follow the instructions in the next sections depending on whether you installed ECK through the Helm chart or the manifests. + +=== Using the operator Helm chart + +If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `config.metrics.port` to a value greater than 0 in your values file and the metrics endpoint will be enabled. + +=== Using the operator manifests + +If you installed ECK using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable the metrics endpoint. + +* Enable the metrics endpoint in the `ConfigMap`. + +[source,shell,subs="attributes,+macros"] +---- +cat $$<<$$EOF | kubectl apply -f - +kind: ConfigMap +apiVersion: v1 +metadata: + name: elastic-operator + namespace: elastic-system +data: + eck.yaml: |- + log-verbosity: 0 + metrics-port: 8080 + metrics-host: 0.0.0.0 + container-registry: docker.elastic.co + max-concurrent-reconciles: 3 + ca-cert-validity: 8760h + ca-cert-rotate-before: 24h + cert-validity: 8760h + cert-rotate-before: 24h + disable-config-watch: false + exposed-node-labels: [topology.kubernetes.io/.*,failure-domain.beta.kubernetes.io/.*] + set-default-security-context: auto-detect + kube-client-timeout: 60s + elasticsearch-client-timeout: 180s + disable-telemetry: false + distribution-channel: all-in-one + validate-storage-class: true + enable-webhook: true + webhook-name: elastic-webhook.k8s.elastic.co + webhook-port: 9443 + operator-namespace: elastic-system + enable-leader-election: true + elasticsearch-observation-interval: 10s + ubi-only: false +EOF +---- + +* Patch the `StatefulSet` to expose the metrics port. + +[source,shell,subs="attributes,+macros"] +---- +kubectl patch sts -n elastic-system elastic-operator --patch-file=/dev/stdin <<-EOF +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: https-webhook + protocol: TCP + - containerPort: 8080 + protocol: TCP + name: metrics +EOF +---- + +* Restart the ECK operator. + +[source,sh] +---- +kubectl delete pod -n elastic-system elastic-operator-0 +---- + +* If using the Prometheus operator, install a `PodMonitor` to allow scraping of the metrics endpoint by Prometheus. + +[source,shell,subs="attributes,+macros"] +---- +cat $$<<$$EOF | kubectl apply -f - +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: elastic-operator + namespace: elastic-system + labels: + control-plane: elastic-operator + app.kubernetes.io/component: metrics +spec: + podMetricsEndpoints: + - port: metrics + path: /metrics + interval: 1m + scrapeTimeout: 30s + namespaceSelector: + matchNames: + - elastic-system + selector: + matchLabels: + control-plane: elastic-operator +EOF +---- + +[id="{p}-securing-the-metrics-endpoint"] +== Securing the metrics endpoint + +NOTE: The ECK operator metrics endpoint will be secured by default beginning in version 2.14.0. + +The ECK operator provides a metrics endpoint that can be used to monitor the operator's performance and health. By default, the metrics endpoint is not enabled and is not secured. To enable the metrics endpoint follow the previous instructions. To enable RBAC and TLS on the metrics endpoint, follow the instructions in the following sections depending on whether you installed ECK through the Helm chart or the manifests. + +=== Using the operator Helm chart + +If you installed ECK through the Helm chart commands listed in <<{p}-install-helm>>, you can now set `config.metrics.secureMode.enabled` to `true` and both RBAC and TLS/HTTPs will be enabled for the metrics endpoint. + +==== Using your own TLS certificate for the metrics endpoint when using the Helm chart + +By default a self-signed certificate will be generated for use by the metrics endpoint. If you want to use your own TLS certificate for the metrics endpoint you can provide the `config.metrics.secureMode.tls.certificateSecret` to the Helm chart. The `certificateSecret` should be the name of an existing Kubernetes `Secret` that contains both the TLS certificate and the TLS private key. The following keys are supported within the secret: + +* `tls.crt` - The PEM-encoded TLS certificate +* `tls.key` - The PEM-encoded TLS private key + +The easiest way to create this secret is to use the `kubectl create secret tls` command. For example: + +[source,sh] +---- +kubectl create secret tls eck-metrics-tls-certificate -n elastic-system --cert=/path/to/tls.crt --key=/path/to/tls.key +---- + +Providing this secret is sufficient to use your own certificate if it is from a trusted Certificate Authority. If the certificate is not signed by a trusted CA you have 2 options: + +* Disable TLS verification. + ** Set `config.metrics.secureMode.tls.insecureSkipVerify` to `true` to disable TLS validation. +* Provide the Certificate Authority to Prometheus. + ** Set `config.metrics.secureMode.tls.insecureSkipVerify` to `false` to enable TLS validation. + ** Set `config.metrics.secureMode.tls.caSecret` to the name of an existing Kubernetes secret within the Prometheus namespace that contains the CA in PEM format. + ** Set the `spec.secrets` field of the `Prometheus` custom resource such that the CA secret is mounted into the Prometheus pod at `config.metrics.secureMode.tls.caMountDirectory` (assuming you are using the Prometheus operator). See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm chart values file] for more information. + +See the <<{p}-prometheus-requirements,prometheus requirements section>> for more information on creating the CA secret. + +=== Using the operator manifests + +If you installed ECK through using the manifests using the commands listed in <<{p}-deploy-eck>> some additional changes will be required to enable secure metrics. + +* Enable the metrics port in the `ConfigMap`, and set the metrics host to `127.0.0.1` to force communication through `kube-rbac-proxy`. + +[source,shell,subs="attributes,+macros"] +---- +cat $$<<$$EOF | kubectl apply -f - +kind: ConfigMap +apiVersion: v1 +metadata: + name: elastic-operator + namespace: elastic-system +data: + eck.yaml: |- + log-verbosity: 0 + metrics-port: 8081 + metrics-host: 127.0.0.1 + container-registry: docker.elastic.co + max-concurrent-reconciles: 3 + ca-cert-validity: 8760h + ca-cert-rotate-before: 24h + cert-validity: 8760h + cert-rotate-before: 24h + disable-config-watch: false + exposed-node-labels: [topology.kubernetes.io/.*,failure-domain.beta.kubernetes.io/.*] + set-default-security-context: auto-detect + kube-client-timeout: 60s + elasticsearch-client-timeout: 180s + disable-telemetry: false + distribution-channel: all-in-one + validate-storage-class: true + enable-webhook: true + webhook-name: elastic-webhook.k8s.elastic.co + webhook-port: 9443 + operator-namespace: elastic-system + enable-leader-election: true + elasticsearch-observation-interval: 10s + ubi-only: false +EOF +---- + +* Add an additional `ClusterRole` and `ClusterRoleBinding` for the ECK operator. + +[source,shell,subs="attributes,+macros"] +---- +cat $$<<$$EOF | kubectl apply -f - +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: elastic-operator-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: elastic-operator-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: elastic-operator-proxy-role +subjects: +- kind: ServiceAccount + name: elastic-operator + namespace: elastic-system +EOF +---- + +* Add a `Service` to expose the metrics endpoint. + +[source,shell,subs="attributes,+macros"] +---- +cat $$<<$$EOF | kubectl apply -f - +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: elastic-operator + app.kubernetes.io/component: metrics + name: elastic-operator-metrics + namespace: elastic-system +spec: + ports: + - name: https + port: 8080 + protocol: TCP + targetPort: metrics + selector: + control-plane: elastic-operator +EOF +---- + +* Patch the `StatefulSet` to include a sidecar container for `kube-rbac-proxy` to secure the metrics endpoint. + +[source,shell,subs="attributes,+macros"] +---- +kubectl patch sts -n elastic-system elastic-operator --patch-file=/dev/stdin <<-EOF +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 + args: + - "--secure-listen-address=0.0.0.0:8080" + - "--upstream=http://127.0.0.1:8081" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8080 + protocol: TCP + name: metrics + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi +EOF +---- + +* If using the Prometheus operator, add a `ServiceMonitor` to allow scraping of the metrics endpoint by Prometheus. + +[source,shell,subs="attributes,+macros"] +---- +cat $$<<$$EOF | kubectl apply -f - +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: elastic-operator + namespace: elastic-system +spec: + namespaceSelector: + matchNames: + - elastic-system + selector: + matchLabels: + control-plane: elastic-operator + app.kubernetes.io/component: metrics + endpoints: + - port: https + path: /metrics + scheme: https + interval: 30s + tlsConfig: + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +EOF +---- + +==== Using your own TLS certificate for the metrics endpoint when using the manifests + +By default a self-signed certificate will be generated for use by the metrics endpoint. If you want to use your own TLS certificate for the metrics endpoint you will need to follow the previous instructions to enable secure metrics as well as the following steps: + +* Create a `Secret` containing the TLS certificate and TLS private key. The following keys are supported within the secret: + + * `tls.crt` - The PEM-encoded TLS certificate + * `tls.key` - The PEM-encoded TLS private key + +The easiest way to create this secret is to use the `kubectl create secret tls` command. For example: + +[source,sh] +---- +kubectl create secret tls my-tls-secret -n elastic-system --cert=/path/to/tls.crt --key=/path/to/tls.key +---- + +* Patch the `StatefulSet` to include the `tls.crt` and `tls.key` as a volume and mount it into the `kube-rbac-proxy` container. + +[source,shell,subs="attributes,+macros"] +---- +kubectl patch sts -n elastic-system elastic-operator --patch-file=/dev/stdin <<-EOF +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 + args: + - "--secure-listen-address=0.0.0.0:8080" + - "--upstream=http://127.0.0.1:8081" + - "--logtostderr=true" + - "--v=0" + - "--tls-cert-file=/tls/tls.crt" + - "--tls-private-key-file=/tls/tls.key" + volumeMounts: + - mountPath: "/tls" + name: tls-certificate + readOnly: true + ports: + - containerPort: 8080 + protocol: TCP + name: metrics + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + volumes: + - name: conf + configMap: + name: elastic-operator + - name: cert + secret: + defaultMode: 420 + secretName: elastic-webhook-server-cert + - name: tls-certificate + secret: + defaultMode: 420 + secretName: eck-metrics-tls-certificate +EOF +---- + +* Potentially patch the `ServiceMonitor`. This will only need to be done if you are adjusting the `insecureSkipVerify` field to `false`. + +[source,shell,subs="attributes,+macros,callouts"] +---- +kubectl patch servicemonitor -n elastic-system elastic-operator --patch-file=/dev/stdin <<-EOF +spec: + endpoints: + - port: https + path: /metrics + scheme: https + interval: 30s + tlsConfig: + insecureSkipVerify: false + caFile: /etc/prometheus/secrets/{secret-name}/ca.crt <1> + serverName: elastic-operator-metrics.elastic-system.svc + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +EOF +---- + +<1> See the <<{p}-prometheus-requirements,prometheus requirements section>> for more information on creating the CA secret. + +[id="{p}-prometheus-requirements"] +== Prometheus requirements + +The previous options requires the following settings within Prometheus to function properly: + +=== RBAC settings for scraping the metrics + +Configure the RBAC settings for the Prometheus instance to access the metrics endpoint similar to the following: (These typically will be set automatically when using the Prometheus operator) + +[source,yaml,subs="attributes"] +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +---- + +=== Optional Prometheus operator Helm settings to allow reading PodMonitor and ServiceMonitor across namespaces + +* If using the Prometheus operator and your Prometheus instance is not in the same namespace as the ECK operator you will need the Prometheus operator configured with the following Helm values: + +[source,yaml,subs="attributes"] +---- +prometheus: + prometheusSpec: + podMonitorNamespaceSelector: {} + podMonitorSelectorNilUsesHelmValues: false + serviceMonitorNamespaceSelector: {} + serviceMonitorSelectorNilUsesHelmValues: false +---- + +=== Optional settings to allow full TLS verification when using a custom TLS certificate + +If you are using a custom TLS certificate and you need to set `insecureSkipVerify` to `false` you will need to do the following: + +* Create a Kubernetes secret within the Prometheus namespace that contains the Certificate Authority in PEM format. + +The easiest way to create the CA secret within the Prometheus namespace is to use the `kubectl create secret generic` command. For example: + +[source,sh] +---- +kubectl create secret generic eck-metrics-tls-ca -n monitoring --from-file=ca.crt=/path/to/ca.pem +---- + +* Ensure that the CA secret is mounted within the Prometheus Pod. + +This will vary between Prometheus installations, but if using the Prometheus operator you can set the `spec.secrets` field of the `Prometheus` custom resource to the name of the previously created Kubernetes Secret. See the link:{eck_github}/tree/{eck_release_branch}/deploy/eck-operator/values.yaml[ECK Helm chart values file] for more information. diff --git a/docs/operating-eck/operating-eck.asciidoc b/docs/operating-eck/operating-eck.asciidoc index 1b6ac63edb..ec60f1a4eb 100644 --- a/docs/operating-eck/operating-eck.asciidoc +++ b/docs/operating-eck/operating-eck.asciidoc @@ -12,6 +12,7 @@ endif::[] - <<{p}-operator-config>> - <<{p}-eck-permissions>> - <<{p}-webhook>> +- <<{p}-configure-operator-metrics>> - <<{p}-restrict-cross-namespace-associations>> - <<{p}-licensing>> - <<{p}-troubleshooting>> @@ -24,6 +25,7 @@ endif::[] include::operator-config.asciidoc[leveloffset=+1] include::eck-permissions.asciidoc[leveloffset=+1] include::webhook.asciidoc[leveloffset=+1] +include::configure-operator-metrics.asciidoc[leveloffset=+1] include::restrict-cross-namespace-associations.asciidoc[leveloffset=+1] include::licensing.asciidoc[leveloffset=+1] include::troubleshooting.asciidoc[leveloffset=+1] diff --git a/pkg/controller/common/operator/flags.go b/pkg/controller/common/operator/flags.go index f7df112bc1..c965d81a8d 100644 --- a/pkg/controller/common/operator/flags.go +++ b/pkg/controller/common/operator/flags.go @@ -33,6 +33,7 @@ const ( ManageWebhookCertsFlag = "manage-webhook-certs" MaxConcurrentReconcilesFlag = "max-concurrent-reconciles" MetricsPortFlag = "metrics-port" + MetricsHostFlag = "metrics-host" NamespacesFlag = "namespaces" OperatorNamespaceFlag = "operator-namespace" SetDefaultSecurityContextFlag = "set-default-security-context"