diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 1a6df9a96f..a06e23d37e 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -679,6 +679,7 @@ func registerControllers(mgr manager.Manager, params operator.Parameters, access {name: "AGENT-ES", registerFunc: associationctl.AddAgentES}, {name: "EMS-ES", registerFunc: associationctl.AddMapsES}, {name: "ES-MONITORING", registerFunc: associationctl.AddEsMonitoring}, + {name: "KB-MONITORING", registerFunc: associationctl.AddKbMonitoring}, } for _, c := range assocControllers { diff --git a/config/crds/v1/all-crds.yaml b/config/crds/v1/all-crds.yaml index 1701d5fdaa..92e3ccec68 100644 --- a/config/crds/v1/all-crds.yaml +++ b/config/crds/v1/all-crds.yaml @@ -6738,6 +6738,80 @@ spec: image: description: Image is the Kibana Docker image to deploy. type: string + monitoring: + description: Monitoring enables you to collect and ship log and monitoring + data of this Kibana. See https://www.elastic.co/guide/en/kibana/current/xpack-monitoring.html. + Metricbeat and Filebeat are deployed in the same Pod as sidecars + and each one sends data to one or two different Elasticsearch monitoring + clusters running in the same Kubernetes cluster. + properties: + logs: + description: Logs holds references to Elasticsearch clusters which + will receive log data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of + monitoring Elasticsearch clusters running in the same Kubernetes + cluster. Due to existing limitations, only a single Elasticsearch + cluster is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If + empty, defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing + Kubernetes service which is used to make requests + to the referenced object. It has to be in the same + namespace as the referenced resource. If left empty, + the default HTTP service of the referenced resource + is used. + type: string + required: + - name + type: object + type: array + type: object + metrics: + description: Metrics holds references to Elasticsearch clusters + which will receive monitoring data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of + monitoring Elasticsearch clusters running in the same Kubernetes + cluster. Due to existing limitations, only a single Elasticsearch + cluster is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If + empty, defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing + Kubernetes service which is used to make requests + to the referenced object. It has to be in the same + namespace as the referenced resource. If left empty, + the default HTTP service of the referenced resource + is used. + type: string + required: + - name + type: object + type: array + type: object + type: object podTemplate: description: PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods @@ -6819,6 +6893,13 @@ spec: health: description: Health of the deployment. type: string + monitoringAssociationStatus: + additionalProperties: + description: AssociationStatus is the status of an association resource. + type: string + description: MonitoringAssociationStatus is the status of any auto-linking + to monitoring Elasticsearch clusters. + type: object selector: description: Selector is the label selector used to find all pods. type: string diff --git a/config/crds/v1/bases/kibana.k8s.elastic.co_kibanas.yaml b/config/crds/v1/bases/kibana.k8s.elastic.co_kibanas.yaml index bc721bb2b1..e4acc48ce7 100644 --- a/config/crds/v1/bases/kibana.k8s.elastic.co_kibanas.yaml +++ b/config/crds/v1/bases/kibana.k8s.elastic.co_kibanas.yaml @@ -544,6 +544,80 @@ spec: image: description: Image is the Kibana Docker image to deploy. type: string + monitoring: + description: Monitoring enables you to collect and ship log and monitoring + data of this Kibana. See https://www.elastic.co/guide/en/kibana/current/xpack-monitoring.html. + Metricbeat and Filebeat are deployed in the same Pod as sidecars + and each one sends data to one or two different Elasticsearch monitoring + clusters running in the same Kubernetes cluster. + properties: + logs: + description: Logs holds references to Elasticsearch clusters which + will receive log data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of + monitoring Elasticsearch clusters running in the same Kubernetes + cluster. Due to existing limitations, only a single Elasticsearch + cluster is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If + empty, defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing + Kubernetes service which is used to make requests + to the referenced object. It has to be in the same + namespace as the referenced resource. If left empty, + the default HTTP service of the referenced resource + is used. + type: string + required: + - name + type: object + type: array + type: object + metrics: + description: Metrics holds references to Elasticsearch clusters + which will receive monitoring data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of + monitoring Elasticsearch clusters running in the same Kubernetes + cluster. Due to existing limitations, only a single Elasticsearch + cluster is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If + empty, defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing + Kubernetes service which is used to make requests + to the referenced object. It has to be in the same + namespace as the referenced resource. If left empty, + the default HTTP service of the referenced resource + is used. + type: string + required: + - name + type: object + type: array + type: object + type: object podTemplate: description: PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods @@ -7310,6 +7384,13 @@ spec: health: description: Health of the deployment. type: string + monitoringAssociationStatus: + additionalProperties: + description: AssociationStatus is the status of an association resource. + type: string + description: MonitoringAssociationStatus is the status of any auto-linking + to monitoring Elasticsearch clusters. + type: object selector: description: Selector is the label selector used to find all pods. type: string diff --git a/config/crds/v1beta1/all-crds.yaml b/config/crds/v1beta1/all-crds.yaml index 39c75934d8..917a3646cc 100644 --- a/config/crds/v1beta1/all-crds.yaml +++ b/config/crds/v1beta1/all-crds.yaml @@ -4359,6 +4359,78 @@ spec: image: description: Image is the Kibana Docker image to deploy. type: string + monitoring: + description: Monitoring enables you to collect and ship log and monitoring + data of this Kibana. See https://www.elastic.co/guide/en/kibana/current/xpack-monitoring.html. + Metricbeat and Filebeat are deployed in the same Pod as sidecars and + each one sends data to one or two different Elasticsearch monitoring + clusters running in the same Kubernetes cluster. + properties: + logs: + description: Logs holds references to Elasticsearch clusters which + will receive log data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of monitoring + Elasticsearch clusters running in the same Kubernetes cluster. + Due to existing limitations, only a single Elasticsearch cluster + is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, + defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes + service which is used to make requests to the referenced + object. It has to be in the same namespace as the referenced + resource. If left empty, the default HTTP service of + the referenced resource is used. + type: string + required: + - name + type: object + type: array + type: object + metrics: + description: Metrics holds references to Elasticsearch clusters + which will receive monitoring data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of monitoring + Elasticsearch clusters running in the same Kubernetes cluster. + Due to existing limitations, only a single Elasticsearch cluster + is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, + defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes + service which is used to make requests to the referenced + object. It has to be in the same namespace as the referenced + resource. If left empty, the default HTTP service of + the referenced resource is used. + type: string + required: + - name + type: object + type: array + type: object + type: object podTemplate: description: PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods @@ -4439,6 +4511,13 @@ spec: health: description: Health of the deployment. type: string + monitoringAssociationStatus: + additionalProperties: + description: AssociationStatus is the status of an association resource. + type: string + description: MonitoringAssociationStatus is the status of any auto-linking + to monitoring Elasticsearch clusters. + type: object selector: description: Selector is the label selector used to find all pods. type: string diff --git a/config/crds/v1beta1/bases/kibana.k8s.elastic.co_kibanas.yaml b/config/crds/v1beta1/bases/kibana.k8s.elastic.co_kibanas.yaml index b1e1eed99a..89daf5b5c0 100644 --- a/config/crds/v1beta1/bases/kibana.k8s.elastic.co_kibanas.yaml +++ b/config/crds/v1beta1/bases/kibana.k8s.elastic.co_kibanas.yaml @@ -527,6 +527,80 @@ spec: image: description: Image is the Kibana Docker image to deploy. type: string + monitoring: + description: Monitoring enables you to collect and ship log and monitoring + data of this Kibana. See https://www.elastic.co/guide/en/kibana/current/xpack-monitoring.html. + Metricbeat and Filebeat are deployed in the same Pod as sidecars + and each one sends data to one or two different Elasticsearch monitoring + clusters running in the same Kubernetes cluster. + properties: + logs: + description: Logs holds references to Elasticsearch clusters which + will receive log data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of + monitoring Elasticsearch clusters running in the same Kubernetes + cluster. Due to existing limitations, only a single Elasticsearch + cluster is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If + empty, defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing + Kubernetes service which is used to make requests + to the referenced object. It has to be in the same + namespace as the referenced resource. If left empty, + the default HTTP service of the referenced resource + is used. + type: string + required: + - name + type: object + type: array + type: object + metrics: + description: Metrics holds references to Elasticsearch clusters + which will receive monitoring data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of + monitoring Elasticsearch clusters running in the same Kubernetes + cluster. Due to existing limitations, only a single Elasticsearch + cluster is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If + empty, defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing + Kubernetes service which is used to make requests + to the referenced object. It has to be in the same + namespace as the referenced resource. If left empty, + the default HTTP service of the referenced resource + is used. + type: string + required: + - name + type: object + type: array + type: object + type: object podTemplate: description: PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods @@ -7256,6 +7330,13 @@ spec: health: description: Health of the deployment. type: string + monitoringAssociationStatus: + additionalProperties: + description: AssociationStatus is the status of an association resource. + type: string + description: MonitoringAssociationStatus is the status of any auto-linking + to monitoring Elasticsearch clusters. + type: object selector: description: Selector is the label selector used to find all pods. type: string diff --git a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds-legacy.yaml b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds-legacy.yaml index a105bf72d8..9e27b5b53c 100644 --- a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds-legacy.yaml +++ b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds-legacy.yaml @@ -4404,6 +4404,78 @@ spec: image: description: Image is the Kibana Docker image to deploy. type: string + monitoring: + description: Monitoring enables you to collect and ship log and monitoring + data of this Kibana. See https://www.elastic.co/guide/en/kibana/current/xpack-monitoring.html. + Metricbeat and Filebeat are deployed in the same Pod as sidecars and + each one sends data to one or two different Elasticsearch monitoring + clusters running in the same Kubernetes cluster. + properties: + logs: + description: Logs holds references to Elasticsearch clusters which + will receive log data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of monitoring + Elasticsearch clusters running in the same Kubernetes cluster. + Due to existing limitations, only a single Elasticsearch cluster + is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, + defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes + service which is used to make requests to the referenced + object. It has to be in the same namespace as the referenced + resource. If left empty, the default HTTP service of + the referenced resource is used. + type: string + required: + - name + type: object + type: array + type: object + metrics: + description: Metrics holds references to Elasticsearch clusters + which will receive monitoring data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of monitoring + Elasticsearch clusters running in the same Kubernetes cluster. + Due to existing limitations, only a single Elasticsearch cluster + is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, + defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes + service which is used to make requests to the referenced + object. It has to be in the same namespace as the referenced + resource. If left empty, the default HTTP service of + the referenced resource is used. + type: string + required: + - name + type: object + type: array + type: object + type: object podTemplate: description: PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods @@ -4484,6 +4556,13 @@ spec: health: description: Health of the deployment. type: string + monitoringAssociationStatus: + additionalProperties: + description: AssociationStatus is the status of an association resource. + type: string + description: MonitoringAssociationStatus is the status of any auto-linking + to monitoring Elasticsearch clusters. + type: object selector: description: Selector is the label selector used to find all pods. type: string diff --git a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml index 30bf37cbde..de7d743189 100644 --- a/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml +++ b/deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml @@ -6783,6 +6783,80 @@ spec: image: description: Image is the Kibana Docker image to deploy. type: string + monitoring: + description: Monitoring enables you to collect and ship log and monitoring + data of this Kibana. See https://www.elastic.co/guide/en/kibana/current/xpack-monitoring.html. + Metricbeat and Filebeat are deployed in the same Pod as sidecars + and each one sends data to one or two different Elasticsearch monitoring + clusters running in the same Kubernetes cluster. + properties: + logs: + description: Logs holds references to Elasticsearch clusters which + will receive log data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of + monitoring Elasticsearch clusters running in the same Kubernetes + cluster. Due to existing limitations, only a single Elasticsearch + cluster is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If + empty, defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing + Kubernetes service which is used to make requests + to the referenced object. It has to be in the same + namespace as the referenced resource. If left empty, + the default HTTP service of the referenced resource + is used. + type: string + required: + - name + type: object + type: array + type: object + metrics: + description: Metrics holds references to Elasticsearch clusters + which will receive monitoring data from this Kibana. + properties: + elasticsearchRefs: + description: ElasticsearchRefs is a reference to a list of + monitoring Elasticsearch clusters running in the same Kubernetes + cluster. Due to existing limitations, only a single Elasticsearch + cluster is currently supported. + items: + description: ObjectSelector defines a reference to a Kubernetes + object. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If + empty, defaults to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing + Kubernetes service which is used to make requests + to the referenced object. It has to be in the same + namespace as the referenced resource. If left empty, + the default HTTP service of the referenced resource + is used. + type: string + required: + - name + type: object + type: array + type: object + type: object podTemplate: description: PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods @@ -6864,6 +6938,13 @@ spec: health: description: Health of the deployment. type: string + monitoringAssociationStatus: + additionalProperties: + description: AssociationStatus is the status of an association resource. + type: string + description: MonitoringAssociationStatus is the status of any auto-linking + to monitoring Elasticsearch clusters. + type: object selector: description: Selector is the label selector used to find all pods. type: string diff --git a/docs/reference/api-docs.asciidoc b/docs/reference/api-docs.asciidoc index a904037e7a..b9adb0a3ba 100644 --- a/docs/reference/api-docs.asciidoc +++ b/docs/reference/api-docs.asciidoc @@ -443,8 +443,10 @@ ObjectSelector defines a reference to a Kubernetes object. - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-enterprisesearch-v1beta1-enterprisesearchspec[$$EnterpriseSearchSpec$$] - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-kibanaspec[$$KibanaSpec$$] - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-elasticsearch-v1-logsmonitoring[$$LogsMonitoring$$] +- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-logsmonitoring[$$LogsMonitoring$$] - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-maps-v1alpha1-mapsspec[$$MapsSpec$$] - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-elasticsearch-v1-metricsmonitoring[$$MetricsMonitoring$$] +- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-metricsmonitoring[$$MetricsMonitoring$$] - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-agent-v1alpha1-output[$$Output$$] - xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-elasticsearch-v1-remotecluster[$$RemoteCluster$$] **** @@ -1382,6 +1384,59 @@ KibanaSpec holds the specification of a Kibana instance. | *`podTemplate`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#podtemplatespec-v1-core[$$PodTemplateSpec$$]__ | PodTemplate provides customisation options (labels, annotations, affinity rules, resource requests, and so on) for the Kibana pods | *`secureSettings`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-common-v1-secretsource[$$SecretSource$$]__ | SecureSettings is a list of references to Kubernetes secrets containing sensitive configuration options for Kibana. | *`serviceAccountName`* __string__ | ServiceAccountName is used to check access from the current resource to a resource (eg. Elasticsearch) in a different namespace. Can only be used if ECK is enforcing RBAC on references. +| *`monitoring`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-monitoring[$$Monitoring$$]__ | Monitoring enables you to collect and ship log and monitoring data of this Kibana. See https://www.elastic.co/guide/en/kibana/current/xpack-monitoring.html. Metricbeat and Filebeat are deployed in the same Pod as sidecars and each one sends data to one or two different Elasticsearch monitoring clusters running in the same Kubernetes cluster. +|=== + + +[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-logsmonitoring"] +=== LogsMonitoring + + + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-monitoring[$$Monitoring$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`elasticsearchRefs`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-common-v1-objectselector[$$ObjectSelector$$]__ | ElasticsearchRefs is a reference to a list of monitoring Elasticsearch clusters running in the same Kubernetes cluster. Due to existing limitations, only a single Elasticsearch cluster is currently supported. +|=== + + +[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-metricsmonitoring"] +=== MetricsMonitoring + + + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-monitoring[$$Monitoring$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`elasticsearchRefs`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-common-v1-objectselector[$$ObjectSelector$$]__ | ElasticsearchRefs is a reference to a list of monitoring Elasticsearch clusters running in the same Kubernetes cluster. Due to existing limitations, only a single Elasticsearch cluster is currently supported. +|=== + + +[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-monitoring"] +=== Monitoring + + + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-kibanaspec[$$KibanaSpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`metrics`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-metricsmonitoring[$$MetricsMonitoring$$]__ | Metrics holds references to Elasticsearch clusters which will receive monitoring data from this Kibana. +| *`logs`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-pkg-apis-kibana-v1-logsmonitoring[$$LogsMonitoring$$]__ | Logs holds references to Elasticsearch clusters which will receive log data from this Kibana. |=== diff --git a/pkg/apis/common/v1/association.go b/pkg/apis/common/v1/association.go index 72067e2cc2..f37ac3429a 100644 --- a/pkg/apis/common/v1/association.go +++ b/pkg/apis/common/v1/association.go @@ -99,6 +99,7 @@ const ( KibanaConfigAnnotationNameBase = "association.k8s.elastic.co/kb-conf" KibanaAssociationType = "kibana" + KbMonitoringAssociationType = "kb-monitoring" EntConfigAnnotationNameBase = "association.k8s.elastic.co/ent-conf" EntAssociationType = "ent" diff --git a/pkg/apis/elasticsearch/v1/elasticsearch_types.go b/pkg/apis/elasticsearch/v1/elasticsearch_types.go index 2c3c8bc4db..564411d139 100644 --- a/pkg/apis/elasticsearch/v1/elasticsearch_types.go +++ b/pkg/apis/elasticsearch/v1/elasticsearch_types.go @@ -20,10 +20,22 @@ const ( ElasticsearchContainerName = "elasticsearch" // Kind is inferred from the struct name using reflection in SchemeBuilder.Register() // we duplicate it as a constant here for practical purposes. - Kind = "Elasticsearch" - ShortKind = "es" + Kind = "Elasticsearch" ) +// +kubebuilder:object:root=true + +// ElasticsearchList contains a list of Elasticsearch clusters +type ElasticsearchList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Elasticsearch `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Elasticsearch{}, &ElasticsearchList{}) +} + // ElasticsearchSpec holds the specification of an Elasticsearch cluster. type ElasticsearchSpec struct { // Version of Elasticsearch. @@ -108,138 +120,6 @@ type LogsMonitoring struct { ElasticsearchRefs []commonv1.ObjectSelector `json:"elasticsearchRefs,omitempty"` } -// EsMonitoringAssociation helps to manage Elasticsearch+Metricbeat+Filebeat <-> Elasticsearch(es) associations -type EsMonitoringAssociation struct { - // The monitored Elasticsearch cluster from where are collected logs and monitoring metrics - *Elasticsearch - // ref is the namespaced name of the Elasticsearch referenced in the Association used to send and store monitoring data - ref types.NamespacedName -} - -var _ commonv1.Association = &EsMonitoringAssociation{} - -func (ema *EsMonitoringAssociation) Associated() commonv1.Associated { - if ema == nil { - return nil - } - if ema.Elasticsearch == nil { - ema.Elasticsearch = &Elasticsearch{} - } - return ema.Elasticsearch -} - -func (ema *EsMonitoringAssociation) AssociationConfAnnotationName() string { - return commonv1.ElasticsearchConfigAnnotationName(ema.ref) -} - -func (ema *EsMonitoringAssociation) AssociationType() commonv1.AssociationType { - return commonv1.EsMonitoringAssociationType -} - -func (ema *EsMonitoringAssociation) AssociationRef() commonv1.ObjectSelector { - return commonv1.ObjectSelector{ - Name: ema.ref.Name, - Namespace: ema.ref.Namespace, - } -} - -func (ema *EsMonitoringAssociation) AssociationConf() *commonv1.AssociationConf { - if ema.AssocConfs == nil { - return nil - } - assocConf, found := ema.AssocConfs[ema.ref] - if !found { - return nil - } - - return &assocConf -} - -func (ema *EsMonitoringAssociation) SetAssociationConf(assocConf *commonv1.AssociationConf) { - if ema.AssocConfs == nil { - ema.AssocConfs = make(map[types.NamespacedName]commonv1.AssociationConf) - } - if assocConf != nil { - ema.AssocConfs[ema.ref] = *assocConf - } -} - -func (ema *EsMonitoringAssociation) AssociationID() string { - return fmt.Sprintf("%s-%s", ema.ref.Namespace, ema.ref.Name) -} - -// Associated methods - -var _ commonv1.Associated = &Elasticsearch{} - -func (es *Elasticsearch) ServiceAccountName() string { - return es.Spec.ServiceAccountName -} - -func (es *Elasticsearch) GetAssociations() []commonv1.Association { - associations := make([]commonv1.Association, 0) - for _, ref := range es.Spec.Monitoring.Metrics.ElasticsearchRefs { - if ref.IsDefined() { - associations = append(associations, &EsMonitoringAssociation{ - Elasticsearch: es, - ref: ref.WithDefaultNamespace(es.Namespace).NamespacedName(), - }) - } - } - for _, ref := range es.Spec.Monitoring.Logs.ElasticsearchRefs { - if ref.IsDefined() { - associations = append(associations, &EsMonitoringAssociation{ - Elasticsearch: es, - ref: ref.WithDefaultNamespace(es.Namespace).NamespacedName(), - }) - } - } - return associations -} - -func (es *Elasticsearch) GetMonitoringMetricsAssociation() []commonv1.Association { - associations := make([]commonv1.Association, 0) - for _, ref := range es.Spec.Monitoring.Metrics.ElasticsearchRefs { - if ref.IsDefined() { - associations = append(associations, &EsMonitoringAssociation{ - Elasticsearch: es, - ref: ref.WithDefaultNamespace(es.Namespace).NamespacedName(), - }) - } - } - return associations -} - -func (es *Elasticsearch) GetMonitoringLogsAssociation() []commonv1.Association { - associations := make([]commonv1.Association, 0) - for _, ref := range es.Spec.Monitoring.Logs.ElasticsearchRefs { - if ref.IsDefined() { - associations = append(associations, &EsMonitoringAssociation{ - Elasticsearch: es, - ref: ref.WithDefaultNamespace(es.Namespace).NamespacedName(), - }) - } - } - return associations -} - -func (es *Elasticsearch) AssociationStatusMap(typ commonv1.AssociationType) commonv1.AssociationStatusMap { - if typ != commonv1.EsMonitoringAssociationType { - return commonv1.AssociationStatusMap{} - } - - return es.Status.MonitoringAssociationsStatus -} - -func (es *Elasticsearch) SetAssociationStatusMap(typ commonv1.AssociationType, status commonv1.AssociationStatusMap) error { - if typ != commonv1.EsMonitoringAssociationType { - return fmt.Errorf("association type %s not known", typ) - } - - es.Status.MonitoringAssociationsStatus = status - return nil -} - // VolumeClaimDeletePolicy describes the delete policy for handling PersistentVolumeClaims that hold Elasticsearch data. // Inspired by https://github.com/kubernetes/enhancements/pull/2440 type VolumeClaimDeletePolicy string @@ -573,6 +453,10 @@ func (es Elasticsearch) IsMarkedForDeletion() bool { return !es.DeletionTimestamp.IsZero() } +func (es *Elasticsearch) ServiceAccountName() string { + return es.Spec.ServiceAccountName +} + // IsAutoscalingDefined returns true if there is an autoscaling configuration in the annotations. func (es Elasticsearch) IsAutoscalingDefined() bool { _, ok := es.Annotations[ElasticsearchAutoscalingSpecAnnotationName] @@ -588,15 +472,123 @@ func (es Elasticsearch) SecureSettings() []commonv1.SecretSource { return es.Spec.SecureSettings } -// +kubebuilder:object:root=true +// -- associations -// ElasticsearchList contains a list of Elasticsearch clusters -type ElasticsearchList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Elasticsearch `json:"items"` +var _ commonv1.Associated = &Elasticsearch{} + +func (es *Elasticsearch) GetAssociations() []commonv1.Association { + associations := make([]commonv1.Association, 0) + for _, ref := range es.Spec.Monitoring.Metrics.ElasticsearchRefs { + if ref.IsDefined() { + associations = append(associations, &EsMonitoringAssociation{ + Elasticsearch: es, + ref: ref.WithDefaultNamespace(es.Namespace).NamespacedName(), + }) + } + } + for _, ref := range es.Spec.Monitoring.Logs.ElasticsearchRefs { + if ref.IsDefined() { + associations = append(associations, &EsMonitoringAssociation{ + Elasticsearch: es, + ref: ref.WithDefaultNamespace(es.Namespace).NamespacedName(), + }) + } + } + return associations } -func init() { - SchemeBuilder.Register(&Elasticsearch{}, &ElasticsearchList{}) +func (es *Elasticsearch) AssociationStatusMap(typ commonv1.AssociationType) commonv1.AssociationStatusMap { + if typ != commonv1.EsMonitoringAssociationType { + return commonv1.AssociationStatusMap{} + } + + return es.Status.MonitoringAssociationsStatus +} + +func (es *Elasticsearch) SetAssociationStatusMap(typ commonv1.AssociationType, status commonv1.AssociationStatusMap) error { + if typ != commonv1.EsMonitoringAssociationType { + return fmt.Errorf("association type %s not known", typ) + } + + es.Status.MonitoringAssociationsStatus = status + return nil +} + +// -- association with monitoring Elasticsearch clusters + +// EsMonitoringAssociation helps to manage Elasticsearch+Metricbeat+Filebeat <-> Elasticsearch(es) associations +type EsMonitoringAssociation struct { + // The monitored Elasticsearch cluster from where are collected logs and monitoring metrics + *Elasticsearch + // ref is the namespaced name of the Elasticsearch referenced in the Association used to send and store monitoring data + ref types.NamespacedName +} + +var _ commonv1.Association = &EsMonitoringAssociation{} + +func (ema *EsMonitoringAssociation) Associated() commonv1.Associated { + if ema == nil { + return nil + } + if ema.Elasticsearch == nil { + ema.Elasticsearch = &Elasticsearch{} + } + return ema.Elasticsearch +} + +func (ema *EsMonitoringAssociation) AssociationConfAnnotationName() string { + return commonv1.ElasticsearchConfigAnnotationName(ema.ref) +} + +func (ema *EsMonitoringAssociation) AssociationType() commonv1.AssociationType { + return commonv1.EsMonitoringAssociationType +} + +func (ema *EsMonitoringAssociation) AssociationRef() commonv1.ObjectSelector { + return commonv1.ObjectSelector{ + Name: ema.ref.Name, + Namespace: ema.ref.Namespace, + } +} + +func (ema *EsMonitoringAssociation) AssociationConf() *commonv1.AssociationConf { + if ema.AssocConfs == nil { + return nil + } + assocConf, found := ema.AssocConfs[ema.ref] + if !found { + return nil + } + + return &assocConf +} + +func (ema *EsMonitoringAssociation) SetAssociationConf(assocConf *commonv1.AssociationConf) { + if ema.AssocConfs == nil { + ema.AssocConfs = make(map[types.NamespacedName]commonv1.AssociationConf) + } + if assocConf != nil { + ema.AssocConfs[ema.ref] = *assocConf + } +} + +func (ema *EsMonitoringAssociation) AssociationID() string { + return fmt.Sprintf("%s-%s", ema.ref.Namespace, ema.ref.Name) +} + +// HasMonitoring methods + +func (es *Elasticsearch) GetMonitoringMetricsRefs() []commonv1.ObjectSelector { + return es.Spec.Monitoring.Metrics.ElasticsearchRefs +} + +func (es *Elasticsearch) GetMonitoringLogsRefs() []commonv1.ObjectSelector { + return es.Spec.Monitoring.Logs.ElasticsearchRefs +} + +func (es *Elasticsearch) MonitoringAssociation(ref commonv1.ObjectSelector) commonv1.Association { + return &EsMonitoringAssociation{ + Elasticsearch: es, + ref: ref.WithDefaultNamespace(es.Namespace).NamespacedName(), + } } diff --git a/pkg/apis/kibana/v1/kibana_types.go b/pkg/apis/kibana/v1/kibana_types.go index f656b6aa85..0ea0110b32 100644 --- a/pkg/apis/kibana/v1/kibana_types.go +++ b/pkg/apis/kibana/v1/kibana_types.go @@ -7,9 +7,11 @@ package v1 import ( "fmt" - commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" ) const ( @@ -40,6 +42,8 @@ type Kibana struct { assocConf *commonv1.AssociationConf `json:"-"` // entAssocConf holds the configuration for the Enterprise Search association entAssocConf *commonv1.AssociationConf `json:"-"` + // monitoringAssocConf holds the configuration for the monitoring Elasticsearch clusters association + monitoringAssocConfs map[types.NamespacedName]commonv1.AssociationConf `json:"-"` } // +kubebuilder:object:root=true @@ -92,6 +96,36 @@ type KibanaSpec struct { // Can only be used if ECK is enforcing RBAC on references. // +optional ServiceAccountName string `json:"serviceAccountName,omitempty"` + + // Monitoring enables you to collect and ship log and monitoring data of this Kibana. + // See https://www.elastic.co/guide/en/kibana/current/xpack-monitoring.html. + // Metricbeat and Filebeat are deployed in the same Pod as sidecars and each one sends data to one or two different + // Elasticsearch monitoring clusters running in the same Kubernetes cluster. + // +kubebuilder:validation:Optional + Monitoring Monitoring `json:"monitoring,omitempty"` +} + +type Monitoring struct { + // Metrics holds references to Elasticsearch clusters which will receive monitoring data from this Kibana. + // +kubebuilder:validation:Optional + Metrics MetricsMonitoring `json:"metrics,omitempty"` + // Logs holds references to Elasticsearch clusters which will receive log data from this Kibana. + // +kubebuilder:validation:Optional + Logs LogsMonitoring `json:"logs,omitempty"` +} + +type MetricsMonitoring struct { + // ElasticsearchRefs is a reference to a list of monitoring Elasticsearch clusters running in the same Kubernetes cluster. + // Due to existing limitations, only a single Elasticsearch cluster is currently supported. + // +kubebuilder:validation:Required + ElasticsearchRefs []commonv1.ObjectSelector `json:"elasticsearchRefs,omitempty"` +} + +type LogsMonitoring struct { + // ElasticsearchRefs is a reference to a list of monitoring Elasticsearch clusters running in the same Kubernetes cluster. + // Due to existing limitations, only a single Elasticsearch cluster is currently supported. + // +kubebuilder:validation:Required + ElasticsearchRefs []commonv1.ObjectSelector `json:"elasticsearchRefs,omitempty"` } // KibanaStatus defines the observed state of Kibana @@ -104,6 +138,8 @@ type KibanaStatus struct { ElasticsearchAssociationStatus commonv1.AssociationStatus `json:"elasticsearchAssociationStatus,omitempty"` // EnterpriseSearchAssociationStatus is the status of any auto-linking to Enterprise Search. EnterpriseSearchAssociationStatus commonv1.AssociationStatus `json:"enterpriseSearchAssociationStatus,omitempty"` + // MonitoringAssociationStatus is the status of any auto-linking to monitoring Elasticsearch clusters. + MonitoringAssociationStatus commonv1.AssociationStatusMap `json:"monitoringAssociationStatus,omitempty"` } // IsMarkedForDeletion returns true if the Kibana is going to be deleted @@ -140,6 +176,22 @@ func (k *Kibana) GetAssociations() []commonv1.Association { Kibana: k, }) } + for _, ref := range k.Spec.Monitoring.Metrics.ElasticsearchRefs { + if ref.IsDefined() { + associations = append(associations, &KbMonitoringAssociation{ + Kibana: k, + ref: ref.WithDefaultNamespace(k.Namespace).NamespacedName(), + }) + } + } + for _, ref := range k.Spec.Monitoring.Logs.ElasticsearchRefs { + if ref.IsDefined() { + associations = append(associations, &KbMonitoringAssociation{ + Kibana: k, + ref: ref.WithDefaultNamespace(k.Namespace).NamespacedName(), + }) + } + } return associations } @@ -154,27 +206,44 @@ func (k *Kibana) AssociationStatusMap(typ commonv1.AssociationType) commonv1.Ass if k.Spec.EnterpriseSearchRef.IsDefined() { return commonv1.NewSingleAssociationStatusMap(k.Status.EnterpriseSearchAssociationStatus) } + case commonv1.KbMonitoringAssociationType: + for _, esRef := range k.Spec.Monitoring.Metrics.ElasticsearchRefs { + if esRef.IsDefined() { + return k.Status.MonitoringAssociationStatus + } + } + for _, esRef := range k.Spec.Monitoring.Logs.ElasticsearchRefs { + if esRef.IsDefined() { + return k.Status.MonitoringAssociationStatus + } + } } return commonv1.AssociationStatusMap{} } func (k *Kibana) SetAssociationStatusMap(typ commonv1.AssociationType, status commonv1.AssociationStatusMap) error { - single, err := status.Single() - if err != nil { - return err - } - switch typ { case commonv1.ElasticsearchAssociationType: + single, err := status.Single() + if err != nil { + return err + } k.Status.ElasticsearchAssociationStatus = single // also set Status.AssociationStatus to report the status of the association with es, // for backward compatibility reasons k.Status.AssociationStatus = single return nil case commonv1.EntAssociationType: + single, err := status.Single() + if err != nil { + return err + } k.Status.EnterpriseSearchAssociationStatus = single return nil + case commonv1.KbMonitoringAssociationType: + k.Status.MonitoringAssociationStatus = status + return nil default: return fmt.Errorf("association type %s not known", typ) } @@ -273,3 +342,81 @@ func (kbent *KibanaEntAssociation) SetAssociationConf(assocConf *commonv1.Associ func (kbent *KibanaEntAssociation) AssociationID() string { return commonv1.SingletonAssociationID } + +// -- association with monitoring Elasticsearch clusters + +// KbMonitoringAssociation helps to manage the Kibana / monitoring Elasticsearch clusters association. +type KbMonitoringAssociation struct { + // The associated Kibana + *Kibana + // ref is the namespaced name of the monitoring Elasticsearch referenced in the Association + ref types.NamespacedName +} + +var _ commonv1.Association = &KbMonitoringAssociation{} + +func (kbmon *KbMonitoringAssociation) Associated() commonv1.Associated { + if kbmon == nil { + return nil + } + if kbmon.Kibana == nil { + kbmon.Kibana = &Kibana{} + } + return kbmon.Kibana +} + +func (kbmon *KbMonitoringAssociation) AssociationConfAnnotationName() string { + return commonv1.ElasticsearchConfigAnnotationName(kbmon.ref) +} + +func (kbmon *KbMonitoringAssociation) AssociationType() commonv1.AssociationType { + return commonv1.KbMonitoringAssociationType +} + +func (kbmon *KbMonitoringAssociation) AssociationRef() commonv1.ObjectSelector { + return commonv1.ObjectSelector{ + Name: kbmon.ref.Name, + Namespace: kbmon.ref.Namespace, + } +} + +func (kbmon *KbMonitoringAssociation) AssociationConf() *commonv1.AssociationConf { + if kbmon.monitoringAssocConfs == nil { + return nil + } + assocConf, found := kbmon.monitoringAssocConfs[kbmon.ref] + if !found { + return nil + } + return &assocConf +} + +func (kbmon *KbMonitoringAssociation) SetAssociationConf(assocConf *commonv1.AssociationConf) { + if kbmon.monitoringAssocConfs == nil { + kbmon.monitoringAssocConfs = make(map[types.NamespacedName]commonv1.AssociationConf) + } + if assocConf != nil { + kbmon.monitoringAssocConfs[kbmon.ref] = *assocConf + } +} + +func (kbmon *KbMonitoringAssociation) AssociationID() string { + return fmt.Sprintf("%s-%s", kbmon.ref.Namespace, kbmon.ref.Name) +} + +// -- HasMonitoring methods + +func (k *Kibana) GetMonitoringMetricsRefs() []commonv1.ObjectSelector { + return k.Spec.Monitoring.Metrics.ElasticsearchRefs +} + +func (k *Kibana) GetMonitoringLogsRefs() []commonv1.ObjectSelector { + return k.Spec.Monitoring.Logs.ElasticsearchRefs +} + +func (k *Kibana) MonitoringAssociation(esRef commonv1.ObjectSelector) commonv1.Association { + return &KbMonitoringAssociation{ + Kibana: k, + ref: esRef.WithDefaultNamespace(k.Namespace).NamespacedName(), + } +} diff --git a/pkg/controller/kibana/name.go b/pkg/apis/kibana/v1/name.go similarity index 65% rename from pkg/controller/kibana/name.go rename to pkg/apis/kibana/v1/name.go index 17b438bfd8..a836e92398 100644 --- a/pkg/controller/kibana/name.go +++ b/pkg/apis/kibana/v1/name.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package kibana +package v1 import ( common_name "github.com/elastic/cloud-on-k8s/pkg/controller/common/name" @@ -10,13 +10,13 @@ import ( const httpServiceSuffix = "http" -// Namer is a Namer that is configured with the defaults for resources related to a Kibana resource. -var Namer = common_name.NewNamer("kb") +// KBNamer is a KBNamer that is configured with the defaults for resources related to a Kibana resource. +var KBNamer = common_name.NewNamer("kb") func HTTPService(kbName string) string { - return Namer.Suffix(kbName, httpServiceSuffix) + return KBNamer.Suffix(kbName, httpServiceSuffix) } func Deployment(kbName string) string { - return Namer.Suffix(kbName) + return KBNamer.Suffix(kbName) } diff --git a/pkg/controller/kibana/name_test.go b/pkg/apis/kibana/v1/name_test.go similarity index 97% rename from pkg/controller/kibana/name_test.go rename to pkg/apis/kibana/v1/name_test.go index 1603e13337..18ddf636dc 100644 --- a/pkg/controller/kibana/name_test.go +++ b/pkg/apis/kibana/v1/name_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package kibana +package v1 import ( "testing" diff --git a/pkg/apis/kibana/v1/webhook.go b/pkg/apis/kibana/v1/webhook.go index 2aded68632..257703bd83 100644 --- a/pkg/apis/kibana/v1/webhook.go +++ b/pkg/apis/kibana/v1/webhook.go @@ -8,6 +8,8 @@ import ( "errors" commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/validations" "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" ulog "github.com/elastic/cloud-on-k8s/pkg/utils/log" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -26,6 +28,7 @@ var ( checkNoUnknownFields, checkNameLength, checkSupportedVersion, + checkMonitoring, } updateChecks = []func(old, curr *Kibana) field.ErrorList{ @@ -104,3 +107,13 @@ func checkSupportedVersion(k *Kibana) field.ErrorList { func checkNoDowngrade(prev, curr *Kibana) field.ErrorList { return commonv1.CheckNoDowngrade(prev.Spec.Version, curr.Spec.Version) } + +func checkMonitoring(k *Kibana) field.ErrorList { + errs := validations.Validate(k, k.Spec.Version) + // Kibana must be associated to an Elasticsearch when monitoring metrics are enabled + if monitoring.IsMetricsDefined(k) && !k.Spec.ElasticsearchRef.IsDefined() { + errs = append(errs, field.Invalid(field.NewPath("spec").Child("elasticsearchRef"), k.Spec.ElasticsearchRef, + validations.InvalidKibanaElasticsearchRefForStackMonitoringMsg)) + } + return errs +} diff --git a/pkg/apis/kibana/v1/zz_generated.deepcopy.go b/pkg/apis/kibana/v1/zz_generated.deepcopy.go index ca7fb38fa2..88e8c8b34c 100644 --- a/pkg/apis/kibana/v1/zz_generated.deepcopy.go +++ b/pkg/apis/kibana/v1/zz_generated.deepcopy.go @@ -11,15 +11,37 @@ package v1 import ( commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KbMonitoringAssociation) DeepCopyInto(out *KbMonitoringAssociation) { + *out = *in + if in.Kibana != nil { + in, out := &in.Kibana, &out.Kibana + *out = new(Kibana) + (*in).DeepCopyInto(*out) + } + out.ref = in.ref +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KbMonitoringAssociation. +func (in *KbMonitoringAssociation) DeepCopy() *KbMonitoringAssociation { + if in == nil { + return nil + } + out := new(KbMonitoringAssociation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Kibana) DeepCopyInto(out *Kibana) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) if in.assocConf != nil { in, out := &in.assocConf, &out.assocConf *out = new(commonv1.AssociationConf) @@ -30,6 +52,13 @@ func (in *Kibana) DeepCopyInto(out *Kibana) { *out = new(commonv1.AssociationConf) **out = **in } + if in.monitoringAssocConfs != nil { + in, out := &in.monitoringAssocConfs, &out.monitoringAssocConfs + *out = make(map[types.NamespacedName]commonv1.AssociationConf, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kibana. @@ -140,6 +169,7 @@ func (in *KibanaSpec) DeepCopyInto(out *KibanaSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.Monitoring.DeepCopyInto(&out.Monitoring) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KibanaSpec. @@ -156,6 +186,13 @@ func (in *KibanaSpec) DeepCopy() *KibanaSpec { func (in *KibanaStatus) DeepCopyInto(out *KibanaStatus) { *out = *in out.DeploymentStatus = in.DeploymentStatus + if in.MonitoringAssociationStatus != nil { + in, out := &in.MonitoringAssociationStatus, &out.MonitoringAssociationStatus + *out = make(commonv1.AssociationStatusMap, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KibanaStatus. @@ -167,3 +204,60 @@ func (in *KibanaStatus) DeepCopy() *KibanaStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogsMonitoring) DeepCopyInto(out *LogsMonitoring) { + *out = *in + if in.ElasticsearchRefs != nil { + in, out := &in.ElasticsearchRefs, &out.ElasticsearchRefs + *out = make([]commonv1.ObjectSelector, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogsMonitoring. +func (in *LogsMonitoring) DeepCopy() *LogsMonitoring { + if in == nil { + return nil + } + out := new(LogsMonitoring) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsMonitoring) DeepCopyInto(out *MetricsMonitoring) { + *out = *in + if in.ElasticsearchRefs != nil { + in, out := &in.ElasticsearchRefs, &out.ElasticsearchRefs + *out = make([]commonv1.ObjectSelector, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsMonitoring. +func (in *MetricsMonitoring) DeepCopy() *MetricsMonitoring { + if in == nil { + return nil + } + out := new(MetricsMonitoring) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Monitoring) DeepCopyInto(out *Monitoring) { + *out = *in + in.Metrics.DeepCopyInto(&out.Metrics) + in.Logs.DeepCopyInto(&out.Logs) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Monitoring. +func (in *Monitoring) DeepCopy() *Monitoring { + if in == nil { + return nil + } + out := new(Monitoring) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controller/association/controller/apm_kibana.go b/pkg/controller/association/controller/apm_kibana.go index de60a9c995..62a016098e 100644 --- a/pkg/controller/association/controller/apm_kibana.go +++ b/pkg/controller/association/controller/apm_kibana.go @@ -29,7 +29,7 @@ func AddApmKibana(mgr manager.Manager, accessReviewer rbac.AccessReviewer, param ReferencedObjTemplate: func() client.Object { return &kbv1.Kibana{} }, ExternalServiceURL: getKibanaExternalURL, ReferencedResourceVersion: referencedKibanaStatusVersion, - ReferencedResourceNamer: kibana.Namer, + ReferencedResourceNamer: kbv1.KBNamer, AssociationName: "apm-kibana", AssociationType: commonv1.KibanaAssociationType, Labels: func(associated types.NamespacedName) map[string]string { @@ -64,7 +64,7 @@ func getKibanaExternalURL(c k8s.Client, assoc commonv1.Association) (string, err } serviceName := kibanaRef.ServiceName if serviceName == "" { - serviceName = kibana.HTTPService(kb.Name) + serviceName = kbv1.HTTPService(kb.Name) } nsn := types.NamespacedName{Namespace: kb.Namespace, Name: serviceName} return association.ServiceURL(c, nsn, kb.Spec.HTTP.Protocol()) diff --git a/pkg/controller/association/controller/beat_kibana.go b/pkg/controller/association/controller/beat_kibana.go index 75194276ea..be5f832d9b 100644 --- a/pkg/controller/association/controller/beat_kibana.go +++ b/pkg/controller/association/controller/beat_kibana.go @@ -29,7 +29,7 @@ func AddBeatKibana(mgr manager.Manager, accessReviewer rbac.AccessReviewer, para ReferencedObjTemplate: func() client.Object { return &kbv1.Kibana{} }, ExternalServiceURL: getKibanaExternalURL, ReferencedResourceVersion: referencedKibanaStatusVersion, - ReferencedResourceNamer: kibana.Namer, + ReferencedResourceNamer: kbv1.KBNamer, AssociationName: "beat-kibana", AssociatedShortName: "beat", AssociationType: commonv1.KibanaAssociationType, diff --git a/pkg/controller/association/controller/es_monitoring.go b/pkg/controller/association/controller/es_monitoring.go index dc9723963f..fca0c47f54 100644 --- a/pkg/controller/association/controller/es_monitoring.go +++ b/pkg/controller/association/controller/es_monitoring.go @@ -20,14 +20,15 @@ import ( ) const ( - // EsAssociationLabelName marks resources created by this controller for easier retrieval. + // EsAssociationLabelName marks resources created for an association originating from Elasticsearch with the + // Elasticsearch name. EsAssociationLabelName = "esassociation.k8s.elastic.co/name" - // EsAssociationLabelNamespace marks resources created by this controller for easier retrieval. + // EsAssociationLabelNamespace marks resources created for an association originating from Elasticsearch with the + // Elasticsearch namespace. EsAssociationLabelNamespace = "esassociation.k8s.elastic.co/namespace" - // EsAssociationLabelType marks the type of association. + // EsAssociationLabelType marks resources created for an association originating from Elasticsearch + // with the target resource type (e.g. "elasticsearch"). EsAssociationLabelType = "esassociation.k8s.elastic.co/type" - - EsMonitoringAssociationType = "es-monitoring" ) // AddEsMonitoring reconciles an association between two Elasticsearch clusters for Stack Monitoring. @@ -47,7 +48,7 @@ func AddEsMonitoring(mgr manager.Manager, accessReviewer rbac.AccessReviewer, pa return map[string]string{ EsAssociationLabelName: associated.Name, EsAssociationLabelNamespace: associated.Namespace, - EsAssociationLabelType: EsMonitoringAssociationType, + EsAssociationLabelType: commonv1.EsMonitoringAssociationType, } }, AssociationConfAnnotationNameBase: commonv1.ElasticsearchConfigAnnotationNameBase, diff --git a/pkg/controller/association/controller/kb_monitoring.go b/pkg/controller/association/controller/kb_monitoring.go new file mode 100644 index 0000000000..4f1e639e79 --- /dev/null +++ b/pkg/controller/association/controller/kb_monitoring.go @@ -0,0 +1,57 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package controller + +import ( + kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/association" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/operator" + eslabel "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/user" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + "github.com/elastic/cloud-on-k8s/pkg/utils/rbac" +) + +// AddKbMonitoring reconciles an association between Kibana and Elasticsearch clusters for Stack Monitoring. +// Beats are configured to collect monitoring metrics and logs data of the associated Kibana and send +// them to the Elasticsearch referenced in the association. +func AddKbMonitoring(mgr manager.Manager, accessReviewer rbac.AccessReviewer, params operator.Parameters) error { + return association.AddAssociationController(mgr, accessReviewer, params, association.AssociationInfo{ + AssociatedObjTemplate: func() commonv1.Associated { return &kbv1.Kibana{} }, + ReferencedObjTemplate: func() client.Object { return &esv1.Elasticsearch{} }, + ReferencedResourceVersion: referencedElasticsearchStatusVersion, + ExternalServiceURL: getElasticsearchExternalURL, + AssociationType: commonv1.KbMonitoringAssociationType, + ReferencedResourceNamer: esv1.ESNamer, + AssociationName: "kb-monitoring", + AssociatedShortName: "kb-mon", + Labels: func(associated types.NamespacedName) map[string]string { + return map[string]string{ + KibanaAssociationLabelName: associated.Name, + KibanaAssociationLabelNamespace: associated.Namespace, + KibanaAssociationLabelType: commonv1.KbMonitoringAssociationType, + } + }, + AssociationConfAnnotationNameBase: commonv1.ElasticsearchConfigAnnotationNameBase, + AssociationResourceNameLabelName: eslabel.ClusterNameLabelName, + AssociationResourceNamespaceLabelName: eslabel.ClusterNamespaceLabelName, + + ElasticsearchUserCreation: &association.ElasticsearchUserCreation{ + ElasticsearchRef: func(c k8s.Client, association commonv1.Association) (bool, commonv1.ObjectSelector, error) { + return true, association.AssociationRef(), nil + }, + UserSecretSuffix: "beat-kb-mon-user", + ESUserRole: func(associated commonv1.Associated) (string, error) { + return user.StackMonitoringUserRole, nil + }, + }, + }) +} diff --git a/pkg/controller/common/stackmon/config.go b/pkg/controller/common/stackmon/config.go index 8e21ba2f3f..f6c7794d5d 100644 --- a/pkg/controller/common/stackmon/config.go +++ b/pkg/controller/common/stackmon/config.go @@ -5,21 +5,27 @@ package stackmon import ( + "bytes" "crypto/sha256" "fmt" "hash" "path/filepath" + "text/template" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/association" "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/name" "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings" "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/user" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ) @@ -31,7 +37,7 @@ type beatConfig struct { volumes []volume.VolumeLike } -func newBeatConfig(client k8s.Client, beatName string, resource HasMonitoring, associations []commonv1.Association, baseConfig string) (beatConfig, error) { +func newBeatConfig(client k8s.Client, beatName string, resource monitoring.HasMonitoring, associations []commonv1.Association, baseConfig string) (beatConfig, error) { if len(associations) != 1 { // should never happen because of the pre-creation validation return beatConfig{}, errors.New("only one Elasticsearch reference is supported for Stack Monitoring") @@ -50,12 +56,13 @@ func newBeatConfig(client k8s.Client, beatName string, resource HasMonitoring, a } // name for the config secret and the associated config volume for the es pod + configSecretName := fmt.Sprintf("%s-%s-%s-config", resource.GetName(), string(assoc.AssociationType()), beatName) configName := configVolumeName(resource.GetName(), beatName) configFilename := fmt.Sprintf("%s.yml", beatName) configDirPath := fmt.Sprintf("/etc/%s-config", beatName) // add the config volume - configVolume := volume.NewSecretVolumeWithMountPath(configName, configName, configDirPath) + configVolume := volume.NewSecretVolumeWithMountPath(configSecretName, configName, configDirPath) configFilepath := filepath.Join(configDirPath, configFilename) volumes := []volume.VolumeLike{configVolume} @@ -75,7 +82,7 @@ func newBeatConfig(client k8s.Client, beatName string, resource HasMonitoring, a configSecret := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: configName, + Name: configSecretName, Namespace: resource.GetNamespace(), Labels: label.NewLabels(k8s.ExtractNamespacedName(resource)), }, @@ -145,3 +152,60 @@ func mergeConfig(rawConfig string, config map[string]interface{}) ([]byte, error return cfgBytes, nil } + +// inputConfigData holds data to configure the Metricbeat Elasticsearch and Kibana modules used +// to collect metrics for Stack Monitoring +type inputConfigData struct { + URL string + Username string + Password string + IsSSL bool + SSLPath string + SSLMode string +} + +// buildMetricbeatBaseConfig builds the base configuration for Metricbeat with the Elasticsearch or Kibana modules used +// to collect metrics for Stack Monitoring +func buildMetricbeatBaseConfig( + client k8s.Client, + associationType commonv1.AssociationType, + nsn types.NamespacedName, + esNsn types.NamespacedName, + namer name.Namer, + url string, + isTLS bool, + configTemplate string, +) (string, volume.VolumeLike, error) { + password, err := user.GetMonitoringUserPassword(client, esNsn) + if err != nil { + return "", nil, err + } + + configData := inputConfigData{ + URL: url, + Username: user.MonitoringUserName, + Password: password, + IsSSL: isTLS, + } + + var caVolume volume.VolumeLike + if configData.IsSSL { + caVolume = volume.NewSecretVolumeWithMountPath( + certificates.PublicCertsSecretName(namer, nsn.Name), + fmt.Sprintf("%s-local-ca", string(associationType)), + fmt.Sprintf("/mnt/elastic-internal/%s/%s/%s/certs", string(associationType), nsn.Namespace, nsn.Name), + ) + + configData.SSLPath = filepath.Join(caVolume.VolumeMount().MountPath, certificates.CAFileName) + configData.SSLMode = "certificate" + } + + // render the config template with the config data + var metricbeatConfig bytes.Buffer + err = template.Must(template.New("").Parse(configTemplate)).Execute(&metricbeatConfig, configData) + if err != nil { + return "", nil, err + } + + return metricbeatConfig.String(), caVolume, nil +} diff --git a/pkg/controller/common/stackmon/config_test.go b/pkg/controller/common/stackmon/config_test.go new file mode 100644 index 0000000000..46a8307e85 --- /dev/null +++ b/pkg/controller/common/stackmon/config_test.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stackmon + +import ( + "testing" + + "github.com/elastic/cloud-on-k8s/pkg/controller/common/name" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func TestBuildMetricbeatBaseConfig(t *testing.T) { + tests := []struct { + name string + isTLS bool + baseConfig string + }{ + { + name: "with tls", + isTLS: true, + baseConfig: ` + hosts: ["scheme://localhost:1234"] + username: elastic-internal-monitoring + password: 1234567890 + ssl.certificate_authorities: ["/mnt/elastic-internal/xx-monitoring/namespace/name/certs/ca.crt"] + ssl.verification_mode: "certificate"`, + }, + { + name: "without tls", + isTLS: false, + baseConfig: ` + hosts: ["scheme://localhost:1234"] + username: elastic-internal-monitoring + password: 1234567890`, + }, + } + + baseConfigTemplate := ` + hosts: ["{{ .URL }}"] + username: {{ .Username }} + password: {{ .Password }} + {{- if .IsSSL }} + ssl.certificate_authorities: ["{{ .SSLPath }}"] + ssl.verification_mode: "{{ .SSLMode }}" + {{- end }}` + sampleURL := "scheme://localhost:1234" + + fakeClient := k8s.NewFakeClient(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "name-es-internal-users", Namespace: "namespace"}, + Data: map[string][]byte{"elastic-internal-monitoring": []byte("1234567890")}, + }) + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + baseConfig, _, err := buildMetricbeatBaseConfig( + fakeClient, + "xx-monitoring", + types.NamespacedName{Namespace: "namespace", Name: "name"}, + types.NamespacedName{Namespace: "namespace", Name: "name"}, + name.NewNamer("xx"), + sampleURL, + tc.isTLS, + baseConfigTemplate, + ) + assert.NoError(t, err) + assert.Equal(t, tc.baseConfig, baseConfig) + }) + } +} diff --git a/pkg/controller/common/stackmon/monitoring.go b/pkg/controller/common/stackmon/monitoring.go deleted file mode 100644 index 41d6cfbf61..0000000000 --- a/pkg/controller/common/stackmon/monitoring.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package stackmon - -import ( - commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// HasMonitoring is the interface implemented by an Elastic Stack application that supports Stack Monitoring () -type HasMonitoring interface { - metav1.Object - GetMonitoringMetricsAssociation() []commonv1.Association - GetMonitoringLogsAssociation() []commonv1.Association -} diff --git a/pkg/controller/common/stackmon/monitoring/monitoring.go b/pkg/controller/common/stackmon/monitoring/monitoring.go new file mode 100644 index 0000000000..5d70135723 --- /dev/null +++ b/pkg/controller/common/stackmon/monitoring/monitoring.go @@ -0,0 +1,60 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package monitoring + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" +) + +// HasMonitoring is the interface implemented by an Elastic Stack application that supports Stack Monitoring +type HasMonitoring interface { + client.Object + GetMonitoringMetricsRefs() []commonv1.ObjectSelector + GetMonitoringLogsRefs() []commonv1.ObjectSelector + MonitoringAssociation(ref commonv1.ObjectSelector) commonv1.Association +} + +func IsDefined(resource HasMonitoring) bool { + return IsMetricsDefined(resource) || IsLogsDefined(resource) +} + +func IsMetricsDefined(resource HasMonitoring) bool { + return AreEsRefsDefined(resource.GetMonitoringMetricsRefs()) +} + +func IsLogsDefined(resource HasMonitoring) bool { + return AreEsRefsDefined(resource.GetMonitoringLogsRefs()) +} + +func AreEsRefsDefined(esRefs []commonv1.ObjectSelector) bool { + for _, ref := range esRefs { + if !ref.IsDefined() { + return false + } + } + return len(esRefs) > 0 +} + +func GetMetricsAssociation(resource HasMonitoring) []commonv1.Association { + associations := make([]commonv1.Association, 0) + for _, ref := range resource.GetMonitoringMetricsRefs() { + if ref.IsDefined() { + associations = append(associations, resource.MonitoringAssociation(ref)) + } + } + return associations +} + +func GetLogsAssociation(resource HasMonitoring) []commonv1.Association { + associations := make([]commonv1.Association, 0) + for _, ref := range resource.GetMonitoringLogsRefs() { + if ref.IsDefined() { + associations = append(associations, resource.MonitoringAssociation(ref)) + } + } + return associations +} diff --git a/pkg/controller/common/stackmon/monitoring/monitoring_test.go b/pkg/controller/common/stackmon/monitoring/monitoring_test.go new file mode 100644 index 0000000000..f0fda47e89 --- /dev/null +++ b/pkg/controller/common/stackmon/monitoring/monitoring_test.go @@ -0,0 +1,58 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package monitoring + +import ( + "testing" + + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + "github.com/stretchr/testify/assert" +) + +var ( + sampleEs = esv1.Elasticsearch{} + monitoringEsRef = commonv1.ObjectSelector{Name: "monitoring", Namespace: "observability"} + sampleMonitoredEs = esv1.Elasticsearch{ + Spec: esv1.ElasticsearchSpec{ + Monitoring: esv1.Monitoring{ + Metrics: esv1.MetricsMonitoring{ + ElasticsearchRefs: []commonv1.ObjectSelector{monitoringEsRef}, + }, + }, + }, + } +) + +func TestIsDefined(t *testing.T) { + assert.False(t, IsDefined(&sampleEs)) + assert.True(t, IsDefined(&sampleMonitoredEs)) +} + +func TestIsMetricsDefined(t *testing.T) { + assert.False(t, IsMetricsDefined(&sampleEs)) + assert.True(t, IsMetricsDefined(&sampleMonitoredEs)) +} + +func TestIsLogsDefined(t *testing.T) { + assert.False(t, IsLogsDefined(&sampleEs)) + assert.False(t, IsLogsDefined(&sampleMonitoredEs)) +} + +func TestAreEsRefsDefined(t *testing.T) { + assert.False(t, AreEsRefsDefined(sampleEs.Spec.Monitoring.Metrics.ElasticsearchRefs)) + assert.True(t, AreEsRefsDefined(sampleMonitoredEs.Spec.Monitoring.Metrics.ElasticsearchRefs)) + assert.False(t, AreEsRefsDefined(sampleMonitoredEs.Spec.Monitoring.Logs.ElasticsearchRefs)) +} + +func TestGetMetricsAssociation(t *testing.T) { + assert.Equal(t, 0, len(GetMetricsAssociation(&sampleEs))) + assert.Equal(t, 1, len(GetMetricsAssociation(&sampleMonitoredEs))) +} + +func TestGetLogsAssociation(t *testing.T) { + assert.Equal(t, 0, len(GetLogsAssociation(&sampleEs))) + assert.Equal(t, 0, len(GetLogsAssociation(&sampleMonitoredEs))) +} diff --git a/pkg/controller/common/stackmon/name_test.go b/pkg/controller/common/stackmon/name_test.go index dea2129147..6f05a9ab81 100644 --- a/pkg/controller/common/stackmon/name_test.go +++ b/pkg/controller/common/stackmon/name_test.go @@ -9,6 +9,7 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -45,16 +46,16 @@ func TestCAVolumeName(t *testing.T) { }, } - name := caVolumeName(es.GetMonitoringMetricsAssociation()[0]) + name := caVolumeName(monitoring.GetMetricsAssociation(&es)[0]) assert.LessOrEqual(t, len(name), maxVolumeNameLength) assert.Equal(t, "es-monitoring-954c60-ca", name) - name = caVolumeName(es.GetMonitoringLogsAssociation()[0]) + name = caVolumeName(monitoring.GetLogsAssociation(&es)[0]) assert.LessOrEqual(t, len(name), maxVolumeNameLength) assert.Equal(t, "es-monitoring-954c60-ca", name) es.Spec.Monitoring.Logs.ElasticsearchRefs[0].Name = "another-name" - newName := caVolumeName(es.GetMonitoringLogsAssociation()[0]) + newName := caVolumeName(monitoring.GetLogsAssociation(&es)[0]) assert.NotEqual(t, name, newName) assert.Equal(t, "es-monitoring-ae0f57-ca", newName) } diff --git a/pkg/controller/common/stackmon/sidecar.go b/pkg/controller/common/stackmon/sidecar.go index 2be074fe3a..2f07bda1c9 100644 --- a/pkg/controller/common/stackmon/sidecar.go +++ b/pkg/controller/common/stackmon/sidecar.go @@ -7,20 +7,49 @@ package stackmon import ( "hash" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/pkg/controller/common/defaults" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/name" "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ) -func NewMetricBeatSidecar(client k8s.Client, resource HasMonitoring, image string, baseConfig string, additionalVolume volume.VolumeLike) (BeatSidecar, error) { - return NewBeatSidecar(client, "metricbeat", image, resource, resource.GetMonitoringMetricsAssociation(), baseConfig, additionalVolume) +func NewMetricBeatSidecar( + client k8s.Client, + associationType commonv1.AssociationType, + resource monitoring.HasMonitoring, + version string, + esNsn types.NamespacedName, + baseConfigTemplate string, + namer name.Namer, + url string, + isTLS bool, +) (BeatSidecar, error) { + baseConfig, sourceCaVolume, err := buildMetricbeatBaseConfig( + client, + associationType, + k8s.ExtractNamespacedName(resource), + esNsn, + namer, + url, + isTLS, + baseConfigTemplate, + ) + if err != nil { + return BeatSidecar{}, err + } + image := container.ImageRepository(container.MetricbeatImage, version) + return NewBeatSidecar(client, "metricbeat", image, resource, monitoring.GetMetricsAssociation(resource), baseConfig, sourceCaVolume) } -func NewFileBeatSidecar(client k8s.Client, resource HasMonitoring, image string, baseConfig string, additionalVolume volume.VolumeLike) (BeatSidecar, error) { - return NewBeatSidecar(client, "filebeat", image, resource, resource.GetMonitoringLogsAssociation(), baseConfig, additionalVolume) +func NewFileBeatSidecar(client k8s.Client, resource monitoring.HasMonitoring, version string, baseConfig string, additionalVolume volume.VolumeLike) (BeatSidecar, error) { + image := container.ImageRepository(container.FilebeatImage, version) + return NewBeatSidecar(client, "filebeat", image, resource, monitoring.GetLogsAssociation(resource), baseConfig, additionalVolume) } // BeatSidecar helps with building a beat sidecar container to monitor an Elastic Stack application. It focuses on @@ -32,7 +61,7 @@ type BeatSidecar struct { Volumes []corev1.Volume } -func NewBeatSidecar(client k8s.Client, beatName string, image string, resource HasMonitoring, +func NewBeatSidecar(client k8s.Client, beatName string, image string, resource monitoring.HasMonitoring, associations []commonv1.Association, baseConfig string, additionalVolume volume.VolumeLike, ) (BeatSidecar, error) { // build the beat config diff --git a/pkg/controller/common/stackmon/validations/validations.go b/pkg/controller/common/stackmon/validations/validations.go new file mode 100644 index 0000000000..9087be9d74 --- /dev/null +++ b/pkg/controller/common/stackmon/validations/validations.go @@ -0,0 +1,65 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package validations + +import ( + "fmt" + + "github.com/blang/semver/v4" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +const ( + unsupportedVersionMsg = "Unsupported version for Stack Monitoring. Required >= %s." + invalidElasticsearchRefsMsg = "Only one Elasticsearch reference is supported for %s Stack Monitoring" + + InvalidKibanaElasticsearchRefForStackMonitoringMsg = "Kibana must be associated to an Elasticsearch cluster through elasticsearchRef in order to enable monitoring metrics features" +) + +var ( + // MinStackVersion is the minimum Stack version to enable Stack Monitoring on an Elastic Stack application.. + // This requirement comes from the fact that we configure Elasticsearch to write logs to disk for Filebeat + // via the env var ES_LOG_STYLE available from this version. + MinStackVersion = version.MustParse("7.14.0-SNAPSHOT") +) + +// Validate validates that the resource version is supported for Stack Monitoring and that there is exactly one +// Elasticsearch reference defined to send monitoring data when Stack Monitoring is defined +func Validate(resource monitoring.HasMonitoring, version string) field.ErrorList { + var errs field.ErrorList + if monitoring.IsDefined(resource) { + err := IsSupportedVersion(version) + if err != nil { + finalMinStackVersion, _ := semver.FinalizeVersion(MinStackVersion.String()) // discards prerelease suffix + errs = append(errs, field.Invalid(field.NewPath("spec").Child("version"), version, + fmt.Sprintf(unsupportedVersionMsg, finalMinStackVersion))) + } + } + refs := resource.GetMonitoringMetricsRefs() + if monitoring.AreEsRefsDefined(refs) && len(refs) != 1 { + errs = append(errs, field.Invalid(field.NewPath("spec").Child("monitoring").Child("metrics").Child("elasticsearchRefs"), + refs, fmt.Sprintf(invalidElasticsearchRefsMsg, "Metrics"))) + } + refs = resource.GetMonitoringLogsRefs() + if monitoring.AreEsRefsDefined(refs) && len(refs) != 1 { + errs = append(errs, field.Invalid(field.NewPath("spec").Child("monitoring").Child("logs").Child("elasticsearchRefs"), + refs, fmt.Sprintf(invalidElasticsearchRefsMsg, "Logs"))) + } + return errs +} + +// IsSupportedVersion returns true if the resource version is supported for Stack Monitoring, else returns false +func IsSupportedVersion(v string) error { + ver, err := version.Parse(v) + if err != nil { + return err + } + if ver.LT(MinStackVersion) { + return fmt.Errorf("unsupported version for Stack Monitoring: required >= %s", MinStackVersion) + } + return nil +} diff --git a/pkg/controller/elasticsearch/stackmon/validations_test.go b/pkg/controller/common/stackmon/validations/validations_test.go similarity index 97% rename from pkg/controller/elasticsearch/stackmon/validations_test.go rename to pkg/controller/common/stackmon/validations/validations_test.go index 3883e320e6..d1d38b34f0 100644 --- a/pkg/controller/elasticsearch/stackmon/validations_test.go +++ b/pkg/controller/common/stackmon/validations/validations_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package stackmon +package validations import ( "testing" @@ -97,7 +97,7 @@ func TestValidate(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - err := Validate(tc.es) + err := Validate(&tc.es, tc.es.Spec.Version) if len(err) > 0 { require.True(t, tc.isErr) } else { diff --git a/pkg/controller/elasticsearch/stackmon/beat_config.go b/pkg/controller/elasticsearch/stackmon/beat_config.go index 3dd06bb15e..7d8692eb62 100644 --- a/pkg/controller/elasticsearch/stackmon/beat_config.go +++ b/pkg/controller/elasticsearch/stackmon/beat_config.go @@ -5,18 +5,11 @@ package stackmon import ( - "bytes" _ "embed" // for the beats config files - "fmt" - "path/filepath" - "text/template" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/network" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/user" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ) @@ -32,7 +25,7 @@ var ( // ReconcileConfigSecrets reconciles the secrets holding beats configuration func ReconcileConfigSecrets(client k8s.Client, es esv1.Elasticsearch) error { - if IsMonitoringMetricsDefined(es) { + if monitoring.IsMetricsDefined(&es) { b, err := Metricbeat(client, es) if err != nil { return err @@ -43,7 +36,7 @@ func ReconcileConfigSecrets(client k8s.Client, es esv1.Elasticsearch) error { } } - if IsMonitoringLogsDefined(es) { + if monitoring.IsLogsDefined(&es) { b, err := Filebeat(client, es) if err != nil { return err @@ -56,49 +49,3 @@ func ReconcileConfigSecrets(client k8s.Client, es esv1.Elasticsearch) error { return nil } - -// esConfigData holds data to configure the Metricbeat Elasticsearch module -type esConfigData struct { - URL string - Username string - Password string - IsSSL bool - SSLPath string - SSLMode string -} - -// buildMetricbeatConfig builds the base Metricbeat config with the associated volume holding the CA of the monitored ES -func buildMetricbeatBaseConfig(client k8s.Client, es esv1.Elasticsearch) (string, volume.VolumeLike, error) { - password, err := user.GetMonitoringUserPassword(client, es) - if err != nil { - return "", nil, err - } - - configData := esConfigData{ - URL: fmt.Sprintf("%s://localhost:%d", es.Spec.HTTP.Protocol(), network.HTTPPort), - Username: user.MonitoringUserName, - Password: password, - IsSSL: es.Spec.HTTP.TLS.Enabled(), - } - - var caVolume volume.VolumeLike - if configData.IsSSL { - caVolume = volume.NewSecretVolumeWithMountPath( - certificates.PublicCertsSecretName(esv1.ESNamer, es.Name), - "es-monitoring-local-ca", - fmt.Sprintf("/mnt/elastic-internal/es-monitoring/%s/%s/certs", es.Namespace, es.Name), - ) - - configData.SSLPath = filepath.Join(caVolume.VolumeMount().MountPath, certificates.CAFileName) - configData.SSLMode = "certificate" - } - - // render the config template with the config data - var metricbeatConfig bytes.Buffer - err = template.Must(template.New("").Parse(metricbeatConfigTemplate)).Execute(&metricbeatConfig, configData) - if err != nil { - return "", nil, err - } - - return metricbeatConfig.String(), caVolume, nil -} diff --git a/pkg/controller/elasticsearch/stackmon/es_config.go b/pkg/controller/elasticsearch/stackmon/es_config.go index 2222ddb209..5ef568f9c7 100644 --- a/pkg/controller/elasticsearch/stackmon/es_config.go +++ b/pkg/controller/elasticsearch/stackmon/es_config.go @@ -9,11 +9,12 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" ) // MonitoringConfig returns the Elasticsearch settings to enable the collection of monitoring data func MonitoringConfig(es esv1.Elasticsearch) commonv1.Config { - if !IsMonitoringMetricsDefined(es) { + if !monitoring.IsMetricsDefined(&es) { return commonv1.Config{} } return commonv1.Config{Data: map[string]interface{}{ diff --git a/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml b/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml index aee4ca8b17..6663d10170 100644 --- a/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml +++ b/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml @@ -1,4 +1,5 @@ metricbeat.modules: + # https://www.elastic.co/guide/en/beats/metricbeat/7.13/metricbeat-module-elasticsearch.html - module: elasticsearch metricsets: - ccr @@ -19,7 +20,7 @@ metricbeat.modules: {{- if .IsSSL }} ssl.certificate_authorities: ["{{ .SSLPath }}"] ssl.verification_mode: "{{ .SSLMode }}" - {{ end }} + {{- end }} processors: - add_cloud_metadata: {} diff --git a/pkg/controller/elasticsearch/stackmon/sidecar.go b/pkg/controller/elasticsearch/stackmon/sidecar.go index f1e380914d..c8364afeca 100644 --- a/pkg/controller/elasticsearch/stackmon/sidecar.go +++ b/pkg/controller/elasticsearch/stackmon/sidecar.go @@ -10,10 +10,12 @@ import ( corev1 "k8s.io/api/core/v1" + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/pkg/controller/common/defaults" - common "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/network" esvolume "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ) @@ -24,48 +26,28 @@ const ( cfgHashLabel = "elasticsearch.k8s.elastic.co/monitoring-config-hash" ) -func IsMonitoringMetricsDefined(es esv1.Elasticsearch) bool { - for _, ref := range es.Spec.Monitoring.Metrics.ElasticsearchRefs { - if !ref.IsDefined() { - return false - } - } - return len(es.Spec.Monitoring.Metrics.ElasticsearchRefs) > 0 -} - -func IsMonitoringLogsDefined(es esv1.Elasticsearch) bool { - for _, ref := range es.Spec.Monitoring.Logs.ElasticsearchRefs { - if !ref.IsDefined() { - return false - } - } - return len(es.Spec.Monitoring.Logs.ElasticsearchRefs) > 0 -} - -func isMonitoringDefined(es esv1.Elasticsearch) bool { - return IsMonitoringMetricsDefined(es) || IsMonitoringLogsDefined(es) -} - -func Metricbeat(client k8s.Client, es esv1.Elasticsearch) (common.BeatSidecar, error) { - metricbeatConfig, sourceEsCaVolume, err := buildMetricbeatBaseConfig(client, es) +func Metricbeat(client k8s.Client, es esv1.Elasticsearch) (stackmon.BeatSidecar, error) { + metricbeat, err := stackmon.NewMetricBeatSidecar( + client, + commonv1.KbMonitoringAssociationType, + &es, + es.Spec.Version, + k8s.ExtractNamespacedName(&es), + metricbeatConfigTemplate, + esv1.ESNamer, + fmt.Sprintf("%s://localhost:%d", es.Spec.HTTP.Protocol(), network.HTTPPort), + es.Spec.HTTP.TLS.Enabled(), + ) if err != nil { - return common.BeatSidecar{}, err + return stackmon.BeatSidecar{}, err } - - image := container.ImageRepository(container.MetricbeatImage, es.Spec.Version) - metricbeat, err := common.NewMetricBeatSidecar(client, &es, image, metricbeatConfig, sourceEsCaVolume) - if err != nil { - return common.BeatSidecar{}, err - } - return metricbeat, nil } -func Filebeat(client k8s.Client, es esv1.Elasticsearch) (common.BeatSidecar, error) { - image := container.ImageRepository(container.FilebeatImage, es.Spec.Version) - filebeat, err := common.NewFileBeatSidecar(client, &es, image, filebeatConfig, nil) +func Filebeat(client k8s.Client, es esv1.Elasticsearch) (stackmon.BeatSidecar, error) { + filebeat, err := stackmon.NewFileBeatSidecar(client, &es, es.Spec.Version, filebeatConfig, nil) if err != nil { - return common.BeatSidecar{}, err + return stackmon.BeatSidecar{}, err } return filebeat, nil @@ -75,14 +57,14 @@ func Filebeat(client k8s.Client, es esv1.Elasticsearch) (common.BeatSidecar, err // in the Elasticsearch pod and injects the volumes for the beat configurations and the ES CA certificates. func WithMonitoring(client k8s.Client, builder *defaults.PodTemplateBuilder, es esv1.Elasticsearch) (*defaults.PodTemplateBuilder, error) { // no monitoring defined, skip - if !isMonitoringDefined(es) { + if !monitoring.IsDefined(&es) { return builder, nil } configHash := sha256.New224() volumes := make([]corev1.Volume, 0) - if IsMonitoringMetricsDefined(es) { + if monitoring.IsMetricsDefined(&es) { b, err := Metricbeat(client, es) if err != nil { return nil, err @@ -93,7 +75,7 @@ func WithMonitoring(client k8s.Client, builder *defaults.PodTemplateBuilder, es configHash.Write(b.ConfigHash.Sum(nil)) } - if IsMonitoringLogsDefined(es) { + if monitoring.IsLogsDefined(&es) { // enable Stack logging to write Elasticsearch logs to disk builder.WithEnv(fileLogStyleEnvVar()) diff --git a/pkg/controller/elasticsearch/stackmon/sidecar_test.go b/pkg/controller/elasticsearch/stackmon/sidecar_test.go index ef3047c50e..9960a7df57 100644 --- a/pkg/controller/elasticsearch/stackmon/sidecar_test.go +++ b/pkg/controller/elasticsearch/stackmon/sidecar_test.go @@ -14,6 +14,7 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/defaults" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ) @@ -84,7 +85,7 @@ func TestWithMonitoring(t *testing.T) { name: "with metrics monitoring", es: func() esv1.Elasticsearch { sampleEs.Spec.Monitoring.Metrics.ElasticsearchRefs = monitoringEsRef - sampleEs.GetMonitoringMetricsAssociation()[0].SetAssociationConf(&monitoringAssocConf) + monitoring.GetMetricsAssociation(&sampleEs)[0].SetAssociationConf(&monitoringAssocConf) return sampleEs }, containersLength: 2, @@ -97,7 +98,7 @@ func TestWithMonitoring(t *testing.T) { es: func() esv1.Elasticsearch { sampleEs.Spec.Monitoring.Metrics.ElasticsearchRefs = nil sampleEs.Spec.Monitoring.Logs.ElasticsearchRefs = monitoringEsRef - sampleEs.GetMonitoringLogsAssociation()[0].SetAssociationConf(&monitoringAssocConf) + monitoring.GetLogsAssociation(&sampleEs)[0].SetAssociationConf(&monitoringAssocConf) return sampleEs }, containersLength: 2, @@ -109,9 +110,9 @@ func TestWithMonitoring(t *testing.T) { name: "with metrics and logs monitoring", es: func() esv1.Elasticsearch { sampleEs.Spec.Monitoring.Metrics.ElasticsearchRefs = monitoringEsRef - sampleEs.GetMonitoringMetricsAssociation()[0].SetAssociationConf(&monitoringAssocConf) + monitoring.GetMetricsAssociation(&sampleEs)[0].SetAssociationConf(&monitoringAssocConf) sampleEs.Spec.Monitoring.Logs.ElasticsearchRefs = monitoringEsRef - sampleEs.GetMonitoringLogsAssociation()[0].SetAssociationConf(&logsAssocConf) + monitoring.GetLogsAssociation(&sampleEs)[0].SetAssociationConf(&logsAssocConf) return sampleEs }, containersLength: 3, @@ -123,9 +124,9 @@ func TestWithMonitoring(t *testing.T) { name: "with metrics and logs monitoring with different es ref", es: func() esv1.Elasticsearch { sampleEs.Spec.Monitoring.Metrics.ElasticsearchRefs = monitoringEsRef - sampleEs.GetMonitoringMetricsAssociation()[0].SetAssociationConf(&monitoringAssocConf) + monitoring.GetMetricsAssociation(&sampleEs)[0].SetAssociationConf(&monitoringAssocConf) sampleEs.Spec.Monitoring.Logs.ElasticsearchRefs = logsEsRef - sampleEs.GetMonitoringLogsAssociation()[0].SetAssociationConf(&logsAssocConf) + monitoring.GetLogsAssociation(&sampleEs)[0].SetAssociationConf(&logsAssocConf) return sampleEs }, containersLength: 3, @@ -146,15 +147,14 @@ func TestWithMonitoring(t *testing.T) { assert.Equal(t, tc.esEnvVarsLength, len(builder.PodTemplate.Spec.Containers[0].Env)) assert.Equal(t, tc.podVolumesLength, len(builder.PodTemplate.Spec.Volumes)) - if IsMonitoringMetricsDefined(es) { + if monitoring.IsMetricsDefined(&es) { for _, c := range builder.PodTemplate.Spec.Containers { if c.Name == "metricbeat" { assert.Equal(t, tc.beatVolumeMountsLength, len(c.VolumeMounts)) } } } - - if IsMonitoringLogsDefined(es) { + if monitoring.IsLogsDefined(&es) { for _, c := range builder.PodTemplate.Spec.Containers { if c.Name == "filebeat" { assert.Equal(t, tc.beatVolumeMountsLength, len(c.VolumeMounts)) diff --git a/pkg/controller/elasticsearch/stackmon/validations.go b/pkg/controller/elasticsearch/stackmon/validations.go deleted file mode 100644 index 45dd417557..0000000000 --- a/pkg/controller/elasticsearch/stackmon/validations.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package stackmon - -import ( - "fmt" - - "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" - "k8s.io/apimachinery/pkg/util/validation/field" - - esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" -) - -const ( - unsupportedVersionForStackMonitoringMsg = "Unsupported version for Stack Monitoring. Required >= %s." - invalidStackMonitoringElasticsearchRefsMsg = "Only one Elasticsearch reference is supported for %s Stack Monitoring" -) - -var ( - // MinStackVersion is the minimum Stack version to enable Stack Monitoring on an Elastic Stack application.. - // This requirement comes from the fact that we configure Elasticsearch to write logs to disk for Filebeat - // via the env var ES_LOG_STYLE available from this version. - MinStackVersion = version.MustParse("7.14.0-SNAPSHOT") -) - -// Validate validates that the Elasticsearch version is supported for Stack Monitoring and that there is exactly one -// Elasticsearch reference defined when Stack Monitoring is defined -func Validate(es esv1.Elasticsearch) field.ErrorList { - var errs field.ErrorList - if isMonitoringDefined(es) { - err := IsSupportedVersion(es.Spec.Version) - if err != nil { - errs = append(errs, field.Invalid(field.NewPath("spec").Child("version"), es.Spec.Version, - fmt.Sprintf(unsupportedVersionForStackMonitoringMsg, MinStackVersion))) - } - } - if IsMonitoringMetricsDefined(es) && len(es.Spec.Monitoring.Metrics.ElasticsearchRefs) != 1 { - errs = append(errs, field.Invalid(field.NewPath("spec").Child("monitoring").Child("metrics").Child("elasticsearchRefs"), - es.Spec.Monitoring.Metrics.ElasticsearchRefs, - fmt.Sprintf(invalidStackMonitoringElasticsearchRefsMsg, "Metrics"))) - } - if IsMonitoringLogsDefined(es) && len(es.Spec.Monitoring.Logs.ElasticsearchRefs) != 1 { - errs = append(errs, field.Invalid(field.NewPath("spec").Child("monitoring").Child("logs").Child("elasticsearchRefs"), - es.Spec.Monitoring.Logs.ElasticsearchRefs, - fmt.Sprintf(invalidStackMonitoringElasticsearchRefsMsg, "Logs"))) - } - return errs -} - -// IsSupportedVersion returns true if the Elasticsearch version is supported for Stack Monitoring, else returns false -func IsSupportedVersion(esVersion string) error { - ver, err := version.Parse(esVersion) - if err != nil { - return err - } - if ver.LT(MinStackVersion) { - return fmt.Errorf("unsupported version for Stack Monitoring: required >= %s", MinStackVersion) - } - return nil -} diff --git a/pkg/controller/elasticsearch/user/predefined.go b/pkg/controller/elasticsearch/user/predefined.go index db2dc4118b..a03eb887de 100644 --- a/pkg/controller/elasticsearch/user/predefined.go +++ b/pkg/controller/elasticsearch/user/predefined.go @@ -154,8 +154,8 @@ func reuseOrGenerateHash(users users, fileRealm filerealm.Realm) (users, error) return users, nil } -func GetMonitoringUserPassword(c k8s.Client, es esv1.Elasticsearch) (string, error) { - secretObjKey := types.NamespacedName{Namespace: es.Namespace, Name: esv1.InternalUsersSecret(es.Name)} +func GetMonitoringUserPassword(c k8s.Client, nsn types.NamespacedName) (string, error) { + secretObjKey := types.NamespacedName{Namespace: nsn.Namespace, Name: esv1.InternalUsersSecret(nsn.Name)} var secret corev1.Secret if err := c.Get(context.Background(), secretObjKey, &secret); err != nil { return "", err diff --git a/pkg/controller/elasticsearch/validation/validations.go b/pkg/controller/elasticsearch/validation/validations.go index a782eecb7f..0ee26f8224 100644 --- a/pkg/controller/elasticsearch/validation/validations.go +++ b/pkg/controller/elasticsearch/validation/validations.go @@ -11,8 +11,8 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + stackmon "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/validations" "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/stackmon" esversion "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/version" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ulog "github.com/elastic/cloud-on-k8s/pkg/utils/log" @@ -52,7 +52,7 @@ var validations = []validation{ validSanIP, validAutoscalingConfiguration, validPVCNaming, - stackmon.Validate, + validMonitoring, } type updateValidation func(esv1.Elasticsearch, esv1.Elasticsearch) field.ErrorList @@ -272,3 +272,7 @@ func validUpgradePath(current, proposed esv1.Elasticsearch) field.ErrorList { } return errs } + +func validMonitoring(es esv1.Elasticsearch) field.ErrorList { + return stackmon.Validate(&es, es.Spec.Version) +} diff --git a/pkg/controller/kibana/config_settings.go b/pkg/controller/kibana/config_settings.go index e43b773790..55050c0d57 100644 --- a/pkg/controller/kibana/config_settings.go +++ b/pkg/controller/kibana/config_settings.go @@ -18,6 +18,7 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing" "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana/stackmon" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/pkg/utils/net" "github.com/elastic/go-ucfg" @@ -104,6 +105,10 @@ func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v kibanaTLSCfg := settings.MustCanonicalConfig(kibanaTLSSettings(kb)) versionSpecificCfg := VersionDefaults(&kb, v) entSearchCfg := settings.MustCanonicalConfig(enterpriseSearchSettings(kb)) + monitoringCfg, err := settings.NewCanonicalConfigFrom(stackmon.MonitoringConfig(kb).Data) + if err != nil { + return CanonicalConfig{}, err + } if !kb.EsAssociation().AssociationConf().IsConfigured() { // merge the configuration with userSettings last so they take precedence @@ -112,6 +117,7 @@ func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v versionSpecificCfg, kibanaTLSCfg, entSearchCfg, + monitoringCfg, userSettings); err != nil { return CanonicalConfig{}, err } @@ -129,6 +135,7 @@ func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v versionSpecificCfg, kibanaTLSCfg, entSearchCfg, + monitoringCfg, settings.MustCanonicalConfig(elasticsearchTLSSettings(kb)), settings.MustCanonicalConfig( map[string]interface{}{ diff --git a/pkg/controller/kibana/controller.go b/pkg/controller/kibana/controller.go index b5fdeb525f..50636bea60 100644 --- a/pkg/controller/kibana/controller.go +++ b/pkg/controller/kibana/controller.go @@ -37,17 +37,17 @@ import ( ) const ( - name = "kibana-controller" + controllerName = "kibana-controller" configChecksumLabel = "kibana.k8s.elastic.co/config-checksum" ) -var log = ulog.Log.WithName(name) +var log = ulog.Log.WithName(controllerName) // Add creates a new Kibana Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(mgr manager.Manager, params operator.Parameters) error { reconciler := newReconciler(mgr, params) - c, err := common.NewController(mgr, name, reconciler, params) + c, err := common.NewController(mgr, controllerName, reconciler, params) if err != nil { return err } @@ -59,7 +59,7 @@ func newReconciler(mgr manager.Manager, params operator.Parameters) *ReconcileKi client := mgr.GetClient() return &ReconcileKibana{ Client: client, - recorder: mgr.GetEventRecorderFor(name), + recorder: mgr.GetEventRecorderFor(controllerName), dynamicWatches: watches.NewDynamicWatches(), params: params, } @@ -245,7 +245,7 @@ func (r *ReconcileKibana) onDelete(obj types.NamespacedName) error { // Clean up watches set on secure settings r.dynamicWatches.Secrets.RemoveHandlerForKey(keystore.SecureSettingsWatchName(obj)) // Clean up watches set on custom http tls certificates - r.dynamicWatches.Secrets.RemoveHandlerForKey(certificates.CertificateWatchKey(Namer, obj.Name)) + r.dynamicWatches.Secrets.RemoveHandlerForKey(certificates.CertificateWatchKey(kbv1.KBNamer, obj.Name)) return reconciler.GarbageCollectSoftOwnedSecrets(r.Client, obj, kbv1.Kind) } diff --git a/pkg/controller/kibana/driver.go b/pkg/controller/kibana/driver.go index cb346bfd23..c15771dba3 100644 --- a/pkg/controller/kibana/driver.go +++ b/pkg/controller/kibana/driver.go @@ -33,6 +33,8 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" commonvolume "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" "github.com/elastic/cloud-on-k8s/pkg/controller/common/watches" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana/network" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana/stackmon" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" ) @@ -114,7 +116,7 @@ func (d *driver) Reconcile( DynamicWatches: d.DynamicWatches(), Owner: kb, TLSOptions: kb.Spec.HTTP.TLS, - Namer: Namer, + Namer: kbv1.KBNamer, Labels: NewLabels(kb.Name), Services: []corev1.Service{*svc}, CACertRotation: params.CACertRotation, @@ -142,6 +144,11 @@ func (d *driver) Reconcile( return results.WithError(err) } + err = stackmon.ReconcileConfigSecrets(d.client, *kb) + if err != nil { + return results.WithError(err) + } + span, _ := apm.StartSpan(ctx, "reconcile_deployment", tracing.SpanTypeApp) defer span.End() @@ -201,7 +208,7 @@ func (d *driver) deploymentParams(kb *kbv1.Kibana) (deployment.Params, error) { keystoreResources, err := keystore.NewResources( d, kb, - Namer, + kbv1.KBNamer, NewLabels(kb.Name), initContainersParameters, ) @@ -209,7 +216,10 @@ func (d *driver) deploymentParams(kb *kbv1.Kibana) (deployment.Params, error) { return deployment.Params{}, err } - kibanaPodSpec := NewPodTemplateSpec(*kb, keystoreResources, d.buildVolumes(kb)) + kibanaPodSpec, err := NewPodTemplateSpec(d.client, *kb, keystoreResources, d.buildVolumes(kb)) + if err != nil { + return deployment.Params{}, err + } // Build a checksum of the configuration, which we can use to cause the Deployment to roll Kibana // instances in case of any change in the CA file, secure settings or credentials contents. @@ -229,7 +239,7 @@ func (d *driver) deploymentParams(kb *kbv1.Kibana) (deployment.Params, error) { var httpCerts corev1.Secret err := d.client.Get(context.Background(), types.NamespacedName{ Namespace: kb.Namespace, - Name: certificates.InternalCertsSecretName(Namer, kb.Name), + Name: certificates.InternalCertsSecretName(kbv1.KBNamer, kb.Name), }, &httpCerts) if err != nil { return deployment.Params{}, err @@ -258,7 +268,7 @@ func (d *driver) deploymentParams(kb *kbv1.Kibana) (deployment.Params, error) { } return deployment.Params{ - Name: Namer.Suffix(kb.Name), + Name: kbv1.KBNamer.Suffix(kb.Name), Namespace: kb.Namespace, Replicas: kb.Spec.Count, Selector: NewLabels(kb.Name), @@ -282,7 +292,7 @@ func (d *driver) buildVolumes(kb *kbv1.Kibana) []commonvolume.VolumeLike { } if kb.Spec.HTTP.TLS.Enabled() { - httpCertsVolume := certificates.HTTPCertSecretVolume(Namer, kb.Name) + httpCertsVolume := certificates.HTTPCertSecretVolume(kbv1.KBNamer, kb.Name) volumes = append(volumes, httpCertsVolume) } return volumes @@ -295,14 +305,14 @@ func NewService(kb kbv1.Kibana) *corev1.Service { } svc.ObjectMeta.Namespace = kb.Namespace - svc.ObjectMeta.Name = HTTPService(kb.Name) + svc.ObjectMeta.Name = kbv1.HTTPService(kb.Name) labels := NewLabels(kb.Name) ports := []corev1.ServicePort{ { Name: kb.Spec.HTTP.Protocol(), Protocol: corev1.ProtocolTCP, - Port: HTTPPort, + Port: network.HTTPPort, }, } return defaults.SetServiceDefaults(&svc, labels, labels, ports) diff --git a/pkg/controller/kibana/driver_test.go b/pkg/controller/kibana/driver_test.go index 124251380b..ae08caef4d 100644 --- a/pkg/controller/kibana/driver_test.go +++ b/pkg/controller/kibana/driver_test.go @@ -9,6 +9,7 @@ import ( "fmt" "testing" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana/network" "github.com/go-test/deep" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -777,7 +778,7 @@ func mkService() corev1.Service { { Name: "http", Protocol: corev1.ProtocolTCP, - Port: HTTPPort, + Port: network.HTTPPort, }, }, Selector: map[string]string{ diff --git a/pkg/controller/kibana/network/ports.go b/pkg/controller/kibana/network/ports.go new file mode 100644 index 0000000000..56ec3a9a0d --- /dev/null +++ b/pkg/controller/kibana/network/ports.go @@ -0,0 +1,10 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package network + +const ( + // HTTPPort is the (default) port used by Kibana + HTTPPort = 5601 +) diff --git a/pkg/controller/kibana/pod.go b/pkg/controller/kibana/pod.go index 503efb3a0b..c2a1a6bafa 100644 --- a/pkg/controller/kibana/pod.go +++ b/pkg/controller/kibana/pod.go @@ -5,9 +5,12 @@ package kibana import ( + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana/network" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana/stackmon" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/annotation" @@ -19,8 +22,6 @@ import ( ) const ( - // HTTPPort is the (default) port used by Kibana - HTTPPort = 5601 DataVolumeName = "kibana-data" DataVolumeMountPath = "/usr/share/kibana/data" ) @@ -61,7 +62,7 @@ func readinessProbe(useTLS bool) corev1.Probe { TimeoutSeconds: 5, Handler: corev1.Handler{ HTTPGet: &corev1.HTTPGetAction{ - Port: intstr.FromInt(HTTPPort), + Port: intstr.FromInt(network.HTTPPort), Path: "/login", Scheme: scheme, }, @@ -69,7 +70,7 @@ func readinessProbe(useTLS bool) corev1.Probe { } } -func NewPodTemplateSpec(kb kbv1.Kibana, keystore *keystore.Resources, volumes []volume.VolumeLike) corev1.PodTemplateSpec { +func NewPodTemplateSpec(client k8sclient.Client, kb kbv1.Kibana, keystore *keystore.Resources, volumes []volume.VolumeLike) (corev1.PodTemplateSpec, error) { labels := NewLabels(kb.Name) labels[KibanaVersionLabelName] = kb.Spec.Version @@ -93,7 +94,12 @@ func NewPodTemplateSpec(kb kbv1.Kibana, keystore *keystore.Resources, volumes [] WithInitContainers(keystore.InitContainer) } - return builder.WithInitContainerDefaults().PodTemplate + builder, err := stackmon.WithMonitoring(client, builder, kb) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + + return builder.WithInitContainerDefaults().PodTemplate, nil } // GetKibanaContainer returns the Kibana container from the given podSpec. @@ -102,5 +108,5 @@ func GetKibanaContainer(podSpec corev1.PodSpec) *corev1.Container { } func getDefaultContainerPorts(kb kbv1.Kibana) []corev1.ContainerPort { - return []corev1.ContainerPort{{Name: kb.Spec.HTTP.Protocol(), ContainerPort: int32(HTTPPort), Protocol: corev1.ProtocolTCP}} + return []corev1.ContainerPort{{Name: kb.Spec.HTTP.Protocol(), ContainerPort: int32(network.HTTPPort), Protocol: corev1.ProtocolTCP}} } diff --git a/pkg/controller/kibana/pod_test.go b/pkg/controller/kibana/pod_test.go index d9f141dfe0..409551554c 100644 --- a/pkg/controller/kibana/pod_test.go +++ b/pkg/controller/kibana/pod_test.go @@ -8,6 +8,8 @@ import ( "testing" commonvolume "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana/network" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -218,7 +220,8 @@ func TestNewPodTemplateSpec(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := NewPodTemplateSpec(tt.kb, tt.keystore, []commonvolume.VolumeLike{}) + got, err := NewPodTemplateSpec(k8s.NewFakeClient(), tt.kb, tt.keystore, []commonvolume.VolumeLike{}) + assert.NoError(t, err) tt.assertions(got) }) } @@ -238,7 +241,7 @@ func Test_getDefaultContainerPorts(t *testing.T) { }, }, want: []corev1.ContainerPort{ - {Name: "https", HostPort: 0, ContainerPort: int32(HTTPPort), Protocol: "TCP", HostIP: ""}, + {Name: "https", HostPort: 0, ContainerPort: int32(network.HTTPPort), Protocol: "TCP", HostIP: ""}, }, }, { @@ -255,7 +258,7 @@ func Test_getDefaultContainerPorts(t *testing.T) { }, }, want: []corev1.ContainerPort{ - {Name: "http", HostPort: 0, ContainerPort: int32(HTTPPort), Protocol: "TCP", HostIP: ""}, + {Name: "http", HostPort: 0, ContainerPort: int32(network.HTTPPort), Protocol: "TCP", HostIP: ""}, }, }, } diff --git a/pkg/controller/kibana/stackmon/beat_config.go b/pkg/controller/kibana/stackmon/beat_config.go new file mode 100644 index 0000000000..023fbd1673 --- /dev/null +++ b/pkg/controller/kibana/stackmon/beat_config.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stackmon + +import ( + _ "embed" // for the beats config files + + kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" +) + +var ( + // metricbeatConfigTemplate is a configuration template for Metricbeat to collect monitoring data about Kibana + //go:embed metricbeat.tpl.yml + metricbeatConfigTemplate string + + // filebeatConfig is a static configuration for Filebeat to collect Kibana logs + //go:embed filebeat.yml + filebeatConfig string +) + +// ReconcileConfigSecrets reconciles the secrets holding beats configuration +func ReconcileConfigSecrets(client k8s.Client, kb kbv1.Kibana) error { + if monitoring.IsMetricsDefined(&kb) { + b, err := Metricbeat(client, kb) + if err != nil { + return err + } + + if _, err := reconciler.ReconcileSecret(client, b.ConfigSecret, &kb); err != nil { + return err + } + } + + if monitoring.IsLogsDefined(&kb) { + b, err := Filebeat(client, kb) + if err != nil { + return err + } + + if _, err := reconciler.ReconcileSecret(client, b.ConfigSecret, &kb); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/controller/kibana/stackmon/filebeat.yml b/pkg/controller/kibana/stackmon/filebeat.yml new file mode 100644 index 0000000000..78661023f2 --- /dev/null +++ b/pkg/controller/kibana/stackmon/filebeat.yml @@ -0,0 +1,21 @@ +filebeat.modules: + # https://www.elastic.co/guide/en/beats/filebeat/7.13/filebeat-module-kibana.html + - module: kibana + log: + enabled: true + var.paths: + - /usr/share/kibana/logs/kibana.json + close_timeout: 2h + fields_under_root: true + audit: + enabled: true + var.paths: + - /usr/share/kibana/logs/*_audit.json + close_timeout: 2h + fields_under_root: true + +processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + +# Elasticsearch output configuration is generated diff --git a/pkg/controller/kibana/stackmon/kb_config.go b/pkg/controller/kibana/stackmon/kb_config.go new file mode 100644 index 0000000000..fc9ca275c1 --- /dev/null +++ b/pkg/controller/kibana/stackmon/kb_config.go @@ -0,0 +1,63 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stackmon + +import ( + "path/filepath" + + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" +) + +var ( + MonitoringKibanaCollectionEnabled = "monitoring.kibana.collection.enabled" + + LoggingAppendersJSONFileAppenderType = "logging.appenders.rolling-file.type" + LoggingAppendersJSONFileAppenderFilename = "logging.appenders.rolling-file.fileName" + LoggingAppendersJSONFileAppenderLayoutType = "logging.appenders.rolling-file.layout.type" + LoggingAppendersJSONFileAppenderPolicyType = "logging.appenders.rolling-file.policy.type" + LoggingAppendersJSONFileAppenderPolicySize = "logging.appenders.rolling-file.policy.size" + LoggingRootAppenders = "logging.root.appenders" + + XPackSecurityAuditAppenderType = "xpack.security.audit.appender.type" + XPackSecurityAuditAppenderFileName = "xpack.security.audit.appender.fileName" + XPackSecurityAuditAppenderLayoutType = "xpack.security.audit.appender.layout.type" + XPackSecurityAuditAppenderPolicyType = "xpack.security.audit.appender.policy.type" + XPackSecurityAuditAppenderPolicySize = "xpack.security.audit.appender.policy.size" + + kibanaLogFilename = "kibana.json" + kibanaAuditLogFilename = "kibana_audit.json" +) + +// MonitoringConfig returns the Kibana settings required to enable the collection of monitoring data and disk logging +func MonitoringConfig(kb kbv1.Kibana) commonv1.Config { + cfg := commonv1.Config{} + if monitoring.IsMetricsDefined(&kb) { + cfg.Data = map[string]interface{}{ + MonitoringKibanaCollectionEnabled: false, + } + } + if monitoring.IsLogsDefined(&kb) { + if cfg.Data == nil { + cfg.Data = map[string]interface{}{} + } + // configure the main Kibana log to be written to disk and stdout + cfg.Data[LoggingAppendersJSONFileAppenderType] = "rolling-file" + cfg.Data[LoggingAppendersJSONFileAppenderFilename] = filepath.Join(kibanaLogsMountPath, kibanaLogFilename) + cfg.Data[LoggingAppendersJSONFileAppenderLayoutType] = "json" + cfg.Data[LoggingAppendersJSONFileAppenderPolicyType] = "size-limit" + cfg.Data[LoggingAppendersJSONFileAppenderPolicySize] = "50mb" + cfg.Data[LoggingRootAppenders] = []string{"default", "rolling-file"} + + // configure audit logs to be written to disk so that user has just to enable audit logs to collect them + cfg.Data[XPackSecurityAuditAppenderType] = "rolling-file" + cfg.Data[XPackSecurityAuditAppenderFileName] = filepath.Join(kibanaLogsMountPath, kibanaAuditLogFilename) + cfg.Data[XPackSecurityAuditAppenderLayoutType] = "json" + cfg.Data[XPackSecurityAuditAppenderPolicyType] = "size-limit" + cfg.Data[XPackSecurityAuditAppenderPolicySize] = "50mb" + } + return cfg +} diff --git a/pkg/controller/kibana/stackmon/metricbeat.tpl.yml b/pkg/controller/kibana/stackmon/metricbeat.tpl.yml new file mode 100644 index 0000000000..0a02371124 --- /dev/null +++ b/pkg/controller/kibana/stackmon/metricbeat.tpl.yml @@ -0,0 +1,21 @@ +metricbeat.modules: + # https://www.elastic.co/guide/en/beats/metricbeat/7.13/metricbeat-module-kibana.html + - module: kibana + metricsets: + - stats + - status + period: 10s + xpack.enabled: true + hosts: ["{{ .URL }}"] + username: {{ .Username }} + password: {{ .Password }} + {{- if .IsSSL }} + ssl.certificate_authorities: ["{{ .SSLPath }}"] + ssl.verification_mode: "{{ .SSLMode }}" + {{- end }} + +processors: + - add_cloud_metadata: {} + - add_host_metadata: {} + +# Elasticsearch output configuration is generated diff --git a/pkg/controller/kibana/stackmon/sidecar.go b/pkg/controller/kibana/stackmon/sidecar.go new file mode 100644 index 0000000000..ee2f027295 --- /dev/null +++ b/pkg/controller/kibana/stackmon/sidecar.go @@ -0,0 +1,115 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stackmon + +import ( + "crypto/sha256" + "errors" + "fmt" + + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/defaults" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/validations" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/volume" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana/network" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + corev1 "k8s.io/api/core/v1" +) + +const ( + // cfgHashLabel is used to store a hash of the Metricbeat and Filebeat configurations. + // Using only one label for both configs to save labels. + cfgHashLabel = "kibana.k8s.elastic.co/monitoring-config-hash" + + kibanaLogsVolumeName = "kibana-logs" + kibanaLogsMountPath = "/usr/share/kibana/logs" +) + +func Metricbeat(client k8s.Client, kb kbv1.Kibana) (stackmon.BeatSidecar, error) { + if !kb.Spec.ElasticsearchRef.IsDefined() { + // should never happen because of the pre-creation validation + return stackmon.BeatSidecar{}, errors.New(validations.InvalidKibanaElasticsearchRefForStackMonitoringMsg) + } + associatedEsNsn := kb.Spec.ElasticsearchRef.NamespacedName() + if associatedEsNsn.Namespace == "" { + associatedEsNsn.Namespace = kb.Namespace + } + + metricbeat, err := stackmon.NewMetricBeatSidecar( + client, + commonv1.KbMonitoringAssociationType, + &kb, + kb.Spec.Version, + associatedEsNsn, + metricbeatConfigTemplate, + kbv1.KBNamer, + fmt.Sprintf("%s://localhost:%d", kb.Spec.HTTP.Protocol(), network.HTTPPort), + kb.Spec.HTTP.TLS.Enabled(), + ) + if err != nil { + return stackmon.BeatSidecar{}, err + } + return metricbeat, nil +} + +func Filebeat(client k8s.Client, kb kbv1.Kibana) (stackmon.BeatSidecar, error) { + filebeat, err := stackmon.NewFileBeatSidecar(client, &kb, kb.Spec.Version, filebeatConfig, nil) + if err != nil { + return stackmon.BeatSidecar{}, err + } + + return filebeat, nil +} + +// WithMonitoring updates the Kibana Pod template builder to deploy Metricbeat and Filebeat in sidecar containers +// in the Kibana pod and injects the volumes for the beat configurations and the ES CA certificates. +func WithMonitoring(client k8s.Client, builder *defaults.PodTemplateBuilder, kb kbv1.Kibana) (*defaults.PodTemplateBuilder, error) { + // no monitoring defined, skip + if !monitoring.IsDefined(&kb) { + return builder, nil + } + + configHash := sha256.New224() + volumes := make([]corev1.Volume, 0) + + if monitoring.IsMetricsDefined(&kb) { + b, err := Metricbeat(client, kb) + if err != nil { + return nil, err + } + + volumes = append(volumes, b.Volumes...) + builder.WithContainers(b.Container) + configHash.Write(b.ConfigHash.Sum(nil)) + } + + if monitoring.IsLogsDefined(&kb) { + b, err := Filebeat(client, kb) + if err != nil { + return nil, err + } + + // create a logs volume shared between Kibana and Filebeat + logsVolume := volume.NewEmptyDirVolume(kibanaLogsVolumeName, kibanaLogsMountPath) + volumes = append(volumes, logsVolume.Volume()) + filebeat := b.Container + filebeat.VolumeMounts = append(filebeat.VolumeMounts, logsVolume.VolumeMount()) + builder.WithVolumeMounts(logsVolume.VolumeMount()) + + volumes = append(volumes, b.Volumes...) + builder.WithContainers(filebeat) + configHash.Write(b.ConfigHash.Sum(nil)) + } + + // add the config hash label to ensure pod rotation when an ES password or a CA are rotated + builder.WithLabels(map[string]string{cfgHashLabel: fmt.Sprintf("%x", configHash.Sum(nil))}) + // inject all volumes + builder.WithVolumes(volumes...) + + return builder, nil +} diff --git a/pkg/controller/kibana/stackmon/sidecar_test.go b/pkg/controller/kibana/stackmon/sidecar_test.go new file mode 100644 index 0000000000..c9d0e45237 --- /dev/null +++ b/pkg/controller/kibana/stackmon/sidecar_test.go @@ -0,0 +1,165 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stackmon + +import ( + "fmt" + "testing" + + commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" + kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/defaults" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/monitoring" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestWithMonitoring(t *testing.T) { + esRef := commonv1.ObjectSelector{Name: "sample", Namespace: "aerospace"} + sampleKb := kbv1.Kibana{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample", + Namespace: "aerospace", + }, + Spec: kbv1.KibanaSpec{ + Version: "7.14.0", + ElasticsearchRef: esRef, + }, + } + monitoringEsRef := []commonv1.ObjectSelector{{Name: "monitoring", Namespace: "observability"}} + logsEsRef := []commonv1.ObjectSelector{{Name: "logs", Namespace: "observability"}} + + fakeElasticUserSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "sample-es-internal-users", Namespace: "aerospace"}, + Data: map[string][]byte{"elastic-internal-monitoring": []byte("1234567890")}, + } + fakeMetricsBeatUserSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "sample-observability-monitoring-beat-es-mon-user", Namespace: "aerospace"}, + Data: map[string][]byte{"aerospace-sample-observability-monitoring-beat-es-mon-user": []byte("1234567890")}, + } + fakeLogsBeatUserSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "sample-observability-logs-beat-es-mon-user", Namespace: "aerospace"}, + Data: map[string][]byte{"aerospace-sample-observability-logs-beat-es-mon-user": []byte("1234567890")}, + } + fakeEsHTTPCertSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "sample-es-http-certs-public", Namespace: "aerospace"}, + Data: map[string][]byte{"ca.crt": []byte("7H1515N074r341C3r71F1C473")}, + } + fakeClient := k8s.NewFakeClient(&fakeElasticUserSecret, &fakeMetricsBeatUserSecret, &fakeLogsBeatUserSecret, &fakeEsHTTPCertSecret) + + monitoringAssocConf := commonv1.AssociationConf{ + AuthSecretName: "sample-observability-monitoring-beat-es-mon-user", + AuthSecretKey: "aerospace-sample-observability-monitoring-beat-es-mon-user", + CACertProvided: true, + CASecretName: "sample-es-monitoring-observability-monitoring-ca", + URL: "https://monitoring-es-http.observability.svc:9200", + Version: "7.14.0", + } + logsAssocConf := commonv1.AssociationConf{ + AuthSecretName: "sample-observability-logs-beat-es-mon-user", + AuthSecretKey: "aerospace-sample-observability-logs-beat-es-mon-user", + CACertProvided: true, + CASecretName: "sample-es-logs-observability-monitoring-ca", + URL: "https://logs-es-http.observability.svc:9200", + Version: "7.14.0", + } + + tests := []struct { + name string + kb func() kbv1.Kibana + containersLength int + podVolumesLength int + beatVolumeMountsLength int + }{ + { + name: "without monitoring", + kb: func() kbv1.Kibana { + return sampleKb + }, + containersLength: 1, + }, + { + name: "with metrics monitoring", + kb: func() kbv1.Kibana { + sampleKb.Spec.Monitoring.Metrics.ElasticsearchRefs = monitoringEsRef + monitoring.GetMetricsAssociation(&sampleKb)[0].SetAssociationConf(&monitoringAssocConf) + return sampleKb + }, + containersLength: 2, + podVolumesLength: 3, + beatVolumeMountsLength: 3, + }, + { + name: "with logs monitoring", + kb: func() kbv1.Kibana { + sampleKb.Spec.Monitoring.Metrics.ElasticsearchRefs = nil + sampleKb.Spec.Monitoring.Logs.ElasticsearchRefs = monitoringEsRef + monitoring.GetLogsAssociation(&sampleKb)[0].SetAssociationConf(&monitoringAssocConf) + return sampleKb + }, + containersLength: 2, + podVolumesLength: 3, + beatVolumeMountsLength: 3, + }, + { + name: "with metrics and logs monitoring", + kb: func() kbv1.Kibana { + sampleKb.Spec.Monitoring.Metrics.ElasticsearchRefs = monitoringEsRef + monitoring.GetMetricsAssociation(&sampleKb)[0].SetAssociationConf(&monitoringAssocConf) + sampleKb.Spec.Monitoring.Logs.ElasticsearchRefs = monitoringEsRef + monitoring.GetLogsAssociation(&sampleKb)[0].SetAssociationConf(&logsAssocConf) + return sampleKb + }, + containersLength: 3, + podVolumesLength: 5, + beatVolumeMountsLength: 3, + }, + { + name: "with metrics and logs monitoring with different es ref", + kb: func() kbv1.Kibana { + sampleKb.Spec.Monitoring.Metrics.ElasticsearchRefs = monitoringEsRef + monitoring.GetMetricsAssociation(&sampleKb)[0].SetAssociationConf(&monitoringAssocConf) + sampleKb.Spec.Monitoring.Logs.ElasticsearchRefs = logsEsRef + monitoring.GetLogsAssociation(&sampleKb)[0].SetAssociationConf(&logsAssocConf) + return sampleKb + }, + containersLength: 3, + podVolumesLength: 6, + beatVolumeMountsLength: 3, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + kb := tc.kb() + builder := defaults.NewPodTemplateBuilder(corev1.PodTemplateSpec{}, kbv1.KibanaContainerName) + _, err := WithMonitoring(fakeClient, builder, kb) + assert.NoError(t, err) + + assert.Equal(t, tc.containersLength, len(builder.PodTemplate.Spec.Containers)) + for _, v := range builder.PodTemplate.Spec.Volumes { + fmt.Println(v) + } + assert.Equal(t, tc.podVolumesLength, len(builder.PodTemplate.Spec.Volumes)) + + if monitoring.IsMetricsDefined(&kb) { + for _, c := range builder.PodTemplate.Spec.Containers { + if c.Name == "metricbeat" { + assert.Equal(t, tc.beatVolumeMountsLength, len(c.VolumeMounts)) + } + } + } + if monitoring.IsLogsDefined(&kb) { + for _, c := range builder.PodTemplate.Spec.Containers { + if c.Name == "filebeat" { + assert.Equal(t, tc.beatVolumeMountsLength, len(c.VolumeMounts)) + } + } + } + }) + } +} diff --git a/test/e2e/beat/recipes_test.go b/test/e2e/beat/recipes_test.go index 2cec3131bb..5da858ff8d 100644 --- a/test/e2e/beat/recipes_test.go +++ b/test/e2e/beat/recipes_test.go @@ -15,10 +15,10 @@ import ( beatv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/beat/v1beta1" commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" beatcommon "github.com/elastic/cloud-on-k8s/pkg/controller/beat/common" "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings" "github.com/elastic/cloud-on-k8s/pkg/controller/common/version" - "github.com/elastic/cloud-on-k8s/pkg/controller/kibana" "github.com/elastic/cloud-on-k8s/pkg/utils/net" "github.com/elastic/cloud-on-k8s/test/e2e/test" "github.com/elastic/cloud-on-k8s/test/e2e/test/beat" @@ -154,7 +154,7 @@ func TestHeartbeatEsKbHealthRecipe(t *testing.T) { spec := builder.Beat.Spec newEsHost := fmt.Sprintf("%s.%s.svc", esv1.HTTPService(spec.ElasticsearchRef.Name), builder.Beat.Namespace) - newKbHost := fmt.Sprintf("%s.%s.svc", kibana.HTTPService(spec.KibanaRef.Name), builder.Beat.Namespace) + newKbHost := fmt.Sprintf("%s.%s.svc", kbv1.HTTPService(spec.KibanaRef.Name), builder.Beat.Namespace) yaml := string(yamlBytes) yaml = strings.ReplaceAll(yaml, "elasticsearch-es-http.default.svc", newEsHost) diff --git a/test/e2e/es/stack_monitoring_test.go b/test/e2e/es/stack_monitoring_test.go index cf9c9b2875..aa783f07e8 100644 --- a/test/e2e/es/stack_monitoring_test.go +++ b/test/e2e/es/stack_monitoring_test.go @@ -2,31 +2,24 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build es e2e + package es import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strconv" "testing" - esClient "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/client" - "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/stackmon" - "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/validations" "github.com/elastic/cloud-on-k8s/test/e2e/test" + "github.com/elastic/cloud-on-k8s/test/e2e/test/checks" "github.com/elastic/cloud-on-k8s/test/e2e/test/elasticsearch" ) // TestESStackMonitoring tests that when an Elasticsearch cluster is configured with monitoring, its log and metrics are // correctly delivered to the referenced monitoring Elasticsearch clusters. -// It tests that the monitored ES pods have 3 containers ready and that there are documents indexed in the beat indexes -// of the monitoring Elasticsearch clusters. func TestESStackMonitoring(t *testing.T) { // only execute this test on supported version - err := stackmon.IsSupportedVersion(test.Ctx().ElasticStackVersion) + err := validations.IsSupportedVersion(test.Ctx().ElasticStackVersion) if err != nil { t.SkipNow() } @@ -42,114 +35,8 @@ func TestESStackMonitoring(t *testing.T) { // checks that the sidecar beats have sent data in the monitoring clusters steps := func(k *test.K8sClient) test.StepList { - checks := stackMonitoringChecks{monitored, metrics, logs, k} - return test.StepList{ - checks.CheckBeatSidecars(), - checks.CheckMetricbeatIndex(), - checks.CheckFilebeatIndex(), - } + return checks.MonitoredSteps(&monitored, k) } test.Sequence(nil, steps, metrics, logs, monitored).RunSequential(t) } - -type stackMonitoringChecks struct { - monitored elasticsearch.Builder - metrics elasticsearch.Builder - logs elasticsearch.Builder - k *test.K8sClient -} - -func (c *stackMonitoringChecks) CheckBeatSidecars() test.Step { - return test.Step{ - Name: "Check that beat sidecars are running", - Test: test.Eventually(func() error { - pods, err := c.k.GetPods(test.ESPodListOptions(c.monitored.Elasticsearch.Namespace, c.monitored.Elasticsearch.Name)...) - if err != nil { - return err - } - for _, pod := range pods { - if len(pod.Spec.Containers) != 3 { - return fmt.Errorf("expected %d containers, got %d", 3, len(pod.Spec.Containers)) - } - if !k8s.IsPodReady(pod) { - return fmt.Errorf("pod %s not ready", pod.Name) - } - } - return nil - })} -} - -func (c *stackMonitoringChecks) CheckMetricbeatIndex() test.Step { - return test.Step{ - Name: "Check that documents are indexed in one metricbeat-* index", - Test: test.Eventually(func() error { - client, err := elasticsearch.NewElasticsearchClient(c.metrics.Elasticsearch, c.k) - if err != nil { - return err - } - err = AreIndexedDocs(client, "metricbeat-*") - if err != nil { - return err - } - return nil - })} -} - -func (c *stackMonitoringChecks) CheckFilebeatIndex() test.Step { - return test.Step{ - Name: "Check that documents are indexed in one filebeat-* index", - Test: test.Eventually(func() error { - client, err := elasticsearch.NewElasticsearchClient(c.logs.Elasticsearch, c.k) - if err != nil { - return err - } - err = AreIndexedDocs(client, "filebeat*") - if err != nil { - return err - } - return nil - })} -} - -// Index partially models Elasticsearch cluster index returned by /_cat/indices -type Index struct { - Index string `json:"index"` - DocsCount string `json:"docs.count"` -} - -func AreIndexedDocs(esClient esClient.Client, indexPattern string) error { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/_cat/indices/%s?format=json", indexPattern), nil) //nolint:noctx - if err != nil { - return err - } - resp, err := esClient.Request(context.Background(), req) - if err != nil { - return err - } - defer resp.Body.Close() - resultBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - var indices []Index - err = json.Unmarshal(resultBytes, &indices) - if err != nil { - return err - } - - // 1 index must exist - if len(indices) != 1 { - return fmt.Errorf("expected [%d] index [%s], found [%d]", len(indices), indexPattern, 1) - } - docsCount, err := strconv.Atoi(indices[0].DocsCount) - if err != nil { - return err - } - // with at least 1 doc - if docsCount < 0 { - return fmt.Errorf("index [%s] empty", indexPattern) - } - - return nil -} diff --git a/test/e2e/kb/failure_test.go b/test/e2e/kb/failure_test.go index a9d297098d..8b0b58bda9 100644 --- a/test/e2e/kb/failure_test.go +++ b/test/e2e/kb/failure_test.go @@ -10,7 +10,7 @@ import ( "context" "testing" - kibana2 "github.com/elastic/cloud-on-k8s/pkg/controller/kibana" + kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" "github.com/elastic/cloud-on-k8s/test/e2e/test" "github.com/elastic/cloud-on-k8s/test/e2e/test/elasticsearch" "github.com/elastic/cloud-on-k8s/test/e2e/test/kibana" @@ -52,7 +52,7 @@ func TestKillKibanaDeployment(t *testing.T) { var dep appsv1.Deployment err := k.Client.Get(context.Background(), types.NamespacedName{ Namespace: test.Ctx().ManagedNamespace(0), - Name: kibana2.Deployment(kbBuilder.Kibana.Name), + Name: kbv1.Deployment(kbBuilder.Kibana.Name), }, &dep) if apierrors.IsNotFound(err) { // already deleted diff --git a/test/e2e/kb/stack_monitoring_test.go b/test/e2e/kb/stack_monitoring_test.go new file mode 100644 index 0000000000..f2b685ddb2 --- /dev/null +++ b/test/e2e/kb/stack_monitoring_test.go @@ -0,0 +1,46 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build kb e2e + +package kb + +import ( + "testing" + + "github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/validations" + "github.com/elastic/cloud-on-k8s/test/e2e/test" + "github.com/elastic/cloud-on-k8s/test/e2e/test/checks" + "github.com/elastic/cloud-on-k8s/test/e2e/test/elasticsearch" + "github.com/elastic/cloud-on-k8s/test/e2e/test/kibana" +) + +// TestKBStackMonitoring tests that when a Kibana is configured with monitoring, its log and metrics are +// correctly delivered to the referenced monitoring Elasticsearch clusters. +func TestKBStackMonitoring(t *testing.T) { + // only execute this test on supported version + err := validations.IsSupportedVersion(test.Ctx().ElasticStackVersion) + if err != nil { + t.SkipNow() + } + + // create 1 monitored and 2 monitoring clusters to collect separately metrics and logs + metrics := elasticsearch.NewBuilder("test-kb-mon-metrics"). + WithESMasterDataNodes(2, elasticsearch.DefaultResources) + logs := elasticsearch.NewBuilder("test-kb-mon-logs"). + WithESMasterDataNodes(2, elasticsearch.DefaultResources) + assocEs := elasticsearch.NewBuilder("test-kb-mon-a"). + WithESMasterDataNodes(1, elasticsearch.DefaultResources) + monitored := kibana.NewBuilder("test-kb-mon-a"). + WithElasticsearchRef(assocEs.Ref()). + WithNodeCount(1). + WithMonitoring(metrics.Ref(), logs.Ref()) + + // checks that the sidecar beats have sent data in the monitoring clusters + steps := func(k *test.K8sClient) test.StepList { + return checks.MonitoredSteps(&monitored, k) + } + + test.Sequence(nil, steps, metrics, logs, assocEs, monitored).RunSequential(t) +} diff --git a/test/e2e/test/apmserver/checks_k8s.go b/test/e2e/test/apmserver/checks_k8s.go index 0a777d365f..62e820f969 100644 --- a/test/e2e/test/apmserver/checks_k8s.go +++ b/test/e2e/test/apmserver/checks_k8s.go @@ -12,14 +12,15 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/test/e2e/test" + "github.com/elastic/cloud-on-k8s/test/e2e/test/checks" ) func (b Builder) CheckK8sTestSteps(k *test.K8sClient) test.StepList { return test.StepList{ - test.CheckDeployment(b, k, b.ApmServer.Name+"-apm-server"), - test.CheckPods(b, k), - test.CheckServices(b, k), - test.CheckServicesEndpoints(b, k), + checks.CheckDeployment(b, k, b.ApmServer.Name+"-apm-server"), + checks.CheckPods(b, k), + checks.CheckServices(b, k), + checks.CheckServicesEndpoints(b, k), CheckSecrets(b, k), CheckStatus(b, k), } diff --git a/test/e2e/test/common_checks.go b/test/e2e/test/checks/common.go similarity index 82% rename from test/e2e/test/common_checks.go rename to test/e2e/test/checks/common.go index f360419fb3..f1c62ec476 100644 --- a/test/e2e/test/common_checks.go +++ b/test/e2e/test/checks/common.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package test +package checks import ( "context" @@ -10,6 +10,7 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/common/hash" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + "github.com/elastic/cloud-on-k8s/test/e2e/test" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -17,10 +18,10 @@ import ( ) // CheckDeployment checks the Deployment resource exists -func CheckDeployment(subj Subject, k *K8sClient, deploymentName string) Step { - return Step{ +func CheckDeployment(subj test.Subject, k *test.K8sClient, deploymentName string) test.Step { + return test.Step{ Name: subj.Kind() + " deployment should be created", - Test: Eventually(func() error { + Test: test.Eventually(func() error { var dep appsv1.Deployment err := k.Client.Get(context.Background(), types.NamespacedName{ Namespace: subj.NSN().Namespace, @@ -41,14 +42,14 @@ func CheckDeployment(subj Subject, k *K8sClient, deploymentName string) Step { } // CheckPods checks that the test subject's expected pods are eventually ready. -func CheckPods(subj Subject, k *K8sClient) Step { +func CheckPods(subj test.Subject, k *test.K8sClient) test.Step { // This is a shared test but it is common for Enterprise Search Pods to take some time to be ready, especially // during the initial bootstrap, or during version upgrades. Let's increase the timeout // for this particular step. - timeout := Ctx().TestTimeout * 2 - return Step{ + timeout := test.Ctx().TestTimeout * 2 + return test.Step{ Name: subj.Kind() + " Pods should eventually be ready", - Test: UntilSuccess(func() error { + Test: test.UntilSuccess(func() error { var pods corev1.PodList if err := k.Client.List(context.Background(), &pods, subj.ListOptions()...); err != nil { return err @@ -57,7 +58,7 @@ func CheckPods(subj Subject, k *K8sClient) Step { // builder hash matches expectedBuilderHash := hash.HashObject(subj.Spec()) for _, pod := range pods.Items { - if err := ValidateBuilderHashAnnotation(pod, expectedBuilderHash); err != nil { + if err := test.ValidateBuilderHashAnnotation(pod, expectedBuilderHash); err != nil { return err } } @@ -87,10 +88,10 @@ func CheckPods(subj Subject, k *K8sClient) Step { } // CheckServices checks that all expected services have been created -func CheckServices(subj Subject, k *K8sClient) Step { - return Step{ +func CheckServices(subj test.Subject, k *test.K8sClient) test.Step { + return test.Step{ Name: subj.Kind() + " services should be created", - Test: Eventually(func() error { + Test: test.Eventually(func() error { for _, s := range []string{ subj.ServiceName(), } { @@ -104,10 +105,10 @@ func CheckServices(subj Subject, k *K8sClient) Step { } // CheckServicesEndpoints checks that services have the expected number of endpoints -func CheckServicesEndpoints(subj Subject, k *K8sClient) Step { - return Step{ +func CheckServicesEndpoints(subj test.Subject, k *test.K8sClient) test.Step { + return test.Step{ Name: subj.Kind() + " services should have endpoints", - Test: Eventually(func() error { + Test: test.Eventually(func() error { for endpointName, addrCount := range map[string]int{ subj.ServiceName(): int(subj.Count()), } { diff --git a/test/e2e/test/checks/monitoring.go b/test/e2e/test/checks/monitoring.go new file mode 100644 index 0000000000..d8cc54f686 --- /dev/null +++ b/test/e2e/test/checks/monitoring.go @@ -0,0 +1,170 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package checks + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strconv" + + esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" + esClient "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/client" + "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" + "github.com/elastic/cloud-on-k8s/test/e2e/test" + "github.com/elastic/cloud-on-k8s/test/e2e/test/elasticsearch" + "k8s.io/apimachinery/pkg/types" +) + +type Monitored interface { + Name() string + Namespace() string + GetMetricsIndexPattern() string + GetLogsCluster() *types.NamespacedName + GetMetricsCluster() *types.NamespacedName +} + +func MonitoredSteps(monitored Monitored, k8sClient *test.K8sClient) test.StepList { + return stackMonitoringChecks{ + monitored: monitored, + k8sClient: k8sClient, + }.Steps() +} + +// stackMonitoringChecks tests that the monitored resource pods have 3 containers ready and that there are documents indexed in the beat indexes +// of the monitoring Elasticsearch clusters. +type stackMonitoringChecks struct { + monitored Monitored + k8sClient *test.K8sClient +} + +func (c stackMonitoringChecks) Steps() test.StepList { + return test.StepList{ + c.CheckBeatSidecars(), + c.CheckMonitoringMetricsIndex(), + c.CheckFilebeatIndex(), + } +} + +func (c stackMonitoringChecks) CheckBeatSidecars() test.Step { + return test.Step{ + Name: "Check that beat sidecars are running", + Test: test.Eventually(func() error { + pods, err := c.k8sClient.GetPods( + test.ESPodListOptions( + c.monitored.Namespace(), + c.monitored.Name())..., + ) + if err != nil { + return err + } + for _, pod := range pods { + if len(pod.Spec.Containers) != 3 { + return fmt.Errorf("expected %d containers, got %d", 3, len(pod.Spec.Containers)) + } + if !k8s.IsPodReady(pod) { + return fmt.Errorf("pod %s not ready", pod.Name) + } + } + return nil + })} +} + +func (c stackMonitoringChecks) CheckMonitoringMetricsIndex() test.Step { + indexPattern := c.monitored.GetMetricsIndexPattern() + return test.Step{ + Name: fmt.Sprintf("Check that documents are indexed in index %s", indexPattern), + Test: test.Eventually(func() error { + if c.monitored.GetMetricsCluster() == nil { + return nil + } + esMetricsRef := *c.monitored.GetMetricsCluster() + // Get Elasticsearch + esMetrics := esv1.Elasticsearch{} + if err := c.k8sClient.Client.Get(context.Background(), esMetricsRef, &esMetrics); err != nil { + return err + } + // Create a new Elasticsearch client + client, err := elasticsearch.NewElasticsearchClient(esMetrics, c.k8sClient) + if err != nil { + return err + } + // Check that there is at least one document + err = containsDocuments(client, indexPattern) + if err != nil { + return err + } + return nil + })} +} + +func (c stackMonitoringChecks) CheckFilebeatIndex() test.Step { + return test.Step{ + Name: "Check that documents are indexed in one filebeat-* index", + Test: test.Eventually(func() error { + if c.monitored.GetMetricsCluster() == nil { + return nil + } + esLogsRef := *c.monitored.GetLogsCluster() + // Get Elasticsearch + esLogs := esv1.Elasticsearch{} + if err := c.k8sClient.Client.Get(context.Background(), esLogsRef, &esLogs); err != nil { + return err + } + // Create a new Elasticsearch client + client, err := elasticsearch.NewElasticsearchClient(esLogs, c.k8sClient) + if err != nil { + return err + } + err = containsDocuments(client, "filebeat-*") + if err != nil { + return err + } + return nil + })} +} + +// Index partially models Elasticsearch cluster index returned by /_cat/indices +type Index struct { + Index string `json:"index"` + DocsCount string `json:"docs.count"` +} + +func containsDocuments(esClient esClient.Client, indexPattern string) error { + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/_cat/indices/%s?format=json", indexPattern), nil) //nolint:noctx + if err != nil { + return err + } + resp, err := esClient.Request(context.Background(), req) + if err != nil { + return err + } + defer resp.Body.Close() + resultBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + var indices []Index + err = json.Unmarshal(resultBytes, &indices) + if err != nil { + return err + } + + // 1 index must exist + if len(indices) != 1 { + return fmt.Errorf("expected [%d] index [%s], found [%d]", len(indices), indexPattern, 1) + } + docsCount, err := strconv.Atoi(indices[0].DocsCount) + if err != nil { + return err + } + // with at least 1 doc + if docsCount <= 0 { + return fmt.Errorf("index [%s] empty", indexPattern) + } + return nil +} diff --git a/test/e2e/test/elasticsearch/builder.go b/test/e2e/test/elasticsearch/builder.go index 333a4903b6..ee0f48444b 100644 --- a/test/e2e/test/elasticsearch/builder.go +++ b/test/e2e/test/elasticsearch/builder.go @@ -17,6 +17,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -459,6 +460,34 @@ func (b Builder) WithMonitoring(metricsESRef commonv1.ObjectSelector, logsESRef return b } +func (b Builder) GetMetricsIndexPattern() string { + return ".monitoring-es-*" +} + +func (b Builder) Name() string { + return b.Elasticsearch.Name +} + +func (b Builder) Namespace() string { + return b.Elasticsearch.Namespace +} + +func (b Builder) GetLogsCluster() *types.NamespacedName { + if len(b.Elasticsearch.Spec.Monitoring.Logs.ElasticsearchRefs) == 0 { + return nil + } + logsCluster := b.Elasticsearch.Spec.Monitoring.Logs.ElasticsearchRefs[0].NamespacedName() + return &logsCluster +} + +func (b Builder) GetMetricsCluster() *types.NamespacedName { + if len(b.Elasticsearch.Spec.Monitoring.Metrics.ElasticsearchRefs) == 0 { + return nil + } + metricsCluster := b.Elasticsearch.Spec.Monitoring.Metrics.ElasticsearchRefs[0].NamespacedName() + return &metricsCluster +} + // -- Helper functions func (b Builder) RuntimeObjects() []client.Object { diff --git a/test/e2e/test/enterprisesearch/checks_k8s.go b/test/e2e/test/enterprisesearch/checks_k8s.go index acd214ceba..3a099beb8e 100644 --- a/test/e2e/test/enterprisesearch/checks_k8s.go +++ b/test/e2e/test/enterprisesearch/checks_k8s.go @@ -12,14 +12,15 @@ import ( entv1 "github.com/elastic/cloud-on-k8s/pkg/apis/enterprisesearch/v1" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/test/e2e/test" + "github.com/elastic/cloud-on-k8s/test/e2e/test/checks" ) func (b Builder) CheckK8sTestSteps(k *test.K8sClient) test.StepList { return test.StepList{ - test.CheckDeployment(b, k, b.EnterpriseSearch.Name+"-ent"), - test.CheckPods(b, k), - test.CheckServices(b, k), - test.CheckServicesEndpoints(b, k), + checks.CheckDeployment(b, k, b.EnterpriseSearch.Name+"-ent"), + checks.CheckPods(b, k), + checks.CheckServices(b, k), + checks.CheckServicesEndpoints(b, k), CheckSecrets(b, k), CheckStatus(b, k), } diff --git a/test/e2e/test/kibana/builder.go b/test/e2e/test/kibana/builder.go index f7521c9075..ca927a4c3a 100644 --- a/test/e2e/test/kibana/builder.go +++ b/test/e2e/test/kibana/builder.go @@ -173,6 +173,40 @@ func (b Builder) WithTLSDisabled(disabled bool) Builder { return b } +func (b Builder) WithMonitoring(metricsESRef commonv1.ObjectSelector, logsESRef commonv1.ObjectSelector) Builder { + b.Kibana.Spec.Monitoring.Metrics.ElasticsearchRefs = []commonv1.ObjectSelector{metricsESRef} + b.Kibana.Spec.Monitoring.Logs.ElasticsearchRefs = []commonv1.ObjectSelector{logsESRef} + return b +} + +func (b Builder) GetMetricsIndexPattern() string { + return ".monitoring-kibana-*" +} + +func (b Builder) Name() string { + return b.Kibana.Name +} + +func (b Builder) Namespace() string { + return b.Kibana.Namespace +} + +func (b Builder) GetLogsCluster() *types.NamespacedName { + if len(b.Kibana.Spec.Monitoring.Logs.ElasticsearchRefs) == 0 { + return nil + } + logsCluster := b.Kibana.Spec.Monitoring.Logs.ElasticsearchRefs[0].NamespacedName() + return &logsCluster +} + +func (b Builder) GetMetricsCluster() *types.NamespacedName { + if len(b.Kibana.Spec.Monitoring.Metrics.ElasticsearchRefs) == 0 { + return nil + } + metricsCluster := b.Kibana.Spec.Monitoring.Metrics.ElasticsearchRefs[0].NamespacedName() + return &metricsCluster +} + // -- test.Subject impl func (b Builder) NSN() types.NamespacedName { diff --git a/test/e2e/test/kibana/checks_k8s.go b/test/e2e/test/kibana/checks_k8s.go index 2107b8bf8c..c46b75e064 100644 --- a/test/e2e/test/kibana/checks_k8s.go +++ b/test/e2e/test/kibana/checks_k8s.go @@ -10,17 +10,17 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1" kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" - "github.com/elastic/cloud-on-k8s/pkg/controller/kibana" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/test/e2e/test" + "github.com/elastic/cloud-on-k8s/test/e2e/test/checks" ) func (b Builder) CheckK8sTestSteps(k *test.K8sClient) test.StepList { return test.StepList{ - test.CheckDeployment(b, k, kibana.Deployment(b.Kibana.Name)), - test.CheckPods(b, k), - test.CheckServices(b, k), - test.CheckServicesEndpoints(b, k), + checks.CheckDeployment(b, k, kbv1.Deployment(b.Kibana.Name)), + checks.CheckPods(b, k), + checks.CheckServices(b, k), + checks.CheckServicesEndpoints(b, k), CheckSecrets(b, k), CheckStatus(b, k), } diff --git a/test/e2e/test/kibana/http_client.go b/test/e2e/test/kibana/http_client.go index 22ff25bb66..7826508119 100644 --- a/test/e2e/test/kibana/http_client.go +++ b/test/e2e/test/kibana/http_client.go @@ -15,7 +15,7 @@ import ( "github.com/pkg/errors" kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" - "github.com/elastic/cloud-on-k8s/pkg/controller/kibana" + "github.com/elastic/cloud-on-k8s/pkg/controller/kibana/network" "github.com/elastic/cloud-on-k8s/test/e2e/test" ) @@ -31,7 +31,7 @@ func (e *APIError) Error() string { func NewKibanaClient(kb kbv1.Kibana, k *test.K8sClient) (*http.Client, error) { var caCerts []*x509.Certificate if kb.Spec.HTTP.TLS.Enabled() { - crts, err := k.GetHTTPCerts(kibana.Namer, kb.Namespace, kb.Name) + crts, err := k.GetHTTPCerts(kbv1.KBNamer, kb.Namespace, kb.Name) if err != nil { return nil, err } @@ -47,7 +47,7 @@ func DoRequest(k *test.K8sClient, kb kbv1.Kibana, password string, method string scheme = "https" } // add .svc suffix so that requests work when using the port-forwarder during local test runs - u, err := url.Parse(fmt.Sprintf("%s://%s.%s.svc:5601", scheme, kibana.HTTPService(kb.Name), kb.Namespace)) + u, err := url.Parse(fmt.Sprintf("%s://%s.%s.svc:%d", scheme, kbv1.HTTPService(kb.Name), kb.Namespace, network.HTTPPort)) if err != nil { return nil, errors.Wrap(err, "while parsing url") } diff --git a/test/e2e/test/kibana/steps_deletion.go b/test/e2e/test/kibana/steps_deletion.go index 1689c8186d..42e180a75a 100644 --- a/test/e2e/test/kibana/steps_deletion.go +++ b/test/e2e/test/kibana/steps_deletion.go @@ -7,8 +7,8 @@ package kibana import ( "context" + kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" - "github.com/elastic/cloud-on-k8s/pkg/controller/kibana" "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/test/e2e/test" "github.com/pkg/errors" @@ -57,7 +57,7 @@ func (b Builder) DeletionTestSteps(k *test.K8sClient) test.StepList { Test: test.Eventually(func() error { namespace := b.Kibana.Namespace return k.CheckSecretsRemoved([]types.NamespacedName{ - {Namespace: namespace, Name: certificates.PublicCertsSecretName(kibana.Namer, b.Kibana.Name)}, + {Namespace: namespace, Name: certificates.PublicCertsSecretName(kbv1.KBNamer, b.Kibana.Name)}, }) }), }, diff --git a/test/e2e/test/maps/steps.go b/test/e2e/test/maps/steps.go index d540a255df..9f70b65a42 100644 --- a/test/e2e/test/maps/steps.go +++ b/test/e2e/test/maps/steps.go @@ -14,6 +14,7 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/utils/k8s" "github.com/elastic/cloud-on-k8s/test/e2e/cmd/run" "github.com/elastic/cloud-on-k8s/test/e2e/test" + "github.com/elastic/cloud-on-k8s/test/e2e/test/checks" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -82,10 +83,10 @@ func (b Builder) CreationTestSteps(k *test.K8sClient) test.StepList { func (b Builder) CheckK8sTestSteps(k *test.K8sClient) test.StepList { return test.StepList{ - test.CheckDeployment(b, k, maps.Deployment(b.EMS.Name)), - test.CheckPods(b, k), - test.CheckServices(b, k), - test.CheckServicesEndpoints(b, k), + checks.CheckDeployment(b, k, maps.Deployment(b.EMS.Name)), + checks.CheckPods(b, k), + checks.CheckServices(b, k), + checks.CheckServicesEndpoints(b, k), CheckSecrets(b, k), CheckStatus(b, k), }