Skip to content

Commit

Permalink
Introduce Monitored interface
Browse files Browse the repository at this point in the history
  • Loading branch information
barkbay committed Jul 12, 2021
1 parent 7ce7984 commit b600503
Showing 5 changed files with 117 additions and 36 deletions.
6 changes: 1 addition & 5 deletions test/e2e/es/stack_monitoring_test.go
Original file line number Diff line number Diff line change
@@ -10,7 +10,6 @@ import (
"testing"

"github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/validations"
"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
"github.com/elastic/cloud-on-k8s/test/e2e/test"
"github.com/elastic/cloud-on-k8s/test/e2e/test/checks"
"github.com/elastic/cloud-on-k8s/test/e2e/test/elasticsearch"
@@ -36,10 +35,7 @@ func TestESStackMonitoring(t *testing.T) {

// checks that the sidecar beats have sent data in the monitoring clusters
steps := func(k *test.K8sClient) test.StepList {
return checks.StackMonitoringChecks{
MonitoredNsn: k8s.ExtractNamespacedName(&monitored.Elasticsearch),
Metrics: metrics, Logs: logs, K: k,
}.Steps()
return checks.MonitoredSteps(&monitored, k)
}

test.Sequence(nil, steps, metrics, logs, monitored).RunSequential(t)
13 changes: 1 addition & 12 deletions test/e2e/kb/stack_monitoring_test.go
Original file line number Diff line number Diff line change
@@ -10,7 +10,6 @@ import (
"testing"

"github.com/elastic/cloud-on-k8s/pkg/controller/common/stackmon/validations"
"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
"github.com/elastic/cloud-on-k8s/test/e2e/test"
"github.com/elastic/cloud-on-k8s/test/e2e/test/checks"
"github.com/elastic/cloud-on-k8s/test/e2e/test/elasticsearch"
@@ -40,17 +39,7 @@ func TestKBStackMonitoring(t *testing.T) {

// checks that the sidecar beats have sent data in the monitoring clusters
steps := func(k *test.K8sClient) test.StepList {
c := checks.StackMonitoringChecks{
MonitoredNsn: k8s.ExtractNamespacedName(&monitored.Kibana),
Metrics: metrics,
Logs: logs,
K: k,
}
return test.StepList{
c.CheckBeatSidecars(),
//c.CheckMetricbeatIndex(), TODO: investigate if it's normal that there is no document in this index when es monitoring is off
c.CheckFilebeatIndex(),
}
return checks.MonitoredSteps(&monitored, k)
}

test.Sequence(nil, steps, metrics, logs, assocEs, monitored).RunSequential(t)
77 changes: 58 additions & 19 deletions test/e2e/test/checks/monitoring.go
Original file line number Diff line number Diff line change
@@ -12,35 +12,53 @@ import (
"net/http"
"strconv"

esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1"
esClient "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/client"
"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
"github.com/elastic/cloud-on-k8s/test/e2e/test"
"github.com/elastic/cloud-on-k8s/test/e2e/test/elasticsearch"
"k8s.io/apimachinery/pkg/types"
)

// StackMonitoringChecks tests that the monitored resource pods have 3 containers ready and that there are documents indexed in the beat indexes
type Monitored interface {
Name() string
Namespace() string
GetMetricsIndexPattern() string
GetLogsCluster() *types.NamespacedName
GetMetricsCluster() *types.NamespacedName
}

func MonitoredSteps(monitored Monitored, k8sClient *test.K8sClient) test.StepList {
return stackMonitoringChecks{
monitored: monitored,
k8sClient: k8sClient,
}.Steps()
}

// stackMonitoringChecks tests that the monitored resource pods have 3 containers ready and that there are documents indexed in the beat indexes
// of the monitoring Elasticsearch clusters.
type StackMonitoringChecks struct {
MonitoredNsn types.NamespacedName
Metrics elasticsearch.Builder
Logs elasticsearch.Builder
K *test.K8sClient
type stackMonitoringChecks struct {
monitored Monitored
k8sClient *test.K8sClient
}

func (c StackMonitoringChecks) Steps() test.StepList {
func (c stackMonitoringChecks) Steps() test.StepList {
return test.StepList{
c.CheckBeatSidecars(),
c.CheckMetricbeatIndex(),
c.CheckMonitoringMetricsIndex(),
c.CheckFilebeatIndex(),
}
}

func (c StackMonitoringChecks) CheckBeatSidecars() test.Step {
func (c stackMonitoringChecks) CheckBeatSidecars() test.Step {
return test.Step{
Name: "Check that beat sidecars are running",
Test: test.Eventually(func() error {
pods, err := c.K.GetPods(test.ESPodListOptions(c.MonitoredNsn.Namespace, c.MonitoredNsn.Name)...)
pods, err := c.k8sClient.GetPods(
test.ESPodListOptions(
c.monitored.Namespace(),
c.monitored.Name())...,
)
if err != nil {
return err
}
@@ -56,31 +74,53 @@ func (c StackMonitoringChecks) CheckBeatSidecars() test.Step {
})}
}

func (c StackMonitoringChecks) CheckMetricbeatIndex() test.Step {
func (c stackMonitoringChecks) CheckMonitoringMetricsIndex() test.Step {
indexPattern := c.monitored.GetMetricsIndexPattern()
return test.Step{
Name: "Check that documents are indexed in one metricbeat-* index",
Name: fmt.Sprintf("Check that documents are indexed in index %s", indexPattern),
Test: test.Eventually(func() error {
client, err := elasticsearch.NewElasticsearchClient(c.Metrics.Elasticsearch, c.K)
if c.monitored.GetMetricsCluster() == nil {
return nil
}
esMetricsRef := *c.monitored.GetMetricsCluster()
// Get Elasticsearch
esMetrics := esv1.Elasticsearch{}
if err := c.k8sClient.Client.Get(context.Background(), esMetricsRef, &esMetrics); err != nil {
return err
}
// Create a new Elasticsearch client
client, err := elasticsearch.NewElasticsearchClient(esMetrics, c.k8sClient)
if err != nil {
return err
}
err = AreIndexedDocs(client, "metricbeat-*")
// Check that there is at least one document
err = containsDocuments(client, indexPattern)
if err != nil {
return err
}
return nil
})}
}

func (c StackMonitoringChecks) CheckFilebeatIndex() test.Step {
func (c stackMonitoringChecks) CheckFilebeatIndex() test.Step {
return test.Step{
Name: "Check that documents are indexed in one filebeat-* index",
Test: test.Eventually(func() error {
client, err := elasticsearch.NewElasticsearchClient(c.Logs.Elasticsearch, c.K)
if c.monitored.GetMetricsCluster() == nil {
return nil
}
esLogsRef := *c.monitored.GetLogsCluster()
// Get Elasticsearch
esLogs := esv1.Elasticsearch{}
if err := c.k8sClient.Client.Get(context.Background(), esLogsRef, &esLogs); err != nil {
return err
}
// Create a new Elasticsearch client
client, err := elasticsearch.NewElasticsearchClient(esLogs, c.k8sClient)
if err != nil {
return err
}
err = AreIndexedDocs(client, "filebeat*")
err = containsDocuments(client, "filebeat-*")
if err != nil {
return err
}
@@ -94,7 +134,7 @@ type Index struct {
DocsCount string `json:"docs.count"`
}

func AreIndexedDocs(esClient esClient.Client, indexPattern string) error {
func containsDocuments(esClient esClient.Client, indexPattern string) error {
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/_cat/indices/%s?format=json", indexPattern), nil) //nolint:noctx
if err != nil {
return err
@@ -126,6 +166,5 @@ func AreIndexedDocs(esClient esClient.Client, indexPattern string) error {
if docsCount <= 0 {
return fmt.Errorf("index [%s] empty", indexPattern)
}

return nil
}
29 changes: 29 additions & 0 deletions test/e2e/test/elasticsearch/builder.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/rand"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -459,6 +460,34 @@ func (b Builder) WithMonitoring(metricsESRef commonv1.ObjectSelector, logsESRef
return b
}

func (b Builder) GetMetricsIndexPattern() string {
return ".monitoring-es-*"
}

func (b Builder) Name() string {
return b.Elasticsearch.Name
}

func (b Builder) Namespace() string {
return b.Elasticsearch.Namespace
}

func (b Builder) GetLogsCluster() *types.NamespacedName {
if len(b.Elasticsearch.Spec.Monitoring.Logs.ElasticsearchRefs) == 0 {
return nil
}
logsCluster := b.Elasticsearch.Spec.Monitoring.Logs.ElasticsearchRefs[0].NamespacedName()
return &logsCluster
}

func (b Builder) GetMetricsCluster() *types.NamespacedName {
if len(b.Elasticsearch.Spec.Monitoring.Metrics.ElasticsearchRefs) == 0 {
return nil
}
metricsCluster := b.Elasticsearch.Spec.Monitoring.Metrics.ElasticsearchRefs[0].NamespacedName()
return &metricsCluster
}

// -- Helper functions

func (b Builder) RuntimeObjects() []client.Object {
28 changes: 28 additions & 0 deletions test/e2e/test/kibana/builder.go
Original file line number Diff line number Diff line change
@@ -179,6 +179,34 @@ func (b Builder) WithMonitoring(metricsESRef commonv1.ObjectSelector, logsESRef
return b
}

func (b Builder) GetMetricsIndexPattern() string {
return ".monitoring-kibana-*"
}

func (b Builder) Name() string {
return b.Kibana.Name
}

func (b Builder) Namespace() string {
return b.Kibana.Namespace
}

func (b Builder) GetLogsCluster() *types.NamespacedName {
if len(b.Kibana.Spec.Monitoring.Logs.ElasticsearchRefs) == 0 {
return nil
}
logsCluster := b.Kibana.Spec.Monitoring.Logs.ElasticsearchRefs[0].NamespacedName()
return &logsCluster
}

func (b Builder) GetMetricsCluster() *types.NamespacedName {
if len(b.Kibana.Spec.Monitoring.Metrics.ElasticsearchRefs) == 0 {
return nil
}
metricsCluster := b.Kibana.Spec.Monitoring.Metrics.ElasticsearchRefs[0].NamespacedName()
return &metricsCluster
}

// -- test.Subject impl

func (b Builder) NSN() types.NamespacedName {

0 comments on commit b600503

Please sign in to comment.