From a677aa70f2f85d79ceac2d0506054794c28000e1 Mon Sep 17 00:00:00 2001 From: kvalliyurnatt <135071014+kvalliyurnatt@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:35:41 -0400 Subject: [PATCH] Add missing metricsets (#8048) * add missing metricsets, enable ingest pipeline metricset only if version is 8.7.0 and higher * Update config/recipes/beats/stack_monitoring.yaml --------- Co-authored-by: Thibault Richard --- config/recipes/beats/stack_monitoring.yaml | 3 +++ pkg/controller/common/stackmon/config.go | 21 +++++++++++++--- pkg/controller/common/stackmon/config_test.go | 25 ++++++++++++++++++- pkg/controller/common/stackmon/sidecar.go | 10 +++++--- .../elasticsearch/stackmon/metricbeat.tpl.yml | 5 ++++ 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/config/recipes/beats/stack_monitoring.yaml b/config/recipes/beats/stack_monitoring.yaml index 4bd957cbee..98e5bfabe7 100644 --- a/config/recipes/beats/stack_monitoring.yaml +++ b/config/recipes/beats/stack_monitoring.yaml @@ -29,8 +29,11 @@ spec: - index - index_recovery - index_summary + - ingest_pipeline - ml_job + - node - node_stats + - pending_tasks - shard period: 10s hosts: "https://${data.host}:${data.ports.https}" diff --git a/pkg/controller/common/stackmon/config.go b/pkg/controller/common/stackmon/config.go index 73874500e3..50b84d538b 100644 --- a/pkg/controller/common/stackmon/config.go +++ b/pkg/controller/common/stackmon/config.go @@ -13,6 +13,7 @@ import ( "path/filepath" "text/template" + "github.com/blang/semver/v4" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -173,6 +174,7 @@ type inputConfigData struct { IsSSL bool HasCA bool CAPath string + Version semver.Version } // buildMetricbeatBaseConfig builds the base configuration for Metricbeat with the Elasticsearch or Kibana modules used @@ -187,6 +189,7 @@ func buildMetricbeatBaseConfig( password string, isTLS bool, configTemplate string, + version semver.Version, ) (string, volume.VolumeLike, error) { hasCA := false if isTLS { @@ -200,9 +203,10 @@ func buildMetricbeatBaseConfig( configData := inputConfigData{ Username: username, Password: password, - URL: url, // Metricbeat in the sidecar connects to the monitored resource using `localhost` - IsSSL: isTLS, // enable SSL configuration based on whether the monitored resource has TLS enabled - HasCA: hasCA, // the CA is optional to support custom certificate issued by a well-known CA, so without provided CA to configure + URL: url, // Metricbeat in the sidecar connects to the monitored resource using `localhost` + IsSSL: isTLS, // enable SSL configuration based on whether the monitored resource has TLS enabled + HasCA: hasCA, // the CA is optional to support custom certificate issued by a well-known CA, so without provided CA to configure + Version: version, // Version of the monitored resource } var caVolume volume.VolumeLike @@ -216,9 +220,18 @@ func buildMetricbeatBaseConfig( configData.CAPath = filepath.Join(caVolume.VolumeMount().MountPath, certificates.CAFileName) } + templateFuncMap := template.FuncMap{ + "isVersionGTE": func(minAllowedVersion string) (bool, error) { + minAllowedSemver, err := semver.Parse(minAllowedVersion) + if err != nil { + return false, err + } + return version.GTE(minAllowedSemver), nil + }, + } // render the config template with the config data var metricbeatConfig bytes.Buffer - err := template.Must(template.New("").Parse(configTemplate)).Execute(&metricbeatConfig, configData) + err := template.Must(template.New("").Funcs(templateFuncMap).Parse(configTemplate)).Execute(&metricbeatConfig, configData) if err != nil { return "", nil, err } diff --git a/pkg/controller/common/stackmon/config_test.go b/pkg/controller/common/stackmon/config_test.go index 7effa601b1..8f1cee7238 100644 --- a/pkg/controller/common/stackmon/config_test.go +++ b/pkg/controller/common/stackmon/config_test.go @@ -8,6 +8,7 @@ import ( "context" "testing" + "github.com/blang/semver/v4" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -131,6 +132,7 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) { certsSecret *corev1.Secret hasCA bool baseConfig string + version semver.Version }{ { name: "with TLS and a CA", @@ -148,7 +150,9 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) { password: 1234567890 ssl.enabled: true ssl.verification_mode: "certificate" + ingest_pipeline: "enabled" ssl.certificate_authorities: ["/mnt/elastic-internal/xx-monitoring/namespace/name/certs/ca.crt"]`, + version: semver.MustParse("8.7.0"), }, { name: "with TLS and no CA", @@ -164,17 +168,32 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) { username: elastic-internal-monitoring password: 1234567890 ssl.enabled: true - ssl.verification_mode: "certificate"`, + ssl.verification_mode: "certificate" + ingest_pipeline: "enabled"`, + version: semver.MustParse("8.7.0"), }, { name: "without TLS", isTLS: false, + baseConfig: ` + hosts: ["scheme://localhost:1234"] + username: elastic-internal-monitoring + password: 1234567890 + ssl.enabled: false + ssl.verification_mode: "certificate" + ingest_pipeline: "enabled"`, + version: semver.MustParse("8.7.0"), + }, + { + name: "with version less than 8.7.0", + isTLS: false, baseConfig: ` hosts: ["scheme://localhost:1234"] username: elastic-internal-monitoring password: 1234567890 ssl.enabled: false ssl.verification_mode: "certificate"`, + version: semver.MustParse("8.6.0"), }, } baseConfigTemplate := ` @@ -183,6 +202,9 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) { password: {{ .Password }} ssl.enabled: {{ .IsSSL }} ssl.verification_mode: "certificate" + {{- if isVersionGTE "8.7.0" }} + ingest_pipeline: "enabled" + {{- end }} {{- if .HasCA }} ssl.certificate_authorities: ["{{ .CAPath }}"] {{- end }}` @@ -210,6 +232,7 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) { "1234567890", tc.isTLS, baseConfigTemplate, + tc.version, ) assert.NoError(t, err) assert.Equal(t, tc.baseConfig, baseConfig) diff --git a/pkg/controller/common/stackmon/sidecar.go b/pkg/controller/common/stackmon/sidecar.go index 615237a64d..b42ec88caf 100644 --- a/pkg/controller/common/stackmon/sidecar.go +++ b/pkg/controller/common/stackmon/sidecar.go @@ -33,6 +33,11 @@ func NewMetricBeatSidecar( password string, isTLS bool, ) (BeatSidecar, error) { + v, err := version.Parse(imageVersion) + if err != nil { + return BeatSidecar{}, err // error unlikely and should have been caught during validation + } + baseConfig, sourceCaVolume, err := buildMetricbeatBaseConfig( client, associationType, @@ -43,15 +48,12 @@ func NewMetricBeatSidecar( password, isTLS, baseConfigTemplate, + v, ) if err != nil { return BeatSidecar{}, err } - v, err := version.Parse(imageVersion) - if err != nil { - return BeatSidecar{}, err // error unlikely and should have been caught during validation - } image := container.ImageRepository(container.MetricbeatImage, v) // EmptyDir volume so that MetricBeat does not write in the container image, which allows ReadOnlyRootFilesystem: true diff --git a/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml b/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml index 07bc21758e..e7a781a742 100644 --- a/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml +++ b/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml @@ -8,10 +8,15 @@ metricbeat.modules: - index - index_recovery - index_summary + {{- if isVersionGTE "8.7.0" }} + - ingest_pipeline + {{- end }} - ml_job + - node - node_stats - pending_tasks - shard + period: 10s xpack.enabled: true hosts: ["{{ .URL }}"]