Add missing metricsets (#8048)

* add missing metricsets, enable ingest pipeline metricset only if version is 8.7.0 and higher

* Update config/recipes/beats/stack_monitoring.yaml

---------

Co-authored-by: Thibault Richard <thbkrkr@users.noreply.github.com>

config/recipes/beats/stack_monitoring.yaml

@@ -29,8 +29,11 @@ spec:
                       - index
                       - index_recovery
                       - index_summary
+                      - ingest_pipeline
                       - ml_job
+                      - node
                       - node_stats
+                      - pending_tasks
                       - shard
                     period: 10s
                     hosts: "https://${data.host}:${data.ports.https}"

pkg/controller/common/stackmon/config.go

@@ -13,6 +13,7 @@ import (
 	"path/filepath"
 	"text/template"
 
+	"github.com/blang/semver/v4"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -173,6 +174,7 @@ type inputConfigData struct {
 	IsSSL    bool
 	HasCA    bool
 	CAPath   string
+	Version  semver.Version
 }
 
 // buildMetricbeatBaseConfig builds the base configuration for Metricbeat with the Elasticsearch or Kibana modules used
@@ -187,6 +189,7 @@ func buildMetricbeatBaseConfig(
 	password string,
 	isTLS bool,
 	configTemplate string,
+	version semver.Version,
 ) (string, volume.VolumeLike, error) {
 	hasCA := false
 	if isTLS {
@@ -200,9 +203,10 @@ func buildMetricbeatBaseConfig(
 	configData := inputConfigData{
 		Username: username,
 		Password: password,
-		URL:      url,   // Metricbeat in the sidecar connects to the monitored resource using `localhost`
-		IsSSL:    isTLS, // enable SSL configuration based on whether the monitored resource has TLS enabled
-		HasCA:    hasCA, // the CA is optional to support custom certificate issued by a well-known CA, so without provided CA to configure
+		URL:      url,     // Metricbeat in the sidecar connects to the monitored resource using `localhost`
+		IsSSL:    isTLS,   // enable SSL configuration based on whether the monitored resource has TLS enabled
+		HasCA:    hasCA,   // the CA is optional to support custom certificate issued by a well-known CA, so without provided CA to configure
+		Version:  version, // Version of the monitored resource
 	}
 
 	var caVolume volume.VolumeLike
@@ -216,9 +220,18 @@ func buildMetricbeatBaseConfig(
 		configData.CAPath = filepath.Join(caVolume.VolumeMount().MountPath, certificates.CAFileName)
 	}
 
+	templateFuncMap := template.FuncMap{
+		"isVersionGTE": func(minAllowedVersion string) (bool, error) {
+			minAllowedSemver, err := semver.Parse(minAllowedVersion)
+			if err != nil {
+				return false, err
+			}
+			return version.GTE(minAllowedSemver), nil
+		},
+	}
 	// render the config template with the config data
 	var metricbeatConfig bytes.Buffer
-	err := template.Must(template.New("").Parse(configTemplate)).Execute(&metricbeatConfig, configData)
+	err := template.Must(template.New("").Funcs(templateFuncMap).Parse(configTemplate)).Execute(&metricbeatConfig, configData)
 	if err != nil {
 		return "", nil, err
 	}

pkg/controller/common/stackmon/config_test.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/blang/semver/v4"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -131,6 +132,7 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
 		certsSecret *corev1.Secret
 		hasCA       bool
 		baseConfig  string
+		version     semver.Version
 	}{
 		{
 			name:  "with TLS and a CA",
@@ -148,7 +150,9 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
 				password: 1234567890
 				ssl.enabled: true
 				ssl.verification_mode: "certificate"
+				ingest_pipeline: "enabled"
 				ssl.certificate_authorities: ["/mnt/elastic-internal/xx-monitoring/namespace/name/certs/ca.crt"]`,
+			version: semver.MustParse("8.7.0"),
 		},
 		{
 			name:  "with TLS and no CA",
@@ -164,17 +168,32 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
 				username: elastic-internal-monitoring
 				password: 1234567890
 				ssl.enabled: true
-				ssl.verification_mode: "certificate"`,
+				ssl.verification_mode: "certificate"
+				ingest_pipeline: "enabled"`,
+			version: semver.MustParse("8.7.0"),
 		},
 		{
 			name:  "without TLS",
 			isTLS: false,
+			baseConfig: `
+				hosts: ["scheme://localhost:1234"]
+				username: elastic-internal-monitoring
+				password: 1234567890
+				ssl.enabled: false
+				ssl.verification_mode: "certificate"
+				ingest_pipeline: "enabled"`,
+			version: semver.MustParse("8.7.0"),
+		},
+		{
+			name:  "with version less than 8.7.0",
+			isTLS: false,
 			baseConfig: `
 				hosts: ["scheme://localhost:1234"]
 				username: elastic-internal-monitoring
 				password: 1234567890
 				ssl.enabled: false
 				ssl.verification_mode: "certificate"`,
+			version: semver.MustParse("8.6.0"),
 		},
 	}
 	baseConfigTemplate := `
@@ -183,6 +202,9 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
 				password: {{ .Password }}
 				ssl.enabled: {{ .IsSSL }}
 				ssl.verification_mode: "certificate"
+				{{- if isVersionGTE "8.7.0" }}
+				ingest_pipeline: "enabled"
+				{{- end }}
 				{{- if .HasCA }}
 				ssl.certificate_authorities: ["{{ .CAPath }}"]
 				{{- end }}`
@@ -210,6 +232,7 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
 				"1234567890",
 				tc.isTLS,
 				baseConfigTemplate,
+				tc.version,
 			)
 			assert.NoError(t, err)
 			assert.Equal(t, tc.baseConfig, baseConfig)

pkg/controller/common/stackmon/sidecar.go

@@ -33,6 +33,11 @@ func NewMetricBeatSidecar(
 	password string,
 	isTLS bool,
 ) (BeatSidecar, error) {
+	v, err := version.Parse(imageVersion)
+	if err != nil {
+		return BeatSidecar{}, err // error unlikely and should have been caught during validation
+	}
+
 	baseConfig, sourceCaVolume, err := buildMetricbeatBaseConfig(
 		client,
 		associationType,
@@ -43,15 +48,12 @@ func NewMetricBeatSidecar(
 		password,
 		isTLS,
 		baseConfigTemplate,
+		v,
 	)
 	if err != nil {
 		return BeatSidecar{}, err
 	}
 
-	v, err := version.Parse(imageVersion)
-	if err != nil {
-		return BeatSidecar{}, err // error unlikely and should have been caught during validation
-	}
 	image := container.ImageRepository(container.MetricbeatImage, v)
 
 	// EmptyDir volume so that MetricBeat does not write in the container image, which allows ReadOnlyRootFilesystem: true

pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml

@@ -8,10 +8,15 @@ metricbeat.modules:
       - index
       - index_recovery
       - index_summary
+      {{- if isVersionGTE "8.7.0" }}
+      - ingest_pipeline
+      {{- end }}
       - ml_job
+      - node
       - node_stats
       - pending_tasks
       - shard
+
     period: 10s
     xpack.enabled: true
     hosts: ["{{ .URL }}"]