Set basePath in Metricbeat when using stack monitoring (elastic#8311) (…

…elastic#8315)

Ensure stack monitoring works when Kibana is configured with a basePath parameter.

(cherry picked from commit eadb46d)

pkg/controller/beat/common/stackmon/stackmon.go

@@ -99,6 +99,7 @@ func MetricBeat(ctx context.Context, client k8s.Client, beat *v1beta1.Beat, vers
 		GetStackMonitoringSocketURL(beat),
 		"",
 		"",
+		"",
 		false,
 	)
 	if err != nil {

pkg/controller/common/stackmon/config.go

@@ -168,6 +168,7 @@ func mergeConfig(rawConfig string, config map[string]interface{}) ([]byte, error
 // inputConfigData holds data to configure the Metricbeat Elasticsearch and Kibana modules used
 // to collect metrics for Stack Monitoring
 type inputConfigData struct {
+	BasePath         string
 	URL              string
 	Username         string
 	Password         string
@@ -186,6 +187,7 @@ func buildMetricbeatBaseConfig(
 	nsn types.NamespacedName,
 	namer name.Namer,
 	url string,
+	basePath string,
 	username string,
 	password string,
 	isTLS bool,
@@ -204,10 +206,11 @@ func buildMetricbeatBaseConfig(
 	configData := inputConfigData{
 		Username: username,
 		Password: password,
-		URL:      url,     // Metricbeat in the sidecar connects to the monitored resource using `localhost`
-		IsSSL:    isTLS,   // enable SSL configuration based on whether the monitored resource has TLS enabled
-		HasCA:    hasCA,   // the CA is optional to support custom certificate issued by a well-known CA, so without provided CA to configure
-		Version:  version, // Version of the monitored resource
+		URL:      url,      // Metricbeat in the sidecar connects to the monitored resource using `localhost`
+		BasePath: basePath, // for the Metricbeat Kibana module
+		IsSSL:    isTLS,    // enable SSL configuration based on whether the monitored resource has TLS enabled
+		HasCA:    hasCA,    // the CA is optional to support custom certificate issued by a well-known CA, so without provided CA to configure
+		Version:  version,  // Version of the monitored resource
 	}
 
 	// See https://github.com/elastic/cloud-on-k8s/pull/8284

pkg/controller/common/stackmon/config_test.go

@@ -132,6 +132,7 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
 		certsSecret *corev1.Secret
 		hasCA       bool
 		baseConfig  string
+		basePath    string
 		version     semver.Version
 	}{
 		{
@@ -195,9 +196,26 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
 				ssl.verification_mode: "certificate"`,
 			version: semver.MustParse("8.6.0"),
 		},
+		{
+			name:     "with basepath",
+			isTLS:    false,
+			basePath: "/kibana",
+			baseConfig: `
+				hosts: ["scheme://localhost:1234"]
+				basepath: /kibana
+				username: elastic-internal-monitoring
+				password: 1234567890
+				ssl.enabled: false
+				ssl.verification_mode: "certificate"
+				ingest_pipeline: "enabled"`,
+			version: semver.MustParse("8.7.0"),
+		},
 	}
 	baseConfigTemplate := `
 				hosts: ["{{ .URL }}"]
+				{{- with .BasePath }}
+				basepath: {{ . }}
+				{{- end }}
 				username: {{ .Username }}
 				password: {{ .Password }}
 				ssl.enabled: {{ .IsSSL }}
@@ -228,6 +246,7 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
 				types.NamespacedName{Namespace: "namespace", Name: "name"},
 				name.NewNamer("es"),
 				sampleURL,
+				tc.basePath,
 				"elastic-internal-monitoring",
 				"1234567890",
 				tc.isTLS,

pkg/controller/common/stackmon/sidecar.go

@@ -29,6 +29,7 @@ func NewMetricBeatSidecar(
 	baseConfigTemplate string,
 	namer name.Namer,
 	url string,
+	basePath string,
 	username string,
 	password string,
 	isTLS bool,
@@ -44,6 +45,7 @@ func NewMetricBeatSidecar(
 		k8s.ExtractNamespacedName(resource),
 		namer,
 		url,
+		basePath,
 		username,
 		password,
 		isTLS,

pkg/controller/elasticsearch/stackmon/sidecar.go

@@ -44,6 +44,7 @@ func Metricbeat(ctx context.Context, client k8s.Client, es esv1.Elasticsearch) (
 		metricbeatConfigTemplate,
 		esv1.ESNamer,
 		fmt.Sprintf("%s://localhost:%d", es.Spec.HTTP.Protocol(), network.HTTPPort),
+		"",
 		username,
 		password,
 		es.Spec.HTTP.TLS.Enabled(),

pkg/controller/kibana/driver.go

@@ -165,15 +165,20 @@ func (d *driver) Reconcile(
 		return results.WithError(err)
 	}
 
-	err = stackmon.ReconcileConfigSecrets(ctx, d.client, *kb)
+	basePath, err := GetKibanaBasePath(*kb)
+	if err != nil {
+		return results.WithError(err)
+	}
+
+	err = stackmon.ReconcileConfigSecrets(ctx, d.client, *kb, basePath)
 	if err != nil {
 		return results.WithError(err)
 	}
 
 	span, _ := apm.StartSpan(ctx, "reconcile_deployment", tracing.SpanTypeApp)
 	defer span.End()
 
-	deploymentParams, err := d.deploymentParams(ctx, kb, kibanaPolicyCfg.PodAnnotations)
+	deploymentParams, err := d.deploymentParams(ctx, kb, kibanaPolicyCfg.PodAnnotations, basePath)
 	if err != nil {
 		return results.WithError(err)
 	}
@@ -220,7 +225,7 @@ func (d *driver) getStrategyType(kb *kbv1.Kibana) (appsv1.DeploymentStrategyType
 	return appsv1.RollingUpdateDeploymentStrategyType, nil
 }
 
-func (d *driver) deploymentParams(ctx context.Context, kb *kbv1.Kibana, policyAnnotations map[string]string) (deployment.Params, error) {
+func (d *driver) deploymentParams(ctx context.Context, kb *kbv1.Kibana, policyAnnotations map[string]string, basePath string) (deployment.Params, error) {
 	initContainersParameters, err := newInitContainersParameters(kb)
 	if err != nil {
 		return deployment.Params{}, err
@@ -242,7 +247,7 @@ func (d *driver) deploymentParams(ctx context.Context, kb *kbv1.Kibana, policyAn
 	if err != nil {
 		return deployment.Params{}, err
 	}
-	kibanaPodSpec, err := NewPodTemplateSpec(ctx, d.client, *kb, keystoreResources, volumes)
+	kibanaPodSpec, err := NewPodTemplateSpec(ctx, d.client, *kb, keystoreResources, volumes, basePath)
 	if err != nil {
 		return deployment.Params{}, err
 	}

pkg/controller/kibana/driver_test.go

@@ -392,7 +392,7 @@ func TestDriverDeploymentParams(t *testing.T) {
 			d, err := newDriver(client, w, record.NewFakeRecorder(100), kb, corev1.IPv4Protocol)
 			require.NoError(t, err)
 
-			got, err := d.deploymentParams(context.Background(), kb, tt.args.policyAnnotations)
+			got, err := d.deploymentParams(context.Background(), kb, tt.args.policyAnnotations, "")
 			if tt.wantErr {
 				require.Error(t, err)
 				return

pkg/controller/kibana/pod.go

@@ -71,8 +71,8 @@ var (
 	}
 )
 
-// kibanaConfig is used to get the base path from the Kibana configuration.
-type kibanaConfig struct {
+// basePathConfig is used to get the base path from the Kibana configuration.
+type basePathConfig struct {
 	Server struct {
 		RewriteBasePath bool   `config:"rewriteBasePath"`
 		BasePath        string `config:"basePath"`
@@ -101,7 +101,14 @@ func readinessProbe(useTLS bool, basePath string) corev1.Probe {
 	}
 }
 
-func NewPodTemplateSpec(ctx context.Context, client k8sclient.Client, kb kbv1.Kibana, keystore *keystore.Resources, volumes []volume.VolumeLike) (corev1.PodTemplateSpec, error) {
+func NewPodTemplateSpec(
+	ctx context.Context,
+	client k8sclient.Client,
+	kb kbv1.Kibana,
+	keystore *keystore.Resources,
+	volumes []volume.VolumeLike,
+	basePath string,
+) (corev1.PodTemplateSpec, error) {
 	labels := kb.GetIdentityLabels()
 	labels[kblabel.KibanaVersionLabelName] = kb.Spec.Version
 
@@ -111,16 +118,12 @@ func NewPodTemplateSpec(ctx context.Context, client k8sclient.Client, kb kbv1.Ki
 		return corev1.PodTemplateSpec{}, err // error unlikely and should have been caught during validation
 	}
 
-	kibanaBasePath, err := GetKibanaBasePath(kb)
-	if err != nil {
-		return corev1.PodTemplateSpec{}, fmt.Errorf("failed to get kibana base path error:%w", err)
-	}
 	builder := defaults.NewPodTemplateBuilder(kb.Spec.PodTemplate, kbv1.KibanaContainerName).
 		WithResources(DefaultResources).
 		WithLabels(labels).
 		WithAnnotations(DefaultAnnotations).
 		WithDockerImage(kb.Spec.Image, container.ImageRepository(container.KibanaImage, v)).
-		WithReadinessProbe(readinessProbe(kb.Spec.HTTP.TLS.Enabled(), kibanaBasePath)).
+		WithReadinessProbe(readinessProbe(kb.Spec.HTTP.TLS.Enabled(), basePath)).
 		WithPorts(ports).
 		WithInitContainers(initConfigContainer(kb))
 
@@ -146,7 +149,7 @@ func NewPodTemplateSpec(ctx context.Context, client k8sclient.Client, kb kbv1.Ki
 			WithInitContainers(keystore.InitContainer)
 	}
 
-	builder, err = stackmon.WithMonitoring(ctx, client, builder, kb)
+	builder, err = stackmon.WithMonitoring(ctx, client, builder, kb, basePath)
 	if err != nil {
 		return corev1.PodTemplateSpec{}, err
 	}
@@ -211,7 +214,7 @@ func GetKibanaBasePath(kb kbv1.Kibana) (string, error) {
 		return "", err
 	}
 
-	kbCfg := kibanaConfig{}
+	kbCfg := basePathConfig{}
 	if err := kbucfgConfig.Unpack(&kbCfg); err != nil {
 		return "", err
 	}

pkg/controller/kibana/pod_test.go

@@ -371,7 +371,9 @@ func TestNewPodTemplateSpec(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := NewPodTemplateSpec(context.Background(), k8s.NewFakeClient(), tt.kb, tt.keystore, []commonvolume.VolumeLike{})
+			bp, err := GetKibanaBasePath(tt.kb)
+			require.NoError(t, err)
+			got, err := NewPodTemplateSpec(context.Background(), k8s.NewFakeClient(), tt.kb, tt.keystore, []commonvolume.VolumeLike{}, bp)
 			assert.NoError(t, err)
 			tt.assertions(got)
 		})

pkg/controller/kibana/stackmon/beat_config.go

@@ -25,7 +25,7 @@ var (
 )
 
 // ReconcileConfigSecrets reconciles the secrets holding beats configuration
-func ReconcileConfigSecrets(ctx context.Context, client k8s.Client, kb kbv1.Kibana) error {
+func ReconcileConfigSecrets(ctx context.Context, client k8s.Client, kb kbv1.Kibana, basePath string) error {
 	isMonitoringReconcilable, err := monitoring.IsReconcilable(&kb)
 	if err != nil {
 		return err
@@ -35,7 +35,7 @@ func ReconcileConfigSecrets(ctx context.Context, client k8s.Client, kb kbv1.Kiba
 	}
 
 	if monitoring.IsMetricsDefined(&kb) {
-		b, err := Metricbeat(ctx, client, kb)
+		b, err := Metricbeat(ctx, client, kb, basePath)
 		if err != nil {
 			return err
 		}

pkg/controller/kibana/stackmon/metricbeat.tpl.yml

@@ -9,6 +9,9 @@ metricbeat.modules:
     hosts: ["{{ .URL }}"]
     username: {{ .Username }}
     password: {{ .Password }}
+    {{- with .BasePath }}
+    basepath: {{ . }}
+    {{- end }}
     ssl.enabled: {{ .IsSSL }}
     # The ssl verification_mode is set to `certificate` in the config template to verify that the certificate is signed by a trusted authority,
     # but does not perform any hostname verification. This is used when SSL is enabled with or without CA, to support self-signed certificate

pkg/controller/kibana/stackmon/sidecar.go

@@ -33,7 +33,7 @@ const (
 	kibanaLogsMountPath  = "/usr/share/kibana/logs"
 )
 
-func Metricbeat(ctx context.Context, client k8s.Client, kb kbv1.Kibana) (stackmon.BeatSidecar, error) {
+func Metricbeat(ctx context.Context, client k8s.Client, kb kbv1.Kibana, basePath string) (stackmon.BeatSidecar, error) {
 	if !kb.Spec.ElasticsearchRef.IsDefined() {
 		// should never happen because of the pre-creation validation
 		return stackmon.BeatSidecar{}, errors.New(validations.InvalidKibanaElasticsearchRefForStackMonitoringMsg)
@@ -69,6 +69,7 @@ func Metricbeat(ctx context.Context, client k8s.Client, kb kbv1.Kibana) (stackmo
 		metricbeatConfigTemplate,
 		kbv1.KBNamer,
 		fmt.Sprintf("%s://localhost:%d", kb.Spec.HTTP.Protocol(), network.HTTPPort),
+		basePath,
 		username,
 		password,
 		kb.Spec.HTTP.TLS.Enabled(),
@@ -85,7 +86,7 @@ func Filebeat(ctx context.Context, client k8s.Client, kb kbv1.Kibana) (stackmon.
 
 // WithMonitoring updates the Kibana Pod template builder to deploy Metricbeat and Filebeat in sidecar containers
 // in the Kibana pod and injects the volumes for the beat configurations and the ES CA certificates.
-func WithMonitoring(ctx context.Context, client k8s.Client, builder *defaults.PodTemplateBuilder, kb kbv1.Kibana) (*defaults.PodTemplateBuilder, error) {
+func WithMonitoring(ctx context.Context, client k8s.Client, builder *defaults.PodTemplateBuilder, kb kbv1.Kibana, basePath string) (*defaults.PodTemplateBuilder, error) {
 	isMonitoringReconcilable, err := monitoring.IsReconcilable(&kb)
 	if err != nil {
 		return nil, err
@@ -98,7 +99,7 @@ func WithMonitoring(ctx context.Context, client k8s.Client, builder *defaults.Po
 	volumes := make([]corev1.Volume, 0)
 
 	if monitoring.IsMetricsDefined(&kb) {
-		b, err := Metricbeat(ctx, client, kb)
+		b, err := Metricbeat(ctx, client, kb, basePath)
 		if err != nil {
 			return nil, err
 		}

pkg/controller/kibana/stackmon/sidecar_test.go

@@ -149,7 +149,7 @@ func TestWithMonitoring(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			kb := tc.kb()
 			builder := defaults.NewPodTemplateBuilder(corev1.PodTemplateSpec{}, kbv1.KibanaContainerName)
-			_, err := WithMonitoring(context.Background(), fakeClient, builder, kb)
+			_, err := WithMonitoring(context.Background(), fakeClient, builder, kb, "")
 			assert.NoError(t, err)
 
 			assert.Equal(t, tc.containersLength, len(builder.PodTemplate.Spec.Containers))

pkg/controller/logstash/stackmon/sidecar.go

@@ -44,6 +44,7 @@ func Metricbeat(ctx context.Context, client k8s.Client, logstash logstashv1alpha
 		metricbeatConfigTemplate,
 		logstashv1alpha1.Namer,
 		fmt.Sprintf("%s://localhost:%d", protocol, network.HTTPPort),
+		"",
 		apiServer.Username,
 		apiServer.Password,
 		useTLS,

test/e2e/kb/stack_monitoring_test.go

@@ -44,3 +44,35 @@ func TestKBStackMonitoring(t *testing.T) {
 
 	test.Sequence(nil, steps, metrics, logs, assocEs, monitored).RunSequential(t)
 }
+
+// TestKBStackMonitoringWithBasePath tests if Kibana when configured with a basePath can be monitored by the
+// stack monitoring feature in ECK. Almost identical to the previous test but we need a fresh monitoring cluster
+// for the document assertions to work.
+func TestKBStackMonitoringWithBasePath(t *testing.T) {
+	// only execute this test on supported version
+	err := validations.IsSupportedVersion(test.Ctx().ElasticStackVersion, validations.MinStackVersion)
+	if err != nil {
+		t.SkipNow()
+	}
+
+	// create 1 monitored and 2 monitoring clusters to collect separately monitor and logs
+	monitor := elasticsearch.NewBuilder("test-kb-monitor").
+		WithESMasterDataNodes(2, elasticsearch.DefaultResources)
+	assocEs := elasticsearch.NewBuilder("test-kb-mon-b").
+		WithESMasterDataNodes(1, elasticsearch.DefaultResources)
+	monitored := kibana.NewBuilder("test-kb-mon-b").
+		WithElasticsearchRef(assocEs.Ref()).
+		WithNodeCount(1).
+		WithConfig(map[string]interface{}{
+			"server.basePath":        "/monitoring/kibana",
+			"server.rewriteBasePath": true,
+		}).
+		WithMonitoring(monitor.Ref(), monitor.Ref())
+
+	// checks that the sidecar beats have sent data in the monitoring clusters
+	steps := func(k *test.K8sClient) test.StepList {
+		return checks.MonitoredSteps(&monitored, k)
+	}
+
+	test.Sequence(nil, steps, monitor, assocEs, monitored).RunSequential(t)
+}