From 65d9a188beac321483e798649494a061138c1769 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 2 Jul 2024 20:40:42 +0200 Subject: [PATCH 1/3] Add flag to enable logsdb mode in logs data streams --- README.md | 2 ++ internal/profile/_static/config.yml.example | 4 ++++ internal/stack/_static/elasticsearch.yml.tmpl | 6 ++++++ internal/stack/resources.go | 6 ++++-- tools/readme/readme.md.tmpl | 2 ++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 27dd981f7..e4f8a10c0 100644 --- a/README.md +++ b/README.md @@ -650,6 +650,8 @@ The following settings are available per profile: an absolute path, out of the `.elastic-package` directory. * `stack.kibana_http2_enabled` can be used to control if HTTP/2 should be used in versions of kibana that support it. Defaults to true. +* `stack.logsdb_enabled` can be set to true to activate the feature flag in Elasticsearch that + enables logs index mode in all data streams that support it. Defaults to false. * `stack.logstash_enabled` can be set to true to start Logstash and configure it as the default output for tests using elastic-package. Supported only by the compose provider. Defaults to false. diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index c693c57fc..2d8d21893 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -14,6 +14,10 @@ # Flag to enable apm-server in elastic-package stack profile config # stack.apm_enabled: true +## Logs DB +# Flag to enable the logs index mode in logs data stream. +# stack.logsdb_enabled: true + ## Enable logstash for testing # Flag to enable logstash in elastic-package stack profile config # stack.logstash_enabled: true diff --git a/internal/stack/_static/elasticsearch.yml.tmpl b/internal/stack/_static/elasticsearch.yml.tmpl index 693d7731f..a981bd034 100644 --- a/internal/stack/_static/elasticsearch.yml.tmpl +++ b/internal/stack/_static/elasticsearch.yml.tmpl @@ -13,6 +13,12 @@ xpack.security.http.ssl.certificate: "certs/cert.pem" ingest.geoip.downloader.enabled: false +{{- $version := fact "elasticsearch_version" -}} +{{- $logsdb_enabled := fact "logsdb_enabled" -}} +{{ if (and (eq $logsdb_enabled "true") (not (semverLessThan $version "8.15.0-SNAPSHOT"))) }} +cluster.logsdb.enabled: true +{{- end -}} + {{ $version := fact "elasticsearch_version" }} {{ if semverLessThan $version "8.0.0" }} script.max_compilations_rate: "use-context" diff --git a/internal/stack/resources.go b/internal/stack/resources.go index 25ccc5180..a7587e663 100644 --- a/internal/stack/resources.go +++ b/internal/stack/resources.go @@ -60,6 +60,7 @@ const ( configAPMEnabled = "stack.apm_enabled" configGeoIPDir = "stack.geoip_dir" configKibanaHTTP2Enabled = "stack.kibana_http2_enabled" + configLogsDBEnabled = "stack.logsdb_enabled" configLogstashEnabled = "stack.logstash_enabled" configSelfMonitorEnabled = "stack.self_monitor_enabled" ) @@ -158,12 +159,13 @@ func applyResources(profile *profile.Profile, stackVersion string) error { "username": elasticsearchUsername, "password": elasticsearchPassword, + "agent_publish_ports": strings.Join(agentPorts, ","), "apm_enabled": profile.Config(configAPMEnabled, "false"), "geoip_dir": profile.Config(configGeoIPDir, "./ingest-geoip"), + "kibana_http2_enabled": profile.Config(configKibanaHTTP2Enabled, "true"), + "logsdb_enabled": profile.Config(configLogsDBEnabled, "false"), "logstash_enabled": profile.Config(configLogstashEnabled, "false"), "self_monitor_enabled": profile.Config(configSelfMonitorEnabled, "false"), - "agent_publish_ports": strings.Join(agentPorts, ","), - "kibana_http2_enabled": profile.Config(configKibanaHTTP2Enabled, "true"), }) if err := os.MkdirAll(stackDir, 0755); err != nil { diff --git a/tools/readme/readme.md.tmpl b/tools/readme/readme.md.tmpl index 875cd245d..fd9c30ef7 100644 --- a/tools/readme/readme.md.tmpl +++ b/tools/readme/readme.md.tmpl @@ -192,6 +192,8 @@ The following settings are available per profile: an absolute path, out of the `.elastic-package` directory. * `stack.kibana_http2_enabled` can be used to control if HTTP/2 should be used in versions of kibana that support it. Defaults to true. +* `stack.logsdb_enabled` can be set to true to activate the feature flag in Elasticsearch that + enables logs index mode in all data streams that support it. Defaults to false. * `stack.logstash_enabled` can be set to true to start Logstash and configure it as the default output for tests using elastic-package. Supported only by the compose provider. Defaults to false. From aec9a0959442cf1c406358d7a845d176a359ad3d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 3 Jul 2024 08:58:55 +0200 Subject: [PATCH 2/3] Remove duplicated line --- internal/stack/_static/elasticsearch.yml.tmpl | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/stack/_static/elasticsearch.yml.tmpl b/internal/stack/_static/elasticsearch.yml.tmpl index a981bd034..9007ed558 100644 --- a/internal/stack/_static/elasticsearch.yml.tmpl +++ b/internal/stack/_static/elasticsearch.yml.tmpl @@ -19,7 +19,6 @@ ingest.geoip.downloader.enabled: false cluster.logsdb.enabled: true {{- end -}} -{{ $version := fact "elasticsearch_version" }} {{ if semverLessThan $version "8.0.0" }} script.max_compilations_rate: "use-context" script.context.template.max_compilations_rate: "unlimited" From fd68bfcb5d9f7321d4290f70a17ce9287e66b087 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 3 Jul 2024 09:57:28 +0200 Subject: [PATCH 3/3] Check if synthetics is enabled by simulating the index template that applies to the data stream --- internal/testrunner/runners/system/tester.go | 63 ++++++-------------- 1 file changed, 18 insertions(+), 45 deletions(-) diff --git a/internal/testrunner/runners/system/tester.go b/internal/testrunner/runners/system/tester.go index c7f98999d..1ab4a2ac7 100644 --- a/internal/testrunner/runners/system/tester.go +++ b/internal/testrunner/runners/system/tester.go @@ -607,60 +607,38 @@ func (r *tester) runTestPerVariant(ctx context.Context, result *testrunner.Resul return partial, nil } -func (r *tester) isSyntheticsEnabled(ctx context.Context, dataStream, componentTemplatePackage string) (bool, error) { - resp, err := r.esAPI.Cluster.GetComponentTemplate( - r.esAPI.Cluster.GetComponentTemplate.WithContext(ctx), - r.esAPI.Cluster.GetComponentTemplate.WithName(componentTemplatePackage), +func (r *tester) isSyntheticsEnabled(ctx context.Context, dataStreamName string) (bool, error) { + resp, err := r.esAPI.Indices.SimulateIndexTemplate(dataStreamName, + r.esAPI.Indices.SimulateIndexTemplate.WithContext(ctx), ) if err != nil { - return false, fmt.Errorf("could not get component template %s from data stream %s: %w", componentTemplatePackage, dataStream, err) + return false, fmt.Errorf("could not simulate index template for %s: %w", dataStreamName, err) } defer resp.Body.Close() - if resp.StatusCode == http.StatusNotFound { - // @package component template doesn't exist before 8.2. On these versions synthetics was not supported - // in any case, so just return false. - logger.Debugf("no component template %s found for data stream %s", componentTemplatePackage, dataStream) - return false, nil - } if resp.IsError() { - return false, fmt.Errorf("could not get component template %s for data stream %s: %s", componentTemplatePackage, dataStream, resp.String()) + return false, fmt.Errorf("could not simulate index template for %s: %s", dataStreamName, resp.String()) } var results struct { - ComponentTemplates []struct { - Name string `json:"name"` - ComponentTemplate struct { - Template struct { - Mappings struct { - Source *struct { - Mode string `json:"mode"` - } `json:"_source,omitempty"` - } `json:"mappings"` - } `json:"template"` - } `json:"component_template"` - } `json:"component_templates"` + Template struct { + Mappings struct { + Source *struct { + Mode string `json:"mode"` + } `json:"_source,omitempty"` + } `json:"mappings"` + } `json:"template"` } if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { - return false, fmt.Errorf("could not decode search results response: %w", err) - } - - if len(results.ComponentTemplates) == 0 { - logger.Debugf("no component template %s found for data stream %s", componentTemplatePackage, dataStream) - return false, nil + return false, fmt.Errorf("could not decode index template simulation response: %w", err) } - if len(results.ComponentTemplates) != 1 { - return false, fmt.Errorf("ambiguous response, expected one component template for %s, found %d", componentTemplatePackage, len(results.ComponentTemplates)) - } - - template := results.ComponentTemplates[0] - if template.ComponentTemplate.Template.Mappings.Source == nil { + if results.Template.Mappings.Source == nil { return false, nil } - return template.ComponentTemplate.Template.Mappings.Source.Mode == "synthetic", nil + return results.Template.Mappings.Source.Mode == "synthetic", nil } type hits struct { @@ -962,11 +940,6 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, svcInf dataStreamDataset, ds.Namespace, ) - componentTemplatePackage := fmt.Sprintf( - "%s-%s@package", - ds.Inputs[0].Streams[0].DataStream.Type, - dataStreamDataset, - ) r.cleanTestScenarioHandler = func(ctx context.Context) error { logger.Debugf("Deleting data stream for testing %s", scenario.dataStream) @@ -1130,10 +1103,10 @@ func (r *tester) prepareScenario(ctx context.Context, config *testConfig, svcInf return nil, testrunner.ErrTestCaseFailed{Reason: fmt.Sprintf("could not find hits in %s data stream", scenario.dataStream)} } - logger.Debugf("check whether or not synthetics is enabled (component template %s)...", componentTemplatePackage) - scenario.syntheticEnabled, err = r.isSyntheticsEnabled(ctx, scenario.dataStream, componentTemplatePackage) + logger.Debugf("check whether or not synthetics is enabled (data stream %s)...", scenario.dataStream) + scenario.syntheticEnabled, err = r.isSyntheticsEnabled(ctx, scenario.dataStream) if err != nil { - return nil, fmt.Errorf("failed to check if synthetic source is enabled: %w", err) + return nil, fmt.Errorf("failed to check if synthetic source is enabled for data stream %s: %w", scenario.dataStream, err) } logger.Debugf("data stream %s has synthetics enabled: %t", scenario.dataStream, scenario.syntheticEnabled)