From f5e41023c6ec48f85ec247f55c35cf6c7ddb4068 Mon Sep 17 00:00:00 2001 From: Eric Lin Date: Fri, 19 Apr 2024 09:21:17 -0700 Subject: [PATCH 1/6] add batcher config to splunkhec exporter --- .../exporter-splunkhec-addbatcherconfig.yaml | 27 +++++++++ exporter/splunkhecexporter/README.md | 1 + exporter/splunkhecexporter/config.go | 2 + exporter/splunkhecexporter/config_test.go | 11 ++++ exporter/splunkhecexporter/factory.go | 55 ++++++++++++------- .../splunkhecexporter/testdata/config.yaml | 5 ++ 6 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 .chloggen/exporter-splunkhec-addbatcherconfig.yaml diff --git a/.chloggen/exporter-splunkhec-addbatcherconfig.yaml b/.chloggen/exporter-splunkhec-addbatcherconfig.yaml new file mode 100644 index 000000000000..b45eefa70e5f --- /dev/null +++ b/.chloggen/exporter-splunkhec-addbatcherconfig.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: exporter/splunkhec + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: add exporter batcher config in splunk hec exporter + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [32545] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md index 57d17eff7466..0531a6730168 100644 --- a/exporter/splunkhecexporter/README.md +++ b/exporter/splunkhecexporter/README.md @@ -66,6 +66,7 @@ The following configuration options can also be configured: - `telemetry/enabled` (default: false): Specifies whether to enable telemetry inside splunk hec exporter. - `telemetry/override_metrics_names` (default: empty map): Specifies the metrics name to overrides in splunk hec exporter. - `telemetry/extra_attributes` (default: empty map): Specifies the extra metrics attributes in splunk hec exporter. +- `batcher`: Specifies batching configuration on the exporter. Information about the configuration can be found [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) In addition, this exporter offers queued retry which is enabled by default. Information about queued retry configuration parameters can be found diff --git a/exporter/splunkhecexporter/config.go b/exporter/splunkhecexporter/config.go index 9d2e73a722bc..a6f5fd1b6053 100644 --- a/exporter/splunkhecexporter/config.go +++ b/exporter/splunkhecexporter/config.go @@ -13,6 +13,7 @@ import ( "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/config/configretry" + "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk" @@ -67,6 +68,7 @@ type Config struct { confighttp.ClientConfig `mapstructure:",squash"` exporterhelper.QueueSettings `mapstructure:"sending_queue"` configretry.BackOffConfig `mapstructure:"retry_on_failure"` + BatcherConfig exporterbatcher.Config `mapstructure:"batcher"` // LogDataEnabled can be used to disable sending logs by the exporter. LogDataEnabled bool `mapstructure:"log_data_enabled"` diff --git a/exporter/splunkhecexporter/config_test.go b/exporter/splunkhecexporter/config_test.go index 996c59a27c39..5fcfd021cd18 100644 --- a/exporter/splunkhecexporter/config_test.go +++ b/exporter/splunkhecexporter/config_test.go @@ -16,6 +16,7 @@ import ( "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/config/configtls" "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter/internal/metadata" @@ -90,6 +91,16 @@ func TestLoadConfig(t *testing.T) { NumConsumers: 2, QueueSize: 10, }, + BatcherConfig: exporterbatcher.Config{ + Enabled: true, + FlushTimeout: time.Second, + MinSizeConfig: exporterbatcher.MinSizeConfig{ + MinSizeItems: 1, + }, + MaxSizeConfig: exporterbatcher.MaxSizeConfig{ + MaxSizeItems: 10, + }, + }, HecToOtelAttrs: splunk.HecToOtelAttrs{ Source: "mysource", SourceType: "mysourcetype", diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go index b67baa3d4dce..d6e4d70e57eb 100644 --- a/exporter/splunkhecexporter/factory.go +++ b/exporter/splunkhecexporter/factory.go @@ -12,6 +12,7 @@ import ( "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/exporter/exporterbatcher" "go.opentelemetry.io/collector/exporter/exporterhelper" conventions "go.opentelemetry.io/collector/semconv/v1.6.1" @@ -57,6 +58,23 @@ func NewFactory() exporter.Factory { exporter.WithLogs(createLogsExporter, metadata.LogsStability)) } +func getExporterOptions(cfg *Config, startFunc component.StartFunc, shutdownFunc component.ShutdownFunc) []exporterhelper.Option { + options := []exporterhelper.Option{ + // explicitly disable since we rely on http.Client timeout logic. + exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), + exporterhelper.WithRetry(cfg.BackOffConfig), + exporterhelper.WithQueue(cfg.QueueSettings), + exporterhelper.WithStart(startFunc), + exporterhelper.WithShutdown(shutdownFunc), + } + + if cfg.BatcherConfig.Enabled { + options = append(options, exporterhelper.WithBatcher(cfg.BatcherConfig)) + } + + return options +} + func createDefaultConfig() component.Config { defaultMaxConns := defaultMaxIdleCons defaultIdleConnTimeout := defaultIdleConnTimeout @@ -71,9 +89,13 @@ func createDefaultConfig() component.Config { HTTP2ReadIdleTimeout: defaultHTTP2ReadIdleTimeout, HTTP2PingTimeout: defaultHTTP2PingTimeout, }, - SplunkAppName: defaultSplunkAppName, - BackOffConfig: configretry.NewDefaultBackOffConfig(), - QueueSettings: exporterhelper.NewDefaultQueueSettings(), + SplunkAppName: defaultSplunkAppName, + BackOffConfig: configretry.NewDefaultBackOffConfig(), + QueueSettings: exporterhelper.NewDefaultQueueSettings(), + BatcherConfig: exporterbatcher.Config{ + Enabled: false, + FlushTimeout: 200 * time.Millisecond, // need default value to prevent validation failures + }, DisableCompression: false, MaxContentLengthLogs: defaultContentLengthLogsLimit, MaxContentLengthMetrics: defaultContentLengthMetricsLimit, @@ -109,17 +131,14 @@ func createTracesExporter( c := newTracesClient(set, cfg) + exporterOptions := getExporterOptions(cfg, c.start, c.stop) + e, err := exporterhelper.NewTracesExporter( ctx, set, cfg, c.pushTraceData, - // explicitly disable since we rely on http.Client timeout logic. - exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), - exporterhelper.WithRetry(cfg.BackOffConfig), - exporterhelper.WithQueue(cfg.QueueSettings), - exporterhelper.WithStart(c.start), - exporterhelper.WithShutdown(c.stop)) + exporterOptions...) if err != nil { return nil, err @@ -142,17 +161,14 @@ func createMetricsExporter( c := newMetricsClient(set, cfg) + exporterOptions := getExporterOptions(cfg, c.start, c.stop) + e, err := exporterhelper.NewMetricsExporter( ctx, set, cfg, c.pushMetricsData, - // explicitly disable since we rely on http.Client timeout logic. - exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), - exporterhelper.WithRetry(cfg.BackOffConfig), - exporterhelper.WithQueue(cfg.QueueSettings), - exporterhelper.WithStart(c.start), - exporterhelper.WithShutdown(c.stop)) + exporterOptions...) if err != nil { return nil, err } @@ -174,17 +190,14 @@ func createLogsExporter( c := newLogsClient(set, cfg) + exporterOptions := getExporterOptions(cfg, c.start, c.stop) + logsExporter, err := exporterhelper.NewLogsExporter( ctx, set, cfg, c.pushLogData, - // explicitly disable since we rely on http.Client timeout logic. - exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), - exporterhelper.WithRetry(cfg.BackOffConfig), - exporterhelper.WithQueue(cfg.QueueSettings), - exporterhelper.WithStart(c.start), - exporterhelper.WithShutdown(c.stop)) + exporterOptions...) if err != nil { return nil, err diff --git a/exporter/splunkhecexporter/testdata/config.yaml b/exporter/splunkhecexporter/testdata/config.yaml index c8d8b59489e2..54b7040dbf8d 100644 --- a/exporter/splunkhecexporter/testdata/config.yaml +++ b/exporter/splunkhecexporter/testdata/config.yaml @@ -26,6 +26,11 @@ splunk_hec/allsettings: initial_interval: 10s max_interval: 60s max_elapsed_time: 10m + batcher: + enabled: true + flush_timeout: 1s + min_size_items: 1 + max_size_items: 10 splunk_app_name: "OpenTelemetry-Collector Splunk Exporter" splunk_app_version: "v0.0.1" hec_metadata_to_otel_attrs: From cdc2cd63394d35b04fe0d0b3060cb099863844c7 Mon Sep 17 00:00:00 2001 From: Eric Lin Date: Mon, 22 Apr 2024 09:31:39 -0700 Subject: [PATCH 2/6] add experimental documentation for batcher config --- .chloggen/exporter-splunkhec-addbatcherconfig.yaml | 2 +- exporter/splunkhecexporter/README.md | 2 +- exporter/splunkhecexporter/config.go | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.chloggen/exporter-splunkhec-addbatcherconfig.yaml b/.chloggen/exporter-splunkhec-addbatcherconfig.yaml index b45eefa70e5f..0ed1f9461eb0 100644 --- a/.chloggen/exporter-splunkhec-addbatcherconfig.yaml +++ b/.chloggen/exporter-splunkhec-addbatcherconfig.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: exporter/splunkhec # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: add exporter batcher config in splunk hec exporter +note: add experimental exporter batcher config in splunk hec exporter # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [32545] diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md index 0531a6730168..d8f0ce0a02f9 100644 --- a/exporter/splunkhecexporter/README.md +++ b/exporter/splunkhecexporter/README.md @@ -66,7 +66,7 @@ The following configuration options can also be configured: - `telemetry/enabled` (default: false): Specifies whether to enable telemetry inside splunk hec exporter. - `telemetry/override_metrics_names` (default: empty map): Specifies the metrics name to overrides in splunk hec exporter. - `telemetry/extra_attributes` (default: empty map): Specifies the extra metrics attributes in splunk hec exporter. -- `batcher`: Specifies batching configuration on the exporter. Information about the configuration can be found [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) +- `batcher`(Experimental): Specifies batching configuration on the exporter. This configuration is experimental and may change until the [issue](https://github.com/open-telemetry/opentelemetry-collector/issues/8122) is resolved. Information about the configuration can be found [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) In addition, this exporter offers queued retry which is enabled by default. Information about queued retry configuration parameters can be found diff --git a/exporter/splunkhecexporter/config.go b/exporter/splunkhecexporter/config.go index a6f5fd1b6053..168fb6aced26 100644 --- a/exporter/splunkhecexporter/config.go +++ b/exporter/splunkhecexporter/config.go @@ -68,7 +68,10 @@ type Config struct { confighttp.ClientConfig `mapstructure:",squash"` exporterhelper.QueueSettings `mapstructure:"sending_queue"` configretry.BackOffConfig `mapstructure:"retry_on_failure"` - BatcherConfig exporterbatcher.Config `mapstructure:"batcher"` + + // Experimental: This configuration is at the early stage of development and may change without backward compatibility + // until https://github.com/open-telemetry/opentelemetry-collector/issues/8122 is resolved. + BatcherConfig exporterbatcher.Config `mapstructure:"batcher"` // LogDataEnabled can be used to disable sending logs by the exporter. LogDataEnabled bool `mapstructure:"log_data_enabled"` From 2fd6071bf7131f48b188071c0686f65087b967d1 Mon Sep 17 00:00:00 2001 From: Eric Lin Date: Wed, 24 Apr 2024 17:05:25 -0700 Subject: [PATCH 3/6] add documentation --- .chloggen/exporter-splunkhec-addbatcherconfig.yaml | 2 +- exporter/splunkhecexporter/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.chloggen/exporter-splunkhec-addbatcherconfig.yaml b/.chloggen/exporter-splunkhec-addbatcherconfig.yaml index 0ed1f9461eb0..19d2c256485d 100644 --- a/.chloggen/exporter-splunkhec-addbatcherconfig.yaml +++ b/.chloggen/exporter-splunkhec-addbatcherconfig.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: exporter/splunkhec # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: add experimental exporter batcher config in splunk hec exporter +note: add experimental exporter batcher config # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [32545] diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md index d8f0ce0a02f9..954df48f2d7c 100644 --- a/exporter/splunkhecexporter/README.md +++ b/exporter/splunkhecexporter/README.md @@ -66,7 +66,7 @@ The following configuration options can also be configured: - `telemetry/enabled` (default: false): Specifies whether to enable telemetry inside splunk hec exporter. - `telemetry/override_metrics_names` (default: empty map): Specifies the metrics name to overrides in splunk hec exporter. - `telemetry/extra_attributes` (default: empty map): Specifies the extra metrics attributes in splunk hec exporter. -- `batcher`(Experimental): Specifies batching configuration on the exporter. This configuration is experimental and may change until the [issue](https://github.com/open-telemetry/opentelemetry-collector/issues/8122) is resolved. Information about the configuration can be found [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) +- `batcher`(Experimental, disabled by default): Specifies batching configuration on the exporter. This configuration is experimental and may change until the [issue](https://github.com/open-telemetry/opentelemetry-collector/issues/8122) is resolved. Information about the configuration can be found [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) In addition, this exporter offers queued retry which is enabled by default. Information about queued retry configuration parameters can be found From 7431cf27c5d451dc782cf627edc67e5a76c788e9 Mon Sep 17 00:00:00 2001 From: Eric Lin Date: Tue, 30 Apr 2024 09:27:14 -0700 Subject: [PATCH 4/6] remove experimental github issue from readme --- exporter/splunkhecexporter/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md index 954df48f2d7c..51065bd7fd24 100644 --- a/exporter/splunkhecexporter/README.md +++ b/exporter/splunkhecexporter/README.md @@ -66,7 +66,7 @@ The following configuration options can also be configured: - `telemetry/enabled` (default: false): Specifies whether to enable telemetry inside splunk hec exporter. - `telemetry/override_metrics_names` (default: empty map): Specifies the metrics name to overrides in splunk hec exporter. - `telemetry/extra_attributes` (default: empty map): Specifies the extra metrics attributes in splunk hec exporter. -- `batcher`(Experimental, disabled by default): Specifies batching configuration on the exporter. This configuration is experimental and may change until the [issue](https://github.com/open-telemetry/opentelemetry-collector/issues/8122) is resolved. Information about the configuration can be found [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) +- `batcher`(Experimental, disabled by default): Specifies batching configuration on the exporter. Information about the configuration can be found [here](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) In addition, this exporter offers queued retry which is enabled by default. Information about queued retry configuration parameters can be found From a117868ffe29a790120728d99807a1b48de39ee5 Mon Sep 17 00:00:00 2001 From: Eric Lin Date: Fri, 3 May 2024 11:57:35 -0700 Subject: [PATCH 5/6] remove if-statement check for exporter batching enablement --- exporter/splunkhecexporter/factory.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go index d6e4d70e57eb..04bd30263f34 100644 --- a/exporter/splunkhecexporter/factory.go +++ b/exporter/splunkhecexporter/factory.go @@ -66,10 +66,7 @@ func getExporterOptions(cfg *Config, startFunc component.StartFunc, shutdownFunc exporterhelper.WithQueue(cfg.QueueSettings), exporterhelper.WithStart(startFunc), exporterhelper.WithShutdown(shutdownFunc), - } - - if cfg.BatcherConfig.Enabled { - options = append(options, exporterhelper.WithBatcher(cfg.BatcherConfig)) + exporterhelper.WithBatcher(cfg.BatcherConfig), } return options From 1b5d22e864d1350752320298f1806f2f3f8800c7 Mon Sep 17 00:00:00 2001 From: Eric Lin Date: Fri, 3 May 2024 14:44:52 -0700 Subject: [PATCH 6/6] reduce changelog with new batcher change --- exporter/splunkhecexporter/factory.go | 61 ++++++++++++++------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go index 04bd30263f34..56435c64ef6f 100644 --- a/exporter/splunkhecexporter/factory.go +++ b/exporter/splunkhecexporter/factory.go @@ -58,21 +58,10 @@ func NewFactory() exporter.Factory { exporter.WithLogs(createLogsExporter, metadata.LogsStability)) } -func getExporterOptions(cfg *Config, startFunc component.StartFunc, shutdownFunc component.ShutdownFunc) []exporterhelper.Option { - options := []exporterhelper.Option{ - // explicitly disable since we rely on http.Client timeout logic. - exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), - exporterhelper.WithRetry(cfg.BackOffConfig), - exporterhelper.WithQueue(cfg.QueueSettings), - exporterhelper.WithStart(startFunc), - exporterhelper.WithShutdown(shutdownFunc), - exporterhelper.WithBatcher(cfg.BatcherConfig), - } - - return options -} - func createDefaultConfig() component.Config { + batcherCfg := exporterbatcher.NewDefaultConfig() + batcherCfg.Enabled = false + defaultMaxConns := defaultMaxIdleCons defaultIdleConnTimeout := defaultIdleConnTimeout return &Config{ @@ -86,13 +75,10 @@ func createDefaultConfig() component.Config { HTTP2ReadIdleTimeout: defaultHTTP2ReadIdleTimeout, HTTP2PingTimeout: defaultHTTP2PingTimeout, }, - SplunkAppName: defaultSplunkAppName, - BackOffConfig: configretry.NewDefaultBackOffConfig(), - QueueSettings: exporterhelper.NewDefaultQueueSettings(), - BatcherConfig: exporterbatcher.Config{ - Enabled: false, - FlushTimeout: 200 * time.Millisecond, // need default value to prevent validation failures - }, + SplunkAppName: defaultSplunkAppName, + BackOffConfig: configretry.NewDefaultBackOffConfig(), + QueueSettings: exporterhelper.NewDefaultQueueSettings(), + BatcherConfig: batcherCfg, DisableCompression: false, MaxContentLengthLogs: defaultContentLengthLogsLimit, MaxContentLengthMetrics: defaultContentLengthMetricsLimit, @@ -128,14 +114,19 @@ func createTracesExporter( c := newTracesClient(set, cfg) - exporterOptions := getExporterOptions(cfg, c.start, c.stop) - e, err := exporterhelper.NewTracesExporter( ctx, set, cfg, c.pushTraceData, - exporterOptions...) + // explicitly disable since we rely on http.Client timeout logic. + exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), + exporterhelper.WithRetry(cfg.BackOffConfig), + exporterhelper.WithQueue(cfg.QueueSettings), + exporterhelper.WithStart(c.start), + exporterhelper.WithShutdown(c.stop), + exporterhelper.WithBatcher(cfg.BatcherConfig), + ) if err != nil { return nil, err @@ -158,14 +149,19 @@ func createMetricsExporter( c := newMetricsClient(set, cfg) - exporterOptions := getExporterOptions(cfg, c.start, c.stop) - e, err := exporterhelper.NewMetricsExporter( ctx, set, cfg, c.pushMetricsData, - exporterOptions...) + // explicitly disable since we rely on http.Client timeout logic. + exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), + exporterhelper.WithRetry(cfg.BackOffConfig), + exporterhelper.WithQueue(cfg.QueueSettings), + exporterhelper.WithStart(c.start), + exporterhelper.WithShutdown(c.stop), + exporterhelper.WithBatcher(cfg.BatcherConfig), + ) if err != nil { return nil, err } @@ -187,14 +183,19 @@ func createLogsExporter( c := newLogsClient(set, cfg) - exporterOptions := getExporterOptions(cfg, c.start, c.stop) - logsExporter, err := exporterhelper.NewLogsExporter( ctx, set, cfg, c.pushLogData, - exporterOptions...) + // explicitly disable since we rely on http.Client timeout logic. + exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}), + exporterhelper.WithRetry(cfg.BackOffConfig), + exporterhelper.WithQueue(cfg.QueueSettings), + exporterhelper.WithStart(c.start), + exporterhelper.WithShutdown(c.stop), + exporterhelper.WithBatcher(cfg.BatcherConfig), + ) if err != nil { return nil, err