From 1da566a0912d1b203cc36987e2b91939a7f6b894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Sun, 9 Jan 2022 19:02:43 +0100 Subject: [PATCH 1/3] feat: add batching to otelcol log collector batch_size is set to the same value as the metadata enricher. timeout is set to the same value as fluent-bit's Flush. --- CHANGELOG.md | 2 ++ deploy/helm/sumologic/values.yaml | 11 +++++++++++ tests/helm/logs_otc/static/basic.output.yaml | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16c583c88c..2531ca3e45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- feat: add batching to otelcol log collector [#2018][#2018] - feat: add experimental otelcol log collector [#1986][#1986] - feat: add option to disable pod owners enrichment [#1959][#1959] @@ -33,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#1992]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/1992 [#1993]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/1993 [#2013]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2013 +[#2018]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2018 ## [v2.3.1][v2_3_1] - 2021-12-14 diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 5968aab4c1..176641136c 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -4178,6 +4178,8 @@ otellogs: - filelog/containers exporters: - otlphttp + processors: + - batch receivers: filelog/containers: include: @@ -4322,6 +4324,15 @@ otellogs: exporters: otlphttp: endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.cluster.local:4318 + processors: + ## The batch processor accepts spans and places them into batches grouped by node and resource + batch: + ## Number of spans after which a batch will be sent regardless of time + ## This is set to the same value as in the metadata enricher + ## TODO: Figure out what the optimal values should be for different configurations + send_batch_size: 256 + ## Time duration after which a batch will be sent regardless of size + timeout: 1s daemonset: ## Set securityContext for containers running in pods in log collector daemonset securityContext: diff --git a/tests/helm/logs_otc/static/basic.output.yaml b/tests/helm/logs_otc/static/basic.output.yaml index 72b3d80cba..f79a50e648 100644 --- a/tests/helm/logs_otc/static/basic.output.yaml +++ b/tests/helm/logs_otc/static/basic.output.yaml @@ -20,6 +20,10 @@ data: directory: /var/lib/storage/otc timeout: 10s health_check: {} + processors: + batch: + send_batch_size: 256 + timeout: 1s receivers: filelog/containers: include: @@ -60,6 +64,8 @@ data: logs/containers: exporters: - otlphttp + processors: + - batch receivers: - filelog/containers telemetry: From 3fbc303e63a94bb73d3dcb1abcd57cdbcb947434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Tue, 11 Jan 2022 11:36:33 +0000 Subject: [PATCH 2/3] chore: change argument order in OTC pipeline config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Patryk Małek --- deploy/helm/sumologic/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 176641136c..eb5fce31a7 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -4176,10 +4176,10 @@ otellogs: logs/containers: receivers: - filelog/containers - exporters: - - otlphttp processors: - batch + exporters: + - otlphttp receivers: filelog/containers: include: From dfe21d3c425b38d9ea613caf1d64279e3e0610f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20=C5=9Awi=C4=85tek?= Date: Tue, 11 Jan 2022 12:00:59 +0000 Subject: [PATCH 3/3] chore: update changelog.md Co-authored-by: Marcin Stozek <58700054+perk-sumo@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2531ca3e45..76691f7410 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- feat: add batching to otelcol log collector [#2018][#2018] +- feat: add batching to experimental otelcol log collector [#2018][#2018] - feat: add experimental otelcol log collector [#1986][#1986] - feat: add option to disable pod owners enrichment [#1959][#1959]