From 3d0b29bbaad616531df48c0c474496e6b939443a Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Thu, 3 Feb 2022 10:50:02 +0100 Subject: [PATCH] feat: add `fluentd.apiServerUrl` property --- CHANGELOG.md | 2 + deploy/helm/sumologic/README.md | 1 + deploy/helm/sumologic/conf/events/events.conf | 6 ++ .../logs.enhance.k8s.metadata.filter.conf | 5 +- .../logs.kubernetes.metadata.filter.conf | 5 +- .../conf/metrics/fluentd/metrics.conf | 3 + deploy/helm/sumologic/values.yaml | 4 + .../events/static/apiServerUrl.input.yaml | 2 + .../events/static/apiServerUrl.output.yaml | 71 ++++++++++++++++ .../static/apiServerUrl_null.input.yaml | 2 + .../static/apiServerUrl_null.output.yaml | 70 ++++++++++++++++ ...rceEventsOverrides_apiServerUrl.input.yaml | 7 ++ ...ceEventsOverrides_apiServerUrl.output.yaml | 80 +++++++++++++++++++ 13 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 tests/helm/events/static/apiServerUrl.input.yaml create mode 100644 tests/helm/events/static/apiServerUrl.output.yaml create mode 100644 tests/helm/events/static/apiServerUrl_null.input.yaml create mode 100644 tests/helm/events/static/apiServerUrl_null.output.yaml create mode 100644 tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.input.yaml create mode 100644 tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.output.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 10421ef2aa..6fc73896d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - docs(readme): add support for AKS 1.22 [#2075][#2075] +- feat: add `fluentd.apiServerUrl` property [#2077] ### Changed @@ -28,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#2058]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2058 [#2063]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2063 [#2075]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2075 +[#2077]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2077 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.4.1...main ## [v2.4.1][v2_4_1] diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md index 542943fe5b..e6fa1219d0 100644 --- a/deploy/helm/sumologic/README.md +++ b/deploy/helm/sumologic/README.md @@ -77,6 +77,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an | `fluentd.metadata.addService` | Option to control the enrichment of logs and metrics with `service` metadata. | `true` | | `fluentd.metadata.annotation_match` | Option to control capturing of annotations by metadata filter plugin. | `['sumologic\.com.*']` | | `fluentd.metadata.apiGroups` | List of supported kubernetes API groups. | `['apps/v1']` | +| `fluentd.metadata.apiServerUrl` | Option to specify custom API server URL instead of the default, that is taken from KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT environment variables. Example: `"https://kubernetes.default.svc:443"`. | `""` | | `fluentd.metadata.coreApiVersions` | List of supported kubernetes API versions. | `['v1']` | | `fluentd.metadata.cacheSize` | Option to control the enabling of metadata filter plugin cache_size. | `10000` | | `fluentd.metadata.cacheTtl` | Option to control the enabling of metadata filter plugin cache_ttl (in seconds). | `7200` | diff --git a/deploy/helm/sumologic/conf/events/events.conf b/deploy/helm/sumologic/conf/events/events.conf index a6268c7a4d..6fc9401e1b 100644 --- a/deploy/helm/sumologic/conf/events/events.conf +++ b/deploy/helm/sumologic/conf/events/events.conf @@ -5,12 +5,18 @@ deploy_namespace {{ $.Release.Namespace }} resource_name {{ $name }} api_version {{ $version }} +{{- if $.Values.fluentd.apiServerUrl }} + kubernetes_url {{ $.Values.fluentd.apiServerUrl | quote }} +{{- end }} {{- end }} {{- else }} @type events deploy_namespace {{ .Release.Namespace }} +{{- if .Values.fluentd.apiServerUrl }} + kubernetes_url {{ .Values.fluentd.apiServerUrl | quote }} +{{- end }} {{- end }} # Prevent fluentd from handling records containing its own logs. diff --git a/deploy/helm/sumologic/conf/logs/fluentd/logs.enhance.k8s.metadata.filter.conf b/deploy/helm/sumologic/conf/logs/fluentd/logs.enhance.k8s.metadata.filter.conf index 9dfe61e40d..1872850ba3 100644 --- a/deploy/helm/sumologic/conf/logs/fluentd/logs.enhance.k8s.metadata.filter.conf +++ b/deploy/helm/sumologic/conf/logs/fluentd/logs.enhance.k8s.metadata.filter.conf @@ -9,4 +9,7 @@ api_groups {{ join "," .Values.fluentd.metadata.apiGroups }} data_type logs add_owners {{ .Values.fluentd.metadata.addOwners }} add_service {{ .Values.fluentd.metadata.addService }} -cache_refresh_apiserver_request_delay {{ .Values.fluentd.metadata.cacheRefreshApiserverRequestDelay | quote }} \ No newline at end of file +cache_refresh_apiserver_request_delay {{ .Values.fluentd.metadata.cacheRefreshApiserverRequestDelay | quote }} +{{- if .Values.fluentd.apiServerUrl }} + kubernetes_url {{ .Values.fluentd.apiServerUrl | quote }} +{{- end }} diff --git a/deploy/helm/sumologic/conf/logs/fluentd/logs.kubernetes.metadata.filter.conf b/deploy/helm/sumologic/conf/logs/fluentd/logs.kubernetes.metadata.filter.conf index dc6feb1f24..12068a7ad8 100644 --- a/deploy/helm/sumologic/conf/logs/fluentd/logs.kubernetes.metadata.filter.conf +++ b/deploy/helm/sumologic/conf/logs/fluentd/logs.kubernetes.metadata.filter.conf @@ -8,4 +8,7 @@ client_key {{ .Values.fluentd.logs.containers.k8sMetadataFilter.clientKey | quot bearer_token_file {{ .Values.fluentd.logs.containers.k8sMetadataFilter.bearerTokenFile | quote }} cache_size {{ .Values.fluentd.metadata.cacheSize | quote }} cache_ttl {{ .Values.fluentd.metadata.cacheTtl | quote }} -tag_to_kubernetes_name_regexp '.+?\.containers\.(?[^_]+)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$' \ No newline at end of file +tag_to_kubernetes_name_regexp '.+?\.containers\.(?[^_]+)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$' +{{- if .Values.fluentd.apiServerUrl }} + kubernetes_url {{ .Values.fluentd.apiServerUrl | quote }} +{{- end }} diff --git a/deploy/helm/sumologic/conf/metrics/fluentd/metrics.conf b/deploy/helm/sumologic/conf/metrics/fluentd/metrics.conf index 0fe28ad0d3..eaf967137f 100644 --- a/deploy/helm/sumologic/conf/metrics/fluentd/metrics.conf +++ b/deploy/helm/sumologic/conf/metrics/fluentd/metrics.conf @@ -48,6 +48,9 @@ api_groups {{ join "," .Values.fluentd.metadata.apiGroups }} add_owners {{ .Values.fluentd.metadata.addOwners }} add_service {{ .Values.fluentd.metadata.addService }} +{{- if .Values.fluentd.apiServerUrl }} + kubernetes_url {{ .Values.fluentd.apiServerUrl | quote }} +{{- end }} {{- .Values.fluentd.metrics.extraFilterPluginConf | nindent 4 }} diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 33948e3404..0fd6ac90c1 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -361,6 +361,10 @@ fluentd: addOwners: true ## Option to control the enrichment of logs and metrics with `service` metadata. addService: true + ## Option to specify custom API server URL instead of the default, + ## that is taken from KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT environment variables. + ## Example: "https://kubernetes.default.svc:443". + apiServerUrl: "" logs: enabled: true diff --git a/tests/helm/events/static/apiServerUrl.input.yaml b/tests/helm/events/static/apiServerUrl.input.yaml new file mode 100644 index 0000000000..4bac0a822e --- /dev/null +++ b/tests/helm/events/static/apiServerUrl.input.yaml @@ -0,0 +1,2 @@ +fluentd: + apiServerUrl: "https://kubernetes.default.svc:443" diff --git a/tests/helm/events/static/apiServerUrl.output.yaml b/tests/helm/events/static/apiServerUrl.output.yaml new file mode 100644 index 0000000000..ab5dcf9ec2 --- /dev/null +++ b/tests/helm/events/static/apiServerUrl.output.yaml @@ -0,0 +1,71 @@ +--- +# Source: sumologic/templates/events/fluentd/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: RELEASE-NAME-sumologic-fluentd-events + labels: + app: RELEASE-NAME-sumologic-fluentd-events + chart: "sumologic-%CURRENT_CHART_VERSION%" + release: "RELEASE-NAME" + heritage: "Helm" +data: + fluent.conf: |- + @include events.conf + events.conf: | + + @type events + deploy_namespace sumologic + kubernetes_url "https://kubernetes.default.svc:443" + + # Prevent fluentd from handling records containing its own logs. + + @type null + + # expose the Fluentd metrics to Prometheus + + @type prometheus + metrics_path /metrics + port 24231 + + + @type prometheus_output_monitor + + + @type http + port 9880 + bind 0.0.0.0 + + + @type copy + + @type sumologic + @id sumologic.endpoint.events + sumo_client "k8s_2.4.0" + endpoint "#{ENV['SUMO_ENDPOINT_DEFAULT_EVENTS_SOURCE']}" + data_type logs + disable_cookies true + verify_ssl "true" + proxy_uri "" + compress "true" + compress_encoding "gzip" + + @type file + path /fluentd/buffer/events + @include buffer.output.conf + + + + + log_level info + + buffer.output.conf: | + compress "gzip" + flush_interval "5s" + flush_thread_count "8" + chunk_limit_size "1m" + total_limit_size "128m" + queued_chunks_limit_size "128" + overflow_action drop_oldest_chunk + retry_max_interval "10m" + retry_forever "true" diff --git a/tests/helm/events/static/apiServerUrl_null.input.yaml b/tests/helm/events/static/apiServerUrl_null.input.yaml new file mode 100644 index 0000000000..1cdfa3a507 --- /dev/null +++ b/tests/helm/events/static/apiServerUrl_null.input.yaml @@ -0,0 +1,2 @@ +fluentd: + apiServerUrl: null diff --git a/tests/helm/events/static/apiServerUrl_null.output.yaml b/tests/helm/events/static/apiServerUrl_null.output.yaml new file mode 100644 index 0000000000..5342f7a2be --- /dev/null +++ b/tests/helm/events/static/apiServerUrl_null.output.yaml @@ -0,0 +1,70 @@ +--- +# Source: sumologic/templates/events/fluentd/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: RELEASE-NAME-sumologic-fluentd-events + labels: + app: RELEASE-NAME-sumologic-fluentd-events + chart: "sumologic-%CURRENT_CHART_VERSION%" + release: "RELEASE-NAME" + heritage: "Helm" +data: + fluent.conf: |- + @include events.conf + events.conf: | + + @type events + deploy_namespace sumologic + + # Prevent fluentd from handling records containing its own logs. + + @type null + + # expose the Fluentd metrics to Prometheus + + @type prometheus + metrics_path /metrics + port 24231 + + + @type prometheus_output_monitor + + + @type http + port 9880 + bind 0.0.0.0 + + + @type copy + + @type sumologic + @id sumologic.endpoint.events + sumo_client "k8s_2.4.0" + endpoint "#{ENV['SUMO_ENDPOINT_DEFAULT_EVENTS_SOURCE']}" + data_type logs + disable_cookies true + verify_ssl "true" + proxy_uri "" + compress "true" + compress_encoding "gzip" + + @type file + path /fluentd/buffer/events + @include buffer.output.conf + + + + + log_level info + + buffer.output.conf: | + compress "gzip" + flush_interval "5s" + flush_thread_count "8" + chunk_limit_size "1m" + total_limit_size "128m" + queued_chunks_limit_size "128" + overflow_action drop_oldest_chunk + retry_max_interval "10m" + retry_forever "true" diff --git a/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.input.yaml b/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.input.yaml new file mode 100644 index 0000000000..9a548301a7 --- /dev/null +++ b/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.input.yaml @@ -0,0 +1,7 @@ +fluentd: + apiServerUrl: "https://kubernetes.default.svc:443" + events: + enabled: true + watchResourceEventsOverrides: + pods: "v2" + events: "events.k8s.io/v3" diff --git a/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.output.yaml b/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.output.yaml new file mode 100644 index 0000000000..86340cc027 --- /dev/null +++ b/tests/helm/events/static/watchResourceEventsOverrides_apiServerUrl.output.yaml @@ -0,0 +1,80 @@ +--- +# Source: sumologic/templates/events/fluentd/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: RELEASE-NAME-sumologic-fluentd-events + labels: + app: RELEASE-NAME-sumologic-fluentd-events + chart: "sumologic-%CURRENT_CHART_VERSION%" + release: "RELEASE-NAME" + heritage: "Helm" +data: + fluent.conf: |- + @include events.conf + events.conf: | + + @type events + deploy_namespace sumologic + resource_name events + api_version events.k8s.io/v3 + kubernetes_url "https://kubernetes.default.svc:443" + + + @type events + deploy_namespace sumologic + resource_name pods + api_version v2 + kubernetes_url "https://kubernetes.default.svc:443" + + # Prevent fluentd from handling records containing its own logs. + + @type null + + # expose the Fluentd metrics to Prometheus + + @type prometheus + metrics_path /metrics + port 24231 + + + @type prometheus_output_monitor + + + @type http + port 9880 + bind 0.0.0.0 + + + @type copy + + @type sumologic + @id sumologic.endpoint.events + sumo_client "k8s_2.4.0" + endpoint "#{ENV['SUMO_ENDPOINT_DEFAULT_EVENTS_SOURCE']}" + data_type logs + disable_cookies true + verify_ssl "true" + proxy_uri "" + compress "true" + compress_encoding "gzip" + + @type file + path /fluentd/buffer/events + @include buffer.output.conf + + + + + log_level info + + buffer.output.conf: | + compress "gzip" + flush_interval "5s" + flush_thread_count "8" + chunk_limit_size "1m" + total_limit_size "128m" + queued_chunks_limit_size "128" + overflow_action drop_oldest_chunk + retry_max_interval "10m" + retry_forever "true"