Enable compression in fluentd output plugin by default (#1062)

* Enable compression in fluentd output plugin by default * Generate new 'fluentd-sumologic.yaml.tmpl' * Update deploy/helm/sumologic/README.md Co-authored-by: Patryk Małek <69143962+pmalek-sumo@users.noreply.github.com>
SumoLogic · Nov 3, 2020 · 5ad3ef0 · 5ad3ef0
1 parent 2b3ca63
commit 5ad3ef0
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 9 deletions.
diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md
@@ -38,6 +38,8 @@ Parameter | Description | Default
 `sumologic.setup.job.podAnnotations` | Additional annotations for the setup Job pod. | `{}`
 `sumologic.setup.serviceAccount.annotations` | Annotations for the ServiceAccount. | `[{"helm.sh/hook":"pre-install,pre-upgrade","helm.sh/hook-delete-policy":"before-hook-creation,hook-succeeded","helm.sh/hook-weight":"0"}]`
 `fluentd.additionalPlugins` | Additional Fluentd plugins to install from RubyGems. Please see our [documentation](./Additional_Fluentd_Plugins.md) for more information. | `[]`
+`fluentd.compression.enabled` | Flag to control if data is sent to Sumo Logic compressed or not | `true`
+`fluentd.compression.encoding` | Specifies which encoding should be used to compress data (either `gzip` or `deflate`) | `gzip`
 `fluentd.logLevel` | Sets the fluentd log level. The default log level, if not specified, is info.  Sumo will only ingest the error log level and some specific warnings, the info logs can be seen in kubectl logs. | `info`
 `fluentd.verifySsl` | Verify SumoLogic HTTPS certificates. | `true`
 `fluentd.proxyUri` | Proxy URI for sumologic output plugin. | `Nil`

diff --git a/deploy/helm/sumologic/conf/events/events.conf b/deploy/helm/sumologic/conf/events/events.conf
@@ -45,6 +45,8 @@
     disable_cookies true
     verify_ssl {{ .Values.fluentd.verifySsl | quote }}
     proxy_uri {{ .Values.fluentd.proxyUri | quote }}
+    compress {{ .Values.fluentd.compression.enabled | quote }}
+    compress_encoding {{ .Values.fluentd.compression.encoding | quote }}
     <buffer>
       {{- if or .Values.fluentd.persistence.enabled (eq .Values.fluentd.buffer.type "file") }}
       @type file
@@ -63,4 +65,4 @@
 <system>
   log_level {{ .Values.fluentd.logLevel }}
 </system>
-{{- end }}
+{{- end }}
diff --git a/deploy/helm/sumologic/conf/logs/logs.output.conf b/deploy/helm/sumologic/conf/logs/logs.output.conf
@@ -6,4 +6,6 @@ log_format {{ .Values.fluentd.logs.output.logFormat | quote }}
 add_timestamp {{ .Values.fluentd.logs.output.addTimestamp | quote }}
 timestamp_key {{ .Values.fluentd.logs.output.timestampKey | quote }}
 proxy_uri {{ .Values.fluentd.proxyUri | quote }}
-{{- .Values.fluentd.logs.output.extraConf | nindent 2 }}
+compress {{ .Values.fluentd.compression.enabled | quote }}
+compress_encoding {{ .Values.fluentd.compression.encoding | quote }}
+{{- .Values.fluentd.logs.output.extraConf | nindent 2 }}
diff --git a/deploy/helm/sumologic/conf/metrics/metrics.output.conf b/deploy/helm/sumologic/conf/metrics/metrics.output.conf
@@ -1,4 +1,6 @@
 data_type metrics
 metric_data_format prometheus
 disable_cookies true
-proxy_uri {{ .Values.fluentd.proxyUri | quote }}
+proxy_uri {{ .Values.fluentd.proxyUri | quote }}
+compress {{ .Values.fluentd.compression.enabled | quote }}
+compress_encoding {{ .Values.fluentd.compression.encoding | quote }}
diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -187,6 +187,11 @@ fluentd:
   ## Proxy URI for sumologic output plugin
   proxyUri: ""
 
+  ## Enable and set compression encoding for fluentd output plugin
+  compression:
+    enabled: true
+    encoding: gzip
+
   securityContext:
     ## The group ID of all processes in the statefulset containers. By default this needs to be fluent(999).
     fsGroup: 999

diff --git a/deploy/kubernetes/fluentd-sumologic.yaml.tmpl b/deploy/kubernetes/fluentd-sumologic.yaml.tmpl
@@ -88,7 +88,7 @@ data:
     exclude_pod_regex ""
     exclude_container_regex ""
     exclude_host_regex ""
-  logs.output.conf: |-
+  logs.output.conf: |
     data_type logs
     log_key log
     endpoint "#{ENV['SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE']}"
@@ -97,6 +97,8 @@ data:
     add_timestamp "true"
     timestamp_key "timestamp"
     proxy_uri ""
+    compress "true"
+    compress_encoding "gzip"
 
   logs.source.containers.conf: |
 
@@ -301,7 +303,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: 32b7e94e9eef43a4232d6bf110fafe661cdbc2aa1951683ad60c415af11775ec
+        checksum/config: 1518b8eaf0e673b37ab13e6e072f050a3b45ecca786dfb3842f4b386cb798bae
       labels:
         app: collection-sumologic-fluentd-events
 
@@ -589,11 +591,13 @@ data:
       </match>
 
     </label>
-  metrics.output.conf: |-
+  metrics.output.conf: |
     data_type metrics
     metric_data_format prometheus
     disable_cookies true
     proxy_uri ""
+    compress "true"
+    compress_encoding "gzip"
 ---
 # Source: sumologic/templates/clusterrolebinding.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -661,7 +665,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: ef16142c85545fe389b3d38f760a39fa0414109e484883dcd75e9ab11ccbdc87
+        checksum/config: 5dd0f5537aa0bed112eed180c07b68b7261f97c31313b3a4f16aa513f8adcdea
       labels:
         app: collection-sumologic-fluentd-logs
 
@@ -831,7 +835,7 @@ metadata:
 data:
   fluent.conf: |-
     @include events.conf
-  events.conf: |-
+  events.conf: |
     <source>
       @type events
       deploy_namespace $NAMESPACE
@@ -865,6 +869,8 @@ data:
         disable_cookies true
         verify_ssl "true"
         proxy_uri ""
+        compress "true"
+        compress_encoding "gzip"
         <buffer>
           @type file
           path /fluentd/buffer/events
@@ -927,7 +933,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: 995c20308c88a208d96c6e87815db0382f416612f07ee2c3dec0f95a506e8da5
+        checksum/config: e64629b38e8ad6d13ade338af2239443b24ee4b9d66815aab2755d132f90925d
       labels:
         app: collection-sumologic-fluentd-metrics