feat(EKS Fargate): Add multiline support to EKS Fargate

.changelog/3059.added.txt

@@ -0,0 +1 @@
+feat(EKS Fargate): Add multiline support to EKS Fargate

deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml

@@ -26,24 +26,59 @@ processors:
     send_batch_max_size: 2000
     send_batch_size: 1000
     timeout: 1s
-  transform/cloudwatch_logs:
+  transform/set_source_identifier:
     error_mode: ignore
     log_statements:
       - context: log
         statements:
-          - set(attributes["k8s.container.name"], resource.attributes["cloudwatch.log.stream"])
-          - set(attributes["k8s.pod.name"], resource.attributes["cloudwatch.log.stream"])
-          - set(attributes["k8s.namespace.name"], resource.attributes["cloudwatch.log.stream"])
-          - replace_pattern(attributes["k8s.pod.name"], "^.*kube\\.var\\.log\\.containers\\.([0-9a-zA-Z\\-]+)\\_.*", "$$1")
-          - replace_pattern(attributes["k8s.container.name"], "^.*kube\\.var\\.log\\.containers\\.[0-9a-zA-Z\\-]+\\_[a-zA-Z\\-]*\\_([a-zA-Z]*).*", "$$1")
-          - replace_pattern(attributes["k8s.namespace.name"], "^.*kube\\.var\\.log\\.containers\\.[0-9a-zA-Z\\-]+\\_([a-zA-Z\\-]*)_.*", "$$1")
-  transform/parse:
+          - set(attributes["cloudwatch.log.stream"], resource.attributes["cloudwatch.log.stream"])
+  groupbyattrs/stream:
+    keys:
+      - cloudwatch.log.stream
+  ## need to reset the source identifier after grouping
+  transform/reset_source_identifier:
+    error_mode: ignore
+    log_statements:
+      - context: log
+        statements:
+          - set(attributes["cloudwatch.log.stream"], resource.attributes["cloudwatch.log.stream"])
+  transform/parsejson:
+    error_mode: ignore
     log_statements:
       - context: log
         statements:
           - set(body, ParseJSON(body)) where IsMatch(body, "^{") == true
           - merge_maps(attributes, body, "insert")
           - set(body, "") where IsMatch(body, "^{") == true
+  transform/metadata:
+    error_mode: ignore
+    log_statements:
+      - context: log
+        statements:
+          - set(attributes["k8s.container.name"], resource.attributes["cloudwatch.log.stream"])
+          - set(attributes["k8s.pod.name"], resource.attributes["cloudwatch.log.stream"])
+          - set(attributes["k8s.namespace.name"], resource.attributes["cloudwatch.log.stream"])
+          - replace_pattern(attributes["k8s.pod.name"], "^.*kube\\.var\\.log\\.containers\\.([0-9a-zA-Z\\-]+)\\_.*", "$$1")
+          - replace_pattern(attributes["k8s.container.name"], "^.*kube\\.var\\.log\\.containers\\.[0-9a-zA-Z\\-]+\\_[a-zA-Z\\-]*\\_([a-zA-Z]*).*", "$$1")
+          - replace_pattern(attributes["k8s.namespace.name"], "^.*kube\\.var\\.log\\.containers\\.[0-9a-zA-Z\\-]+\\_([a-zA-Z\\-]*)_.*", "$$1")
+  logstransform/cloudwatch:
+    operators:
+      - id: merge-cri-lines
+        combine_field: attributes.log
+        combine_with: ""
+        is_last_entry: attributes.logtag == "F"
+        output: "merge-multiline-logs"
+        overwrite_with: newest
+        source_identifier: attributes["cloudwatch.log.stream"]
+        type: recombine
+      - id: merge-multiline-logs
+        combine_field: attributes.log
+        combine_with: "\n"
+        is_first_entry: attributes.log matches {{ .Values.sumologic.logs.multiline.first_line_regex | quote }}
+        source_identifier: attributes["cloudwatch.log.stream"]
+        type: recombine
+      - field: attributes["cloudwatch.log.stream"]
+        type: remove
 receivers:
   awscloudwatch:
     region: {{ .Values.sumologic.logs.collector.otelcloudwatch.region }}
@@ -63,8 +98,12 @@ service:
       receivers:
         - awscloudwatch
       processors:
-        - transform/parse
-        - transform/cloudwatch_logs
+        - transform/set_source_identifier
+        - groupbyattrs/stream
+        - transform/reset_source_identifier
+        - transform/parsejson
+        - logstransform/cloudwatch
+        - transform/metadata
         - batch
       exporters:
         - otlphttp

docs/fargate.md

@@ -1,11 +1,8 @@
 # Fargate
 
-**NOTE: This is the alpha release of EKS Fargate.**
+**Release Note: Generally Available (GA) Release of EKS Fargate.**
 
-The following are some limitations of deploying this helm chart on EKS fargate
-
-- Only supports EKS version 1.24 and above
-- Does not support multiline logs
+Supports EKS version 1.24 and above
 
 The following documentation assumes that you are using eksctl to manage Fargate cluster. Code snippets are using environment variables in
 order to make them as generic and reusable.