feat(forwarder): add x-aws-elasticloadbalancing content type (#274)

This commit adds support for a new content type, `x-aws-elasticloadbalancing`. ELB logs are space delimited CSV files, much like VPC Flow Logs. The advantage of adding a custom content type is we are able to segregate this traffic more efficiently.
observeinc · May 23, 2024 · e446533 · e446533
1 parent 051e93b
commit e446533
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 14 deletions.
diff --git a/handler/forwarder/override/presets/aws/v1.yaml b/handler/forwarder/override/presets/aws/v1.yaml
@@ -46,3 +46,9 @@
     source: 'aws-programmatic-access-test-object$'
   override:
     content-type: 'text/x-ignore'
+
+- id: elasticLoadBalancing
+  match:
+    source: '\d{12}_elasticloadbalancing_[a-z\d-]+_[a-zA-Z0-9-]+_\d{8}T\d{4}Z_[0-9.]+_[a-zA-Z0-9]+\.log\.gz$'
+  override:
+    content-type: 'application/x-aws-elasticloadbalancing'
diff --git a/handler/forwarder/override/presets_test.go b/handler/forwarder/override/presets_test.go
@@ -83,6 +83,16 @@ func TestPresets(t *testing.T) {
 						Key: nil,
 					},
 				},
+				{
+					Input: &s3.CopyObjectInput{
+						CopySource: aws.String("test-bucket/AWSLogs/123456789012/elasticloadbalancing/us-west-2/2024/05/23/123456789012_elasticloadbalancing_us-west-2_ac5f85808922711e98f8e02481e016be_20240523T0015Z_127.69.70.85_5cpyxp74.log.gz"),
+					},
+					Expect: &s3.CopyObjectInput{
+						CopySource:        aws.String("test-bucket/AWSLogs/123456789012/elasticloadbalancing/us-west-2/2024/05/23/123456789012_elasticloadbalancing_us-west-2_ac5f85808922711e98f8e02481e016be_20240523T0015Z_127.69.70.85_5cpyxp74.log.gz"),
+						ContentType:       aws.String("application/x-aws-elasticloadbalancing"),
+						MetadataDirective: types.MetadataDirectiveReplace,
+					},
+				},
 			},
 		},
 		{

diff --git a/handler/forwarder/s3http/internal/decoders/csv.go b/handler/forwarder/s3http/internal/decoders/csv.go
@@ -38,7 +38,8 @@ func CSVDecoderFactory(r io.Reader, params map[string]string) Decoder {
 	return csvDecoder
 }
 
-func VPCFlowLogDecoderFactory(r io.Reader, params map[string]string) Decoder {
+// SSVDecoderFactory handles space separated values.
+func SSVDecoderFactory(r io.Reader, params map[string]string) Decoder {
 	if _, ok := params["delimiter"]; !ok {
 		params["delimiter"] = "space"
 	}

diff --git a/handler/forwarder/s3http/internal/decoders/decoder.go b/handler/forwarder/s3http/internal/decoders/decoder.go
@@ -13,19 +13,20 @@ var (
 )
 
 var decoders = map[string]DecoderFactory{
-	"":                                    JSONDecoderFactory,
-	"application/json":                    JSONDecoderFactory,
-	"application/x-csv":                   CSVDecoderFactory,
-	"application/x-ndjson":                JSONDecoderFactory,
-	"text/plain":                          TextDecoderFactory,
-	"text/csv":                            CSVDecoderFactory,
-	"application/x-aws-cloudwatchlogs":    CloudWatchLogsDecoderFactory,
-	"application/x-aws-cloudwatchmetrics": JSONDecoderFactory,
-	"application/x-aws-config":            FilteredNestedJSONDecoderFactory(ConfigurationItem{}),
-	"application/x-aws-change":            FilteredJSONDecoderFactory(ConfigurationDiff{}),
-	"application/x-aws-cloudtrail":        NestedJSONDecoderFactory,
-	"application/x-aws-sqs":               JSONDecoderFactory,
-	"application/x-aws-vpcflowlogs":       VPCFlowLogDecoderFactory,
+	"":                                       JSONDecoderFactory,
+	"application/json":                       JSONDecoderFactory,
+	"application/x-csv":                      CSVDecoderFactory,
+	"application/x-ndjson":                   JSONDecoderFactory,
+	"text/plain":                             TextDecoderFactory,
+	"text/csv":                               CSVDecoderFactory,
+	"application/x-aws-cloudwatchlogs":       CloudWatchLogsDecoderFactory,
+	"application/x-aws-cloudwatchmetrics":    JSONDecoderFactory,
+	"application/x-aws-config":               FilteredNestedJSONDecoderFactory(ConfigurationItem{}),
+	"application/x-aws-change":               FilteredJSONDecoderFactory(ConfigurationDiff{}),
+	"application/x-aws-cloudtrail":           NestedJSONDecoderFactory,
+	"application/x-aws-sqs":                  JSONDecoderFactory,
+	"application/x-aws-vpcflowlogs":          SSVDecoderFactory,
+	"application/x-aws-elasticloadbalancing": SSVDecoderFactory,
 }
 
 type Decoder interface {