Skip to content

Commit

Permalink
feat(forwarder): add x-aws-elasticloadbalancing content type (#274)
Browse files Browse the repository at this point in the history
This commit adds support for a new content type,
`x-aws-elasticloadbalancing`. ELB logs are space delimited CSV files,
much like VPC Flow Logs. The advantage of adding a custom content type
is we are able to segregate this traffic more efficiently.
  • Loading branch information
jta authored May 23, 2024
1 parent 051e93b commit e446533
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 14 deletions.
6 changes: 6 additions & 0 deletions handler/forwarder/override/presets/aws/v1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,9 @@
source: 'aws-programmatic-access-test-object$'
override:
content-type: 'text/x-ignore'

- id: elasticLoadBalancing
match:
source: '\d{12}_elasticloadbalancing_[a-z\d-]+_[a-zA-Z0-9-]+_\d{8}T\d{4}Z_[0-9.]+_[a-zA-Z0-9]+\.log\.gz$'
override:
content-type: 'application/x-aws-elasticloadbalancing'
10 changes: 10 additions & 0 deletions handler/forwarder/override/presets_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,16 @@ func TestPresets(t *testing.T) {
Key: nil,
},
},
{
Input: &s3.CopyObjectInput{
CopySource: aws.String("test-bucket/AWSLogs/123456789012/elasticloadbalancing/us-west-2/2024/05/23/123456789012_elasticloadbalancing_us-west-2_ac5f85808922711e98f8e02481e016be_20240523T0015Z_127.69.70.85_5cpyxp74.log.gz"),
},
Expect: &s3.CopyObjectInput{
CopySource: aws.String("test-bucket/AWSLogs/123456789012/elasticloadbalancing/us-west-2/2024/05/23/123456789012_elasticloadbalancing_us-west-2_ac5f85808922711e98f8e02481e016be_20240523T0015Z_127.69.70.85_5cpyxp74.log.gz"),
ContentType: aws.String("application/x-aws-elasticloadbalancing"),
MetadataDirective: types.MetadataDirectiveReplace,
},
},
},
},
{
Expand Down
3 changes: 2 additions & 1 deletion handler/forwarder/s3http/internal/decoders/csv.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ func CSVDecoderFactory(r io.Reader, params map[string]string) Decoder {
return csvDecoder
}

func VPCFlowLogDecoderFactory(r io.Reader, params map[string]string) Decoder {
// SSVDecoderFactory handles space separated values.
func SSVDecoderFactory(r io.Reader, params map[string]string) Decoder {
if _, ok := params["delimiter"]; !ok {
params["delimiter"] = "space"
}
Expand Down
27 changes: 14 additions & 13 deletions handler/forwarder/s3http/internal/decoders/decoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,20 @@ var (
)

var decoders = map[string]DecoderFactory{
"": JSONDecoderFactory,
"application/json": JSONDecoderFactory,
"application/x-csv": CSVDecoderFactory,
"application/x-ndjson": JSONDecoderFactory,
"text/plain": TextDecoderFactory,
"text/csv": CSVDecoderFactory,
"application/x-aws-cloudwatchlogs": CloudWatchLogsDecoderFactory,
"application/x-aws-cloudwatchmetrics": JSONDecoderFactory,
"application/x-aws-config": FilteredNestedJSONDecoderFactory(ConfigurationItem{}),
"application/x-aws-change": FilteredJSONDecoderFactory(ConfigurationDiff{}),
"application/x-aws-cloudtrail": NestedJSONDecoderFactory,
"application/x-aws-sqs": JSONDecoderFactory,
"application/x-aws-vpcflowlogs": VPCFlowLogDecoderFactory,
"": JSONDecoderFactory,
"application/json": JSONDecoderFactory,
"application/x-csv": CSVDecoderFactory,
"application/x-ndjson": JSONDecoderFactory,
"text/plain": TextDecoderFactory,
"text/csv": CSVDecoderFactory,
"application/x-aws-cloudwatchlogs": CloudWatchLogsDecoderFactory,
"application/x-aws-cloudwatchmetrics": JSONDecoderFactory,
"application/x-aws-config": FilteredNestedJSONDecoderFactory(ConfigurationItem{}),
"application/x-aws-change": FilteredJSONDecoderFactory(ConfigurationDiff{}),
"application/x-aws-cloudtrail": NestedJSONDecoderFactory,
"application/x-aws-sqs": JSONDecoderFactory,
"application/x-aws-vpcflowlogs": SSVDecoderFactory,
"application/x-aws-elasticloadbalancing": SSVDecoderFactory,
}

type Decoder interface {
Expand Down

0 comments on commit e446533

Please sign in to comment.