From 3b13aeda38333c280d120d6dd62c073a71893ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Fri, 14 Jun 2024 10:14:04 -0700 Subject: [PATCH] fix(forwarder): set content-encoding on override We currently rely on the S3 object being written with the correct content-encoding. There are many cases, such as `vpcflowlogs`, where the content-type is set to `gz` and the content-encoding is left blank. This commit ensures we always set the content-encoding correctly for filenames that end in `.gz` --- pkg/handler/forwarder/override/presets/aws/v1.yaml | 6 ++++++ pkg/handler/forwarder/override/presets_test.go | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/handler/forwarder/override/presets/aws/v1.yaml b/pkg/handler/forwarder/override/presets/aws/v1.yaml index b9faed3c..68c9a9bb 100644 --- a/pkg/handler/forwarder/override/presets/aws/v1.yaml +++ b/pkg/handler/forwarder/override/presets/aws/v1.yaml @@ -10,24 +10,28 @@ source: '/cloudwatchmetrics/[a-z\d-]+/json/\d{4}/\d{2}/\d{2}/\d{2}' override: content-type: 'application/x-aws-cloudwatchmetrics' + # note, cloudwatchmetrics payloads are _not_ gzipped - id: configSnapshot match: source: '\d{12}_Config_[a-z\d-]+_ConfigSnapshot_\d{8}T\d{6}Z_[a-f\d-]+\.json\.gz$' override: content-type: 'application/x-aws-config' + content-encoding: 'gzip' - id: configHistory match: source: '\d{12}_Config_[a-z\d-]+_ConfigHistory_[^\.]+.json\.gz$' override: content-type: 'application/x-aws-config' + content-encoding: 'gzip' - id: configChangeNotification match: source: '\d{12}_Config_[a-z\d-]+_ChangeNotification_AWS\S+_\d{8}T\d{6}Z_[a-f\d-]+\.json\.gz$' override: content-type: 'application/x-aws-change' + content-encoding: 'gzip' - id: cloudtrail match: @@ -40,6 +44,7 @@ source: '\d{12}_vpcflowlogs_[a-z\d-]+_[a-zA-Z0-9-]+_\d{8}T\d{4}Z_[a-zA-Z0-9-]+\.log\.gz$' override: content-type: 'application/x-aws-vpcflowlogs' + content-encoding: 'gzip' - id: awsTestObject match: @@ -52,3 +57,4 @@ source: '\d{12}_elasticloadbalancing_[a-z\d-]+_[a-zA-Z0-9-]+_\d{8}T\d{4}Z_[0-9.]+_[a-zA-Z0-9]+\.log\.gz$' override: content-type: 'application/x-aws-elasticloadbalancing' + content-encoding: 'gzip' diff --git a/pkg/handler/forwarder/override/presets_test.go b/pkg/handler/forwarder/override/presets_test.go index 05bf9464..12377fbd 100644 --- a/pkg/handler/forwarder/override/presets_test.go +++ b/pkg/handler/forwarder/override/presets_test.go @@ -62,8 +62,7 @@ func TestPresets(t *testing.T) { }, { Input: &s3.CopyObjectInput{ - CopySource: aws.String("test-bucket/AWSLogs/123456789012/vpcflowlogs/eu-central-1/2024/04/18/123456789012_vpcflowlogs_eu-central-1_fl-0d867ec290a114c9d_20240418T2155Z_9b1b75d1.log.gz"), - ContentEncoding: aws.String("gzip"), + CopySource: aws.String("test-bucket/AWSLogs/123456789012/vpcflowlogs/eu-central-1/2024/04/18/123456789012_vpcflowlogs_eu-central-1_fl-0d867ec290a114c9d_20240418T2155Z_9b1b75d1.log.gz"), }, Expect: &s3.CopyObjectInput{ CopySource: aws.String("test-bucket/AWSLogs/123456789012/vpcflowlogs/eu-central-1/2024/04/18/123456789012_vpcflowlogs_eu-central-1_fl-0d867ec290a114c9d_20240418T2155Z_9b1b75d1.log.gz"), @@ -90,6 +89,7 @@ func TestPresets(t *testing.T) { Expect: &s3.CopyObjectInput{ CopySource: aws.String("test-bucket/AWSLogs/123456789012/elasticloadbalancing/us-west-2/2024/05/23/123456789012_elasticloadbalancing_us-west-2_ac5f85808922711e98f8e02481e016be_20240523T0015Z_127.69.70.85_5cpyxp74.log.gz"), ContentType: aws.String("application/x-aws-elasticloadbalancing"), + ContentEncoding: aws.String("gzip"), MetadataDirective: types.MetadataDirectiveReplace, }, },