From 075b4c463925e9c23746048ae436320fc25dcdc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Wed, 4 Oct 2023 16:35:46 -0700 Subject: [PATCH] feat: process eventbridge copies Add code to parse custom events from our Eventbridge rule. We use a bespoke struct in order to disambiguate between generic eventbridge events that may also get routed via the forwarder. --- handlers/forwarder/message.go | 27 +++++++++++++++++++++++++++ handlers/forwarder/message_test.go | 23 ++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/handlers/forwarder/message.go b/handlers/forwarder/message.go index 5a696a22..13397ca4 100644 --- a/handlers/forwarder/message.go +++ b/handlers/forwarder/message.go @@ -27,6 +27,11 @@ func (m *SQSMessage) GetObjectCreated() (uris []*url.URL) { if len(uris) == 0 { uris = append(uris, processS3Event(message)...) } + + if len(uris) == 0 { + uris = append(uris, processCopyEvent(message)...) + } + return } @@ -53,3 +58,25 @@ func processS3Event(message []byte) (uris []*url.URL) { } return } + +type CopyEvent struct { + Copy []CopyRecord `"json"` +} + +type CopyRecord struct { + URI string `"uri"` +} + +func processCopyEvent(message []byte) (uris []*url.URL) { + var copyEvent CopyEvent + err := json.Unmarshal(message, ©Event) + + if err == nil { + for _, record := range copyEvent.Copy { + if u, err := url.ParseRequestURI(record.URI); err == nil { + uris = append(uris, u) + } + } + } + return +} diff --git a/handlers/forwarder/message_test.go b/handlers/forwarder/message_test.go index d728bbd4..f7aadb5f 100644 --- a/handlers/forwarder/message_test.go +++ b/handlers/forwarder/message_test.go @@ -43,7 +43,7 @@ func TestObjectCreated(t *testing.T) { { "messageId": "e990e046-8e53-4fdd-8011-379517940223", "receiptHandle": "AQEB6vyZ3VqjMpeOzcd1Simmu+Emi7MnOivmOdS1XEayMhO9RVWI3Ft/hn5YLMEjB5VY2nsSDVCh37gBznhNRvx4AxrdkOHrfn7OOvrTJq/3gG6ecjNEDI/5WpIk6zd3a/rXiN8H7crev336hqxtu4hJSVz66XUkRKda1pfmIlzfBaLMzoBB4hxMKrwpQ1y+IaSx/FXUDMYPBo5r8lG3+sIa/7TpFfFpI8mo0tdnAkF2zeyJ8Hk1YDQZn4Y40cOMxltHGRcIsK2HmT5sa2E0AkAGV5Kd97Pb+Bb+j91+6VJaYQuV0SvkutGlq3aI/8SCZbK2CArqh0gJe32eQKHopRXxi/ihikSa7u47+FzlyPGKnooYOvCAc6zO8rDck+IH9wwCvcCBWc71U6KIG9uiRQu0eemaUDV5dpyRbG4Be9Ep7Io=", - "body": "{\n \"Type\" : \"Notification\",\n \"MessageId\" : \"4999b1ce-cf92-56b2-afe2-ec4076695a9f\",\n \"TopicArn\" : \"arn:aws:sns:us-east-1:123456789012:config-updates\",\n \"Subject\" : \"Amazon S3 Notification\",\n \"Message\" : \"{\\\"Records\\\":[{\\\"eventVersion\\\":\\\"2.1\\\",\\\"eventSource\\\":\\\"aws:s3\\\",\\\"awsRegion\\\":\\\"us-east-1\\\",\\\"eventTime\\\":\\\"2023-09-27T23:16:10.232Z\\\",\\\"eventName\\\":\\\"ObjectCreated:Put\\\",\\\"userIdentity\\\":{\\\"principalId\\\":\\\"AWS:AIDA2YN7JF3XJWA6ZRUAE\\\"},\\\"requestParameters\\\":{\\\"sourceIPAddress\\\":\\\"192.184.182.96\\\"},\\\"responseElements\\\":{\\\"x-amz-request-id\\\":\\\"V3DKPBZWVMCKX1C7\\\",\\\"x-amz-id-2\\\":\\\"b6p+8nV3VGEnXTqnl+3rHvb+UzlTBNPS7Woeq6ftOOMXYStTDociNu4MqUk6BW4hkCInFqMEiuBmTtZC5banA2H9ERIGrbd5\\\"},\\\"s3\\\":{\\\"s3SchemaVersion\\\":\\\"1.0\\\",\\\"configurationId\\\":\\\"tf-s3-topic-20230927215049003100000002\\\",\\\"bucket\\\":{\\\"name\\\":\\\"my-bucket\\\",\\\"ownerIdentity\\\":{\\\"principalId\\\":\\\"A21JCN1A8EHLG1\\\"},\\\"arn\\\":\\\"arn:aws:s3:::my-bucket\\\"},\\\"object\\\":{\\\"key\\\":\\\"test.json\\\",\\\"size\\\":25,\\\"eTag\\\":\\\"d0b8560f261410878a68bbe070d81853\\\",\\\"sequencer\\\":\\\"006514B7BA1ECB9CE0\\\"}}}]}\",\n \"Timestamp\" : \"2023-09-27T23:16:11.551Z\",\n \"SignatureVersion\" : \"1\",\n \"Signature\" : \"R6mj5u0KHmZvcHf83dJyTIExMiApmtsBzEw/dpmIjci7rfiqyE2LtjNZMWbSrXYJ366ZPSI4sX87lFdO1dNoAmUoGshRr1/0vCkva7uv1Zi3SkhEBTFOrCrPKQ13LCvG1sOPWeeFtevjAivLNnwSp3B3kjsGMIbrKFPRsqlnUtEBauKE/hwg1jANTKwZvChNVrfxzqcKvz2TpPACQ4QX9ma3lOWMuI/yY63fXgRAD8y6HSkgYB7OPxut7SEOG0maun+KExL2AocRXGBwb2tFkSHI2p1nlg5agvikKCcvspv3xz5PrEt1Bb9ymCS20Od6WETLjMZS2LlVeQhvfYEI5g==\",\n \"SigningCertURL\" : \"https://sns.us-east-1.amazonaws.com/SimpleNotificationService-01d088a6f77103d0fe307c0069e40ed6.pem\",\n \"UnsubscribeURL\" : \"https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:739672403694:config-updates:e96ef48f-aba9-4b15-96ba-dd72e4f84b25\"\n}", + "body": "{\n \"Type\" : \"Notification\",\n \"MessageId\" : \"4999b1ce-cf92-56b2-afe2-ec4076695a9f\",\n \"TopicArn\" : \"arn:aws:sns:us-east-1:123456789012:config-updates\",\n \"Subject\" : \"Amazon S3 Notification\",\n \"Message\" : \"{\\\"Records\\\":[{\\\"eventVersion\\\":\\\"2.1\\\",\\\"eventSource\\\":\\\"aws:s3\\\",\\\"awsRegion\\\":\\\"us-east-1\\\",\\\"eventTime\\\":\\\"2023-09-27T23:16:10.232Z\\\",\\\"eventName\\\":\\\"ObjectCreated:Put\\\",\\\"userIdentity\\\":{\\\"principalId\\\":\\\"AWS:AIDA2YN7JF3XJWA6ZRUAE\\\"},\\\"requestParameters\\\":{\\\"sourceIPAddress\\\":\\\"192.184.182.96\\\"},\\\"responseElements\\\":{\\\"x-amz-request-id\\\":\\\"V3DKPBZWVMCKX1C7\\\",\\\"x-amz-id-2\\\":\\\"b6p+8nV3VGEnXTqnl+3rHvb+UzlTBNPS7Woeq6ftOOMXYStTDociNu4MqUk6BW4hkCInFqMEiuBmTtZC5banA2H9ERIGrbd5\\\"},\\\"s3\\\":{\\\"s3SchemaVersion\\\":\\\"1.0\\\",\\\"configurationId\\\":\\\"tf-s3-topic-20230927215049003100000002\\\",\\\"bucket\\\":{\\\"name\\\":\\\"my-bucket\\\",\\\"ownerIdentity\\\":{\\\"principalId\\\":\\\"A21JCN1A8EHLG1\\\"},\\\"arn\\\":\\\"arn:aws:s3:::my-bucket\\\"},\\\"object\\\":{\\\"key\\\":\\\"test.json\\\",\\\"size\\\":25,\\\"eTag\\\":\\\"d0b8560f261410878a68bbe070d81853\\\",\\\"sequencer\\\":\\\"006514B7BA1ECB9CE0\\\"}}}]}\",\n \"Timestamp\" : \"2023-09-27T23:16:11.551Z\",\n \"SignatureVersion\" : \"1\",\n \"Signature\" : \"R6mj5u0KHmZvcHf83dJyTIExMiApmtsBzEw/dpmIjci7rfiqyE2LtjNZMWbSrXYJ366ZPSI4sX87lFdO1dNoAmUoGshRr1/0vCkva7uv1Zi3SkhEBTFOrCrPKQ13LCvG1sOPWeeFtevjAivLNnwSp3B3kjsGMIbrKFPRsqlnUtEBauKE/hwg1jANTKwZvChNVrfxzqcKvz2TpPACQ4QX9ma3lOWMuI/yY63fXgRAD8y6HSkgYB7OPxut7SEOG0maun+KExL2AocRXGBwb2tFkSHI2p1nlg5agvikKCcvspv3xz5PrEt1Bb9ymCS20Od6WETLjMZS2LlVeQhvfYEI5g==\",\n \"SigningCertURL\" : \"https://sns.us-east-1.amazonaws.com/SimpleNotificationService-01d088a6f77103d0fe307c0069e40ed6.pem\",\n \"UnsubscribeURL\" : \"https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:123456789012:config-updates:e96ef48f-aba9-4b15-96ba-dd72e4f84b25\"\n}", "md5OfBody": "432f4a36bb2e110e6eed0b3557418012", "md5OfMessageAttributes": "", "attributes": { @@ -61,6 +61,27 @@ func TestObjectCreated(t *testing.T) { "s3://my-bucket/test.json", }, }, + { + Message: ` + { + "attributes": { + "ApproximateFirstReceiveTimestamp": "1696456364266", + "ApproximateReceiveCount": "1", + "SenderId": "AIDAJXNJGGKNS7OSV23OI", + "SentTimestamp": "1696456364253" + }, + "awsRegion": "us-east-1", + "body": "{\"copy\": [{\"uri\": \"s3://my-bucket/test.json\"}]}", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:joao-filedrop-us-east-1", + "md5OfBody": "a231c62bd2ab84f63c85549c8eead615", + "md5OfMessageAttributes": "", + "messageAttributes": {}, + "messageId": "420659fa-599c-4a2d-97fa-da7ade83edc7", + "receiptHandle": "AQEB2kq3hOZLP6rlatTMVY4VOL37Zj7IFEhQeeIJAkZhM5vqCcBwZYgPzTc3QOtLTEg0DIL7okUsbFxz5ba3soihn5wqPM7x8fXuzJ0sBOE1XyYUBSzL5Ot6xjY7SnijCsnMEUc8wYTvx1LfkGkwXqKS4maXA8+R530YEUr1RLZ8EqHYtCG4tI6RU1jd0a0Mzv0DUFOg/NU7TdcMYlL7LjPClFfUoy9Hw/9R9L2aLfpUODQVD6+r86wlKrzzMLUDHw7BYuBXaXGXD/w9KGrCoL1q9IIkzXh0gbiAseC968vIh2xSfFv0l9tokahqPBpL/w6V8awnU9tNUQLafG3WjFzFjB00SuFedbxAhARUjNDGmaFIqoLdUrlYEfkPpVxrfqmwbunCQ0URzOtMMJu2uIp0XA==" + }`, + Expected: []string{"s3://my-bucket/test.json"}, + }, } for i, tc := range testcases {