From 63a167ada0620659dbb15f98002a427c9b04d1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Mon, 17 Jun 2024 14:32:52 -0700 Subject: [PATCH] feat(forwarder): add `source_object_keys` parameter Terraform counterpart to https://github.com/observeinc/aws-sam-apps/pull/308 --- examples/forwarder-s3/README.md | 74 +++++++++++++++++++ examples/forwarder-s3/direct.tf | 12 +++ examples/forwarder-s3/eventbridge.tf | 9 +++ examples/forwarder-s3/main.tf | 26 +++++++ examples/forwarder-s3/outputs.tf | 20 +++++ examples/forwarder-s3/sns.tf | 34 +++++++++ .../forwarder-s3/tests/example.tftest.hcl | 2 + examples/forwarder-s3/variables.tf | 0 examples/forwarder-s3/versions.tf | 10 +++ modules/forwarder/README.md | 1 + modules/forwarder/eventbridge.tf | 3 + modules/forwarder/iam.tf | 6 +- modules/forwarder/lambda.tf | 1 + modules/forwarder/variables.tf | 16 ++++ modules/stack/README.md | 2 +- modules/stack/forwarder.tf | 1 + modules/stack/variables.tf | 1 + 17 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 examples/forwarder-s3/README.md create mode 100644 examples/forwarder-s3/direct.tf create mode 100644 examples/forwarder-s3/eventbridge.tf create mode 100644 examples/forwarder-s3/main.tf create mode 100644 examples/forwarder-s3/outputs.tf create mode 100644 examples/forwarder-s3/sns.tf create mode 100644 examples/forwarder-s3/tests/example.tftest.hcl create mode 100644 examples/forwarder-s3/variables.tf create mode 100644 examples/forwarder-s3/versions.tf diff --git a/examples/forwarder-s3/README.md b/examples/forwarder-s3/README.md new file mode 100644 index 0000000..7e26aad --- /dev/null +++ b/examples/forwarder-s3/README.md @@ -0,0 +1,74 @@ +# Subscribing objects via EventBridge + +This module demonstrates how to forward objects from a source bucket using +EventBridge to trigger the Forwarder module. + +To run this example you need to execute: + +``` +$ terraform init +$ terraform plan +$ terraform apply +``` + +The module will output three source buckets, one for each subscription type. +It will also output a destination bucket, where objects will be copied to by the forwarder. + +``` +→ aws s3 cp ./main.tf s3://`terraform output -raw direct_source` +upload: ./main.tf to s3://forwarder-s3-bucket-notification-src-20240617155010087500000002/main.tf +→ aws s3 ls s3://`terraform output -raw destination` +2024-06-17 08:52:55 869 main.tf +``` + +Note that this example may create resources which can cost money. Run terraform destroy when you don't need these resources. + + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [aws](#requirement\_aws) | >= 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 5.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [forwarder](#module\_forwarder) | ../..//modules/forwarder | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_s3_bucket.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket.direct](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket.eventbridge](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket.sns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [aws_s3_bucket_notification.direct](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource | +| [aws_s3_bucket_notification.eventbridge](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource | +| [aws_s3_bucket_notification.sns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource | +| [aws_sns_topic.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource | +| [aws_sns_topic_policy.s3_to_sns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_policy) | resource | +| [aws_iam_policy_document.s3_to_sns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +No inputs. + +## Outputs + +| Name | Description | +|------|-------------| +| [destination](#output\_destination) | Destination bucket objects are copied to. | +| [direct\_source](#output\_direct\_source) | Source bucket subscribed directly | +| [eventbridge\_source](#output\_eventbridge\_source) | Source bucket subscribed via eventbridge | +| [sns\_source](#output\_sns\_source) | Source bucket subscribed via SNS | + diff --git a/examples/forwarder-s3/direct.tf b/examples/forwarder-s3/direct.tf new file mode 100644 index 0000000..37f896e --- /dev/null +++ b/examples/forwarder-s3/direct.tf @@ -0,0 +1,12 @@ +resource "aws_s3_bucket" "direct" { + bucket_prefix = "${local.name_prefix}direct-" + force_destroy = true +} + +resource "aws_s3_bucket_notification" "direct" { + bucket = aws_s3_bucket.direct.id + queue { + queue_arn = module.forwarder.queue_arn + events = ["s3:ObjectCreated:*"] + } +} diff --git a/examples/forwarder-s3/eventbridge.tf b/examples/forwarder-s3/eventbridge.tf new file mode 100644 index 0000000..eb0b966 --- /dev/null +++ b/examples/forwarder-s3/eventbridge.tf @@ -0,0 +1,9 @@ +resource "aws_s3_bucket" "eventbridge" { + bucket_prefix = "${local.name_prefix}eventbridge-" + force_destroy = true +} + +resource "aws_s3_bucket_notification" "eventbridge" { + bucket = aws_s3_bucket.eventbridge.id + eventbridge = true +} diff --git a/examples/forwarder-s3/main.tf b/examples/forwarder-s3/main.tf new file mode 100644 index 0000000..90bc71f --- /dev/null +++ b/examples/forwarder-s3/main.tf @@ -0,0 +1,26 @@ +locals { + name = basename(abspath(path.root)) + name_prefix = "${local.name}-" +} + +# we'll write data to this bucket +resource "aws_s3_bucket" "destination" { + bucket_prefix = "${local.name_prefix}dst-" + force_destroy = true +} + +module "forwarder" { + # Prefer using the hashicorp registry: + # source = "observeinc/collection/aws//modules/forwarder" + # For validation purposes we will instead refer to a local version of the + # module: + source = "../..//modules/forwarder" + + name = local.name + destination = { + bucket = aws_s3_bucket.destination.id + prefix = "" + } + source_bucket_names = ["${local.name_prefix}*"] + source_object_keys = ["*"] +} diff --git a/examples/forwarder-s3/outputs.tf b/examples/forwarder-s3/outputs.tf new file mode 100644 index 0000000..38aabf3 --- /dev/null +++ b/examples/forwarder-s3/outputs.tf @@ -0,0 +1,20 @@ +output "direct_source" { + description = "Source bucket subscribed directly" + value = aws_s3_bucket.direct.id +} + +output "sns_source" { + description = "Source bucket subscribed via SNS" + value = aws_s3_bucket.sns.id +} + +output "eventbridge_source" { + description = "Source bucket subscribed via eventbridge" + value = aws_s3_bucket.eventbridge.id +} + + +output "destination" { + description = "Destination bucket objects are copied to." + value = aws_s3_bucket.destination.id +} diff --git a/examples/forwarder-s3/sns.tf b/examples/forwarder-s3/sns.tf new file mode 100644 index 0000000..138be40 --- /dev/null +++ b/examples/forwarder-s3/sns.tf @@ -0,0 +1,34 @@ +resource "aws_s3_bucket" "sns" { + bucket_prefix = "${local.name_prefix}sns-" + force_destroy = true +} + +resource "aws_sns_topic" "this" { + name_prefix = local.name_prefix +} + +data "aws_iam_policy_document" "s3_to_sns" { + statement { + actions = ["SNS:Publish"] + resources = [aws_sns_topic.this.arn] + principals { + type = "Service" + identifiers = ["s3.amazonaws.com"] + } + } +} + +resource "aws_sns_topic_policy" "s3_to_sns" { + arn = aws_sns_topic.this.arn + policy = data.aws_iam_policy_document.s3_to_sns.json +} + +resource "aws_s3_bucket_notification" "sns" { + bucket = aws_s3_bucket.sns.id + topic { + topic_arn = aws_sns_topic.this.arn + events = ["s3:ObjectCreated:*"] + } + + depends_on = [aws_sns_topic_policy.s3_to_sns] +} diff --git a/examples/forwarder-s3/tests/example.tftest.hcl b/examples/forwarder-s3/tests/example.tftest.hcl new file mode 100644 index 0000000..db42804 --- /dev/null +++ b/examples/forwarder-s3/tests/example.tftest.hcl @@ -0,0 +1,2 @@ +# only verifies module can be installed and removed correctly +run "install" {} diff --git a/examples/forwarder-s3/variables.tf b/examples/forwarder-s3/variables.tf new file mode 100644 index 0000000..e69de29 diff --git a/examples/forwarder-s3/versions.tf b/examples/forwarder-s3/versions.tf new file mode 100644 index 0000000..29ec41d --- /dev/null +++ b/examples/forwarder-s3/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.3" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} diff --git a/modules/forwarder/README.md b/modules/forwarder/README.md index bf759d2..ee98d53 100644 --- a/modules/forwarder/README.md +++ b/modules/forwarder/README.md @@ -118,6 +118,7 @@ module "forwarder" { | [sam\_release\_version](#input\_sam\_release\_version) | Release version for SAM apps as defined on github.com/observeinc/aws-sam-apps. | `string` | `""` | no | | [source\_bucket\_names](#input\_source\_bucket\_names) | A list of bucket names which the forwarder is allowed to read from. This
list only affects permissions, and supports wildcards. In order to have
files copied to Filedrop, you must also subscribe S3 Bucket Notifications
to the forwarder. | `list(string)` | `[]` | no | | [source\_kms\_key\_arns](#input\_source\_kms\_key\_arns) | A list of KMS Key ARNs the forwarder is allowed to use to decrypt objects in S3. | `list(string)` | `[]` | no | +| [source\_object\_keys](#input\_source\_object\_keys) | A list of object key patterns the forwarder is allowed to read from for
provided source buckets. | `list(string)` | 
[
  "*"
]
| no | | [source\_topic\_arns](#input\_source\_topic\_arns) | A list of SNS topics the forwarder is allowed to be subscribed to. | `list(string)` | `[]` | no | | [verbosity](#input\_verbosity) | Logging verbosity for Lambda. Highest log verbosity is 9. | `number` | `1` | no | diff --git a/modules/forwarder/eventbridge.tf b/modules/forwarder/eventbridge.tf index ac837f3..e6e5d10 100644 --- a/modules/forwarder/eventbridge.tf +++ b/modules/forwarder/eventbridge.tf @@ -17,6 +17,9 @@ resource "aws_cloudwatch_event_rule" "this" { # list must have elements, so we introduce an empty match for name in concat([""], var.source_bucket_names) : { "wildcard" : name } ], + "detail.object.key" = [ + for key in concat([""], var.source_object_keys) : { "wildcard" : key } + ], }) } diff --git a/modules/forwarder/iam.tf b/modules/forwarder/iam.tf index 4444fc9..113b3c7 100644 --- a/modules/forwarder/iam.tf +++ b/modules/forwarder/iam.tf @@ -80,7 +80,11 @@ data "aws_iam_policy_document" "reader" { "s3:GetObject", "s3:GetObjectTagging", ] - resources = [for name in var.source_bucket_names : "arn:aws:s3:::${name}/*"] + # NOTE: this is a deviation from our CloudFormation template. In terraform + # we can compute the product of two lists, allowing us to be strict as to what buckets and keys we grant the function read access for. + resources = [ + for pair in setproduct(var.source_bucket_names, var.source_object_keys) : "arn:aws:s3:::${pair[0]}/${pair[1]}" + ] } } diff --git a/modules/forwarder/lambda.tf b/modules/forwarder/lambda.tf index 038ed04..7598adf 100644 --- a/modules/forwarder/lambda.tf +++ b/modules/forwarder/lambda.tf @@ -15,6 +15,7 @@ resource "aws_lambda_function" "this" { MAX_FILE_SIZE = var.max_file_size != null ? var.max_file_size : local.default_limits.max_file_size CONTENT_TYPE_OVERRIDES = join(",", [for o in var.content_type_overrides : "${o["pattern"]}=${o["content_type"]}"]) SOURCE_BUCKET_NAMES = join(",", var.source_bucket_names) + SOURCE_OBJECT_KEYS = join(",", var.source_object_keys) OTEL_EXPORTER_OTLP_ENDPOINT = var.debug_endpoint OTEL_TRACES_EXPORTER = var.debug_endpoint == "" ? "none" : "otlp" VERBOSITY = var.verbosity diff --git a/modules/forwarder/variables.tf b/modules/forwarder/variables.tf index 16a9f96..6a2eacd 100644 --- a/modules/forwarder/variables.tf +++ b/modules/forwarder/variables.tf @@ -64,6 +64,21 @@ variable "source_bucket_names" { } } +variable "source_object_keys" { + description = <<-EOF + A list of object key patterns the forwarder is allowed to read from for + provided source buckets. + EOF + type = list(string) + nullable = false + default = ["*"] + + validation { + condition = length(var.source_object_keys) > 0 + error_message = "At least one S3 object key match must be provided." + } +} + variable "source_topic_arns" { description = <<-EOF A list of SNS topics the forwarder is allowed to be subscribed to. @@ -220,3 +235,4 @@ variable "code_uri" { default = "" nullable = false } + diff --git a/modules/stack/README.md b/modules/stack/README.md index de7623e..76a5f0e 100644 --- a/modules/stack/README.md +++ b/modules/stack/README.md @@ -170,7 +170,7 @@ You can additionally configure other submodules in this manner: | [configsubscription](#input\_configsubscription) | Variables for AWS Config subscription. | 
object({
    delivery_bucket_name = string
  })
| `null` | no | | [debug\_endpoint](#input\_debug\_endpoint) | Endpoint to send debugging telemetry to. Sets OTEL\_EXPORTER\_OTLP\_ENDPOINT environment variable for supported lambda functions. | `string` | `null` | no | | [destination](#input\_destination) | Destination filedrop | 
object({
    arn    = optional(string, "")
    bucket = optional(string, "")
    prefix = optional(string, "")
    # exclusively for backward compatible HTTP endpoint
    uri = optional(string, "")
  })
| n/a | yes | -| [forwarder](#input\_forwarder) | Variables for forwarder module. | 
object({
    source_bucket_names                      = optional(list(string), [])
    source_topic_arns                        = optional(list(string), [])
    content_type_overrides                   = optional(list(object({ pattern = string, content_type = string })), [])
    max_file_size                            = optional(number)
    lambda_memory_size                       = optional(number)
    lambda_timeout                           = optional(number)
    lambda_env_vars                          = optional(map(string))
    retention_in_days                        = optional(number)
    queue_max_receive_count                  = optional(number)
    queue_delay_seconds                      = optional(number)
    queue_message_retention_seconds          = optional(number)
    queue_batch_size                         = optional(number)
    queue_maximum_batching_window_in_seconds = optional(number)
    code_uri                                 = optional(string)
    sam_release_version                      = optional(string)
  })
| `{}` | no | +| [forwarder](#input\_forwarder) | Variables for forwarder module. | 
object({
    source_bucket_names                      = optional(list(string), [])
    source_object_keys                       = optional(list(string))
    source_topic_arns                        = optional(list(string), [])
    content_type_overrides                   = optional(list(object({ pattern = string, content_type = string })), [])
    max_file_size                            = optional(number)
    lambda_memory_size                       = optional(number)
    lambda_timeout                           = optional(number)
    lambda_env_vars                          = optional(map(string))
    retention_in_days                        = optional(number)
    queue_max_receive_count                  = optional(number)
    queue_delay_seconds                      = optional(number)
    queue_message_retention_seconds          = optional(number)
    queue_batch_size                         = optional(number)
    queue_maximum_batching_window_in_seconds = optional(number)
    code_uri                                 = optional(string)
    sam_release_version                      = optional(string)
  })
| `{}` | no | | [logwriter](#input\_logwriter) | Variables for AWS CloudWatch Logs collection. | 
object({
    log_group_name_patterns         = optional(list(string))
    log_group_name_prefixes         = optional(list(string))
    exclude_log_group_name_prefixes = optional(list(string))
    buffering_interval              = optional(number)
    buffering_size                  = optional(number)
    filter_name                     = optional(string)
    filter_pattern                  = optional(string)
    num_workers                     = optional(number)
    discovery_rate                  = optional(string, "24 hours")
    lambda_memory_size              = optional(number)
    lambda_timeout                  = optional(number)
    code_uri                        = optional(string)
    sam_release_version             = optional(string)
  })
| `null` | no | | [metricstream](#input\_metricstream) | Variables for AWS CloudWatch Metrics Stream collection. | 
object({
    include_filters     = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))
    exclude_filters     = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))
    buffering_interval  = optional(number)
    buffering_size      = optional(number)
    sam_release_version = optional(string)
  })
| `null` | no | | [name](#input\_name) | Name of role. Since this name must be unique within the
account, it will be reused for most of the resources created by this
module. | `string` | n/a | yes | diff --git a/modules/stack/forwarder.tf b/modules/stack/forwarder.tf index e847df0..1bee22a 100644 --- a/modules/stack/forwarder.tf +++ b/modules/stack/forwarder.tf @@ -8,6 +8,7 @@ module "forwarder" { aws_s3_bucket.this.id, var.configsubscription != null ? var.configsubscription.delivery_bucket_name : "", ], var.forwarder.source_bucket_names)) + source_object_keys = var.forwarder.source_object_keys source_topic_arns = concat([aws_sns_topic.this.arn], var.forwarder.source_topic_arns) content_type_overrides = local.content_type_overrides max_file_size = var.forwarder.max_file_size diff --git a/modules/stack/variables.tf b/modules/stack/variables.tf index b22cf1d..76211b4 100644 --- a/modules/stack/variables.tf +++ b/modules/stack/variables.tf @@ -30,6 +30,7 @@ variable "forwarder" { EOF type = object({ source_bucket_names = optional(list(string), []) + source_object_keys = optional(list(string)) source_topic_arns = optional(list(string), []) content_type_overrides = optional(list(object({ pattern = string, content_type = string })), []) max_file_size = optional(number)