From 45b9af44e8e8a28b59a8c667b1647dfa8c4b57d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Wed, 1 Nov 2023 14:18:03 -0700 Subject: [PATCH 1/5] feat: allow running integration tests Remove the hard-coding of a single test when doing integration tests. Fixed simple.tftest.hcl to pass. --- Makefile | 14 +++++++------- integration/tests/simple.tftest.hcl | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 1bf80ad8..db894f0c 100644 --- a/Makefile +++ b/Makefile @@ -42,12 +42,12 @@ go-test: go test -v -race ./... .PHONY: integration-test -integration-test: sam-package +integration-test: sam-package-all cd integration && terraform init && \ if [ "$(DEBUG)" = "1" ]; then \ - CHECK_DEBUG_FILE=debug.sh terraform test -filter=tests/forwarder.tftest.hcl -verbose; \ + CHECK_DEBUG_FILE=debug.sh terraform test $(TEST_ARGS); \ else \ - terraform test -filter=tests/forwarder.tftest.hcl; \ + terraform test $(TEST_ARGS); \ fi ## sam-validate: validate cloudformation templates @@ -60,8 +60,8 @@ sam-validate: ## sam-validate-all: validate all cloudformation templates sam-validate-all: - for dir in $(SUBDIR); do - APP=$$dir $(MAKE) sam-validate || exit 1; + @ for dir in $(SUBDIR); do \ + APP=$$dir $(MAKE) sam-validate || exit 1; \ done .PHONY: sam-build-all @@ -92,8 +92,8 @@ sam-publish: sam-package ## sam-package-all: package all cloudformation templates and push assets to S3 sam-package-all: - for dir in $(SUBDIR); do - APP=$$dir $(MAKE) sam-package || exit 1; + @ for dir in $(SUBDIR); do \ + APP=$$dir $(MAKE) sam-package || exit 1; \ done ## sam-package: package cloudformation templates and push assets to S3 diff --git a/integration/tests/simple.tftest.hcl b/integration/tests/simple.tftest.hcl index fb74b0c7..3d182cb0 100644 --- a/integration/tests/simple.tftest.hcl +++ b/integration/tests/simple.tftest.hcl @@ -12,12 +12,12 @@ run "check" { variables { command = "./scripts/check_bucket_not_empty" env_vars = { - SOURCE = run.setup.source.bucket + SOURCE = run.setup.access_point.bucket } } assert { - condition = output.exitcode == 0 - error_message = "Bucket not empty check failed" + condition = output.exitcode != 0 + error_message = "Bucket isn't empty" } } From a5462f91590ba265121072e51409e8b96a8c3371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Wed, 1 Nov 2023 14:44:59 -0700 Subject: [PATCH 2/5] feat: add firehose app This is a basic stack to setup a firehose pointed at an S3 bucket. We will need Firehose for both streaming cloudwatch logs and metrics. Since these streams have wildly different formats that cannot be interspersed (gzipped vs not), we will need separate delivery streams for each type. --- apps/firehose/README.md | 3 + apps/firehose/template.yaml | 132 +++++++++++++++++++++++++++++ integration/firehose.tftest.hcl | 70 +++++++++++++++ integration/scripts/check_firehose | 57 +++++++++++++ 4 files changed, 262 insertions(+) create mode 100644 apps/firehose/README.md create mode 100644 apps/firehose/template.yaml create mode 100644 integration/firehose.tftest.hcl create mode 100755 integration/scripts/check_firehose diff --git a/apps/firehose/README.md b/apps/firehose/README.md new file mode 100644 index 00000000..51d67f21 --- /dev/null +++ b/apps/firehose/README.md @@ -0,0 +1,3 @@ +# Observe Firehose + +This serverless application forwards data from a Firehose stream to S3. diff --git a/apps/firehose/template.yaml b/apps/firehose/template.yaml new file mode 100644 index 00000000..734b3a91 --- /dev/null +++ b/apps/firehose/template.yaml @@ -0,0 +1,132 @@ +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: 'Stream Firehose Records to S3.' +Metadata: + AWS::ServerlessRepo::Application: + Name: observe-cloudwatchlogs + Description: Stream Firehose Records to S3. + Author: Observe Inc + SpdxLicenseId: Apache-2.0 + ReadmeUrl: README.md + HomePageUrl: https://github.com/observeinc/aws-sam-testing + SemanticVersion: '0.0.1' + SourceCodeUrl: https://github.com/observeinc/aws-sam-testing + +Parameters: + BucketARN: + Type: String + Description: >- + S3 Bucket ARN to write log records to. + Prefix: + Type: String + Description: >- + Optional prefix to write log records to. + Default: '' + NameOverride: + Type: String + Description: >- + Set Firehose Delivery Stream name. In the absence of a value, the stack + name will be used. + Default: '' + BufferingInterval: + Type: Number + Default: 60 + MinValue: 60 + MaxValue: 900 + Description: | + Buffer incoming data for the specified period of time, in seconds, before + delivering it to the destination. + BufferingSize: + Type: Number + Default: 1 + MinValue: 1 + MaxValue: 64 + Description: | + Buffer incoming data to the specified size, in MiBs, before delivering it + to the destination. + +Conditions: + UseStackName: !Equals + - !Ref NameOverride + - '' + +Resources: + Role: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - firehose.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: logging + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - logs:CreateLogStream + - logs:PutLogEvents + Resource: !GetAtt LogGroup.Arn + - PolicyName: s3writer + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - s3:AbortMultipartUpload + - s3:GetBucketLocation + - s3:GetObject + - s3:ListBucket + - s3:ListBucketMultipartUploads + - s3:PutObject + Resource: + - !Ref BucketARN + - !Sub '${BucketARN}/${Prefix}*' + LogGroup: + Type: 'AWS::Logs::LogGroup' + Properties: + LogGroupName: !Join + - '' + - - /aws/firehose/ + - !If + - UseStackName + - !Ref AWS::StackName + - !Ref NameOverride + RetentionInDays: 365 + LogStream: + Type: 'AWS::Logs::LogStream' + Properties: + LogStreamName: s3logs + LogGroupName: !Ref LogGroup + DeliveryStream: + Type: 'AWS::KinesisFirehose::DeliveryStream' + Properties: + DeliveryStreamName: !If + - UseStackName + - !Ref AWS::StackName + - !Ref NameOverride + DeliveryStreamType: DirectPut + S3DestinationConfiguration: + BucketARN: !Ref BucketARN + RoleARN: !GetAtt Role.Arn + Prefix: !Ref Prefix + ErrorOutputPrefix: !Ref Prefix + BufferingHints: + IntervalInSeconds: !Ref BufferingInterval + SizeInMBs: !Ref BufferingSize + CloudWatchLoggingOptions: + Enabled: true + LogGroupName: !Ref LogGroup + LogStreamName: !Ref LogStream + +Outputs: + Firehose: + Description: 'Firehose ARN' + Value: !GetAtt 'DeliveryStream.Arn' diff --git a/integration/firehose.tftest.hcl b/integration/firehose.tftest.hcl new file mode 100644 index 00000000..1c088040 --- /dev/null +++ b/integration/firehose.tftest.hcl @@ -0,0 +1,70 @@ +run "setup" { + module { + source = "./modules/setup/run" + } +} + +run "install" { + variables { + name = run.setup.id + app = "firehose" + parameters = { + BucketARN = "arn:aws:s3:::${run.setup.access_point.bucket}" + } + capabilities = [ + "CAPABILITY_IAM", + ] + } +} + +run "check_firehose" { + module { + source = "./modules/exec" + } + + variables { + command = "./scripts/check_firehose" + env_vars = { + FIREHOSE_ARN = run.install.stack.outputs["Firehose"] + DESTINATION = "s3://${run.setup.access_point.bucket}/" + } + } + + assert { + condition = output.error == "" + error_message = "Failed to write firehose records" + } +} + +run "set_prefix" { + variables { + name = run.setup.id + app = "firehose" + parameters = { + BucketARN = "arn:aws:s3:::${run.setup.access_point.bucket}" + Prefix = "${run.setup.id}/" + } + capabilities = [ + "CAPABILITY_IAM", + ] + } +} + +run "check_firehose_prefix" { + module { + source = "./modules/exec" + } + + variables { + command = "./scripts/check_firehose" + env_vars = { + FIREHOSE_ARN = run.install.stack.outputs["Firehose"] + DESTINATION = "s3://${run.setup.access_point.bucket}/${run.setup.id}/" + } + } + + assert { + condition = output.error == "" + error_message = "Failed to write firehose records" + } +} diff --git a/integration/scripts/check_firehose b/integration/scripts/check_firehose new file mode 100755 index 00000000..12568a03 --- /dev/null +++ b/integration/scripts/check_firehose @@ -0,0 +1,57 @@ +#!/bin/bash +# Write records to firehose, verify data shows up in S3 +set -euo pipefail + +DIE() { echo "$*" 1>&2; exit 1; } + +[[ ! -z "${FIREHOSE_ARN:-}" ]] || DIE "FIREHOSE_ARN not set" +[[ ! -z "${DESTINATION:-}" ]] || DIE "DESTINATION not set" + +cleanup() { + rm -f "$TMPFILE" +} + +trap cleanup EXIT + +TMPFILE=$(mktemp) + +# Assuming a 1MB buffer limit, writing 2 x 500KB records will immediately flush +# a file to S3. +RECORD_SIZE=${RECORD_SIZE:-500k} +RECORD_COUNT=${RECORD_COUNT:-2} + +aws s3 ls ${DESTINATION}`date +%Y` --recursive && DIE "S3 destination already has records" + +FIREHOSE_NAME=$(echo "$FIREHOSE_ARN" | cut -d/ -f2) +AWS_REGION=$(echo "$FIREHOSE_ARN" | cut -d: -f4) + +RANDOM_DATA=$(dd if=/dev/urandom bs=${RECORD_SIZE} count=1 2>/dev/null | base64) + +echo "[" > ${TMPFILE} +for ((i = 1; i <= ${RECORD_COUNT}; i++)); do + if [ $i -gt 1 ]; then + echo "," >> ${TMPFILE} + fi + echo "{\"Data\":\"${RANDOM_DATA}\"}" >> ${TMPFILE} +done +echo "]" >> ${TMPFILE} + +aws firehose put-record-batch \ + --delivery-stream-name "${FIREHOSE_NAME}" \ + --records file://${TMPFILE} \ + --region ${AWS_REGION} \ + --no-cli-pager + +CHECK_INTERVAL=${CHECK_INTERVAL:-5} +CHECK_TIMEOUT=${CHECK_TIMEOUT:-60} + +# Wait up to `CHECK_TIMEOUT` seconds for file to appear +# A file can take quite a long time to flush after reconfiguring a firehose in +# particular. +for i in $(seq 0 ${CHECK_INTERVAL} ${CHECK_TIMEOUT}); do + if [ $i -gt 0 ]; then + sleep ${CHECK_INTERVAL} + fi + aws s3 ls ${DESTINATION}`date +%Y` --recursive && exit +done +DIE "Records not found" From d906f5e95c5e0d88c52661516ffad3161932bc3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Wed, 1 Nov 2023 15:02:36 -0700 Subject: [PATCH 3/5] fix: add verbose flag --- .github/workflows/tests-integration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml index 4c90b91f..41da5acb 100644 --- a/.github/workflows/tests-integration.yaml +++ b/.github/workflows/tests-integration.yaml @@ -84,7 +84,7 @@ jobs: run: | eval $(dce leases login --print-creds $LEASE_ID) aws sts get-caller-identity - make integration-test + TEST_ARGS=-verbose make integration-test env: APP: forwarder From b99c78da01d5e3a202f6f92ae080603582f9ca11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Wed, 1 Nov 2023 15:24:56 -0700 Subject: [PATCH 4/5] fix: remove line wrap --- integration/scripts/check_firehose | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/scripts/check_firehose b/integration/scripts/check_firehose index 12568a03..92d4ea51 100755 --- a/integration/scripts/check_firehose +++ b/integration/scripts/check_firehose @@ -25,7 +25,8 @@ aws s3 ls ${DESTINATION}`date +%Y` --recursive && DIE "S3 destination already ha FIREHOSE_NAME=$(echo "$FIREHOSE_ARN" | cut -d/ -f2) AWS_REGION=$(echo "$FIREHOSE_ARN" | cut -d: -f4) -RANDOM_DATA=$(dd if=/dev/urandom bs=${RECORD_SIZE} count=1 2>/dev/null | base64) +# base64 in linux sets a default line wrap. Using tr makes script agnostic to this behavior. +RANDOM_DATA=$(dd if=/dev/urandom bs=${RECORD_SIZE} count=1 2>/dev/null | base64 | tr -d \\n) echo "[" > ${TMPFILE} for ((i = 1; i <= ${RECORD_COUNT}; i++)); do From e8b9b297634081de4574d1178b22aeccd1b9b84d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Wed, 1 Nov 2023 16:00:47 -0700 Subject: [PATCH 5/5] fix: add debugging to wait step --- integration/scripts/check_firehose | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/scripts/check_firehose b/integration/scripts/check_firehose index 92d4ea51..91a6ff0f 100755 --- a/integration/scripts/check_firehose +++ b/integration/scripts/check_firehose @@ -44,13 +44,14 @@ aws firehose put-record-batch \ --no-cli-pager CHECK_INTERVAL=${CHECK_INTERVAL:-5} -CHECK_TIMEOUT=${CHECK_TIMEOUT:-60} +CHECK_TIMEOUT=${CHECK_TIMEOUT:-120} # Wait up to `CHECK_TIMEOUT` seconds for file to appear # A file can take quite a long time to flush after reconfiguring a firehose in # particular. for i in $(seq 0 ${CHECK_INTERVAL} ${CHECK_TIMEOUT}); do if [ $i -gt 0 ]; then + echo "waiting" sleep ${CHECK_INTERVAL} fi aws s3 ls ${DESTINATION}`date +%Y` --recursive && exit