feat: add collections module

Makefile

@@ -50,7 +50,6 @@ sam-build:
 sam-package: sam-build
 	# requires AWS credentials.
 	# currently dynamically generates bucket. We will want to use a fixed set of buckets for our production artifacts.
-	@ if [ -z "$(AWS_REGION)" ]; then echo >&2 please set AWS_REGION explicitly; exit 2; fi
 	sam package --template apps/$(APP)/.aws-sam/build/template.yaml --output-template-file apps/$(APP)/.aws-sam/build/packaged.yaml --region $(AWS_REGION) --debug --resolve-s3
 
 .PHONY: sam-publish

apps/collection/README.md

@@ -0,0 +1,2 @@
+AWS Collection
+--------------

apps/collection/samconfig.toml

@@ -0,0 +1,19 @@
+# More information about the configuration file can be found here:
+# https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-config.html
+version = 0.1
+
+[default.build.parameters]
+cached = true
+parallel = true
+
+[default.validate.parameters]
+lint = true
+
+[default.sync.parameters]
+watch = true
+
+[default.local_start_api.parameters]
+warm_containers = "EAGER"
+
+[default.local_start_lambda.parameters]
+warm_containers = "EAGER"

apps/collection/template.yaml

@@ -0,0 +1,138 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: >
+  Collect AWS data towards Observe 
+Metadata:
+  AWS::ServerlessRepo::Application:
+    Name: observe-aws-collection
+    Description: Collect resource, logs and metrics data from AWS
+    Author: Observe Inc
+    SpdxLicenseId: Apache-2.0
+    ReadmeUrl: README.md
+    HomePageUrl: https://github.com/observeinc/aws-sam-testing
+    SemanticVersion: 0.0.1
+    SourceCodeUrl: https://github.com/observeinc/aws-sam-testing
+
+Parameters:
+  Name:
+    Type: String
+    Description: >-
+      Name of IAM role expected by Filedrop. This role will be created as part
+      of this stack, and must therefore be unique within the account.
+    Default: ""
+  DataAccessPointArn:
+    Type: String
+    Description: >-
+      The access point ARN for your Filedrop.
+    Default: ""
+  DestinationUri:
+    Type: String
+    Description: >-
+      S3 URI to write data to.
+  InstallConfig:
+    Type: String
+    AllowedValues:
+      - "true"
+      - "false"
+    Default: "true"
+    Description: >-
+      Set to `true` to install AWS Config. AWS Config can only be configured
+      once per region. If you already have an existing AWS Config Recorder
+      installed, set this value to `false` and subscribe the S3 bucket
+      containing the configuration snapshots.
+
+Conditions:
+  EnableConfig: !Equals
+    - !Ref InstallConfig
+    - "true"
+
+Resources:
+  Topic:
+    Type: "AWS::SNS::Topic"
+  TopicPolicy:
+    Type: "AWS::SNS::TopicPolicy"
+    Properties:
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: "Config"
+            Effect: "Allow"
+            Principal:
+              Service:
+                - "config.amazonaws.com"
+            Action:
+              - "sns:Subscribe"
+              - "sns:Receive"
+              - "sns:ListSubscriptionsByTopic"
+            Resource:
+              - !Ref Topic
+          - Sid: "S3"
+            Effect: "Allow"
+            Principal:
+              Service:
+                - "s3.amazonaws.com"
+            Action:
+              - "sns:Publish"
+            Resource:
+              - !Ref Topic
+          - Sid: "Cloudformation"
+            Effect: "Allow"
+            Principal:
+              Service:
+                - "cloudformation.amazonaws.com"
+            Action:
+              - "sns:Publish"
+            Resource:
+              - !Ref Topic
+      Topics:
+        - !Ref Topic
+  Bucket:
+    Type: 'AWS::S3::Bucket'
+    DeletionPolicy: Retain
+    UpdateReplacePolicy: Retain
+    DependsOn: TopicPolicy
+    Properties:
+      LifecycleConfiguration:
+        Rules:
+          - ExpirationInDays: 1
+            Status: Enabled
+      NotificationConfiguration:
+        TopicConfigurations:
+          - Event: "s3:ObjectCreated:*"
+            Topic: !Ref Topic
+  Forwarder:
+    Type: AWS::Serverless::Application
+    Properties:
+      Location:
+        ApplicationId: arn:aws:serverlessrepo:us-east-1:739672403694:applications/observe-filedropper
+        SemanticVersion: 0.0.4
+      NotificationARNs:
+        - !Ref Topic
+      Parameters:
+        Name: !Ref Name
+        DataAccessPointArn: !Ref DataAccessPointArn
+        DestinationUri: !Ref DestinationUri
+        BucketNames: !Ref Bucket
+        SourceTopicArns: !Ref Topic
+  TopicSubscription:
+    Type: AWS::SNS::Subscription
+    Properties: 
+      Endpoint: !GetAtt
+        - Forwarder
+        - Outputs.Queue
+      Protocol: sqs
+      TopicArn: !Ref Topic
+  Config:
+    Type: AWS::Serverless::Application
+    DependsOn: Forwarder
+    Condition: EnableConfig
+    Properties:
+      Location:
+        ApplicationId: arn:aws:serverlessrepo:us-east-1:739672403694:applications/observe-aws-config
+        SemanticVersion: 0.0.4
+      NotificationARNs:
+        - !Ref Topic
+      Parameters:
+        BucketName: !Ref Bucket
+        TopicARN: !Ref Topic
+

apps/config/samconfig.toml

@@ -2,26 +2,13 @@
 # https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-config.html
 version = 0.1
 
-[default]
-[default.global.parameters]
-stack_name = "config"
-
 [default.build.parameters]
 cached = true
 parallel = true
 
 [default.validate.parameters]
 lint = true
 
-[default.deploy.parameters]
-capabilities = "CAPABILITY_IAM"
-confirm_changeset = true
-resolve_s3 = true
-
-[default.package.parameters]
-resolve_s3 = true
-
-
 [default.sync.parameters]
 watch = true
 

apps/config/template.yaml

@@ -11,7 +11,7 @@ Metadata:
     SpdxLicenseId: Apache-2.0
     ReadmeUrl: README.md
     HomePageUrl: https://github.com/observeinc/aws-sam-testing
-    SemanticVersion: 0.0.2
+    SemanticVersion: 0.0.4
     SourceCodeUrl: https://github.com/observeinc/aws-sam-testing
 
   AWS::CloudFormation::Interface:
@@ -101,9 +101,8 @@ Resources:
         AllSupported: true
         IncludeGlobalResourceTypes: true
       RoleARN: !GetAtt ConfigurationRecorderRole.Arn
-  ConfigDeliveryChannel:
+  ConfigurationDeliveryChannel:
     Type: AWS::Config::DeliveryChannel
-    DependsOn: ConfigurationRecorder
     Properties:
       Name: default
       ConfigSnapshotDeliveryProperties:

apps/filedropper/README.md

@@ -1,3 +1,29 @@
-Another readme
---------------
+# Observe Forwarder
 
+This serverless application forwards data to an Observe FileDrop. Data is
+forwarded by a lambda in two ways:
+
+- any `s3:ObjectCreated` events trigger a copy from the source bucket to the destination bucket.
+- all events read out of an SQS queue are written to a file in the destination bucket.
+
+You can use the Observe Forwarder as a cost effective means of loading files
+into Observe or for exporting event streams such as EventBridge or SNS data.
+
+## What it does
+
+## Subscribing an S3 bucket
+
+
+## How it works
+
+## Configuration Options
+
+### How long does setup take?
+
+### How do I subscribe an S3 Bucket?
+
+### How do I subscribe an SNS topic?
+
+### How do I subscribe EventBridge events?
+
+## Troubleshooting 

apps/filedropper/samconfig.toml

@@ -2,23 +2,13 @@
 # https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-config.html
 version = 0.1
 
-[default]
-[default.global.parameters]
-stack_name = "filedropper"
-parameter_overrides="ParameterKey=BucketNames,ParameterValue=* ParameterKey=Name,ParameterValue=filedropper-us-west-2 ParameterKey=DataAccessPointArn,ParameterValue=arn:aws:s3:us-west-2:723346149663:accesspoint/157675308050 ParameterKey=DestinationUri,ParameterValue=s3://157675308050-s1n88tucur4944c6p3ymbu6hr758gusw2a-s3alias/ds1pXOFUn7eUFYpD4dZD/"
-
 [default.build.parameters]
 cached = true
 parallel = true
 
 [default.validate.parameters]
 lint = true
 
-[default.deploy.parameters]
-capabilities = "CAPABILITY_IAM"
-confirm_changeset = true
-resolve_s3 = true
-
 [default.sync.parameters]
 watch = true
 

apps/filedropper/template.yaml

@@ -11,7 +11,7 @@ Metadata:
     SpdxLicenseId: Apache-2.0
     ReadmeUrl: README.md
     HomePageUrl: https://github.com/observeinc/aws-sam-testing
-    SemanticVersion: 0.0.2
+    SemanticVersion: 0.0.4
     SourceCodeUrl: https://github.com/observeinc/aws-sam-testing
 
   AWS::CloudFormation::Interface: