Merge pull request #118 from airbnb/jacknagz-terraform-sns-lambda-sup…

…port

[tf] support SNS as an input, and arbitrary S3/Lambda functions as outputs

conf/inputs.json

@@ -0,0 +1,5 @@
+{
+  "aws-sns": {
+    "sample_sns": "arn:aws:sns:us-region-1:111111111111:topicname"
+  }
+}

conf/outputs.json

@@ -1,14 +1,17 @@
 {
-    "aws-s3": {
-        "sample.bucket": "sample_bucket_name"
-    },
-    "pagerduty": [
-        "sample_integration"
-    ],
-    "phantom": [
-        "sample_integration"
-    ],
-    "slack": [
-        "sample_channel"
-    ]
+  "aws-s3": {
+    "sample.bucket": "sample_bucket_name"
+  },
+  "aws-lambda": {
+    "sample_lambda": "arn:aws:lambda:region:account-id:function:function-name"
+  },
+  "pagerduty": [
+    "sample_integration"
+  ],
+  "phantom": [
+    "sample_integration"
+  ],
+  "slack": [
+    "sample_channel"
+  ]
 }

stream_alert_cli/runner.py

@@ -204,7 +204,8 @@ def tf_runner(**kwargs):
     refresh_state = kwargs.get('refresh_state', True)
     tf_action_index = 1  # The index to the terraform 'action'
 
-    tf_opts = ['-var-file=../{}'.format(CONFIG.filename)]
+    var_files = {CONFIG.filename, 'conf/outputs.json', 'conf/inputs.json'}
+    tf_opts = ['-var-file=../{}'.format(x) for x in var_files]
     tf_targets = ['-target={}'.format(x) for x in targets]
     tf_command = ['terraform', 'plan'] + tf_opts + tf_targets
     if action == 'destroy':

terraform/modules/tf_stream_alert/iam.tf

@@ -146,8 +146,9 @@ EOF
 
 // Allow the Alert Processor to invoke Lambda
 resource "aws_iam_role_policy" "streamalert_alert_processor_lambda" {
-  name = "${var.prefix}_${var.cluster}_streamalert_alert_processor_lambda"
-  role = "${aws_iam_role.streamalert_alert_processor_role.id}"
+  count = "${length(keys(var.output_lambda_functions))}"
+  name  = "${var.prefix}_${var.cluster}_streamalert_alert_processor_lambda_${element(keys(var.output_lambda_functions), count.index)}"
+  role  = "${aws_iam_role.streamalert_alert_processor_role.id}"
 
   policy = <<EOF
 {
@@ -158,7 +159,31 @@ resource "aws_iam_role_policy" "streamalert_alert_processor_lambda" {
         "lambda:InvokeFunction"
       ],
       "Effect": "Allow",
-      "Resource": "*"
+      "Resource": "${lookup(var.output_lambda_functions, element(keys(var.output_lambda_functions), count.index))}"
+    }
+  ]
+}
+EOF
+}
+
+// Allow the Alert Processor to send to arbitrary S3 buckets as outputs
+resource "aws_iam_role_policy" "streamalert_alert_processor_s3_outputs" {
+  count = "${length(keys(var.output_s3_buckets))}"
+  name  = "${var.prefix}_${var.cluster}_streamalert_alert_processor_s3_output_${element(keys(var.output_s3_buckets), count.index)}"
+  role  = "${aws_iam_role.streamalert_alert_processor_role.id}"
+
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "s3:PutObject",
+        "s3:PutObjectAcl",
+        "s3:ListBucket"
+      ],
+      "Effect": "Allow",
+      "Resource": "arn:aws:s3:::${lookup(var.output_s3_buckets, element(keys(var.output_s3_buckets), count.index))}"
     }
   ]
 }

terraform/modules/tf_stream_alert/main.tf

@@ -20,6 +20,17 @@ resource "aws_lambda_alias" "rule_processor_production" {
   function_version = "${var.rule_processor_versions["${var.cluster}"]}"
 }
 
+// Allow SNS to invoke the StreamAlert Output Processor
+resource "aws_lambda_permission" "sns_inputs" {
+  count         = "${length(keys(var.input_sns_topics))}"
+  statement_id  = "AllowExecutionFromSNS_${element(keys(var.input_sns_topics), count.index)}"
+  action        = "lambda:InvokeFunction"
+  function_name = "${aws_lambda_function.streamalert_rule_processor.arn}"
+  principal     = "sns.amazonaws.com"
+  source_arn    = "${lookup(var.input_sns_topics, element(keys(var.input_sns_topics), count.index))}"
+  qualifier     = "production"
+}
+
 // AWS Lambda Function: StreamAlert Alert Processor
 //    Send alerts to declared outputs
 resource "aws_lambda_function" "streamalert_alert_processor" {

terraform/modules/tf_stream_alert/sns.tf

@@ -10,3 +10,11 @@ resource "aws_sns_topic_subscription" "alert_processor" {
   endpoint  = "${aws_lambda_function.streamalert_alert_processor.arn}:production"
   protocol  = "lambda"
 }
+
+// Subscribe the Rule Processor Lambda function to arbitrary SNS topics
+resource "aws_sns_topic_subscription" "input_topic_subscriptions" {
+  count     = "${length(keys(var.input_sns_topics))}"
+  topic_arn = "${lookup(var.input_sns_topics, element(keys(var.input_sns_topics), count.index))}"
+  endpoint  = "${aws_lambda_function.streamalert_rule_processor.arn}:production"
+  protocol  = "lambda"
+}

terraform/modules/tf_stream_alert/variables.tf

@@ -47,3 +47,18 @@ variable "alert_processor_versions" {
   type    = "map"
   default = {}
 }
+
+variable "output_lambda_functions" {
+  type    = "map"
+  default = {}
+}
+
+variable "output_s3_buckets" {
+  type    = "map"
+  default = {}
+}
+
+variable "input_sns_topics" {
+  type    = "map"
+  default = {}
+}

terraform/variables.tf

@@ -1,20 +1,35 @@
 variable "account" {
-  type = "map"
+  type    = "map"
   default = {}
 }
 
 variable "alert_processor_config" {
-  type = "map"
+  type    = "map"
   default = {}
 }
 
 variable "alert_processor_lambda_config" {
-  type = "map"
+  type    = "map"
   default = {}
 }
 
 variable "alert_processor_versions" {
-  type = "map"
+  type    = "map"
+  default = {}
+}
+
+variable "aws-lambda" {
+  type    = "map"
+  default = {}
+}
+
+variable "aws-s3" {
+  type    = "map"
+  default = {}
+}
+
+variable "aws-sns" {
+  type    = "map"
   default = {}
 }
 
@@ -39,21 +54,21 @@ variable "kinesis_streams_config" {
 }
 
 variable "rule_processor_config" {
-  type = "map"
+  type    = "map"
   default = {}
 }
 
 variable "rule_processor_lambda_config" {
-  type = "map"
+  type    = "map"
   default = {}
 }
 
 variable "rule_processor_versions" {
-  type = "map"
+  type    = "map"
   default = {}
 }
 
 variable "terraform" {
-  type = "map"
+  type    = "map"
   default = {}
 }