Merge pull request #168 from airbnb/jacknaglieri-add-vpc-config-to-st…

…ream-alert

Support Alert Processor in VPC, Various Bug Fixes

stream_alert_cli/runner.py

@@ -152,6 +152,13 @@ def terraform_runner(options):
         tf_runner()
 
     elif options.subcommand == 'destroy':
+        if options.target:
+            target = options.target
+            targets = ['module.{}_{}'.format(target, cluster)
+                       for cluster in CONFIG['clusters'].keys()]
+            tf_runner(targets=targets, action='destroy')
+            return
+
         # Migrate back to local state so Terraform can successfully
         # destroy the S3 bucket used by the backend.
         terraform_generate(config=CONFIG, init=True)
@@ -256,21 +263,25 @@ def tf_runner(**kwargs):
 
 def status():
     """Display current AWS infrastructure built by Terraform"""
-    print 'Cluster Info\n'
     for cluster, region in CONFIG['clusters'].iteritems():
-        print '==== {} ==='.format(cluster)
+        print '\n======== {} ========'.format(cluster)
         print 'Region: {}'.format(region)
-        print ('Lambda settings: \n\tTimeout: {}\n\tMemory: {}'
+        print ('Alert Processor Lambda Settings: \n\tTimeout: {}\n\tMemory: {}'
+               '\n\tProd Version: {}').format(
+                   CONFIG['alert_processor_lambda_config'][cluster][0],
+                   CONFIG['alert_processor_lambda_config'][cluster][1],
+                   CONFIG['alert_processor_versions'][cluster])
+        print ('Rule Processor Lambda Settings: \n\tTimeout: {}\n\tMemory: {}'
                '\n\tProd Version: {}').format(
-                   CONFIG['lambda_settings'][cluster][0],
-                   CONFIG['lambda_settings'][cluster][1],
-                   CONFIG['lambda_function_prod_versions'][cluster])
-        print 'Kinesis settings: \n\tShards: {}\n\tRetention: {}\n'.format(
-            CONFIG['kinesis_settings'][cluster][0],
-            CONFIG['kinesis_settings'][cluster][1]
+                   CONFIG['rule_processor_lambda_config'][cluster][0],
+                   CONFIG['rule_processor_lambda_config'][cluster][1],
+                   CONFIG['rule_processor_versions'][cluster])
+        print 'Kinesis settings: \n\tShards: {}\n\tRetention: {}'.format(
+            CONFIG['kinesis_streams_config'][cluster][0],
+            CONFIG['kinesis_streams_config'][cluster][1]
         )
 
-    print 'User access keys'
+    print '\nUser Access Keys:'
     run_command(['terraform', 'output'])
 
 

terraform/modules/tf_stream_alert/README.md

@@ -40,6 +40,24 @@ module "stream_alert" {
     <td>None</td>
     <td>True</td>
   </tr>
+  <tr>
+    <td>alert_processor_vpc_enabled</td>
+    <td>To enable/disable placing the Alert Processor inside a VPC</td>
+    <td>False</td>
+    <td>False/td>
+  </tr>
+  <tr>
+    <td>alert_processor_vpc_subnet_ids</td>
+    <td>The subnet IDs to place the Alert Processor</td>
+    <td>[]</td>
+    <td>False</td>
+  </tr>
+  <tr>
+    <td>alert_processor_vpc_security_group_ids</td>
+    <td>The security group IDs to assign to the Alert Processor</td>
+    <td>[]</td>
+    <td>False</td>
+  </tr>
   <tr>
     <td>region</td>
     <td>The AWS region for your stream</td>

terraform/modules/tf_stream_alert/iam.tf

@@ -1,150 +1,141 @@
+/*
 // Rule Processor Execution Role
+*/
 resource "aws_iam_role" "streamalert_rule_processor_role" {
   name = "${var.prefix}_${var.cluster}_streamalert_rule_processor_role"
 
-  assume_role_policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Action": "sts:AssumeRole",
-      "Principal": {
-        "Service": "lambda.amazonaws.com"
-      },
-      "Effect": "Allow",
-      "Sid": ""
-    }
-  ]
+  assume_role_policy = "${data.aws_iam_policy_document.lambda_assume_role_policy.json}"
 }
-EOF
+
+data "aws_iam_policy_document" "lambda_assume_role_policy" {
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+  }
 }
 
-// Allow the Rule Processor to send alerts to SNS
+// Policy: Allow the Rule Processor to send alerts to SNS
 resource "aws_iam_role_policy" "streamalert_rule_processor_sns" {
   name = "${var.prefix}_${var.cluster}_streamalert_rule_processor_send_to_sns"
   role = "${aws_iam_role.streamalert_rule_processor_role.id}"
 
-  policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Action": [
-        "sns:Publish",
-        "sns:Subscribe"
-      ],
-      "Effect": "Allow",
-      "Resource": "${aws_sns_topic.streamalert.arn}"
-    }
-  ]
+  policy = "${data.aws_iam_policy_document.rule_processor_sns.json}"
 }
-EOF
+
+data "aws_iam_policy_document" "rule_processor_sns" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "sns:Publish",
+      "sns:Subscribe",
+    ]
+
+    resources = [
+      "${aws_sns_topic.streamalert.arn}",
+    ]
+  }
 }
 
+/*
 // Alert Processor Execution Role
+*/
 resource "aws_iam_role" "streamalert_alert_processor_role" {
   name = "${var.prefix}_${var.cluster}_streamalert_alert_processor_role"
 
-  assume_role_policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Action": "sts:AssumeRole",
-      "Principal": {
-        "Service": "lambda.amazonaws.com"
-      },
-      "Effect": "Allow",
-      "Sid": ""
-    }
-  ]
-}
-EOF
+  assume_role_policy = "${data.aws_iam_policy_document.lambda_assume_role_policy.json}"
 }
 
-// Allow the Alert Processor to decrypt secrets
+// Policy: Allow the Alert Processor to decrypt secrets
 resource "aws_iam_role_policy" "streamalert_alert_processor_kms" {
   name = "${var.prefix}_${var.cluster}_streamalert_alert_processor_kms"
   role = "${aws_iam_role.streamalert_alert_processor_role.id}"
 
-  policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Action": [
-        "kms:Decrypt",
-        "kms:DescribeKey"
-      ],
-      "Effect": "Allow",
-      "Resource": "${var.kms_key_arn}"
-    }
-  ]
+  policy = "${data.aws_iam_policy_document.rule_processor_kms_decrypt.json}"
 }
-EOF
+
+data "aws_iam_policy_document" "rule_processor_kms_decrypt" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:DescribeKey",
+    ]
+
+    resources = [
+      "${var.kms_key_arn}",
+    ]
+  }
 }
 
-// Allow the Alert Processor to write objects to S3
-// Default s3 bucket created by this module.
+// Policy: Allow the Alert Processor to write objects to S3.
+//         The default S3 bucket is also created by this module.
 resource "aws_iam_role_policy" "streamalert_alert_processor_s3" {
   name = "${var.prefix}_${var.cluster}_streamalert_alert_processor_s3_default"
   role = "${aws_iam_role.streamalert_alert_processor_role.id}"
 
-  policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Action": [
-        "s3:PutObject",
-        "s3:PutObjectAcl",
-        "s3:ListBucket"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "${aws_s3_bucket.streamalerts.arn}/*"
-      ]
-    },
-    {
-      "Action": [
-        "s3:GetObject"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "arn:aws:s3:::${var.prefix}.streamalert.secrets/*"
-      ]
-    }
-  ]
+  policy = "${data.aws_iam_policy_document.alert_processor_s3.json}"
 }
-EOF
+
+data "aws_iam_policy_document" "alert_processor_s3" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3:PutObject",
+      "s3:PutObjectAcl",
+      "s3:ListBucket",
+    ]
+
+    resources = [
+      "${aws_s3_bucket.streamalerts.arn}/*",
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "s3:GetObject",
+    ]
+
+    resources = [
+      "arn:aws:s3:::${var.prefix}.streamalert.secrets/*",
+    ]
+  }
 }
 
-// Allow the Alert Processor to write cloudwatch logs
+// Policy: Allow the Alert Processor to write cloudwatch logs
 resource "aws_iam_role_policy" "streamalert_alert_processor_cloudwatch" {
   name = "${var.prefix}_${var.cluster}_streamalert_alert_processor_cloudwatch"
   role = "${aws_iam_role.streamalert_alert_processor_role.id}"
 
-  policy = <<EOF
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Action": [
-        "logs:CreateLogGroup",
-        "logs:CreateLogStream",
-        "logs:PutLogEvents"
-      ],
-      "Effect": "Allow",
-      "Resource": [
-        "*"
-      ]
-    }
-  ]
+  policy = "${data.aws_iam_policy_document.alert_processor_cloudwatch.json}"
 }
-EOF
+
+data "aws_iam_policy_document" "alert_processor_cloudwatch" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+    ]
+
+    resources = [
+      "*",
+    ]
+  }
 }
 
-// Allow the Alert Processor to invoke Lambda
+// Policy: Allow the Alert Processor to invoke Lambda functions
 resource "aws_iam_role_policy" "streamalert_alert_processor_lambda" {
   count = "${length(keys(var.output_lambda_functions))}"
   name  = "${var.prefix}_${var.cluster}_streamalert_alert_processor_lambda_${element(keys(var.output_lambda_functions), count.index)}"
@@ -166,7 +157,7 @@ resource "aws_iam_role_policy" "streamalert_alert_processor_lambda" {
 EOF
 }
 
-// Allow the Alert Processor to send to arbitrary S3 buckets as outputs
+// Policy: Allow the Alert Processor to send to arbitrary S3 buckets as outputs
 resource "aws_iam_role_policy" "streamalert_alert_processor_s3_outputs" {
   count = "${length(keys(var.output_s3_buckets))}"
   name  = "${var.prefix}_${var.cluster}_streamalert_alert_processor_s3_output_${element(keys(var.output_s3_buckets), count.index)}"
@@ -189,3 +180,30 @@ resource "aws_iam_role_policy" "streamalert_alert_processor_s3_outputs" {
 }
 EOF
 }
+
+// Policy: Allow the Alert Processor to run in a VPC
+resource "aws_iam_role_policy" "streamalert_alert_processor_vpc" {
+  count = "${var.alert_processor_vpc_enabled ? 1 : 0}"
+  name  = "${var.prefix}_${var.cluster}_streamalert_alert_processor_vpc"
+  role  = "${aws_iam_role.streamalert_alert_processor_role.id}"
+
+  policy = "${data.aws_iam_policy_document.alert_processor_vpc.json}"
+}
+
+data "aws_iam_policy_document" "alert_processor_vpc" {
+  count = "${var.alert_processor_vpc_enabled ? 1 : 0}"
+
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ec2:CreateNetworkInterface",
+      "ec2:DescribeNetworkInterfaces",
+      "ec2:DeleteNetworkInterface",
+    ]
+
+    resources = [
+      "*",
+    ]
+  }
+}