From 7c39837e4a2d639e6b68ee3765b85d3d7944d2f4 Mon Sep 17 00:00:00 2001 From: Jack Naglieri Date: Thu, 13 Jul 2017 11:08:49 -0700 Subject: [PATCH 1/2] [tf] use a single streamalerts bucket for all clusters * terraform generate * unit testing * tf_stream_alert module update --- stream_alert_cli/terraform_generate.py | 26 +++-- terraform/modules/tf_stream_alert/iam.tf | 2 +- terraform/modules/tf_stream_alert/main.tf | 16 --- .../modules/tf_stream_alert/variables.tf | 4 - .../test_terraform_generate.py | 105 ++++++++++++++++-- 5 files changed, 114 insertions(+), 39 deletions(-) diff --git a/stream_alert_cli/terraform_generate.py b/stream_alert_cli/terraform_generate.py index 2ade7ce4e..12364db31 100644 --- a/stream_alert_cli/terraform_generate.py +++ b/stream_alert_cli/terraform_generate.py @@ -138,6 +138,10 @@ def generate_main(**kwargs): acl='log-delivery-write', logging=logging_bucket, lifecycle_rule=logging_bucket_lifecycle + ), + 'streamalerts': generate_s3_bucket( + bucket='{}.streamalerts'.format(config['global']['account']['prefix']), + logging=logging_bucket ) } @@ -219,14 +223,13 @@ def generate_stream_alert(cluster_name, cluster_dict, config): 'alert_processor_config': '${var.alert_processor_config}', 'alert_processor_memory': modules['stream_alert']['alert_processor']['memory'], 'alert_processor_timeout': modules['stream_alert']['alert_processor']['timeout'], - 'alert_processor_version': modules['stream_alert']['alert_processor']['current_version'], - 's3_logging_bucket': '{}.streamalert.s3-logging'.format( - config['global']['account']['prefix']) + 'alert_processor_version': modules['stream_alert']['alert_processor']['current_version'] } - # Add Alert Processor output config + # Add Alert Processor output config from the loaded cluster file output_config = modules['stream_alert']['alert_processor'].get('outputs') if output_config: + # Mapping of Terraform input variables to output config variables output_mapping = { 'output_lambda_functions': 'aws-lambda', 'output_s3_buckets': 'aws-s3' @@ -237,14 +240,19 @@ def generate_stream_alert(cluster_name, cluster_dict, config): tf_key: modules['stream_alert']['alert_processor']['outputs'][output] }) - # Add Rule Processor input config + # Add Rule Processor input config from the loaded cluster file input_config = modules['stream_alert']['rule_processor'].get('inputs') if input_config: - cluster_dict['module']['stream_alert_{}'.format(cluster_name)].update({ - 'input_sns_topics': input_config['aws-sns'] - }) + input_mapping = { + 'input_sns_topics': 'aws-sns' + } + for tf_key, input_key in input_mapping.iteritems(): + if input_key in input_config: + cluster_dict['module']['stream_alert_{}'.format(cluster_name)].update({ + tf_key: input_config[input_key] + }) - # Add the Alert Processor VPC config + # Add the Alert Processor VPC config from the loaded cluster file vpc_config = modules['stream_alert']['alert_processor'].get('vpc_config') if vpc_config: cluster_dict['module']['stream_alert_{}'.format(cluster_name)].update({ diff --git a/terraform/modules/tf_stream_alert/iam.tf b/terraform/modules/tf_stream_alert/iam.tf index c4793dbf4..948e2bfb4 100644 --- a/terraform/modules/tf_stream_alert/iam.tf +++ b/terraform/modules/tf_stream_alert/iam.tf @@ -94,7 +94,7 @@ data "aws_iam_policy_document" "alert_processor_s3" { ] resources = [ - "${aws_s3_bucket.streamalerts.arn}/*", + "arn:aws:s3:::${var.prefix}.streamalerts/*", ] } diff --git a/terraform/modules/tf_stream_alert/main.tf b/terraform/modules/tf_stream_alert/main.tf index 130946fb8..2b9c3b4e0 100644 --- a/terraform/modules/tf_stream_alert/main.tf +++ b/terraform/modules/tf_stream_alert/main.tf @@ -122,19 +122,3 @@ resource "aws_lambda_permission" "with_sns" { qualifier = "production" depends_on = ["aws_lambda_alias.alert_processor_production"] } - -// S3 bucket for S3 outputs -resource "aws_s3_bucket" "streamalerts" { - bucket = "${replace("${var.prefix}.${var.cluster}.streamalerts", "_", ".")}" - acl = "private" - force_destroy = false - - versioning { - enabled = true - } - - logging { - target_bucket = "${var.s3_logging_bucket}" - target_prefix = "${replace("${var.prefix}.${var.cluster}.streamalerts", "_", ".")}/" - } -} diff --git a/terraform/modules/tf_stream_alert/variables.tf b/terraform/modules/tf_stream_alert/variables.tf index 5503bd4b7..3a962ca84 100644 --- a/terraform/modules/tf_stream_alert/variables.tf +++ b/terraform/modules/tf_stream_alert/variables.tf @@ -68,7 +68,3 @@ variable "alert_processor_vpc_security_group_ids" { type = "list" default = [] } - -variable "s3_logging_bucket" { - type = "string" -} diff --git a/test/unit/stream_alert_cli/test_terraform_generate.py b/test/unit/stream_alert_cli/test_terraform_generate.py index dd65930d4..3f807b3b0 100644 --- a/test/unit/stream_alert_cli/test_terraform_generate.py +++ b/test/unit/stream_alert_cli/test_terraform_generate.py @@ -119,12 +119,25 @@ def setup(self): 'security_group_ids': [ 'sg-id-1' ] + }, + 'outputs': { + 'aws-lambda': [ + 'my-lambda-function:production' + ], + 'aws-s3': [ + 'my-s3-bucket.with.data' + ] } }, 'rule_processor': { 'current_version': '$LATEST', 'memory': 128, - 'timeout': 25 + 'timeout': 25, + 'inputs': { + 'aws-sns': [ + 'my-sns-topic-name' + ] + } } }, 'cloudtrail': { @@ -151,10 +164,6 @@ def setup(self): } } - def teardown(self): - """Teardown after each method""" - pass - @staticmethod def test_generate_s3_bucket(): """CLI - Terraform Generate S3 Bucket """ @@ -290,6 +299,18 @@ def test_generate_main(self): 'storage_class': 'GLACIER' } } + }, + 'streamalerts': { + 'bucket': 'unit-testing.streamalerts', + 'acl': 'private', + 'force_destroy': True, + 'versioning': { + 'enabled': True + }, + 'logging': { + 'target_bucket': 'unit-testing.streamalert.s3-logging', + 'target_prefix': 'unit-testing.streamalerts/' + } } } } @@ -300,10 +321,76 @@ def test_generate_main(self): assert_equal(tf_main['resource'], tf_main_expected['resource']) - def test_generate_stream_alert(self): - """CLI - Terraform Generate stream_alert Module""" - # TODO(jacknagz): Write this test - pass + def test_generate_stream_alert_test(self): + """CLI - Terraform Generate stream_alert Module (test cluster)""" + terraform_generate.generate_stream_alert( + 'test', + self.cluster_dict, + self.config + ) + + expected_test_cluster = { + 'module': { + 'stream_alert_test': { + 'source': 'modules/tf_stream_alert', + 'account_id': '12345678910', + 'region': 'us-west-1', + 'prefix': 'unit-testing', + 'cluster': 'test', + 'kms_key_arn': '${aws_kms_key.stream_alert_secrets.arn}', + 'rule_processor_memory': 128, + 'rule_processor_timeout': 25, + 'rule_processor_version': '$LATEST', + 'rule_processor_config': '${var.rule_processor_config}', + 'alert_processor_memory': 128, + 'alert_processor_timeout': 25, + 'alert_processor_version': '$LATEST', + 'alert_processor_config': '${var.alert_processor_config}', + } + } + } + + assert_equal(self.cluster_dict['module']['stream_alert_test'], + expected_test_cluster['module']['stream_alert_test']) + + + def test_generate_stream_alert_advanced(self): + """CLI - Terraform Generate stream_alert Module (advanced cluster)""" + terraform_generate.generate_stream_alert( + 'advanced', + self.cluster_dict, + self.config + ) + + expected_advanced_cluster = { + 'module': { + 'stream_alert_advanced': { + 'source': 'modules/tf_stream_alert', + 'account_id': '12345678910', + 'region': 'us-west-1', + 'prefix': 'unit-testing', + 'cluster': 'advanced', + 'kms_key_arn': '${aws_kms_key.stream_alert_secrets.arn}', + 'rule_processor_memory': 128, + 'rule_processor_timeout': 25, + 'rule_processor_version': '$LATEST', + 'rule_processor_config': '${var.rule_processor_config}', + 'alert_processor_memory': 128, + 'alert_processor_timeout': 25, + 'alert_processor_version': '$LATEST', + 'alert_processor_config': '${var.alert_processor_config}', + 'output_lambda_functions': ['my-lambda-function:production'], + 'output_s3_buckets': ['my-s3-bucket.with.data'], + 'input_sns_topics': ['my-sns-topic-name'], + 'alert_processor_vpc_enabled': True, + 'alert_processor_vpc_subnet_ids': ['subnet-id-1'], + 'alert_processor_vpc_security_group_ids': ['sg-id-1'] + } + } + } + + assert_equal(self.cluster_dict['module']['stream_alert_advanced'], + expected_advanced_cluster['module']['stream_alert_advanced']) def test_generate_flow_logs(self): From cd63e12311f4cb5bbc6cb3da0488b8c710fe5de3 Mon Sep 17 00:00:00 2001 From: Jack Naglieri Date: Thu, 13 Jul 2017 15:09:07 -0700 Subject: [PATCH 2/2] [cli] add the streamalerts bucket to the init command --- stream_alert_cli/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stream_alert_cli/runner.py b/stream_alert_cli/runner.py index cdc1d7a3d..43ab1ff0e 100644 --- a/stream_alert_cli/runner.py +++ b/stream_alert_cli/runner.py @@ -144,6 +144,7 @@ def terraform_handler(options): 'aws_s3_bucket.logging_bucket', 'aws_s3_bucket.stream_alert_secrets', 'aws_s3_bucket.terraform_remote_state', + 'aws_s3_bucket.streamalerts', 'aws_kms_key.stream_alert_secrets', 'aws_kms_alias.stream_alert_secrets' ]