Skip to content

Commit

Permalink
Merge pull request #224 from airbnb/jacknaglieri-single-alert-bucket
Browse files Browse the repository at this point in the history 
Consolidate StreamAlerts Buckets
  • Loading branch information
@jacknagz
jacknagz authored Jul 13, 2017
2 parents 424f6dd + cd63e12 commit c50856c
Show file tree
Hide file tree
Showing 6 changed files with 115 additions and 39 deletions.
1 change: 1 addition & 0 deletions stream_alert_cli/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ def terraform_handler(options):
'aws_s3_bucket.logging_bucket',
'aws_s3_bucket.stream_alert_secrets',
'aws_s3_bucket.terraform_remote_state',
'aws_s3_bucket.streamalerts',
'aws_kms_key.stream_alert_secrets',
'aws_kms_alias.stream_alert_secrets'
]
Expand Down
26 changes: 17 additions & 9 deletions stream_alert_cli/terraform_generate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,10 @@ def generate_main(**kwargs):
acl='log-delivery-write',
logging=logging_bucket,
lifecycle_rule=logging_bucket_lifecycle
),
'streamalerts': generate_s3_bucket(
bucket='{}.streamalerts'.format(config['global']['account']['prefix']),
logging=logging_bucket
)
}

Expand Down Expand Up @@ -219,14 +223,13 @@ def generate_stream_alert(cluster_name, cluster_dict, config):
'alert_processor_config': '${var.alert_processor_config}',
'alert_processor_memory': modules['stream_alert']['alert_processor']['memory'],
'alert_processor_timeout': modules['stream_alert']['alert_processor']['timeout'],
'alert_processor_version': modules['stream_alert']['alert_processor']['current_version'],
's3_logging_bucket': '{}.streamalert.s3-logging'.format(
config['global']['account']['prefix'])
'alert_processor_version': modules['stream_alert']['alert_processor']['current_version']
}

# Add Alert Processor output config
# Add Alert Processor output config from the loaded cluster file
output_config = modules['stream_alert']['alert_processor'].get('outputs')
if output_config:
# Mapping of Terraform input variables to output config variables
output_mapping = {
'output_lambda_functions': 'aws-lambda',
'output_s3_buckets': 'aws-s3'
Expand All @@ -237,14 +240,19 @@ def generate_stream_alert(cluster_name, cluster_dict, config):
tf_key: modules['stream_alert']['alert_processor']['outputs'][output]
})

# Add Rule Processor input config
# Add Rule Processor input config from the loaded cluster file
input_config = modules['stream_alert']['rule_processor'].get('inputs')
if input_config:
cluster_dict['module']['stream_alert_{}'.format(cluster_name)].update({
'input_sns_topics': input_config['aws-sns']
})
input_mapping = {
'input_sns_topics': 'aws-sns'
}
for tf_key, input_key in input_mapping.iteritems():
if input_key in input_config:
cluster_dict['module']['stream_alert_{}'.format(cluster_name)].update({
tf_key: input_config[input_key]
})

# Add the Alert Processor VPC config
# Add the Alert Processor VPC config from the loaded cluster file
vpc_config = modules['stream_alert']['alert_processor'].get('vpc_config')
if vpc_config:
cluster_dict['module']['stream_alert_{}'.format(cluster_name)].update({
Expand Down
2 changes: 1 addition & 1 deletion terraform/modules/tf_stream_alert/iam.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ data "aws_iam_policy_document" "alert_processor_s3" {
]

resources = [
"${aws_s3_bucket.streamalerts.arn}/*",
"arn:aws:s3:::${var.prefix}.streamalerts/*",
]
}

Expand Down
16 changes: 0 additions & 16 deletions terraform/modules/tf_stream_alert/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,19 +122,3 @@ resource "aws_lambda_permission" "with_sns" {
qualifier = "production"
depends_on = ["aws_lambda_alias.alert_processor_production"]
}

// S3 bucket for S3 outputs
resource "aws_s3_bucket" "streamalerts" {
bucket = "${replace("${var.prefix}.${var.cluster}.streamalerts", "_", ".")}"
acl = "private"
force_destroy = false

versioning {
enabled = true
}

logging {
target_bucket = "${var.s3_logging_bucket}"
target_prefix = "${replace("${var.prefix}.${var.cluster}.streamalerts", "_", ".")}/"
}
}
4 changes: 0 additions & 4 deletions terraform/modules/tf_stream_alert/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,3 @@ variable "alert_processor_vpc_security_group_ids" {
type = "list"
default = []
}

variable "s3_logging_bucket" {
type = "string"
}
105 changes: 96 additions & 9 deletions test/unit/stream_alert_cli/test_terraform_generate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,25 @@ def setup(self):
'security_group_ids': [
'sg-id-1'
]
},
'outputs': {
'aws-lambda': [
'my-lambda-function:production'
],
'aws-s3': [
'my-s3-bucket.with.data'
]
}
},
'rule_processor': {
'current_version': '$LATEST',
'memory': 128,
'timeout': 25
'timeout': 25,
'inputs': {
'aws-sns': [
'my-sns-topic-name'
]
}
}
},
'cloudtrail': {
Expand All @@ -151,10 +164,6 @@ def setup(self):
}
}

def teardown(self):
"""Teardown after each method"""
pass

@staticmethod
def test_generate_s3_bucket():
"""CLI - Terraform Generate S3 Bucket """
Expand Down Expand Up @@ -290,6 +299,18 @@ def test_generate_main(self):
'storage_class': 'GLACIER'
}
}
},
'streamalerts': {
'bucket': 'unit-testing.streamalerts',
'acl': 'private',
'force_destroy': True,
'versioning': {
'enabled': True
},
'logging': {
'target_bucket': 'unit-testing.streamalert.s3-logging',
'target_prefix': 'unit-testing.streamalerts/'
}
}
}
}
Expand All @@ -300,10 +321,76 @@ def test_generate_main(self):
assert_equal(tf_main['resource'], tf_main_expected['resource'])


def test_generate_stream_alert(self):
"""CLI - Terraform Generate stream_alert Module"""
# TODO(jacknagz): Write this test
pass
def test_generate_stream_alert_test(self):
"""CLI - Terraform Generate stream_alert Module (test cluster)"""
terraform_generate.generate_stream_alert(
'test',
self.cluster_dict,
self.config
)

expected_test_cluster = {
'module': {
'stream_alert_test': {
'source': 'modules/tf_stream_alert',
'account_id': '12345678910',
'region': 'us-west-1',
'prefix': 'unit-testing',
'cluster': 'test',
'kms_key_arn': '${aws_kms_key.stream_alert_secrets.arn}',
'rule_processor_memory': 128,
'rule_processor_timeout': 25,
'rule_processor_version': '$LATEST',
'rule_processor_config': '${var.rule_processor_config}',
'alert_processor_memory': 128,
'alert_processor_timeout': 25,
'alert_processor_version': '$LATEST',
'alert_processor_config': '${var.alert_processor_config}',
}
}
}

assert_equal(self.cluster_dict['module']['stream_alert_test'],
expected_test_cluster['module']['stream_alert_test'])


def test_generate_stream_alert_advanced(self):
"""CLI - Terraform Generate stream_alert Module (advanced cluster)"""
terraform_generate.generate_stream_alert(
'advanced',
self.cluster_dict,
self.config
)

expected_advanced_cluster = {
'module': {
'stream_alert_advanced': {
'source': 'modules/tf_stream_alert',
'account_id': '12345678910',
'region': 'us-west-1',
'prefix': 'unit-testing',
'cluster': 'advanced',
'kms_key_arn': '${aws_kms_key.stream_alert_secrets.arn}',
'rule_processor_memory': 128,
'rule_processor_timeout': 25,
'rule_processor_version': '$LATEST',
'rule_processor_config': '${var.rule_processor_config}',
'alert_processor_memory': 128,
'alert_processor_timeout': 25,
'alert_processor_version': '$LATEST',
'alert_processor_config': '${var.alert_processor_config}',
'output_lambda_functions': ['my-lambda-function:production'],
'output_s3_buckets': ['my-s3-bucket.with.data'],
'input_sns_topics': ['my-sns-topic-name'],
'alert_processor_vpc_enabled': True,
'alert_processor_vpc_subnet_ids': ['subnet-id-1'],
'alert_processor_vpc_security_group_ids': ['sg-id-1']
}
}
}

assert_equal(self.cluster_dict['module']['stream_alert_advanced'],
expected_advanced_cluster['module']['stream_alert_advanced'])


def test_generate_flow_logs(self):
Expand Down

0 comments on commit c50856c

Please sign in to comment.