From dab26585863d7ff816aca996ab4e8f761fb7c4ca Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Fri, 6 Jul 2018 14:58:10 -0700 Subject: [PATCH] S3 Server-Side Encryption (#784) --- stream_alert_cli/outputs.py | 7 +- stream_alert_cli/terraform/alert_processor.py | 1 + stream_alert_cli/terraform/athena.py | 1 + stream_alert_cli/terraform/cloudtrail.py | 1 + stream_alert_cli/terraform/firehose.py | 6 +- stream_alert_cli/terraform/generate.py | 57 ++++++++- stream_alert_cli/terraform/handler.py | 5 +- .../modules/tf_alert_processor_iam/main.tf | 2 +- .../tf_alert_processor_iam/variables.tf | 6 +- .../modules/tf_stream_alert_athena/main.tf | 9 ++ .../tf_stream_alert_athena/variables.tf | 4 + .../tf_stream_alert_cloudtrail/main.tf | 119 ++++++++++++++++-- .../tf_stream_alert_cloudtrail/variables.tf | 2 + .../alerts_firehose/main.tf | 1 + .../alerts_firehose/variables.tf | 2 + .../modules/tf_stream_alert_globals/main.tf | 9 +- .../tf_stream_alert_globals/variables.tf | 2 + .../main.tf | 1 + .../variables.tf | 4 + .../iam.tf | 12 ++ .../main.tf | 9 ++ .../variables.tf | 2 + .../stream_alert_alert_merger/test_main.py | 4 +- .../test_outputs/test_aws.py | 10 +- .../terraform/test_alert_processor.py | 6 +- .../stream_alert_cli/terraform/test_athena.py | 1 + .../terraform/test_generate.py | 50 +++++++- 27 files changed, 296 insertions(+), 37 deletions(-) diff --git a/stream_alert_cli/outputs.py b/stream_alert_cli/outputs.py index a931ea7f9..f2cb47dbf 100644 --- a/stream_alert_cli/outputs.py +++ b/stream_alert_cli/outputs.py @@ -75,12 +75,7 @@ def send_creds_to_s3(region, bucket, key, blob_data): """ try: client = boto3.client('s3', region_name=region) - client.put_object( - Body=blob_data, - Bucket=bucket, - Key=key, - ServerSideEncryption='AES256' - ) + client.put_object(Body=blob_data, Bucket=bucket, Key=key) return True except ClientError as err: diff --git a/stream_alert_cli/terraform/alert_processor.py b/stream_alert_cli/terraform/alert_processor.py index 6fd434159..fd1fa0cdc 100644 --- a/stream_alert_cli/terraform/alert_processor.py +++ b/stream_alert_cli/terraform/alert_processor.py @@ -40,6 +40,7 @@ def generate_alert_processor(config): 'prefix': prefix, 'role_id': '${module.alert_processor_lambda.role_id}', 'kms_key_arn': '${aws_kms_key.stream_alert_secrets.arn}', + 'sse_kms_key_arn': '${aws_kms_key.server_side_encryption.arn}', 'output_lambda_functions': [ # Strip qualifiers: only the function name is needed for the IAM permissions func.split(':')[0] for func in config['outputs'].get('aws-lambda', {}).values() diff --git a/stream_alert_cli/terraform/athena.py b/stream_alert_cli/terraform/athena.py index d7491f174..0f80d51fb 100644 --- a/stream_alert_cli/terraform/athena.py +++ b/stream_alert_cli/terraform/athena.py @@ -51,6 +51,7 @@ def generate_athena(config): 'database_name': database, 'queue_name': queue_name, 'results_bucket': results_bucket_name, + 'kms_key_id': '${aws_kms_key.server_side_encryption.key_id}', 'lambda_handler': AthenaPackage.lambda_handler, 'lambda_memory': athena_config.get('memory', '128'), 'lambda_timeout': athena_config.get('timeout', '60'), diff --git a/stream_alert_cli/terraform/cloudtrail.py b/stream_alert_cli/terraform/cloudtrail.py index 395f73e1d..5f4c8a36d 100644 --- a/stream_alert_cli/terraform/cloudtrail.py +++ b/stream_alert_cli/terraform/cloudtrail.py @@ -70,6 +70,7 @@ def generate_cloudtrail(cluster_name, cluster_dict, config): module_info = { 'source': 'modules/tf_stream_alert_cloudtrail', + 'primary_account_id': config['global']['account']['aws_account_id'], 'account_ids': account_ids, 'cluster': cluster_name, 'prefix': config['global']['account']['prefix'], diff --git a/stream_alert_cli/terraform/firehose.py b/stream_alert_cli/terraform/firehose.py index ff6ca965d..83409d90f 100644 --- a/stream_alert_cli/terraform/firehose.py +++ b/stream_alert_cli/terraform/firehose.py @@ -42,7 +42,8 @@ def generate_firehose(config, main_dict, logging_bucket): 'prefix': config['global']['account']['prefix'], 'region': config['global']['account']['region'], 's3_logging_bucket': logging_bucket, - 's3_bucket_name': firehose_s3_bucket_name + 's3_bucket_name': firehose_s3_bucket_name, + 'kms_key_id': '${aws_kms_key.server_side_encryption.key_id}' } # Add the Delivery Streams individually @@ -57,5 +58,6 @@ def generate_firehose(config, main_dict, logging_bucket): ['firehose'].get('compression_format', 'GZIP'), 'log_name': enabled_log, 'role_arn': '${module.kinesis_firehose_setup.firehose_role_arn}', - 's3_bucket_name': firehose_s3_bucket_name + 's3_bucket_name': firehose_s3_bucket_name, + 'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}' } diff --git a/stream_alert_cli/terraform/generate.py b/stream_alert_cli/terraform/generate.py index babbe7728..7ca61314d 100644 --- a/stream_alert_cli/terraform/generate.py +++ b/stream_alert_cli/terraform/generate.py @@ -60,12 +60,15 @@ def generate_s3_bucket(bucket, logging, **kwargs): Keyword Args: acl (str): The S3 bucket ACL force_destroy (bool): To enable or disable force destroy of the bucket + sse_algorithm (str): Server-side encryption algorithm 'AES256' or 'aws:kms' (default) versioning (bool): To enable or disable S3 object versioning lifecycle_rule (dict): The S3 bucket lifecycle rule Returns: dict: S3 bucket Terraform dict to be used in clusters/main.tf.json """ + sse_algorithm = kwargs.get('sse_algorithm', 'aws:kms') + s3_bucket = { 'bucket': bucket, 'acl': kwargs.get('acl', 'private'), @@ -76,8 +79,21 @@ def generate_s3_bucket(bucket, logging, **kwargs): 'logging': { 'target_bucket': logging, 'target_prefix': '{}/'.format(bucket) + }, + 'server_side_encryption_configuration': { + 'rule': { + 'apply_server_side_encryption_by_default': { + 'sse_algorithm': sse_algorithm + } + } } } + + if sse_algorithm == 'aws:kms': + s3_bucket['server_side_encryption_configuration']['rule'][ + 'apply_server_side_encryption_by_default']['kms_master_key_id'] = ( + '${aws_kms_key.server_side_encryption.key_id}') + lifecycle_rule = kwargs.get('lifecycle_rule') if lifecycle_rule: s3_bucket['lifecycle_rule'] = lifecycle_rule @@ -146,7 +162,8 @@ def generate_main(config, init=False): 'days': 365, 'storage_class': 'GLACIER' } - } + }, + sse_algorithm='AES256' # SSE-KMS doesn't seem to work with access logs ) # Create bucket for Terraform state (if applicable) @@ -165,6 +182,7 @@ def generate_main(config, init=False): 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': config['global']['account']['prefix'], + 'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}', 'alerts_table_read_capacity': ( config['global']['infrastructure']['alerts_table']['read_capacity']), 'alerts_table_write_capacity': ( @@ -176,6 +194,43 @@ def generate_main(config, init=False): } # KMS Key and Alias creation + main_dict['resource']['aws_kms_key']['server_side_encryption'] = { + 'enable_key_rotation': True, + 'description': 'StreamAlert S3 Server-Side Encryption', + 'policy': json.dumps({ + 'Version': '2012-10-17', + 'Statement': [ + { + 'Sid': 'Enable IAM User Permissions', + 'Effect': 'Allow', + 'Principal': { + 'AWS': 'arn:aws:iam::{}:root'.format( + config['global']['account']['aws_account_id'] + ) + }, + 'Action': 'kms:*', + 'Resource': '*' + }, + { + 'Sid': 'Allow principals in the account to use the key', + 'Effect': 'Allow', + 'Principal': '*', + 'Action': ['kms:Decrypt', 'kms:GenerateDataKey*', 'kms:Encrypt'], + 'Resource': '*', + 'Condition': { + 'StringEquals': { + 'kms:CallerAccount': config['global']['account']['aws_account_id'] + } + } + } + ] + }) + } + main_dict['resource']['aws_kms_alias']['server_side_encryption'] = { + 'name': 'alias/{}_server-side-encryption'.format(config['global']['account']['prefix']), + 'target_key_id': '${aws_kms_key.server_side_encryption.key_id}' + } + main_dict['resource']['aws_kms_key']['stream_alert_secrets'] = { 'enable_key_rotation': True, 'description': 'StreamAlert secret management' diff --git a/stream_alert_cli/terraform/handler.py b/stream_alert_cli/terraform/handler.py index 69f855a6a..4d3a81a52 100644 --- a/stream_alert_cli/terraform/handler.py +++ b/stream_alert_cli/terraform/handler.py @@ -100,8 +100,9 @@ def _terraform_init(config): init_targets = [ 'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket', 'aws_s3_bucket.stream_alert_secrets', 'aws_s3_bucket.terraform_remote_state', - 'aws_s3_bucket.streamalerts', 'aws_kms_key.stream_alert_secrets', - 'aws_kms_alias.stream_alert_secrets' + 'aws_s3_bucket.streamalerts', + 'aws_kms_key.server_side_encryption', 'aws_kms_alias.server_side_encryption', + 'aws_kms_key.stream_alert_secrets', 'aws_kms_alias.stream_alert_secrets' ] if not tf_runner(targets=init_targets): LOGGER_CLI.error('An error occurred while running StreamAlert init') diff --git a/terraform/modules/tf_alert_processor_iam/main.tf b/terraform/modules/tf_alert_processor_iam/main.tf index 1bf62bed5..6622c06eb 100644 --- a/terraform/modules/tf_alert_processor_iam/main.tf +++ b/terraform/modules/tf_alert_processor_iam/main.tf @@ -56,7 +56,7 @@ data "aws_iam_policy_document" "output_secrets" { "kms:DescribeKey", ] - resources = ["${var.kms_key_arn}"] + resources = ["${var.kms_key_arn}", "${var.sse_kms_key_arn}"] } // Allow retrieving encrypted output secrets diff --git a/terraform/modules/tf_alert_processor_iam/variables.tf b/terraform/modules/tf_alert_processor_iam/variables.tf index 02803c81d..9d8528ef2 100644 --- a/terraform/modules/tf_alert_processor_iam/variables.tf +++ b/terraform/modules/tf_alert_processor_iam/variables.tf @@ -15,7 +15,11 @@ variable "role_id" { } variable "kms_key_arn" { - description = "KMS key ARN used for encrypting output secrets" + description = "KMS key ARN used for (client-side) encrypting output secrets" +} + +variable "sse_kms_key_arn" { + description = "KMS key ARN for server-side encryption of the secrets bucket" } variable "output_lambda_functions" { diff --git a/terraform/modules/tf_stream_alert_athena/main.tf b/terraform/modules/tf_stream_alert_athena/main.tf index b8c8e9ca3..4320cf9ab 100644 --- a/terraform/modules/tf_stream_alert_athena/main.tf +++ b/terraform/modules/tf_stream_alert_athena/main.tf @@ -42,6 +42,15 @@ resource "aws_s3_bucket" "athena_results_bucket" { target_bucket = "${var.s3_logging_bucket}" target_prefix = "${var.results_bucket}/" } + + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + kms_master_key_id = "${var.kms_key_id}" + } + } + } } // Athena Database: streamalert diff --git a/terraform/modules/tf_stream_alert_athena/variables.tf b/terraform/modules/tf_stream_alert_athena/variables.tf index 60d6e3d11..aeaabfbbc 100644 --- a/terraform/modules/tf_stream_alert_athena/variables.tf +++ b/terraform/modules/tf_stream_alert_athena/variables.tf @@ -31,6 +31,10 @@ variable "results_bucket" { type = "string" } +variable "kms_key_id" { + type = "string" +} + variable "s3_logging_bucket" { type = "string" } diff --git a/terraform/modules/tf_stream_alert_cloudtrail/main.tf b/terraform/modules/tf_stream_alert_cloudtrail/main.tf index 43b2f22f5..2d73d38dc 100644 --- a/terraform/modules/tf_stream_alert_cloudtrail/main.tf +++ b/terraform/modules/tf_stream_alert_cloudtrail/main.tf @@ -1,3 +1,88 @@ +// KMS key for encrypting CloudTrail logs +resource "aws_kms_key" "cloudtrail_encryption" { + description = "Encrypt Cloudtrail logs for ${var.prefix}.${var.cluster}.streamalert.cloudtrail" + policy = "${data.aws_iam_policy_document.cloudtrail_encryption.json}" + enable_key_rotation = true +} + +// This policy is auto-generated by AWS if you manually encrypt a CloudTrail from the console. +data "aws_iam_policy_document" "cloudtrail_encryption" { + statement { + sid = "Enable IAM User Permissions" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${var.primary_account_id}:root"] + } + + actions = ["kms:*"] + resources = ["*"] + } + + statement { + sid = "Allow CloudTrail to encrypt logs" + + principals { + type = "Service" + identifiers = ["cloudtrail.amazonaws.com"] + } + + actions = ["kms:GenerateDataKey*"] + resources = ["*"] + + condition { + test = "StringLike" + variable = "kms:EncryptionContext:aws:cloudtrail:arn" + values = ["arn:aws:cloudtrail:*:${var.primary_account_id}:trail/*"] + } + } + + statement { + sid = "Allow CloudTrail to describe key" + + principals { + type = "Service" + identifiers = ["cloudtrail.amazonaws.com"] + } + + actions = ["kms:DescribeKey"] + resources = ["*"] + } + + statement { + sid = "Allow principals in the account to decrypt log files" + + principals { + type = "AWS" + identifiers = ["*"] + } + + actions = [ + "kms:Decrypt", + "kms:ReEncryptFrom", + ] + + resources = ["*"] + + condition { + test = "StringEquals" + variable = "kms:CallerAccount" + values = ["${var.primary_account_id}"] + } + + condition { + test = "StringLike" + variable = "kms:EncryptionContext:aws:cloudtrail:arn" + values = ["arn:aws:cloudtrail:*:${var.primary_account_id}:trail/*"] + } + } +} + +resource "aws_kms_alias" "cloudtrail_encryption" { + name = "alias/${var.prefix}-${var.cluster}-streamalert-cloudtrail" + target_key_id = "${aws_kms_key.cloudtrail_encryption.key_id}" +} + // StreamAlert CloudTrail, also sending to CloudWatch Logs group resource "aws_cloudtrail" "streamalert" { count = "${var.send_to_cloudwatch && !var.existing_trail ? 1 : 0}" @@ -9,6 +94,7 @@ resource "aws_cloudtrail" "streamalert" { enable_logging = "${var.enable_logging}" include_global_service_events = true is_multi_region_trail = "${var.is_global_trail}" + kms_key_id = "${aws_kms_key.cloudtrail_encryption.arn}" event_selector { read_write_type = "All" @@ -33,6 +119,7 @@ resource "aws_cloudtrail" "streamalert_no_cloudwatch" { enable_logging = "${var.enable_logging}" include_global_service_events = true is_multi_region_trail = "${var.is_global_trail}" + kms_key_id = "${aws_kms_key.cloudtrail_encryption.arn}" event_selector { read_write_type = "All" @@ -90,25 +177,30 @@ data "aws_iam_policy_document" "cloudtrail_to_cloudwatch_create_logs" { count = "${var.send_to_cloudwatch ? 1 : 0}" statement { - sid = "AWSCloudTrailCreateLogStream" - effect = "Allow" - - actions = [ - "logs:CreateLogStream", - ] + sid = "AWSCloudTrailCreateLogStream" + effect = "Allow" + actions = ["logs:CreateLogStream"] + resources = ["${aws_cloudwatch_log_group.cloudtrail_logging.arn}"] + } + statement { + sid = "AWSCloudTrailPutLogEvents" + effect = "Allow" + actions = ["logs:PutLogEvents"] resources = ["${aws_cloudwatch_log_group.cloudtrail_logging.arn}"] } statement { - sid = "AWSCloudTrailPutLogEvents" + sid = "AWSCloudTrailEncryptLogEvents" effect = "Allow" actions = [ - "logs:PutLogEvents", + "kms:Decrypt", + "kms:Encrypt", + "kms:GenerateDataKey*", ] - resources = ["${aws_cloudwatch_log_group.cloudtrail_logging.arn}"] + resources = ["${aws_kms_key.cloudtrail_encryption.arn}"] } } @@ -145,6 +237,15 @@ resource "aws_s3_bucket" "cloudtrail_bucket" { policy = "${data.aws_iam_policy_document.cloudtrail_bucket.json}" + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + kms_master_key_id = "${aws_kms_key.cloudtrail_encryption.key_id}" + } + } + } + tags { Name = "${var.prefix}.${var.cluster}.streamalert.cloudtrail" Cluster = "${var.cluster}" diff --git a/terraform/modules/tf_stream_alert_cloudtrail/variables.tf b/terraform/modules/tf_stream_alert_cloudtrail/variables.tf index ebb8bac8d..4929f152a 100644 --- a/terraform/modules/tf_stream_alert_cloudtrail/variables.tf +++ b/terraform/modules/tf_stream_alert_cloudtrail/variables.tf @@ -1,3 +1,5 @@ +variable "primary_account_id" {} + variable "account_ids" { type = "list" } diff --git a/terraform/modules/tf_stream_alert_globals/alerts_firehose/main.tf b/terraform/modules/tf_stream_alert_globals/alerts_firehose/main.tf index 1313e68bf..e472c657f 100644 --- a/terraform/modules/tf_stream_alert_globals/alerts_firehose/main.tf +++ b/terraform/modules/tf_stream_alert_globals/alerts_firehose/main.tf @@ -10,6 +10,7 @@ resource "aws_kinesis_firehose_delivery_stream" "stream_alerts" { buffer_size = "${var.buffer_size}" buffer_interval = "${var.buffer_interval}" compression_format = "${var.compression_format}" + kms_key_arn = "${var.kms_key_arn}" cloudwatch_logging_options { enabled = true diff --git a/terraform/modules/tf_stream_alert_globals/alerts_firehose/variables.tf b/terraform/modules/tf_stream_alert_globals/alerts_firehose/variables.tf index 4851a8700..73482384a 100644 --- a/terraform/modules/tf_stream_alert_globals/alerts_firehose/variables.tf +++ b/terraform/modules/tf_stream_alert_globals/alerts_firehose/variables.tf @@ -16,6 +16,8 @@ variable "compression_format" { default = "GZIP" } +variable "kms_key_arn" {} + variable "prefix" {} variable "region" {} diff --git a/terraform/modules/tf_stream_alert_globals/main.tf b/terraform/modules/tf_stream_alert_globals/main.tf index 3bf0fa961..c7e9040b6 100644 --- a/terraform/modules/tf_stream_alert_globals/main.tf +++ b/terraform/modules/tf_stream_alert_globals/main.tf @@ -1,8 +1,9 @@ module "alerts_firehose" { - source = "alerts_firehose" - account_id = "${var.account_id}" - prefix = "${var.prefix}" - region = "${var.region}" + source = "alerts_firehose" + account_id = "${var.account_id}" + prefix = "${var.prefix}" + region = "${var.region}" + kms_key_arn = "${var.kms_key_arn}" } // TODO: Autoscaling diff --git a/terraform/modules/tf_stream_alert_globals/variables.tf b/terraform/modules/tf_stream_alert_globals/variables.tf index e5503184b..d6c48e599 100644 --- a/terraform/modules/tf_stream_alert_globals/variables.tf +++ b/terraform/modules/tf_stream_alert_globals/variables.tf @@ -4,6 +4,8 @@ variable "prefix" {} variable "region" {} +variable "kms_key_arn" {} + variable "alerts_table_read_capacity" {} variable "alerts_table_write_capacity" {} diff --git a/terraform/modules/tf_stream_alert_kinesis_firehose_delivery_stream/main.tf b/terraform/modules/tf_stream_alert_kinesis_firehose_delivery_stream/main.tf index 110cbaf08..78b7acf74 100644 --- a/terraform/modules/tf_stream_alert_kinesis_firehose_delivery_stream/main.tf +++ b/terraform/modules/tf_stream_alert_kinesis_firehose_delivery_stream/main.tf @@ -15,5 +15,6 @@ resource "aws_kinesis_firehose_delivery_stream" "stream_alert_data" { buffer_size = "${var.buffer_size}" buffer_interval = "${var.buffer_interval}" compression_format = "${var.compression_format}" + kms_key_arn = "${var.kms_key_arn}" } } diff --git a/terraform/modules/tf_stream_alert_kinesis_firehose_delivery_stream/variables.tf b/terraform/modules/tf_stream_alert_kinesis_firehose_delivery_stream/variables.tf index c801d8e42..5a57a7c7f 100644 --- a/terraform/modules/tf_stream_alert_kinesis_firehose_delivery_stream/variables.tf +++ b/terraform/modules/tf_stream_alert_kinesis_firehose_delivery_stream/variables.tf @@ -22,3 +22,7 @@ variable "role_arn" { variable "s3_bucket_name" { type = "string" } + +variable "kms_key_arn" { + type = "string" +} diff --git a/terraform/modules/tf_stream_alert_kinesis_firehose_setup/iam.tf b/terraform/modules/tf_stream_alert_kinesis_firehose_setup/iam.tf index a74aa577a..ca3ceaa98 100644 --- a/terraform/modules/tf_stream_alert_kinesis_firehose_setup/iam.tf +++ b/terraform/modules/tf_stream_alert_kinesis_firehose_setup/iam.tf @@ -46,6 +46,18 @@ data "aws_iam_policy_document" "firehose_s3" { "arn:aws:s3:::${var.s3_bucket_name}/*", ] } + + statement { + effect = "Allow" + + actions = [ + "kms:Encrypt", + "kms:Decrypt", + "kms:GenerateDataKey*", + ] + + resources = ["arn:aws:kms:${var.region}:${var.account_id}:key/${var.kms_key_id}"] + } } // IAM Policy: Write logs to CloudWatch diff --git a/terraform/modules/tf_stream_alert_kinesis_firehose_setup/main.tf b/terraform/modules/tf_stream_alert_kinesis_firehose_setup/main.tf index d1e1e964f..4e6e46b15 100644 --- a/terraform/modules/tf_stream_alert_kinesis_firehose_setup/main.tf +++ b/terraform/modules/tf_stream_alert_kinesis_firehose_setup/main.tf @@ -12,6 +12,15 @@ resource "aws_s3_bucket" "stream_alert_data" { target_prefix = "${var.s3_bucket_name}/" } + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + kms_master_key_id = "${var.kms_key_id}" + } + } + } + tags { Name = "${var.s3_bucket_name}" } diff --git a/terraform/modules/tf_stream_alert_kinesis_firehose_setup/variables.tf b/terraform/modules/tf_stream_alert_kinesis_firehose_setup/variables.tf index 8909265c3..80b133453 100644 --- a/terraform/modules/tf_stream_alert_kinesis_firehose_setup/variables.tf +++ b/terraform/modules/tf_stream_alert_kinesis_firehose_setup/variables.tf @@ -11,3 +11,5 @@ variable "region" {} variable "s3_bucket_name" {} variable "s3_logging_bucket" {} + +variable "kms_key_id" {} diff --git a/tests/unit/stream_alert_alert_merger/test_main.py b/tests/unit/stream_alert_alert_merger/test_main.py index 523ea654b..20a194c6f 100644 --- a/tests/unit/stream_alert_alert_merger/test_main.py +++ b/tests/unit/stream_alert_alert_merger/test_main.py @@ -17,7 +17,7 @@ from datetime import datetime, timedelta import os -from mock import ANY, call, patch +from mock import ANY, call, patch, MagicMock from moto import mock_dynamodb2, mock_lambda from nose.tools import assert_equal, assert_false, assert_true @@ -183,6 +183,8 @@ def test_merge_groups_limit_reached(self): @patch.object(main.AlertMerger, 'MAX_LAMBDA_PAYLOAD_SIZE', 600) def test_dispatch(self, mock_logger): """Alert Merger - Dispatch to Alert Processor Lambda""" + self.merger.lambda_client = MagicMock() + self.merger.table.add_alerts([ # An alert without any merge criteria Alert('no_merging', {}, {'output'}), diff --git a/tests/unit/stream_alert_alert_processor/test_outputs/test_aws.py b/tests/unit/stream_alert_alert_processor/test_outputs/test_aws.py index 21f0abf24..5dc087b0f 100644 --- a/tests/unit/stream_alert_alert_processor/test_outputs/test_aws.py +++ b/tests/unit/stream_alert_alert_processor/test_outputs/test_aws.py @@ -15,8 +15,8 @@ """ # pylint: disable=abstract-class-instantiated,protected-access,attribute-defined-outside-init,no-self-use import boto3 -from mock import patch -from moto import mock_kinesis, mock_lambda, mock_s3, mock_sns, mock_sqs +from mock import MagicMock, patch +from moto import mock_kinesis, mock_s3, mock_sns, mock_sqs from nose.tools import ( assert_equal, assert_false, @@ -25,6 +25,7 @@ ) from stream_alert.alert_processor.outputs.output_base import OutputProperty +from stream_alert.alert_processor.outputs import aws as aws_outputs from stream_alert.alert_processor.outputs.aws import ( AWSOutput, KinesisFirehoseOutput, @@ -34,7 +35,6 @@ SQSOutput, CloudwatchLogOutput ) -from stream_alert_cli.helpers import create_lambda_function from tests.unit.stream_alert_alert_processor import ( ACCOUNT_ID, CONFIG, @@ -111,7 +111,7 @@ def test_dispatch_ignore_large_payload(self): assert_false(self._dispatcher.dispatch(alert, self.OUTPUT)) -@mock_lambda +@patch.object(aws_outputs, 'boto3', MagicMock()) class TestLambdaOutput(object): """Test class for LambdaOutput""" DESCRIPTOR = 'unit_test_lambda' @@ -121,7 +121,6 @@ class TestLambdaOutput(object): def setup(self): """Setup before each method""" self._dispatcher = LambdaOutput(REGION, ACCOUNT_ID, FUNCTION_NAME, CONFIG) - create_lambda_function(CONFIG[self.SERVICE][self.DESCRIPTOR], REGION) def test_locals(self): """LambdaOutput local variables""" @@ -140,7 +139,6 @@ def test_dispatch(self, log_mock): def test_dispatch_with_qualifier(self, log_mock): """LambdaOutput - Dispatch Success, With Qualifier""" alt_descriptor = '{}_qual'.format(self.DESCRIPTOR) - create_lambda_function(CONFIG[self.SERVICE][alt_descriptor], REGION) assert_true( self._dispatcher.dispatch(get_alert(), ':'.join([self.SERVICE, alt_descriptor]))) diff --git a/tests/unit/stream_alert_cli/terraform/test_alert_processor.py b/tests/unit/stream_alert_cli/terraform/test_alert_processor.py index 11cdd73a8..e62f7a4e5 100644 --- a/tests/unit/stream_alert_cli/terraform/test_alert_processor.py +++ b/tests/unit/stream_alert_cli/terraform/test_alert_processor.py @@ -47,7 +47,8 @@ def test_generate_all_options(self): 'prefix': 'unit-testing', 'region': 'us-west-1', 'role_id': '${module.alert_processor_lambda.role_id}', - 'source': 'modules/tf_alert_processor_iam' + 'source': 'modules/tf_alert_processor_iam', + 'sse_kms_key_arn': '${aws_kms_key.server_side_encryption.arn}' }, 'alert_processor_lambda': { 'alarm_actions': ['arn:aws:sns:us-west-1:12345678910:stream_alert_monitoring'], @@ -102,7 +103,8 @@ def test_generate_minimal_options(self): 'prefix': 'unit-testing', 'region': 'us-west-1', 'role_id': '${module.alert_processor_lambda.role_id}', - 'source': 'modules/tf_alert_processor_iam' + 'source': 'modules/tf_alert_processor_iam', + 'sse_kms_key_arn': '${aws_kms_key.server_side_encryption.arn}' }, 'alert_processor_lambda': { 'description': 'Unit-Testing Streamalert Alert Processor', diff --git a/tests/unit/stream_alert_cli/terraform/test_athena.py b/tests/unit/stream_alert_cli/terraform/test_athena.py index 1a472f036..dd598d0e1 100644 --- a/tests/unit/stream_alert_cli/terraform/test_athena.py +++ b/tests/unit/stream_alert_cli/terraform/test_athena.py @@ -43,6 +43,7 @@ def test_generate_athena(): 'database_name': '{}_streamalert'.format(prefix), 'queue_name': '{}_streamalert_athena_s3_notifications'.format(prefix), 'results_bucket': '{}.streamalert.athena-results'.format(prefix), + 'kms_key_id': '${aws_kms_key.server_side_encryption.key_id}', 'enable_metrics': False, 'lambda_handler': 'stream_alert.athena_partition_refresh.main.handler', 'lambda_log_level': 'info', diff --git a/tests/unit/stream_alert_cli/terraform/test_generate.py b/tests/unit/stream_alert_cli/terraform/test_generate.py index 47aad17b0..b7c7b5753 100644 --- a/tests/unit/stream_alert_cli/terraform/test_generate.py +++ b/tests/unit/stream_alert_cli/terraform/test_generate.py @@ -23,7 +23,7 @@ streamalert ) -from mock import patch +from mock import ANY, patch from nose.tools import assert_equal, assert_false, assert_true @@ -50,7 +50,8 @@ def test_generate_s3_bucket(): 'acl', 'force_destroy', 'versioning', - 'logging' + 'logging', + 'server_side_encryption_configuration' } assert_equal(type(bucket), dict) @@ -101,12 +102,21 @@ def test_generate_main(self): }, 'resource': { 'aws_kms_key': { + 'server_side_encryption': { + 'enable_key_rotation': True, + 'description': 'StreamAlert S3 Server-Side Encryption', + 'policy': ANY + }, 'stream_alert_secrets': { 'enable_key_rotation': True, 'description': 'StreamAlert secret management' } }, 'aws_kms_alias': { + 'server_side_encryption': { + 'name': 'alias/unit-testing_server-side-encryption', + 'target_key_id': '${aws_kms_key.server_side_encryption.key_id}' + }, 'stream_alert_secrets': { 'name': 'alias/unit-testing', 'target_key_id': '${aws_kms_key.stream_alert_secrets.key_id}' @@ -123,6 +133,15 @@ def test_generate_main(self): 'logging': { 'target_bucket': 'unit-testing.streamalert.s3-logging', 'target_prefix': 'unit-testing.streamalert.secrets/' + }, + 'server_side_encryption_configuration': { + 'rule': { + 'apply_server_side_encryption_by_default': { + 'sse_algorithm': 'aws:kms', + 'kms_master_key_id': ( + '${aws_kms_key.server_side_encryption.key_id}') + } + } } }, 'terraform_remote_state': { @@ -135,6 +154,15 @@ def test_generate_main(self): 'logging': { 'target_bucket': 'unit-testing.streamalert.s3-logging', 'target_prefix': 'unit-testing.streamalert.terraform.state/' + }, + 'server_side_encryption_configuration': { + 'rule': { + 'apply_server_side_encryption_by_default': { + 'sse_algorithm': 'aws:kms', + 'kms_master_key_id': ( + '${aws_kms_key.server_side_encryption.key_id}') + } + } } }, 'logging_bucket': { @@ -155,6 +183,13 @@ def test_generate_main(self): 'days': 365, 'storage_class': 'GLACIER' } + }, + 'server_side_encryption_configuration': { + 'rule': { + 'apply_server_side_encryption_by_default': { + 'sse_algorithm': 'AES256' + } + } } }, 'streamalerts': { @@ -167,6 +202,15 @@ def test_generate_main(self): 'logging': { 'target_bucket': 'unit-testing.streamalert.s3-logging', 'target_prefix': 'unit-testing.streamalerts/' + }, + 'server_side_encryption_configuration': { + 'rule': { + 'apply_server_side_encryption_by_default': { + 'sse_algorithm': 'aws:kms', + 'kms_master_key_id': ( + '${aws_kms_key.server_side_encryption.key_id}') + } + } } } }, @@ -302,6 +346,7 @@ def test_generate_cloudtrail_basic(self): {'enable_logging', 'enable_kinesis'}) assert_equal(self.cluster_dict['module']['cloudtrail_advanced'], { 'account_ids': ['12345678910'], + 'primary_account_id': '12345678910', 'cluster': 'advanced', 'kinesis_arn': '${module.kinesis_advanced.arn}', 'prefix': 'unit-testing', @@ -342,6 +387,7 @@ def test_generate_cloudtrail_all_options(self): assert_equal('cloudtrail_advanced' in self.cluster_dict['module'], True) assert_equal(self.cluster_dict['module']['cloudtrail_advanced'], { 'account_ids': ['12345678910'], + 'primary_account_id': '12345678910', 'cluster': 'advanced', 'existing_trail': False, 'is_global_trail': False,